mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
Merge 090d675e3c into 461a58c40a
This commit is contained in:
tinysolver
GitHub
commit
dae2a8ec0e
4 changed files with 652 additions and 788 deletions
|
|
@ -1,32 +1,23 @@
|
|||
"""
|
||||
OpenRouter Image Generation Support
|
||||
OpenRouter image generation through POST {api_base}/images
|
||||
|
||||
OpenRouter provides image generation through chat completion endpoints.
|
||||
Models like google/gemini-2.5-flash-image return images in the message content.
|
||||
|
||||
Response format:
|
||||
Response shape:
|
||||
{
|
||||
"choices": [{
|
||||
"message": {
|
||||
"content": "Here is a beautiful sunset for you! ",
|
||||
"role": "assistant",
|
||||
"images": [{
|
||||
"image_url": {"url": "data:image/png;base64,..."},
|
||||
"index": 0,
|
||||
"type": "image_url"
|
||||
}]
|
||||
}
|
||||
}],
|
||||
"created": 1790994420,
|
||||
"data": [{"b64_json": "...", "media_type": "image/png"}],
|
||||
"usage": {
|
||||
"completion_tokens": 1299,
|
||||
"prompt_tokens": 6,
|
||||
"total_tokens": 1305,
|
||||
"completion_tokens_details": {"image_tokens": 1290},
|
||||
"cost": 0.0387243
|
||||
"prompt_tokens": 18,
|
||||
"completion_tokens": 272,
|
||||
"total_tokens": 290,
|
||||
"cost": 0.002212,
|
||||
"cost_details": {"upstream_inference_cost": 0.002212, ...},
|
||||
"completion_tokens_details": {"image_tokens": 272}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
import re
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -55,23 +46,38 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
OPENROUTER_API_BASE: Final = "https://openrouter.ai/api/v1"
|
||||
IMAGES_PATH: Final = "/images"
|
||||
LEGACY_CHAT_COMPLETIONS_SUFFIX: Final = "/chat/completions"
|
||||
QUALITY_ALIASES: Final = MappingProxyType({"standard": "low", "hd": "high"})
|
||||
RESOLUTION_TIER_MODEL_AUTHOR: Final = "google/"
|
||||
QUALITY_RESOLUTION_TIERS: Final = MappingProxyType({"auto": "1K", "low": "1K", "medium": "2K", "high": "4K"})
|
||||
OPENAI_SIZE_ASPECT_RATIOS: Final = MappingProxyType(
|
||||
{
|
||||
"256x256": "1:1",
|
||||
"512x512": "1:1",
|
||||
"1024x1024": "1:1",
|
||||
"1536x1024": "3:2",
|
||||
"1024x1536": "2:3",
|
||||
"1792x1024": "16:9",
|
||||
"1024x1792": "9:16",
|
||||
}
|
||||
)
|
||||
LEGACY_IMAGE_CONFIG_FIELDS: Final = MappingProxyType({"aspect_ratio": "aspect_ratio", "image_size": "resolution"})
|
||||
NON_BODY_PARAMS: Final = frozenset(
|
||||
{"model", "prompt", "messages", "modalities", "stream", "image_config", "extra_headers"}
|
||||
)
|
||||
PIXEL_SIZE: Final = re.compile(r"\d+x\d+")
|
||||
SIZE_OVERRIDING_FIELDS: Final = frozenset({"aspect_ratio", "resolution"})
|
||||
|
||||
|
||||
class OpenRouterImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""
|
||||
Configuration for OpenRouter image generation via chat completions.
|
||||
|
||||
OpenRouter uses chat completion endpoints for image generation,
|
||||
so we need to transform image generation requests to chat format
|
||||
and extract images from chat responses.
|
||||
OpenRouter image generation through the dedicated /images endpoint, which serves both
|
||||
image-only models (openai/gpt-image-*) and image+text models (google/gemini-*-image)
|
||||
"""
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
|
||||
"""
|
||||
Get supported OpenAI parameters for OpenRouter image generation.
|
||||
|
||||
Since OpenRouter uses chat completions for image generation,
|
||||
we support standard image generation params.
|
||||
"""
|
||||
return [
|
||||
"size",
|
||||
"quality",
|
||||
|
|
@ -86,104 +92,42 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig):
|
|||
drop_params: bool,
|
||||
) -> dict:
|
||||
"""
|
||||
Map image generation params to OpenRouter chat completion format.
|
||||
|
||||
Maps OpenAI parameters to OpenRouter's image_config format:
|
||||
- size -> image_config.aspect_ratio
|
||||
- quality -> image_config.image_size
|
||||
size and n pass through as is: /images takes explicit pixel sizes and normalizes them per
|
||||
provider. quality is native on /images, so only the dall-e-3 names are translated, except on
|
||||
Google's models, see _map_quality_to_resolution_tier
|
||||
"""
|
||||
supported_params: Final = self.get_supported_openai_params(model)
|
||||
|
||||
for key, value in non_default_params.items():
|
||||
if key in supported_params:
|
||||
if key == "size":
|
||||
# Map OpenAI size to OpenRouter aspect_ratio
|
||||
aspect_ratio = self._map_size_to_aspect_ratio(value)
|
||||
if "image_config" not in optional_params:
|
||||
optional_params["image_config"] = {}
|
||||
optional_params["image_config"]["aspect_ratio"] = aspect_ratio
|
||||
elif key == "quality":
|
||||
# Map OpenAI quality to OpenRouter image_size
|
||||
image_size = self._map_quality_to_image_size(value)
|
||||
if image_size:
|
||||
if "image_config" not in optional_params:
|
||||
optional_params["image_config"] = {}
|
||||
optional_params["image_config"]["image_size"] = image_size
|
||||
else:
|
||||
# Pass through other supported params (like n)
|
||||
optional_params[key] = value
|
||||
elif not drop_params:
|
||||
# If not supported and drop_params is False, pass through
|
||||
optional_params[key] = value
|
||||
|
||||
return optional_params
|
||||
|
||||
def _map_size_to_aspect_ratio(self, size: str) -> str:
|
||||
"""
|
||||
Map OpenAI size format to OpenRouter aspect_ratio format.
|
||||
|
||||
OpenAI sizes:
|
||||
- 1024x1024 (square)
|
||||
- 1536x1024 (landscape)
|
||||
- 1024x1536 (portrait)
|
||||
- 1792x1024 (wide landscape, dall-e-3)
|
||||
- 1024x1792 (tall portrait, dall-e-3)
|
||||
- 256x256, 512x512 (dall-e-2)
|
||||
- auto (default)
|
||||
|
||||
OpenRouter aspect_ratios:
|
||||
- 1:1 → 1024×1024 (default)
|
||||
- 2:3 → 832×1248
|
||||
- 3:2 → 1248×832
|
||||
- 3:4 → 864×1184
|
||||
- 4:3 → 1184×864
|
||||
- 4:5 → 896×1152
|
||||
- 5:4 → 1152×896
|
||||
- 9:16 → 768×1344
|
||||
- 16:9 → 1344×768
|
||||
- 21:9 → 1536×672
|
||||
"""
|
||||
size_to_aspect_ratio: Final = {
|
||||
# Square formats
|
||||
"256x256": "1:1",
|
||||
"512x512": "1:1",
|
||||
"1024x1024": "1:1",
|
||||
# Landscape formats
|
||||
"1536x1024": "3:2", # 1.5:1 ratio, closest to 3:2
|
||||
"1792x1024": "16:9", # 1.75:1 ratio, closest to 16:9
|
||||
# Portrait formats
|
||||
"1024x1536": "2:3", # 0.67:1 ratio, closest to 2:3
|
||||
"1024x1792": "9:16", # 0.57:1 ratio, closest to 9:16
|
||||
# Default
|
||||
"auto": "1:1",
|
||||
mapped_params: Final[dict[str, object]] = {
|
||||
key: QUALITY_ALIASES.get(value, value) if key == "quality" else value
|
||||
for key, value in non_default_params.items()
|
||||
if (key in supported_params or not drop_params) and (key, value) != ("size", "auto")
|
||||
}
|
||||
return size_to_aspect_ratio.get(size, "1:1")
|
||||
if (
|
||||
"quality" in mapped_params
|
||||
and "image_config" not in optional_params
|
||||
and model.removeprefix("openrouter/").startswith(RESOLUTION_TIER_MODEL_AUTHOR)
|
||||
):
|
||||
return {**optional_params, **self._map_quality_to_resolution_tier(mapped_params)}
|
||||
return {**optional_params, **mapped_params}
|
||||
|
||||
def _map_quality_to_image_size(self, quality: str) -> str | None:
|
||||
@staticmethod
|
||||
def _map_quality_to_resolution_tier(mapped_params: dict[str, object]) -> dict[str, object]:
|
||||
"""
|
||||
Map OpenAI quality to OpenRouter image_size format.
|
||||
|
||||
OpenAI quality values:
|
||||
- auto (default) - automatically select best quality
|
||||
- high, medium, low - for GPT image models
|
||||
- hd, standard - for dall-e-3
|
||||
|
||||
OpenRouter image_size values (Gemini only):
|
||||
- 1K → Standard resolution (default)
|
||||
- 2K → Higher resolution
|
||||
- 4K → Highest resolution
|
||||
Google's image models take a resolution tier on /images and ignore quality, so quality keeps the
|
||||
meaning it had on the chat-based path: image_config.image_size (1K, 2K or 4K), next to the aspect
|
||||
ratio of an OpenAI pixel size. A tier size or an image_config set by the caller wins
|
||||
"""
|
||||
quality_to_image_size: Final = {
|
||||
# OpenAI quality mappings
|
||||
"low": "1K",
|
||||
"standard": "1K",
|
||||
"medium": "2K",
|
||||
"high": "4K",
|
||||
"hd": "4K",
|
||||
# Auto defaults to standard
|
||||
"auto": "1K",
|
||||
}
|
||||
return quality_to_image_size.get(quality)
|
||||
size: Final = str(mapped_params.get("size") or "")
|
||||
quality: Final = mapped_params["quality"]
|
||||
tier: Final = QUALITY_RESOLUTION_TIERS.get(quality) if isinstance(quality, str) else None
|
||||
params: Final = {key: value for key, value in mapped_params.items() if key != "quality"}
|
||||
if tier is None or (size and PIXEL_SIZE.fullmatch(size) is None):
|
||||
return params
|
||||
aspect_ratio: Final = OPENAI_SIZE_ASPECT_RATIOS.get(size)
|
||||
image_config: Final = (
|
||||
{"image_size": tier} if aspect_ratio is None else {"aspect_ratio": aspect_ratio, "image_size": tier}
|
||||
)
|
||||
return {**params, "image_config": image_config}
|
||||
|
||||
def _set_usage_and_cost(
|
||||
self,
|
||||
|
|
@ -201,11 +145,13 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig):
|
|||
"""
|
||||
usage_data: Final = response_json.get("usage", {})
|
||||
if usage_data:
|
||||
prompt_tokens: Final = usage_data.get("prompt_tokens", 0)
|
||||
total_tokens: Final = usage_data.get("total_tokens", 0)
|
||||
# The /images usage schema allows null for completion_tokens_details and image_tokens, and
|
||||
# per-image priced models report only completion_tokens
|
||||
prompt_tokens: Final = usage_data.get("prompt_tokens") or 0
|
||||
total_tokens: Final = usage_data.get("total_tokens") or 0
|
||||
|
||||
completion_tokens_details: Final = usage_data.get("completion_tokens_details", {})
|
||||
image_tokens: Final = completion_tokens_details.get("image_tokens", 0)
|
||||
completion_tokens_details: Final = usage_data.get("completion_tokens_details") or {}
|
||||
image_tokens: Final = completion_tokens_details.get("image_tokens")
|
||||
|
||||
model_response.usage = ImageUsage(
|
||||
input_tokens=prompt_tokens,
|
||||
|
|
@ -213,7 +159,7 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig):
|
|||
image_tokens=0, # Input doesn't contain images for generation
|
||||
text_tokens=prompt_tokens,
|
||||
),
|
||||
output_tokens=image_tokens,
|
||||
output_tokens=image_tokens if image_tokens is not None else usage_data.get("completion_tokens") or 0,
|
||||
total_tokens=total_tokens,
|
||||
)
|
||||
|
||||
|
|
@ -244,19 +190,10 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig):
|
|||
litellm_params: dict,
|
||||
stream: bool | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Get the complete URL for OpenRouter image generation.
|
||||
|
||||
OpenRouter uses chat completions endpoint for image generation.
|
||||
Default: https://openrouter.ai/api/v1/chat/completions
|
||||
"""
|
||||
if api_base:
|
||||
if not api_base.endswith("/chat/completions"):
|
||||
api_base = api_base.rstrip("/")
|
||||
return f"{api_base}/chat/completions"
|
||||
return api_base
|
||||
|
||||
return "https://openrouter.ai/api/v1/chat/completions"
|
||||
base_url: Final = (api_base or OPENROUTER_API_BASE).rstrip("/")
|
||||
if base_url.endswith(IMAGES_PATH):
|
||||
return base_url
|
||||
return base_url.removesuffix(LEGACY_CHAT_COMPLETIONS_SUFFIX) + IMAGES_PATH
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
|
|
@ -285,29 +222,27 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig):
|
|||
headers: dict,
|
||||
) -> dict:
|
||||
"""
|
||||
Transform image generation request to OpenRouter chat completion format.
|
||||
image_config is the request shape of the older chat-based path. Its fields map onto the
|
||||
/images names so existing configs keep working, and explicit top-level values win
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
prompt: The image generation prompt
|
||||
optional_params: Optional parameters (including image_config)
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
dict: Request body in chat completion format with image_config
|
||||
A configured aspect_ratio or resolution wins over an OpenAI pixel size, the way image_config
|
||||
won over size on the chat path, because /images answers that pair with a 400
|
||||
"""
|
||||
request_body: Final = {
|
||||
legacy_image_config: Final = optional_params.get("image_config") or {}
|
||||
body: Final[dict[str, object]] = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"prompt": prompt,
|
||||
**{
|
||||
LEGACY_IMAGE_CONFIG_FIELDS[key]: value
|
||||
for key, value in legacy_image_config.items()
|
||||
if key in LEGACY_IMAGE_CONFIG_FIELDS
|
||||
},
|
||||
**{key: value for key, value in optional_params.items() if key not in NON_BODY_PARAMS},
|
||||
}
|
||||
|
||||
# These will be passed through to OpenRouter
|
||||
for key, value in optional_params.items():
|
||||
if key not in ["model", "messages", "modalities"]:
|
||||
request_body[key] = value
|
||||
|
||||
return request_body
|
||||
drop_pixel_size: Final = not SIZE_OVERRIDING_FIELDS.isdisjoint(body) and (
|
||||
PIXEL_SIZE.fullmatch(str(body.get("size", ""))) is not None
|
||||
)
|
||||
return {key: value for key, value in body.items() if not (drop_pixel_size and key == "size")}
|
||||
|
||||
def transform_image_generation_response(
|
||||
self,
|
||||
|
|
@ -322,83 +257,21 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig):
|
|||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
"""
|
||||
Transform OpenRouter chat completion response to ImageResponse format.
|
||||
|
||||
Extracts images from the message content and maps usage/cost information.
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
raw_response: Raw HTTP response from OpenRouter
|
||||
model_response: ImageResponse object to populate
|
||||
logging_obj: Logging object
|
||||
request_data: Original request data
|
||||
optional_params: Optional parameters
|
||||
litellm_params: LiteLLM parameters
|
||||
encoding: Encoding
|
||||
api_key: API key
|
||||
json_mode: JSON mode flag
|
||||
|
||||
Returns:
|
||||
ImageResponse: Populated image response
|
||||
"""
|
||||
try:
|
||||
response_json: Final = raw_response.json()
|
||||
except Exception as e:
|
||||
except ValueError as e:
|
||||
raise OpenRouterException(
|
||||
message=f"Error parsing OpenRouter response: {e}",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
) from e
|
||||
|
||||
if not model_response.data:
|
||||
model_response.data = []
|
||||
|
||||
try:
|
||||
choices: Final = response_json.get("choices", [])
|
||||
|
||||
for choice in choices:
|
||||
message = choice.get("message", {})
|
||||
images = message.get("images", [])
|
||||
|
||||
for image_data in images:
|
||||
image_url_obj = image_data.get("image_url", {})
|
||||
image_url = image_url_obj.get("url")
|
||||
|
||||
if image_url:
|
||||
if image_url.startswith("data:"):
|
||||
# Extract base64 data
|
||||
# Format: data:image/png;base64,<base64_data>
|
||||
parts = image_url.split(",", 1)
|
||||
b64_data = parts[1] if len(parts) > 1 else None
|
||||
|
||||
model_response.data.append(
|
||||
ImageObject(
|
||||
b64_json=b64_data,
|
||||
url=None,
|
||||
revised_prompt=None,
|
||||
)
|
||||
)
|
||||
else:
|
||||
model_response.data.append(
|
||||
ImageObject(
|
||||
b64_json=None,
|
||||
url=image_url,
|
||||
revised_prompt=None,
|
||||
)
|
||||
)
|
||||
|
||||
# Extract and set usage and cost information
|
||||
self._set_usage_and_cost(model_response, response_json, model)
|
||||
|
||||
return model_response
|
||||
|
||||
except Exception as e:
|
||||
raise OpenRouterException(
|
||||
message=f"Error transforming OpenRouter image generation response: {e}",
|
||||
status_code=500,
|
||||
headers={},
|
||||
)
|
||||
image_response: Final = ImageResponse(
|
||||
created=response_json.get("created"),
|
||||
data=[ImageObject(b64_json=item.get("b64_json")) for item in response_json.get("data") or []],
|
||||
)
|
||||
self._set_usage_and_cost(image_response, response_json, model)
|
||||
return image_response
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
|
||||
"""Get the appropriate error class for OpenRouter errors."""
|
||||
|
|
|
|||
|
|
@ -1810,7 +1810,7 @@
|
|||
"messages": true,
|
||||
"responses": true,
|
||||
"embeddings": true,
|
||||
"image_generations": false,
|
||||
"image_generations": true,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"moderations": false,
|
||||
|
|
|
|||
|
|
@ -2009,7 +2009,7 @@
|
|||
"messages": true,
|
||||
"responses": true,
|
||||
"embeddings": true,
|
||||
"image_generations": false,
|
||||
"image_generations": true,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"moderations": false,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Loading…
Add table
Reference in a new issue