diff --git a/litellm/__init__.py b/litellm/__init__.py index d66707f8b3a..fd026ffb9d2 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -899,6 +899,7 @@ from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import from .llms.bedrock.image.amazon_stability1_transformation import AmazonStabilityConfig from .llms.bedrock.image.amazon_stability3_transformation import AmazonStability3Config +from .llms.bedrock.image.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig from .llms.bedrock.embed.amazon_titan_g1_transformation import AmazonTitanG1Config from .llms.bedrock.embed.amazon_titan_multimodal_transformation import ( AmazonTitanMultimodalEmbeddingG1Config, diff --git a/litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py b/litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py new file mode 100644 index 00000000000..de46edb9235 --- /dev/null +++ b/litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py @@ -0,0 +1,106 @@ +import types +from typing import List, Optional + +from openai.types.image import Image + +from litellm.types.llms.bedrock import ( + AmazonNovaCanvasTextToImageRequest, AmazonNovaCanvasTextToImageResponse, + AmazonNovaCanvasTextToImageParams, AmazonNovaCanvasRequestBase, +) +from litellm.types.utils import ImageResponse + + +class AmazonNovaCanvasConfig: + """ + Reference: https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/model-catalog/serverless/amazon.nova-canvas-v1:0 + + """ + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + @classmethod + def get_supported_openai_params(cls, model: Optional[str] = None) -> List: + """ + """ + return ["n", "size", "quality"] + + @classmethod + def _is_nova_model(cls, model: Optional[str] = None) -> bool: + """ + Returns True if the model is a Nova Canvas model + + Nova models follow this pattern: + + """ + if model: + if "amazon.nova-canvas" in model: + return True + return False + + @classmethod + def transform_request_body( + cls, text: str, optional_params: dict + ) -> AmazonNovaCanvasRequestBase: + """ + Transform the request body for Amazon Nova Canvas model + """ + task_type = optional_params.pop("taskType", "TEXT_IMAGE") + image_generation_config = optional_params.pop("imageGenerationConfig", {}) + image_generation_config = {**image_generation_config, **optional_params} + if task_type == "TEXT_IMAGE": + text_to_image_params = image_generation_config.pop("textToImageParams", {}) + text_to_image_params = {"text" :text, **text_to_image_params} + text_to_image_params = AmazonNovaCanvasTextToImageParams(**text_to_image_params) + return AmazonNovaCanvasTextToImageRequest(textToImageParams=text_to_image_params, taskType=task_type, + imageGenerationConfig=image_generation_config) + raise NotImplementedError(f"Task type {task_type} is not supported") + + @classmethod + def map_openai_params(cls, non_default_params: dict, optional_params: dict) -> dict: + """ + Map the OpenAI params to the Bedrock params + """ + _size = non_default_params.get("size") + if _size is not None: + width, height = _size.split("x") + optional_params["width"], optional_params["height"] = int(width), int(height) + if non_default_params.get("n") is not None: + optional_params["numberOfImages"] = non_default_params.get("n") + if non_default_params.get("quality") is not None: + if non_default_params.get("quality") in ("hd", "premium"): + optional_params["quality"] = "premium" + if non_default_params.get("quality") == "standard": + optional_params["quality"] = "standard" + return optional_params + + @classmethod + def transform_response_dict_to_openai_response( + cls, model_response: ImageResponse, response_dict: dict + ) -> ImageResponse: + """ + Transform the response dict to the OpenAI response + """ + + nova_response = AmazonNovaCanvasTextToImageResponse(**response_dict) + openai_images: List[Image] = [] + for _img in nova_response.get("images", []): + openai_images.append(Image(b64_json=_img)) + + model_response.data = openai_images + return model_response diff --git a/litellm/llms/bedrock/image/image_handler.py b/litellm/llms/bedrock/image/image_handler.py index 59a80b22229..8f7762e547a 100644 --- a/litellm/llms/bedrock/image/image_handler.py +++ b/litellm/llms/bedrock/image/image_handler.py @@ -266,6 +266,8 @@ class BedrockImageGeneration(BaseAWSLLM): "text_prompts": [{"text": prompt, "weight": 1}], **inference_params, } + elif provider == "amazon": + return dict(litellm.AmazonNovaCanvasConfig.transform_request_body(text=prompt, optional_params=optional_params)) else: raise BedrockError( status_code=422, message=f"Unsupported model={model}, passed in" @@ -301,6 +303,7 @@ class BedrockImageGeneration(BaseAWSLLM): config_class = ( litellm.AmazonStability3Config if litellm.AmazonStability3Config._is_stability_3_model(model=model) + else litellm.AmazonNovaCanvasConfig if litellm.AmazonNovaCanvasConfig._is_nova_model(model=model) else litellm.AmazonStabilityConfig ) config_class.transform_response_dict_to_openai_response( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cb2322752bb..a34cfb7db94 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6434,7 +6434,7 @@ "supports_response_schema": true }, "us.amazon.nova-micro-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 300000, "max_output_tokens": 4096, "input_cost_per_token": 0.000000035, @@ -6472,7 +6472,7 @@ "supports_response_schema": true }, "us.amazon.nova-lite-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.00000006, @@ -6514,7 +6514,7 @@ "supports_response_schema": true }, "us.amazon.nova-pro-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 300000, "max_output_tokens": 4096, "input_cost_per_token": 0.0000008, @@ -6527,6 +6527,12 @@ "supports_prompt_caching": true, "supports_response_schema": true }, + "1024-x-1024/50-steps/bedrock/amazon.nova-canvas-v1:0": { + "max_input_tokens": 2600, + "output_cost_per_image": 0.06, + "litellm_provider": "bedrock", + "mode": "image_generation" + }, "eu.amazon.nova-pro-v1:0": { "max_tokens": 4096, "max_input_tokens": 300000, @@ -7871,22 +7877,22 @@ "mode": "image_generation" }, "stability.sd3-5-large-v1:0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.08, "litellm_provider": "bedrock", "mode": "image_generation" }, "stability.stable-image-core-v1:0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.04, "litellm_provider": "bedrock", "mode": "image_generation" }, "stability.stable-image-core-v1:1": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.04, "litellm_provider": "bedrock", "mode": "image_generation" @@ -7899,8 +7905,8 @@ "mode": "image_generation" }, "stability.stable-image-ultra-v1:1": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.14, "litellm_provider": "bedrock", "mode": "image_generation" diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index 7013c8a800a..9d276d7d60d 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -365,6 +365,63 @@ class AmazonStability3TextToImageResponse(TypedDict, total=False): finish_reasons: List[str] +class AmazonNovaCanvasRequestBase(TypedDict, total=False): + """ + Base class for Amazon Nova Canvas API requests + """ + + pass + + +class AmazonNovaCanvasImageGenerationConfig(TypedDict, total=False): + """ + Config for Amazon Nova Canvas Text to Image API + + Ref: https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html + """ + + cfgScale: int + seed: int + quality: Literal["standard", "premium"] + width: int + height: int + numberOfImages: int + + +class AmazonNovaCanvasTextToImageParams(TypedDict, total=False): + """ + Params for Amazon Nova Canvas Text to Image API + """ + + text: str + negativeText: str + controlStrength: float + controlMode: Literal["CANNY_EDIT", "SEGMENTATION"] + conditionImage: str + + +class AmazonNovaCanvasTextToImageRequest(AmazonNovaCanvasRequestBase, TypedDict, total=False): + """ + Request for Amazon Nova Canvas Text to Image API + + Ref: https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html + """ + + textToImageParams: AmazonNovaCanvasTextToImageParams + taskType: Literal["TEXT_IMAGE"] + imageGenerationConfig: AmazonNovaCanvasImageGenerationConfig + + +class AmazonNovaCanvasTextToImageResponse(TypedDict, total=False): + """ + Response for Amazon Nova Canvas Text to Image API + + Ref: https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html + """ + + images: List[str] + + if TYPE_CHECKING: from botocore.awsrequest import AWSPreparedRequest else: diff --git a/litellm/utils.py b/litellm/utils.py index ce5acbc694b..2f1cac743c1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2427,6 +2427,7 @@ def get_optional_params_image_gen( config_class = ( litellm.AmazonStability3Config if litellm.AmazonStability3Config._is_stability_3_model(model=model) + else litellm.AmazonNovaCanvasConfig if litellm.AmazonNovaCanvasConfig._is_nova_model(model=model) else litellm.AmazonStabilityConfig ) supported_params = config_class.get_supported_openai_params(model=model) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cb2322752bb..a34cfb7db94 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6434,7 +6434,7 @@ "supports_response_schema": true }, "us.amazon.nova-micro-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 300000, "max_output_tokens": 4096, "input_cost_per_token": 0.000000035, @@ -6472,7 +6472,7 @@ "supports_response_schema": true }, "us.amazon.nova-lite-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 128000, "max_output_tokens": 4096, "input_cost_per_token": 0.00000006, @@ -6514,7 +6514,7 @@ "supports_response_schema": true }, "us.amazon.nova-pro-v1:0": { - "max_tokens": 4096, + "max_tokens": 4096, "max_input_tokens": 300000, "max_output_tokens": 4096, "input_cost_per_token": 0.0000008, @@ -6527,6 +6527,12 @@ "supports_prompt_caching": true, "supports_response_schema": true }, + "1024-x-1024/50-steps/bedrock/amazon.nova-canvas-v1:0": { + "max_input_tokens": 2600, + "output_cost_per_image": 0.06, + "litellm_provider": "bedrock", + "mode": "image_generation" + }, "eu.amazon.nova-pro-v1:0": { "max_tokens": 4096, "max_input_tokens": 300000, @@ -7871,22 +7877,22 @@ "mode": "image_generation" }, "stability.sd3-5-large-v1:0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.08, "litellm_provider": "bedrock", "mode": "image_generation" }, "stability.stable-image-core-v1:0": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.04, "litellm_provider": "bedrock", "mode": "image_generation" }, "stability.stable-image-core-v1:1": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.04, "litellm_provider": "bedrock", "mode": "image_generation" @@ -7899,8 +7905,8 @@ "mode": "image_generation" }, "stability.stable-image-ultra-v1:1": { - "max_tokens": 77, - "max_input_tokens": 77, + "max_tokens": 77, + "max_input_tokens": 77, "output_cost_per_image": 0.14, "litellm_provider": "bedrock", "mode": "image_generation" diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index 544f25bc67f..c2115abeb8d 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -130,6 +130,16 @@ class TestBedrockSd1(BaseImageGenTest): return {"model": "bedrock/stability.sd3-large-v1:0"} +class TestBedrockNovaCanvasTextToImage(BaseImageGenTest): + def get_base_image_generation_call_args(self) -> dict: + litellm.in_memory_llm_clients_cache = InMemoryCache() + return {"model": "bedrock/amazon.nova-canvas-v1:0", + "n": 1, + "size": "320x320", + "imageGenerationConfig": {"cfgScale":6.5,"seed":12}, + "taskType": "TEXT_IMAGE"} + + class TestOpenAIDalle3(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: return {"model": "dall-e-3"}