From 247160277e11b28c59db05feef76c1adc105c886 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 27 Nov 2025 15:46:04 +0530 Subject: [PATCH 1/2] Added support for twelvelabs pegasus --- litellm/__init__.py | 3 + litellm/constants.py | 1 + ...mazon_twelvelabs_pegasus_transformation.py | 133 ++++++++++++++++++ litellm/llms/bedrock/common_utils.py | 2 + .../test_twelvelabs_pegasus_transformation.py | 85 +++++++++++ 5 files changed, 224 insertions(+) create mode 100644 litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py create mode 100644 tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_twelvelabs_pegasus_transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index aebf1404196..e6af8a21ff5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1222,6 +1222,9 @@ from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation imp from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import ( AmazonTitanConfig, ) +from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import ( + AmazonTwelveLabsPegasusConfig, +) from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, ) diff --git a/litellm/constants.py b/litellm/constants.py index cf3d4c6e742..00e17788466 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -851,6 +851,7 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ "nova", "deepseek_r1", "qwen3", + "twelvelabs", ] BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[ diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py new file mode 100644 index 00000000000..7b72968ea3d --- /dev/null +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -0,0 +1,133 @@ +""" +Transforms OpenAI-style requests into TwelveLabs Pegasus 1.2 requests for Bedrock. + +Reference: +https://docs.twelvelabs.io/docs/models/pegasus +""" + +from typing import Any, Dict, List, Optional + +from litellm.llms.base_llm.base_utils import type_to_response_format_param +from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.utils import get_base64_str + + +class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): + """ + Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for + `twelvelabs.pegasus-1-2-v1:0`. + + Pegasus 1.2 requires an `inputPrompt` and a `mediaSource` that either references + an S3 object or a base64-encoded clip. Optional OpenAI params (temperature, + response_format, max_tokens) are translated to the TwelveLabs schema. + """ + + def get_supported_openai_params(self, model: str) -> List[str]: + return [ + "max_tokens", + "max_completion_tokens", + "temperature", + "response_format", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + for param, value in non_default_params.items(): + if param in {"max_tokens", "max_completion_tokens"}: + optional_params["maxOutputTokens"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "response_format": + optional_params["responseFormat"] = self._normalize_response_format( + value + ) + return optional_params + + def _normalize_response_format(self, value: Any) -> Any: + if isinstance(value, dict): + return value + return type_to_response_format_param(response_format=value) or value + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + input_prompt = self._convert_messages_to_prompt(messages=messages) + request_data: Dict[str, Any] = {"inputPrompt": input_prompt} + + media_source = self._build_media_source(optional_params) + if media_source is not None: + request_data["mediaSource"] = media_source + + for key in ("temperature", "maxOutputTokens", "responseFormat"): + if key in optional_params: + request_data[key] = optional_params.get(key) + return request_data + + def _build_media_source(self, optional_params: dict) -> Optional[dict]: + direct_source = optional_params.get("mediaSource") or optional_params.get( + "media_source" + ) + if isinstance(direct_source, dict): + return direct_source + + base64_input = optional_params.get("video_base64") or optional_params.get( + "base64_string" + ) + if base64_input: + return {"base64String": get_base64_str(base64_input)} + + s3_uri = ( + optional_params.get("video_s3_uri") + or optional_params.get("s3_uri") + or optional_params.get("media_source_s3_uri") + ) + if s3_uri: + s3_location = {"uri": s3_uri} + bucket_owner = ( + optional_params.get("video_s3_bucket_owner") + or optional_params.get("s3_bucket_owner") + or optional_params.get("media_source_bucket_owner") + ) + if bucket_owner: + s3_location["bucketOwner"] = bucket_owner + return {"s3Location": s3_location} + return None + + def _convert_messages_to_prompt(self, messages: List[AllMessageValues]) -> str: + prompt_parts: List[str] = [] + for message in messages: + role = message.get("role", "user") + content = message.get("content", "") + if isinstance(content, list): + text_fragments = [] + for item in content: + if isinstance(item, dict): + item_type = item.get("type") + if item_type == "text": + text_fragments.append(item.get("text", "")) + elif item_type == "image_url": + text_fragments.append("") + elif item_type == "video_url": + text_fragments.append("