diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index 21eb3ee6862..9797d678ebb 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -1488,6 +1488,91 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ +### OpenAI GPT OSS + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/converse/openai.gpt-oss-20b-1:0`, `bedrock/converse/openai.gpt-oss-120b-1:0` | +| Provider Documentation | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) | + + + + +```python title="GPT OSS SDK Usage" showLineNumbers +from litellm import completion +import os + +# Set AWS credentials +os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key" +os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key" +os.environ["AWS_REGION_NAME"] = "us-east-1" + +# GPT OSS 20B model +response = completion( + model="bedrock/converse/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "Hello, how are you?"}], +) +print(response.choices[0].message.content) + +# GPT OSS 120B model +response = completion( + model="bedrock/converse/openai.gpt-oss-120b-1:0", + messages=[{"role": "user", "content": "Explain machine learning in simple terms"}], +) +print(response.choices[0].message.content) +``` + + + + + +**1. Add to config** + +```yaml title="config.yaml" showLineNumbers +model_list: + - model_name: gpt-oss-20b + litellm_params: + model: bedrock/converse/openai.gpt-oss-20b-1:0 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: os.environ/AWS_REGION_NAME + + - model_name: gpt-oss-120b + litellm_params: + model: bedrock/converse/openai.gpt-oss-120b-1:0 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: os.environ/AWS_REGION_NAME +``` + +**2. Start proxy** + +```bash title="Start LiteLLM Proxy" showLineNumbers +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash title="Test GPT OSS via Proxy" showLineNumbers +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-oss-20b", + "messages": [ + { + "role": "user", + "content": "What are the key benefits of open source AI?" + } + ] + }' +``` + + + + ## Provisioned throughput models To use provisioned throughput Bedrock models pass - `model=bedrock/`, example `model=bedrock/anthropic.claude-v2`. Set `model` to any of the [Supported AWS models](#supported-aws-bedrock-models) @@ -1522,6 +1607,8 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re | Model Name | Command | |----------------------------|------------------------------------------------------------------| +| GPT-OSS 20B | `completion(model='bedrock/converse/openai.gpt-oss-20b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | +| GPT-OSS 120B | `completion(model='bedrock/converse/openai.gpt-oss-120b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | | Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 900fad3d043..cd351ca16a7 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -276,8 +276,13 @@ class BedrockConverseLLM(BaseAWSLLM): else: modelId = self.encode_model_id(model_id=model) - if stream is True and "ai21" in modelId: - fake_stream = True + fake_stream = litellm.AmazonConverseConfig().should_fake_stream( + fake_stream=fake_stream, + model=model, + stream=stream, + custom_llm_provider="bedrock", + ) + ### SET REGION NAME ### aws_region_name = self._get_aws_region_name( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index e961433b52e..84762e0b99a 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1154,3 +1154,37 @@ class AmazonConverseConfig(BaseConfig): if api_key: headers["Authorization"] = f"Bearer {api_key}" return headers + + + def should_fake_stream( + self, + model: Optional[str], + stream: Optional[bool], + custom_llm_provider: Optional[str] = None, + fake_stream: Optional[bool] = None, + ) -> bool: + """ + Returns True if the model/provider should fake stream + """ + ################################################################### + # If an upstream method already set fake_stream to True, return True + ################################################################### + if fake_stream is True: + return True + + ################################################################### + # Bedrock Converse Specific Logic + ################################################################### + if stream is True: + if model is not None: + ################################################################### + # GPT-OSS models do not support streaming + ################################################################### + if "gpt-oss" in model: + return True + ################################################################### + # AI21 models do not support streaming + ################################################################### + if "ai21" in model: + return True + return False diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 28dec7cce90..1001aca9c09 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12588,8 +12588,6 @@ "output_cost_per_token": 3e-07, "litellm_provider": "bedrock_converse", "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_reasoning": true @@ -12602,8 +12600,6 @@ "output_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_reasoning": true diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 28dec7cce90..1001aca9c09 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12588,8 +12588,6 @@ "output_cost_per_token": 3e-07, "litellm_provider": "bedrock_converse", "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_reasoning": true @@ -12602,8 +12600,6 @@ "output_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "mode": "chat", - "supports_function_calling": true, - "supports_vision": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_reasoning": true diff --git a/tests/llm_translation/test_bedrock_gpt_oss.py b/tests/llm_translation/test_bedrock_gpt_oss.py new file mode 100644 index 00000000000..61bce04e2d0 --- /dev/null +++ b/tests/llm_translation/test_bedrock_gpt_oss.py @@ -0,0 +1,27 @@ +from base_llm_unit_tests import BaseLLMChatTest +import pytest +import sys +import os + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import litellm + + +class TestBedrockGPTOSS(BaseLLMChatTest): + def get_base_completion_call_args(self) -> dict: + litellm._turn_on_debug() + return { + "model": "bedrock/converse/openai.gpt-oss-20b-1:0", + } + + def test_tool_call_no_arguments(self, tool_call_no_arguments): + """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" + pass + + def test_prompt_caching(self): + """ + Remove override once we have access to Bedrock prompt caching + """ + pass