scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

[Feat] Add Streaming support + Docs for bedrock gpt-oss model family (#13346)

Browse source 
* add openai.gpt-oss-20b-1:0

* update BEDROCK_CONVERSE_MODELS

* openai.gpt-oss-20b-1:0 fixes

* fix PDF input

* fix for should_fake_stream

* TestBedrockGPTOSS

* should_fake_stream

* update supports vision field for openai.gpt-oss models

* fixes for bedrock gpt oss

* fixes for should_fake_stream

* docs bedrock gpt oss models
This commit is contained in:
Ishaan Jaff 2025-08-12 08:39:36 -07:00 • committed by GitHub
parent 008ea864a7
commit 828e3e3deb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 155 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
docs/my-website/docs/providers/bedrock.md
View file

@ -1488,6 +1488,91 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
### OpenAI GPT OSS
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/converse/openai.gpt-oss-20b-1:0`, `bedrock/converse/openai.gpt-oss-120b-1:0` |
| Provider Documentation | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
<Tabs>
<TabItem value="sdk" label="SDK">
```python title="GPT OSS SDK Usage" showLineNumbers
from litellm import completion
import os
# Set AWS credentials
os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
os.environ["AWS_REGION_NAME"] = "us-east-1"
# GPT OSS 20B model
response = completion(
model="bedrock/converse/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
)
print(response.choices[0].message.content)
# GPT OSS 120B model
response = completion(
model="bedrock/converse/openai.gpt-oss-120b-1:0",
messages=[{"role": "user", "content": "Explain machine learning in simple terms"}],
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: gpt-oss-20b
litellm_params:
model: bedrock/converse/openai.gpt-oss-20b-1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
- model_name: gpt-oss-120b
litellm_params:
model: bedrock/converse/openai.gpt-oss-120b-1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
```
**2. Start proxy**
```bash title="Start LiteLLM Proxy" showLineNumbers
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash title="Test GPT OSS via Proxy" showLineNumbers
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-oss-20b",
"messages": [
{
"role": "user",
"content": "What are the key benefits of open source AI?"
}
]
}'
```
</TabItem>
</Tabs>
## Provisioned throughput models
To use provisioned throughput Bedrock models pass
- `model=bedrock/<base-model>`, example `model=bedrock/anthropic.claude-v2`. Set `model` to any of the [Supported AWS models](#supported-aws-bedrock-models)
@ -1522,6 +1607,8 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| Model Name | Command |
|----------------------------|------------------------------------------------------------------|
| GPT-OSS 20B | `completion(model='bedrock/converse/openai.gpt-oss-20b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| GPT-OSS 120B | `completion(model='bedrock/converse/openai.gpt-oss-120b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |

9
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -276,8 +276,13 @@ class BedrockConverseLLM(BaseAWSLLM):
else:
modelId = self.encode_model_id(model_id=model)
if stream is True and "ai21" in modelId:
fake_stream = True
fake_stream = litellm.AmazonConverseConfig().should_fake_stream(
fake_stream=fake_stream,
model=model,
stream=stream,
custom_llm_provider="bedrock",
)
### SET REGION NAME ###
aws_region_name = self._get_aws_region_name(

34
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -1154,3 +1154,37 @@ class AmazonConverseConfig(BaseConfig):
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
return headers
def should_fake_stream(
self,
model: Optional[str],
stream: Optional[bool],
custom_llm_provider: Optional[str] = None,
fake_stream: Optional[bool] = None,
) -> bool:
"""
Returns True if the model/provider should fake stream
"""
###################################################################
# If an upstream method already set fake_stream to True, return True
###################################################################
if fake_stream is True:
return True
###################################################################
# Bedrock Converse Specific Logic
###################################################################
if stream is True:
if model is not None:
###################################################################
# GPT-OSS models do not support streaming
###################################################################
if "gpt-oss" in model:
return True
###################################################################
# AI21 models do not support streaming
###################################################################
if "ai21" in model:
return True
return False

4
litellm/model_prices_and_context_window_backup.json
View file

@ -12588,8 +12588,6 @@
"output_cost_per_token": 3e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true
@ -12602,8 +12600,6 @@
"output_cost_per_token": 6e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true

4
model_prices_and_context_window.json
View file

@ -12588,8 +12588,6 @@
"output_cost_per_token": 3e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true
@ -12602,8 +12600,6 @@
"output_cost_per_token": 6e-07,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true

27
tests/llm_translation/test_bedrock_gpt_oss.py Normal file
View file

@ -0,0 +1,27 @@
from base_llm_unit_tests import BaseLLMChatTest
import pytest
import sys
import os
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import litellm
class TestBedrockGPTOSS(BaseLLMChatTest):
def get_base_completion_call_args(self) -> dict:
litellm._turn_on_debug()
return {
"model": "bedrock/converse/openai.gpt-oss-20b-1:0",
}
def test_tool_call_no_arguments(self, tool_call_no_arguments):
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
pass
def test_prompt_caching(self):
"""
Remove override once we have access to Bedrock prompt caching
"""
pass