From 5d326386fb541ff44249b63384b3057c0fa37b21 Mon Sep 17 00:00:00 2001 From: CyrusTC Date: Fri, 12 Dec 2025 00:16:32 +0800 Subject: [PATCH] feat(bedrock): add serviceTier support for Converse API (#17810) Add support for the Bedrock Converse API serviceTier parameter to allow specifying processing tier (priority, default, or flex). Changes: - Add ServiceTierBlock type in litellm/types/llms/bedrock.py - Add serviceTier to CommonRequestObject - Add serviceTier to get_config_blocks() in AmazonConverseConfig - Add comprehensive tests for serviceTier functionality - Add documentation for serviceTier usage This allows users to configure service tier via: - litellm_params in proxy config - optional_params in SDK calls --- docs/my-website/docs/providers/bedrock.md | 59 +++++++ .../bedrock/chat/converse_transformation.py | 1 + litellm/types/llms/bedrock.py | 5 + .../llms/bedrock/chat/test_service_tier.py | 149 ++++++++++++++++++ 4 files changed, 214 insertions(+) create mode 100644 tests/test_litellm/llms/bedrock/chat/test_service_tier.py diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index 17c0d38111d..122554fe8a4 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -957,6 +957,65 @@ curl http://0.0.0.0:4000/v1/chat/completions \ +## Usage - Service Tier + +Control the processing tier for your Bedrock requests using `serviceTier`. Valid values are `priority`, `default`, or `flex`. + +- `priority`: Higher priority processing with guaranteed capacity +- `default`: Standard processing tier +- `flex`: Cost-optimized processing for batch workloads + +[Bedrock ServiceTier API Reference](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ServiceTier.html) + + + + +```python +from litellm import completion + +response = completion( + model="bedrock/converse/qwen.qwen3-235b-a22b-2507-v1:0", + messages=[{"role": "user", "content": "What is the capital of France?"}], + serviceTier={"type": "priority"}, +) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: qwen3-235b-priority + litellm_params: + model: bedrock/converse/qwen.qwen3-235b-a22b-2507-v1:0 + aws_region_name: ap-northeast-1 + serviceTier: + type: priority +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "qwen3-235b-priority", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "serviceTier": {"type": "priority"} + }' +``` + + + ## Usage - Bedrock Guardrails Example of using [Bedrock Guardrails with LiteLLM](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-use-converse-api.html) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 2a1d7f2e3a3..ae0f1baf38b 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -100,6 +100,7 @@ class AmazonConverseConfig(BaseConfig): return { "guardrailConfig": GuardrailConfigBlock, "performanceConfig": PerformanceConfigBlock, + "serviceTier": ServiceTierBlock, } @staticmethod diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index 3696f679640..74853956e60 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -216,6 +216,10 @@ class PerformanceConfigBlock(TypedDict): latency: Literal["optimized", "throughput"] +class ServiceTierBlock(TypedDict): + type: Literal["priority", "default", "flex"] + + class CommonRequestObject( TypedDict, total=False ): # common request object across sync + async flows @@ -226,6 +230,7 @@ class CommonRequestObject( toolConfig: ToolConfigBlock guardrailConfig: Optional[GuardrailConfigBlock] performanceConfig: Optional[PerformanceConfigBlock] + serviceTier: Optional[ServiceTierBlock] requestMetadata: Optional[Dict[str, str]] diff --git a/tests/test_litellm/llms/bedrock/chat/test_service_tier.py b/tests/test_litellm/llms/bedrock/chat/test_service_tier.py new file mode 100644 index 00000000000..f9fedadaaed --- /dev/null +++ b/tests/test_litellm/llms/bedrock/chat/test_service_tier.py @@ -0,0 +1,149 @@ +""" +Tests for Bedrock Converse API serviceTier support. +""" + +import json +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.types.llms.bedrock import ServiceTierBlock + + +def test_service_tier_block_type(): + """Test that ServiceTierBlock is properly defined.""" + # Test valid service tier values + priority_tier: ServiceTierBlock = {"type": "priority"} + default_tier: ServiceTierBlock = {"type": "default"} + flex_tier: ServiceTierBlock = {"type": "flex"} + + assert priority_tier["type"] == "priority" + assert default_tier["type"] == "default" + assert flex_tier["type"] == "flex" + + +def test_service_tier_in_config_blocks(): + """Test that serviceTier is included in get_config_blocks().""" + config_blocks = AmazonConverseConfig.get_config_blocks() + + assert "serviceTier" in config_blocks + assert config_blocks["serviceTier"] == ServiceTierBlock + + +def test_transform_request_with_service_tier(): + """Test that serviceTier is properly included in the transformed request.""" + config = AmazonConverseConfig() + + messages = [{"role": "user", "content": "Hello!"}] + optional_params = { + "serviceTier": {"type": "priority"}, + } + + result = config.transform_request( + model="bedrock/converse/qwen.qwen3-235b-a22b-2507-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + # serviceTier should be a top-level parameter, not in additionalModelRequestFields + assert "serviceTier" in result + assert result["serviceTier"]["type"] == "priority" + + # Verify it's NOT in additionalModelRequestFields + additional_fields = result.get("additionalModelRequestFields", {}) + assert "serviceTier" not in additional_fields + assert "service_tier" not in additional_fields + + +def test_transform_request_with_default_tier(): + """Test serviceTier with default value.""" + config = AmazonConverseConfig() + + messages = [{"role": "user", "content": "Hello!"}] + optional_params = { + "serviceTier": {"type": "default"}, + } + + result = config.transform_request( + model="bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "serviceTier" in result + assert result["serviceTier"]["type"] == "default" + + +def test_transform_request_with_flex_tier(): + """Test serviceTier with flex value.""" + config = AmazonConverseConfig() + + messages = [{"role": "user", "content": "Hello!"}] + optional_params = { + "serviceTier": {"type": "flex"}, + } + + result = config.transform_request( + model="bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "serviceTier" in result + assert result["serviceTier"]["type"] == "flex" + + +def test_transform_request_without_service_tier(): + """Test that requests without serviceTier work correctly.""" + config = AmazonConverseConfig() + + messages = [{"role": "user", "content": "Hello!"}] + optional_params = {} + + result = config.transform_request( + model="bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + # serviceTier should not be present if not specified + assert "serviceTier" not in result + + +def test_service_tier_with_other_config_blocks(): + """Test serviceTier works alongside other config blocks like performanceConfig.""" + config = AmazonConverseConfig() + + messages = [{"role": "user", "content": "Hello!"}] + optional_params = { + "serviceTier": {"type": "priority"}, + "performanceConfig": {"latency": "optimized"}, + } + + result = config.transform_request( + model="bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + # Both should be top-level parameters + assert "serviceTier" in result + assert result["serviceTier"]["type"] == "priority" + assert "performanceConfig" in result + assert result["performanceConfig"]["latency"] == "optimized"