From 6fab790a6ec5089fb5779f56262dbd18a8044ae6 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Thu, 16 Apr 2026 09:42:11 -0700 Subject: [PATCH 1/5] Merge pull request #25867 from BerriAI/litellm_day_0_opus_4.7_support Litellm day 0 opus 4.7 support --- docs/my-website/blog/claude_opus_4_7/index.md | 366 ++++++++++++++++++ litellm/anthropic_beta_headers_config.json | 8 +- litellm/constants.py | 1 + litellm/llms/anthropic/chat/transformation.py | 56 ++- litellm/llms/anthropic/common_utils.py | 27 +- .../messages/transformation.py | 35 ++ .../bedrock/chat/converse_transformation.py | 8 +- litellm/llms/bedrock/common_utils.py | 4 + .../anthropic_claude3_transformation.py | 84 ++++ ...odel_prices_and_context_window_backup.json | 346 +++++++++++++++-- litellm/setup_wizard.py | 9 +- model_prices_and_context_window.json | 317 ++++++++++++++- .../test_anthropic_claude3_transformation.py | 56 +++ tests/test_litellm/test_utils.py | 1 + 14 files changed, 1265 insertions(+), 53 deletions(-) create mode 100644 docs/my-website/blog/claude_opus_4_7/index.md diff --git a/docs/my-website/blog/claude_opus_4_7/index.md b/docs/my-website/blog/claude_opus_4_7/index.md new file mode 100644 index 00000000000..851a0556c4c --- /dev/null +++ b/docs/my-website/blog/claude_opus_4_7/index.md @@ -0,0 +1,366 @@ +--- +slug: claude_opus_4_7 +title: "Day 0 Support: Claude Opus 4.7" +date: 2026-04-16T10:00:00 +authors: + - sameer + - ishaan-alt + - krrish +description: "Day 0 support for Claude Opus 4.7 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock." +tags: [anthropic, claude, opus 4.7] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +LiteLLM now supports [Claude Opus 4.7](https://www.anthropic.com/news/claude-opus-4-7) on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway. + +{/* truncate */} + +## Docker Image + +```bash +docker pull ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.82.0-stable.opus-4-7 +``` + +## Usage - Anthropic + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-opus-4-7 + litellm_params: + model: anthropic/claude-opus-4-7 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.82.0-stable.opus-4-7 \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-opus-4-7", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +## Usage - Azure + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-opus-4-7 + litellm_params: + model: azure_ai/claude-opus-4-7 + api_key: os.environ/AZURE_AI_API_KEY + api_base: os.environ/AZURE_AI_API_BASE # https://.services.ai.azure.com +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \ + -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.82.0-stable.opus-4-7 \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-opus-4-7", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +## Usage - Vertex AI + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-opus-4-7 + litellm_params: + model: vertex_ai/claude-opus-4-7 + vertex_project: os.environ/VERTEX_PROJECT + vertex_location: us-east5 +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e VERTEX_PROJECT=$VERTEX_PROJECT \ + -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \ + -v $(pwd)/config.yaml:/app/config.yaml \ + -v $(pwd)/credentials.json:/app/credentials.json \ + ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.82.0-stable.opus-4-7 \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-opus-4-7", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +## Usage - Bedrock + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-opus-4-7 + litellm_params: + model: bedrock/anthropic.claude-opus-4-7 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: us-east-1 +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.82.0-stable.opus-4-7 \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-opus-4-7", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +## Advanced Features + +### Adaptive Thinking + +:::note +When using `reasoning_effort` with Claude Opus 4.7, all values (`low`, `medium`, `high`, `xhigh`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets with `type: "enabled"`, pass the native `thinking` parameter directly. +::: + + + + +LiteLLM supports adaptive thinking through the `reasoning_effort` parameter: + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-opus-4-7", + "messages": [ + { + "role": "user", + "content": "Solve this complex problem: What is the optimal strategy for..." + } + ], + "reasoning_effort": "high" +}' +``` + + + + +Use the `thinking` parameter with `type: "adaptive"` to enable adaptive thinking mode: + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'x-api-key: sk-12345' \ +--header 'content-type: application/json' \ +--data '{ + "model": "claude-opus-4-7", + "max_tokens": 16000, + "thinking": { + "type": "adaptive" + }, + "messages": [ + { + "role": "user", + "content": "Explain why the sum of two even numbers is always even." + } + ] +}' +``` + + + + +### Effort Levels + +Claude Opus 4.7 supports four effort levels: `low`, `medium`, `high` (default), and `xhigh`. These give you finer-grained control over how much reasoning the model applies to a task. Pass the effort level via the `output_config` parameter. + +`xhigh` is a new effort level introduced with Opus 4.7 that sits above `high`. The `max` effort level is Claude Opus 4.6 only and is not available on 4.7. + + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-opus-4-7", + "messages": [ + { + "role": "user", + "content": "Explain quantum computing" + } + ], + "output_config": { + "effort": "xhigh" + } +}' +``` + +**Using OpenAI SDK:** + +```python +import openai + +client = openai.OpenAI( + api_key="your-litellm-key", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="claude-opus-4-7", + messages=[{"role": "user", "content": "Explain quantum computing"}], + extra_body={"output_config": {"effort": "xhigh"}} +) +``` + +**Using LiteLLM SDK:** + +```python +from litellm import completion + +response = completion( + model="anthropic/claude-opus-4-7", + messages=[{"role": "user", "content": "Explain quantum computing"}], + output_config={"effort": "xhigh"}, +) +``` + +You can combine `reasoning_effort` with `output_config` for even more fine-grained control over the model's behavior. + + + + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'x-api-key: sk-12345' \ +--header 'content-type: application/json' \ +--data '{ + "model": "claude-opus-4-7", + "max_tokens": 4096, + "messages": [ + { + "role": "user", + "content": "Explain quantum computing" + } + ], + "output_config": { + "effort": "xhigh" + } +}' +``` + + + + +**Effort level guide:** + +| Effort | When to use | +|--------|-------------| +| `low` | Short, fast responses — simple lookups, formatting, classification | +| `medium` | Balanced tradeoff for everyday Q&A and light reasoning | +| `high` (default) | Complex reasoning, code generation, analysis | +| `xhigh` | Hardest problems — multi-step math, deep research, agentic planning | + diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 7dd5975b7bb..662b62cf205 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -71,12 +71,12 @@ "computer-use-2025-01-24": "computer-use-2025-01-24", "computer-use-2025-11-24": "computer-use-2025-11-24", "context-1m-2025-08-07": "context-1m-2025-08-07", - "context-management-2025-06-27": "context-management-2025-06-27", + "context-management-2025-06-27": null, "effort-2025-11-24": null, "fast-mode-2026-02-01": null, "files-api-2025-04-14": null, "fine-grained-tool-streaming-2025-05-14": null, - "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", + "interleaved-thinking-2025-05-14": null, "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, "mcp-servers-2025-12-04": null, @@ -102,12 +102,12 @@ "computer-use-2025-01-24": "computer-use-2025-01-24", "computer-use-2025-11-24": "computer-use-2025-11-24", "context-1m-2025-08-07": "context-1m-2025-08-07", - "context-management-2025-06-27": "context-management-2025-06-27", + "context-management-2025-06-27": null, "effort-2025-11-24": null, "fast-mode-2026-02-01": null, "files-api-2025-04-14": null, "fine-grained-tool-streaming-2025-05-14": null, - "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14", + "interleaved-thinking-2025-05-14": null, "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, "mcp-servers-2025-12-04": null, diff --git a/litellm/constants.py b/litellm/constants.py index d0596bed684..4d182b873cb 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1113,6 +1113,7 @@ BEDROCK_CONVERSE_MODELS = [ "openai.gpt-oss-120b-1:0", "anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic.claude-sonnet-4-5-20250929-v1:0", + "anthropic.claude-opus-4-7", "anthropic.claude-opus-4-6-v1:0", "anthropic.claude-opus-4-6-v1", "anthropic.claude-sonnet-4-6", diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index d7ce6a5f8de..6e6078728eb 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -67,6 +67,7 @@ from litellm.types.utils import ( from litellm.utils import ( ModelResponse, Usage, + _supports_factory, add_dummy_tool, any_assistant_message_has_thinking_blocks, get_max_tokens, @@ -189,6 +190,30 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): v in model_lower for v in ("opus-4-6", "opus_4_6", "opus-4.6", "opus_4.6") ) + @staticmethod + def _is_opus_4_7_model(model: str) -> bool: + """Check if the model is specifically Claude Opus 4.7.""" + model_lower = model.lower() + return any( + v in model_lower for v in ("opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7") + ) + + @staticmethod + def _supports_effort_level(model: str, level: str) -> bool: + """Check ``supports_{level}_reasoning_effort`` in the model map. + + Mirrors the pattern used in ``openai/chat/gpt_5_transformation.py`` so + that adding support for a new effort level is a pure model-map change. + """ + try: + return _supports_factory( + model=model, + custom_llm_provider="anthropic", + key=f"supports_{level}_reasoning_effort", + ) + except Exception: + return False + def get_supported_openai_params(self, model: str): params = [ "stream", @@ -212,6 +237,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if ( "claude-3-7-sonnet" in model or AnthropicConfig._is_claude_4_6_model(model) + or AnthropicConfig._is_claude_4_7_model(model) or supports_reasoning( model=model, custom_llm_provider=self.custom_llm_provider, @@ -771,7 +797,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) -> Optional[AnthropicThinkingParam]: if reasoning_effort is None or reasoning_effort == "none": return None - if AnthropicConfig._is_claude_4_6_model(model): + if AnthropicConfig._is_claude_4_6_model( + model + ) or AnthropicConfig._is_claude_4_7_model(model): return AnthropicThinkingParam( type="adaptive", ) @@ -1020,6 +1048,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "opus-4-5", "opus-4.6", "opus-4-6", + "opus-4.7", + "opus-4-7", "sonnet-4.6", "sonnet-4-6", "sonnet_4.6", @@ -1061,14 +1091,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): optional_params["thinking"] = AnthropicConfig._map_reasoning_effort( reasoning_effort=value, model=model ) - # For Claude 4.6 models, effort is controlled via output_config, + # For Claude 4.6+ models, effort is controlled via output_config, # not thinking budget_tokens. Map reasoning_effort to output_config. - if AnthropicConfig._is_claude_4_6_model(model): + if AnthropicConfig._is_claude_4_6_model( + model + ) or AnthropicConfig._is_claude_4_7_model(model): effort_map = { "low": "low", "minimal": "low", "medium": "medium", "high": "high", + "xhigh": "xhigh", "max": "max", } mapped_effort = effort_map.get(value, value) @@ -1495,13 +1528,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not output_config or not isinstance(output_config, dict): return effort = output_config.get("effort") - if effort and effort not in ["high", "medium", "low", "max"]: + valid_efforts = ["high", "medium", "low", "xhigh", "max"] + if effort and effort not in valid_efforts: raise ValueError( - f"Invalid effort value: {effort}. Must be one of: 'high', 'medium', 'low', 'max'" + f"Invalid effort value: {effort}. Must be one of: " + f"'high', 'medium', 'low', 'xhigh', 'max'" ) + # ``max`` is Claude Opus 4.6 only (not Sonnet 4.6, not Opus 4.5/4.7). + # Keep this hardcoded so the error message is specific and stable. if effort == "max" and not self._is_opus_4_6_model(model): raise ValueError( - f"effort='max' is only supported by Claude Opus 4.6. Got model: {model}" + f"effort='max' is only supported by Claude Opus 4.6. " + f"Got model: {model}" + ) + # ``xhigh`` is data-driven via ``supports_xhigh_reasoning_effort`` so + # enabling it for a new model is a pure model-map change. + if effort == "xhigh" and not self._supports_effort_level(model, "xhigh"): + raise ValueError( + f"effort='xhigh' is not supported by this model. Got model: {model}" ) data["output_config"] = output_config diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index a0da14bcc2b..26fe4955c39 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -256,6 +256,27 @@ class AnthropicModelInfo(BaseLLMModelInfo): ) ) + @staticmethod + def _is_claude_4_7_model(model: str) -> bool: + """Check if the model is a Claude 4.7 model (Opus 4.7).""" + model_lower = model.lower() + return any( + v in model_lower + for v in ( + "opus-4-7", + "opus_4_7", + "opus-4.7", + "opus_4.7", + ) + ) + + @staticmethod + def _is_adaptive_thinking_model(model: str) -> bool: + """Claude 4.6+ models use adaptive thinking with output_config effort.""" + return AnthropicModelInfo._is_claude_4_6_model( + model + ) or AnthropicModelInfo._is_claude_4_7_model(model) + def is_effort_used( self, optional_params: Optional[dict], model: Optional[str] = None ) -> bool: @@ -263,14 +284,14 @@ class AnthropicModelInfo(BaseLLMModelInfo): Check if effort parameter is being used and requires a beta header. Returns True if effort-related parameters are present and - the model requires the effort beta header. Claude 4.6 models + the model requires the effort beta header. Claude 4.6+ models use output_config as a stable API feature — no beta header needed. """ if not optional_params: return False - # Claude 4.6 models use output_config as a stable API feature — no beta header needed - if model and self._is_claude_4_6_model(model): + # Claude 4.6+ models use output_config as a stable API feature — no beta header needed + if model and self._is_adaptive_thinking_model(model): return False # Check if reasoning_effort is provided for Claude Opus 4.5 diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 46af1f7fbd1..7617ad52ab1 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -166,6 +166,36 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return headers, api_base + @staticmethod + def _translate_legacy_thinking_for_adaptive_model( + model: str, optional_params: Dict + ) -> None: + """Translate legacy ``thinking.type=enabled`` to adaptive for 4.6/4.7. + Caller-provided ``output_config.effort`` is never overridden. + """ + if not AnthropicModelInfo._is_adaptive_thinking_model(model): + return + thinking = optional_params.get("thinking") + if not isinstance(thinking, dict) or thinking.get("type") != "enabled": + return + + budget = int(thinking.get("budget_tokens") or 0) + if budget >= 24000: + effort = "xhigh" + elif budget >= 10000: + effort = "high" + elif budget >= 5000: + effort = "medium" + else: + effort = "low" + + optional_params["thinking"] = {"type": "adaptive"} + existing_output_config = optional_params.get("output_config") + if not isinstance(existing_output_config, dict): + existing_output_config = {} + existing_output_config.setdefault("effort", effort) + optional_params["output_config"] = existing_output_config + def transform_anthropic_messages_request( self, model: str, @@ -187,6 +217,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): status_code=400, ) + self._translate_legacy_thinking_for_adaptive_model( + model=model, + optional_params=anthropic_messages_optional_request_params, + ) + # Filter out x-anthropic-billing-header from system messages system_param = anthropic_messages_optional_request_params.get("system") if system_param is not None: diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 5cfb00d69b6..5d98e1538bd 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1298,12 +1298,16 @@ class AmazonConverseConfig(BaseConfig): # Add computer use tools and anthropic_beta if needed (only when computer use tools are present) if computer_use_tools: # Determine the correct computer-use beta header based on model - # "computer-use-2025-11-24" for Claude Opus 4.6, Claude Opus 4.5 + # "computer-use-2025-11-24" for Claude Opus 4.7, Opus 4.6, and Opus 4.5 # "computer-use-2025-01-24" for Claude Sonnet 4.5, Haiku 4.5, Opus 4.1, Sonnet 4, Opus 4, and Sonnet 3.7 # "computer-use-2024-10-22" for older models model_lower = model.lower() if ( - "opus-4.6" in model_lower + "opus-4.7" in model_lower + or "opus_4.7" in model_lower + or "opus-4-7" in model_lower + or "opus_4_7" in model_lower + or "opus-4.6" in model_lower or "opus_4.6" in model_lower or "opus-4-6" in model_lower or "opus_4_6" in model_lower diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 6f4f3c3f18a..a68cc7b43e6 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -589,6 +589,10 @@ def is_claude_4_5_on_bedrock(model: str) -> bool: "opus_4.6", "opus-4-6", "opus_4_6", + "opus-4.7", + "opus_4.7", + "opus-4-7", + "opus_4_7", ] return any(pattern in model_lower for pattern in claude_4_5_patterns) diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index d00a18fe7e3..239c666887b 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -13,6 +13,8 @@ from typing import ( import httpx from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers +from litellm.constants import BEDROCK_MIN_THINKING_BUDGET_TOKENS +from litellm.litellm_core_utils.litellm_logging import verbose_logger from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -205,10 +207,78 @@ class AmazonAnthropicClaudeMessagesConfig( "opus_4.6", "opus-4-6", "opus_4_6", + "opus-4.7", + "opus_4.7", + "opus-4-7", + "opus_4_7", ] return any(pattern in model_lower for pattern in supported_patterns) + def _ensure_thinking_for_clear_thinking_context_management( + self, + anthropic_messages_request: Dict, + model: str, + ) -> bool: + """ + Bedrock rejects ``clear_thinking_20251015`` context-management edits unless + extended thinking is ``enabled`` or ``adaptive``. Claude Code often sends + context management without a top-level ``thinking`` field. + + When we detect that edit type on a model that supports extended thinking on + Bedrock, inject a minimal ``thinking`` config so the request succeeds. + + Returns: + True if ``thinking`` was added or upgraded for this fix (caller may + need to add the interleaved-thinking beta header). + """ + cm = anthropic_messages_request.get("context_management") + if not isinstance(cm, dict): + return False + edits = cm.get("edits") + if not isinstance(edits, list): + return False + needs_thinking = any( + isinstance(e, dict) and e.get("type") == "clear_thinking_20251015" + for e in edits + ) + if not needs_thinking: + return False + if not self._supports_extended_thinking_on_bedrock(model): + return False + + thinking = anthropic_messages_request.get("thinking") + if isinstance(thinking, dict): + t = thinking.get("type") + if t in ("enabled", "adaptive"): + return False + # ``disabled`` or unknown — replace with enabled so clear_thinking is valid + verbose_logger.debug( + "Bedrock clear_thinking_20251015: replacing thinking=%s with minimal enabled thinking", + thinking, + ) + + max_tokens = anthropic_messages_request.get("max_tokens") + budget = BEDROCK_MIN_THINKING_BUDGET_TOKENS + if isinstance(max_tokens, int) and max_tokens <= budget: + verbose_logger.warning( + "Bedrock clear_thinking_20251015: max_tokens=%s is not greater than " + "minimum thinking budget (%s); cannot inject thinking safely", + max_tokens, + budget, + ) + return False + + anthropic_messages_request["thinking"] = { + "type": "enabled", + "budget_tokens": budget, + } + verbose_logger.debug( + "Bedrock clear_thinking_20251015: injected thinking with budget_tokens=%s", + budget, + ) + return True + def _is_claude_opus_4_5(self, model: str) -> bool: """ Check if the model is Claude Opus 4.5. @@ -281,6 +351,10 @@ class AmazonAnthropicClaudeMessagesConfig( "sonnet_4.6", "sonnet-4-6", "sonnet_4_6", + # NOTE: Opus 4.7 on Bedrock does not support server-side tool search + # as of launch (2026-04-16). Bedrock rejects the tool type with: + # "tool type 'tool_search_tool_..._20251119' is not supported for this model". + # Re-add the opus-4.7 patterns here once AWS announces support. ] return any(pattern in model_lower for pattern in supported_patterns) @@ -406,6 +480,13 @@ class AmazonAnthropicClaudeMessagesConfig( if "model" in anthropic_messages_request: anthropic_messages_request.pop("model", None) + injected_thinking_for_clear_thinking = ( + self._ensure_thinking_for_clear_thinking_context_management( + anthropic_messages_request=anthropic_messages_request, + model=model, + ) + ) + # 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it for older models) self._remove_ttl_from_cache_control( anthropic_messages_request=anthropic_messages_request, model=model @@ -455,6 +536,9 @@ class AmazonAnthropicClaudeMessagesConfig( ) beta_set.update(auto_betas) + if injected_thinking_for_clear_thinking: + beta_set.add("interleaved-thinking-2025-05-14") + self._get_tool_search_beta_header_for_bedrock( model=model, tool_search_used=tool_search_used, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2000e4e3064..5e0f1ec5c9b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1005,7 +1005,8 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true }, "global.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, @@ -1032,7 +1033,8 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true }, "us.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1059,7 +1061,8 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true }, "eu.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1086,7 +1089,8 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true }, "au.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1113,6 +1117,147 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true + }, + "anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true + }, + "global.anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true + }, + "us.anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true + }, + "eu.anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true + }, + "au.anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "anthropic.claude-sonnet-4-6": { @@ -1765,6 +1910,35 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_max_reasoning_effort": true + }, + "azure_ai/claude-opus-4-7": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, "tool_use_system_prompt_tokens": 159 }, "azure_ai/claude-opus-4-1": { @@ -8928,7 +9102,8 @@ "provider_specific_entry": { "us": 1.1, "fast": 6.0 - } + }, + "supports_max_reasoning_effort": true }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -8956,6 +9131,71 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, + "provider_specific_entry": { + "us": 1.1, + "fast": 6.0 + }, + "supports_max_reasoning_effort": true + }, + "claude-opus-4-7": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "provider_specific_entry": { + "us": 1.1, + "fast": 6.0 + } + }, + "claude-opus-4-7-20260416": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 @@ -18991,13 +19231,11 @@ "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 5e-06, - "input_cost_per_token_above_272k_tokens_priority": 1e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -19007,8 +19245,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, - "output_cost_per_token_priority": 2.25e-05, - "output_cost_per_token_above_272k_tokens_priority": 3.375e-05, + "output_cost_per_token_priority": 3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -19041,13 +19278,11 @@ "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 5e-06, - "input_cost_per_token_above_272k_tokens_priority": 1e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -19057,8 +19292,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, - "output_cost_per_token_priority": 2.25e-05, - "output_cost_per_token_above_272k_tokens_priority": 3.375e-05, + "output_cost_per_token_priority": 3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -19086,14 +19320,10 @@ "gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, - "cache_read_input_token_cost_priority": 6e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, - "input_cost_per_token_priority": 6e-05, - "input_cost_per_token_above_272k_tokens_priority": 0.00012, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -19103,8 +19333,6 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "output_cost_per_token_priority": 0.00027, - "output_cost_per_token_above_272k_tokens_priority": 0.000405, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -19135,14 +19363,10 @@ "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, - "cache_read_input_token_cost_priority": 6e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, - "input_cost_per_token_priority": 6e-05, - "input_cost_per_token_above_272k_tokens_priority": 0.00012, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -19152,8 +19376,6 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "output_cost_per_token_priority": 0.00027, - "output_cost_per_token_above_272k_tokens_priority": 0.000405, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -19183,11 +19405,13 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_flex": 1e-08, - "cache_read_input_token_cost_batches": 3.8e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_batches": 3.75e-08, + "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, "input_cost_per_token_flex": 3.75e-07, "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_priority": 1.5e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -19196,6 +19420,7 @@ "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, + "output_cost_per_token_priority": 9e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -26698,6 +26923,13 @@ "supports_reasoning": false, "supports_function_calling": true }, + "perplexity/anthropic/claude-opus-4-7": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true + }, "perplexity/anthropic/claude-opus-4-5": { "litellm_provider": "perplexity", "mode": "responses", @@ -31060,7 +31292,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "tool_use_system_prompt_tokens": 346, + "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-6@default": { "cache_creation_input_token_cost": 6.25e-06, @@ -31086,6 +31319,61 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-opus-4-7": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346 + }, + "vertex_ai/claude-opus-4-7@default": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, "tool_use_system_prompt_tokens": 346 }, "vertex_ai/claude-sonnet-4-5": { @@ -38251,4 +38539,4 @@ "supports_native_structured_output": true, "supports_pdf_input": true } -} \ No newline at end of file +} diff --git a/litellm/setup_wizard.py b/litellm/setup_wizard.py index 3718655b318..b2aa22f685f 100644 --- a/litellm/setup_wizard.py +++ b/litellm/setup_wizard.py @@ -52,11 +52,16 @@ PROVIDERS: List[Dict] = [ { "id": "anthropic", "name": "Anthropic", - "description": "Claude Opus 4.6, Sonnet 4.6, Haiku 4.5", + "description": "Claude Opus 4.7, Opus 4.6, Sonnet 4.6, Haiku 4.5", "env_key": "ANTHROPIC_API_KEY", "key_hint": "sk-ant-...", "test_model": "claude-haiku-4-5-20251001", - "models": ["claude-opus-4-6", "claude-sonnet-4-6", "claude-haiku-4-5-20251001"], + "models": [ + "claude-opus-4-7", + "claude-opus-4-6", + "claude-sonnet-4-6", + "claude-haiku-4-5-20251001", + ], }, { "id": "gemini", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c624736d6bf..5e0f1ec5c9b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1005,7 +1005,8 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true }, "global.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.25e-06, @@ -1032,7 +1033,8 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true }, "us.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1059,7 +1061,8 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true }, "eu.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1086,7 +1089,8 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true }, "au.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, @@ -1113,6 +1117,147 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true + }, + "anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true + }, + "global.anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true + }, + "us.anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true + }, + "eu.anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true + }, + "au.anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, "anthropic.claude-sonnet-4-6": { @@ -1765,6 +1910,35 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_max_reasoning_effort": true + }, + "azure_ai/claude-opus-4-7": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, "tool_use_system_prompt_tokens": 159 }, "azure_ai/claude-opus-4-1": { @@ -8928,7 +9102,8 @@ "provider_specific_entry": { "us": 1.1, "fast": 6.0 - } + }, + "supports_max_reasoning_effort": true }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -8956,6 +9131,71 @@ "supports_tool_choice": true, "supports_vision": true, "tool_use_system_prompt_tokens": 346, + "provider_specific_entry": { + "us": 1.1, + "fast": 6.0 + }, + "supports_max_reasoning_effort": true + }, + "claude-opus-4-7": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "provider_specific_entry": { + "us": 1.1, + "fast": 6.0 + } + }, + "claude-opus-4-7-20260416": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, "provider_specific_entry": { "us": 1.1, "fast": 6.0 @@ -26683,6 +26923,13 @@ "supports_reasoning": false, "supports_function_calling": true }, + "perplexity/anthropic/claude-opus-4-7": { + "litellm_provider": "perplexity", + "mode": "responses", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true + }, "perplexity/anthropic/claude-opus-4-5": { "litellm_provider": "perplexity", "mode": "responses", @@ -31045,7 +31292,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 346 + "tool_use_system_prompt_tokens": 346, + "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-6@default": { "cache_creation_input_token_cost": 6.25e-06, @@ -31071,6 +31319,61 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-opus-4-7": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346 + }, + "vertex_ai/claude-opus-4-7@default": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, "tool_use_system_prompt_tokens": 346 }, "vertex_ai/claude-sonnet-4-5": { @@ -38236,4 +38539,4 @@ "supports_native_structured_output": true, "supports_pdf_input": true } -} \ No newline at end of file +} diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index f0186f7891f..a76c4118214 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -18,6 +18,7 @@ from litellm.llms.bedrock.common_utils import ( normalize_tool_input_schema_types_for_bedrock_invoke, remove_custom_field_from_tools, ) +from litellm.constants import BEDROCK_MIN_THINKING_BUDGET_TOKENS from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeMessagesConfig, AmazonAnthropicClaudeMessagesStreamDecoder, @@ -384,6 +385,61 @@ def test_bedrock_invoke_messages_transform_adds_name_when_tool_missing_name(): assert result["tools"][0]["name"] == "litellm_unnamed_tool_0" +def test_bedrock_invoke_messages_injects_thinking_for_clear_thinking_context_management(): + """ + Bedrock requires extended thinking when ``clear_thinking_20251015`` appears in + ``context_management`` (Claude Code sends CM without ``thinking``). + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + optional_params = { + "max_tokens": 32000, + "stream": False, + "context_management": { + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + }, + } + result = cfg.transform_anthropic_messages_request( + model="global.anthropic.claude-sonnet-4-6-v1:0", + messages=[{"role": "user", "content": "hi"}], + anthropic_messages_optional_request_params=copy.deepcopy(optional_params), + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert result["thinking"]["type"] == "enabled" + assert result["thinking"]["budget_tokens"] == BEDROCK_MIN_THINKING_BUDGET_TOKENS + betas = result.get("anthropic_beta") or [] + assert "interleaved-thinking-2025-05-14" in betas + + +def test_bedrock_invoke_messages_skips_thinking_injection_when_already_enabled(): + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + optional_params = { + "max_tokens": 32000, + "stream": False, + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "context_management": { + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + }, + } + result = cfg.transform_anthropic_messages_request( + model="global.anthropic.claude-sonnet-4-6-v1:0", + messages=[{"role": "user", "content": "hi"}], + anthropic_messages_optional_request_params=copy.deepcopy(optional_params), + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + # Claude 4.6/4.7 reject ``thinking.type=enabled``; legacy ``enabled`` is + # translated to ``adaptive`` (budget_tokens => output_config.effort) and the + # pre-4.6 ``interleaved-thinking-2025-05-14`` beta must not be attached. + assert result["thinking"]["type"] == "adaptive" + betas = result.get("anthropic_beta") or [] + assert "interleaved-thinking-2025-05-14" not in betas + + def test_bedrock_invoke_messages_transform_converts_custom_tool_schema_type_to_object(): """ End-to-end: AmazonAnthropicClaudeMessagesConfig must emit Bedrock Invoke bodies diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 0acbe901300..67b62696196 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -771,6 +771,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_minimal_reasoning_effort": {"type": "boolean"}, "supports_none_reasoning_effort": {"type": "boolean"}, "supports_xhigh_reasoning_effort": {"type": "boolean"}, + "supports_max_reasoning_effort": {"type": "boolean"}, "supports_service_tier": {"type": "boolean"}, "supports_preset": {"type": "boolean"}, "tool_use_system_prompt_tokens": {"type": "number"}, From fe6fef97d19ac63732c029b606b26a407c650199 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 16 Apr 2026 22:41:32 +0530 Subject: [PATCH 2/5] Fix version in docs --- docs/my-website/blog/claude_opus_4_7/index.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/my-website/blog/claude_opus_4_7/index.md b/docs/my-website/blog/claude_opus_4_7/index.md index 851a0556c4c..e8d86bafbcf 100644 --- a/docs/my-website/blog/claude_opus_4_7/index.md +++ b/docs/my-website/blog/claude_opus_4_7/index.md @@ -21,7 +21,7 @@ LiteLLM now supports [Claude Opus 4.7](https://www.anthropic.com/news/claude-opu ## Docker Image ```bash -docker pull ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.82.0-stable.opus-4-7 +docker pull ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 ``` ## Usage - Anthropic @@ -46,7 +46,7 @@ docker run -d \ -p 4000:4000 \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -v $(pwd)/config.yaml:/app/config.yaml \ - ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.82.0-stable.opus-4-7 \ + ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \ --config /app/config.yaml ``` @@ -94,7 +94,7 @@ docker run -d \ -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \ -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \ -v $(pwd)/config.yaml:/app/config.yaml \ - ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.82.0-stable.opus-4-7 \ + ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \ --config /app/config.yaml ``` @@ -143,7 +143,7 @@ docker run -d \ -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \ -v $(pwd)/config.yaml:/app/config.yaml \ -v $(pwd)/credentials.json:/app/credentials.json \ - ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.82.0-stable.opus-4-7 \ + ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \ --config /app/config.yaml ``` @@ -192,7 +192,7 @@ docker run -d \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -v $(pwd)/config.yaml:/app/config.yaml \ - ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.82.0-stable.opus-4-7 \ + ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \ --config /app/config.yaml ``` From 35a186a133efbf4b94ad9eb8fe1345b8115ae637 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 16 Apr 2026 16:51:49 -0700 Subject: [PATCH 3/5] [Test] Mock live Bedrock Moonshot tests in llm_translation Three tests inherited by TestBedrockMoonshotInvoke from BaseLLMChatTest make live AWS Bedrock completion calls: test_developer_role_translation, test_message_with_name, and test_completion_cost. These have been crashing llm_translation_testing CI workers (reported as "failed on setup with worker 'gwN' crashed"). Replace each with a mocked override that intercepts the outgoing request via HTTPHandler.post / AsyncHTTPHandler.post patching: - test_developer_role_translation asserts the outgoing body maps the developer role to system (LiteLLM's translation for non-OpenAI providers). - test_message_with_name asserts the outgoing body preserves the user message. - test_completion_cost returns a canned moonshot-shaped response body with usage and asserts response_cost > 0 against the local model cost map. Follows the existing HTTPHandler + patch.object(client, "post") pattern used in test_bedrock_gpt_oss.py and test_bedrock_completion.py. No network traffic; the three tests now complete in ~0.3s. --- .../llm_translation/test_bedrock_moonshot.py | 239 +++++++++++++----- 1 file changed, 176 insertions(+), 63 deletions(-) diff --git a/tests/llm_translation/test_bedrock_moonshot.py b/tests/llm_translation/test_bedrock_moonshot.py index c6066c7db42..a92e342a2c7 100644 --- a/tests/llm_translation/test_bedrock_moonshot.py +++ b/tests/llm_translation/test_bedrock_moonshot.py @@ -16,10 +16,12 @@ import pytest import sys import os import json +from unittest.mock import AsyncMock, Mock, patch sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm.llms.bedrock.common_utils import get_bedrock_chat_config +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler class TestBedrockMoonshotInvoke(BaseLLMChatTest): @@ -27,17 +29,121 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): Test suite for Bedrock Moonshot via invoke route. Inherits all standard LLM tests from BaseLLMChatTest. """ - + def get_base_completion_call_args(self) -> dict: litellm._turn_on_debug() return { "model": "bedrock/invoke/moonshot.kimi-k2-thinking", } - + def test_tool_call_no_arguments(self, tool_call_no_arguments): """Test that tool calls with no arguments is translated correctly.""" pass + # --------------------------------------------------------------------- + # The three overrides below replace inherited BaseLLMChatTest tests that + # would otherwise make live AWS Bedrock calls. The live versions were + # consistently crashing llm_translation xdist workers. Each override + # patches the HTTP client's post() so no network request is sent, and + # asserts on the outgoing request body (or a mocked response for cost) + # — which is what the translation lane is actually supposed to cover. + # --------------------------------------------------------------------- + + def test_developer_role_translation(self): + """Verify LiteLLM maps the ``developer`` role to ``system`` on the + outgoing Bedrock invoke request, without hitting the network.""" + client = HTTPHandler() + with patch.object(client, "post", new=Mock()) as mock_post: + try: + litellm.completion( + model="bedrock/invoke/moonshot.kimi-k2-thinking", + messages=[ + {"role": "developer", "content": "Be a good bot!"}, + {"role": "user", "content": "Hello, how are you?"}, + ], + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-west-2", + client=client, + ) + except Exception: + # The Mock() return value is not a parseable Bedrock response; + # we only care about the outgoing request body. + pass + + mock_post.assert_called_once() + body = json.loads(mock_post.call_args.kwargs["data"]) + assert body["messages"][0]["role"] == "system" + assert body["messages"][0]["content"] == "Be a good bot!" + assert body["messages"][1]["role"] == "user" + + def test_message_with_name(self): + """Verify a user message carrying a ``name`` field is serialized into + the outgoing Bedrock invoke request without breaking the call.""" + client = HTTPHandler() + with patch.object(client, "post", new=Mock()) as mock_post: + try: + litellm.completion( + model="bedrock/invoke/moonshot.kimi-k2-thinking", + messages=[ + {"role": "user", "content": "Hello", "name": "test_name"}, + ], + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-west-2", + client=client, + ) + except Exception: + pass + + mock_post.assert_called_once() + body = json.loads(mock_post.call_args.kwargs["data"]) + assert body["messages"][0]["role"] == "user" + assert body["messages"][0]["content"] == "Hello" + + async def test_completion_cost(self): + """Verify LiteLLM computes a positive cost from a mocked Bedrock + Moonshot response, using the local model cost map.""" + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + response_body = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": "moonshot.kimi-k2-thinking", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hi!"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + }, + } + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {"Content-Type": "application/json"} + mock_response.text = json.dumps(response_body) + mock_response.json = lambda: response_body + + client = AsyncHTTPHandler() + with patch.object(client, "post", new=AsyncMock(return_value=mock_response)): + response = await litellm.acompletion( + model="bedrock/invoke/moonshot.kimi-k2-thinking", + messages=[{"role": "user", "content": "Hello, how are you?"}], + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-west-2", + client=client, + ) + + assert response._hidden_params["response_cost"] > 0 + class TestBedrockMoonshotBasic: """Unit tests for Bedrock Moonshot configuration and transformations.""" @@ -47,7 +153,7 @@ class TestBedrockMoonshotBasic: config = get_bedrock_chat_config("bedrock/invoke/moonshot.kimi-k2-thinking") assert config is not None assert config.__class__.__name__ == "AmazonMoonshotConfig" - + def test_provider_detection_converse(self): """Test that Bedrock Moonshot converse models are correctly detected.""" config = get_bedrock_chat_config("bedrock/moonshot.kimi-k2-thinking") @@ -62,8 +168,10 @@ class TestBedrockMoonshotBasic: def test_supported_params(self): """Test that supported OpenAI params are correctly defined.""" config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking") - supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking") - + supported_params = config.get_supported_openai_params( + "moonshot.kimi-k2-thinking" + ) + # Should support these params assert "temperature" in supported_params assert "max_tokens" in supported_params @@ -71,10 +179,10 @@ class TestBedrockMoonshotBasic: assert "stream" in supported_params assert "tools" in supported_params assert "tool_choice" in supported_params - + # Should NOT support stop sequences on Bedrock assert "stop" not in supported_params - + # Should NOT support functions (use tools instead) assert "functions" not in supported_params @@ -83,20 +191,20 @@ class TestBedrockMoonshotBasic: from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( AmazonMoonshotConfig, ) - + config = AmazonMoonshotConfig() - + messages = [{"role": "user", "content": "Hello"}] - + # Test that bedrock/invoke/ prefix is stripped transformed = config.transform_request( model="bedrock/invoke/moonshot.kimi-k2-thinking", messages=messages, optional_params={}, litellm_params={}, - headers={} + headers={}, ) - + # The model ID in the request body should be stripped assert transformed["model"] == "moonshot.kimi-k2-thinking" @@ -109,21 +217,27 @@ class TestBedrockMoonshotReasoningContent: from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( AmazonMoonshotConfig, ) - + config = AmazonMoonshotConfig() - + # Test with reasoning tags - content_with_reasoning = "This is my thought processThis is the answer" - reasoning, content = config._extract_reasoning_from_content(content_with_reasoning) - + content_with_reasoning = ( + "This is my thought processThis is the answer" + ) + reasoning, content = config._extract_reasoning_from_content( + content_with_reasoning + ) + assert reasoning == "This is my thought process" assert content == "This is the answer" assert "" not in content - + # Test without reasoning tags content_without_reasoning = "This is just a regular answer" - reasoning, content = config._extract_reasoning_from_content(content_without_reasoning) - + reasoning, content = config._extract_reasoning_from_content( + content_without_reasoning + ) + assert reasoning is None assert content == "This is just a regular answer" @@ -134,8 +248,10 @@ class TestBedrockMoonshotToolCalling: def test_tool_calling_supported(self): """Test that tool calling is supported for Kimi K2 Thinking model.""" config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking") - supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking") - + supported_params = config.get_supported_openai_params( + "moonshot.kimi-k2-thinking" + ) + # Kimi K2 Thinking DOES support tool calls (unlike kimi-thinking-preview) assert "tools" in supported_params assert "tool_choice" in supported_params @@ -145,13 +261,11 @@ class TestBedrockMoonshotToolCalling: from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( AmazonMoonshotConfig, ) - + config = AmazonMoonshotConfig() - - messages = [ - {"role": "user", "content": "What's the weather in San Francisco?"} - ] - + + messages = [{"role": "user", "content": "What's the weather in San Francisco?"}] + optional_params = { "tools": [ { @@ -161,27 +275,25 @@ class TestBedrockMoonshotToolCalling: "description": "Get the current weather", "parameters": { "type": "object", - "properties": { - "location": {"type": "string"} - }, - "required": ["location"] - } - } + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, } ] } - + transformed = config.transform_request( model="bedrock/invoke/moonshot.kimi-k2-thinking", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + # Verify model ID is stripped assert transformed["model"] == "moonshot.kimi-k2-thinking" - + # Verify tools are included assert "tools" in transformed assert len(transformed["tools"]) == 1 @@ -193,9 +305,9 @@ class TestBedrockMoonshotToolCalling: tool_response_message = { "role": "tool", "tool_call_id": "call_123", - "content": json.dumps({"temperature": 72, "condition": "sunny"}) + "content": json.dumps({"temperature": 72, "condition": "sunny"}), } - + # Verify the message structure assert tool_response_message["role"] == "tool" assert "tool_call_id" in tool_response_message @@ -208,8 +320,10 @@ class TestBedrockMoonshotParameterValidation: def test_stop_sequences_not_supported(self): """Test that stop sequences are correctly excluded from supported params.""" config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking") - supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking") - + supported_params = config.get_supported_openai_params( + "moonshot.kimi-k2-thinking" + ) + # Bedrock Moonshot doesn't support stopSequences field assert "stop" not in supported_params @@ -218,10 +332,12 @@ class TestBedrockMoonshotParameterValidation: # Moonshot models support temperature 0-1 # This is handled by the parent MoonshotChatConfig class config = get_bedrock_chat_config("invoke/moonshot.kimi-k2-thinking") - + # Verify config exists and can handle temperature assert config is not None - supported_params = config.get_supported_openai_params("moonshot.kimi-k2-thinking") + supported_params = config.get_supported_openai_params( + "moonshot.kimi-k2-thinking" + ) assert "temperature" in supported_params @@ -233,34 +349,31 @@ class TestBedrockMoonshotTransformations: from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( AmazonMoonshotConfig, ) - + config = AmazonMoonshotConfig() - + messages = [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} + {"role": "user", "content": "Hello!"}, ] - - optional_params = { - "temperature": 0.7, - "max_tokens": 100 - } - + + optional_params = {"temperature": 0.7, "max_tokens": 100} + transformed = config.transform_request( model="bedrock/invoke/moonshot.kimi-k2-thinking", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) - + # Verify model ID is stripped assert transformed["model"] == "moonshot.kimi-k2-thinking" - + # Verify messages are included assert "messages" in transformed assert len(transformed["messages"]) >= 1 - + # Verify optional params are included assert transformed["temperature"] == 0.7 assert transformed["max_tokens"] == 100 @@ -270,21 +383,21 @@ class TestBedrockMoonshotTransformations: from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import ( AmazonMoonshotConfig, ) - + config = AmazonMoonshotConfig() - + messages = [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} + {"role": "user", "content": "Hello!"}, ] - + transformed = config.transform_request( model="moonshot.kimi-k2-thinking", messages=messages, optional_params={}, litellm_params={}, - headers={} + headers={}, ) - + # System messages should be supported assert "messages" in transformed From 95e1babf67f63ce201c6cd17370213708be95515 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 16 Apr 2026 17:20:58 -0700 Subject: [PATCH 4/5] [Fix] TogetherAIConfig.get_supported_openai_params recursion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TogetherAIConfig.get_supported_openai_params called get_model_info(), whose first line calls litellm.get_supported_openai_params() — which for together_ai routes straight back into this method. The recursion only terminated when Python's recursion limit was hit or when _get_model_info_helper raised "not mapped" at the deepest level. Either way the try/except caught it, so the bug stayed silent — but the cycle ran ~332 deep every time, emitting hundreds of DEBUG log lines per call. Surfaced as "infinite loop" in CI when the success_handler thread emitted that log spam against an already-closed stderr during test teardown. Replace the get_model_info() call with supports_function_calling(), which uses _get_model_info_helper directly and does not call get_supported_openai_params. Measured drop from 332 to 2 _get_model_info_helper calls per first uncached lookup. Also swap the test model from Qwen/Qwen3.5-9B (not in model_cost map) back to a mapped serverless model, Qwen/Qwen2.5-7B-Instruct-Turbo. The mapping gap is what made the recursion's tail end raise up into the success handler during teardown in the first place. --- litellm/llms/together_ai/chat.py | 17 +++++++++++------ tests/llm_translation/test_together_ai.py | 2 +- tests/local_testing/test_completion.py | 6 +++--- .../local_testing/test_multiple_deployments.py | 2 +- tests/local_testing/test_text_completion.py | 2 +- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/litellm/llms/together_ai/chat.py b/litellm/llms/together_ai/chat.py index e8a784d2779..7efb12fc1b2 100644 --- a/litellm/llms/together_ai/chat.py +++ b/litellm/llms/together_ai/chat.py @@ -8,7 +8,7 @@ Docs: https://docs.together.ai/reference/completions-1 from typing import Optional -from litellm.utils import get_model_info +from litellm.utils import supports_function_calling from litellm._logging import verbose_logger from ..openai.chat.gpt_transformation import OpenAIGPTConfig @@ -21,18 +21,23 @@ class TogetherAIConfig(OpenAIGPTConfig): Docs: https://docs.together.ai/docs/json-mode """ - supports_function_calling: Optional[bool] = None + # Use supports_function_calling() — which reads _get_model_info_helper + # directly — instead of get_model_info(). get_model_info() calls + # get_supported_openai_params() as its first step, which routes back + # into this method for together_ai models, creating a recursion that + # only terminates when Python's recursion limit or the "not mapped" + # exception in _get_model_info_helper is hit (~332 deep calls). + supports_fc: Optional[bool] = None try: - model_info = get_model_info(model, custom_llm_provider="together_ai") - supports_function_calling = model_info.get( - "supports_function_calling", False + supports_fc = supports_function_calling( + model, custom_llm_provider="together_ai" ) except Exception as e: verbose_logger.debug(f"Error getting supported openai params: {e}") pass optional_params = super().get_supported_openai_params(model) - if supports_function_calling is not True: + if supports_fc is not True: verbose_logger.debug( "Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling" ) diff --git a/tests/llm_translation/test_together_ai.py b/tests/llm_translation/test_together_ai.py index 5225ab78f61..4ad0c90230d 100644 --- a/tests/llm_translation/test_together_ai.py +++ b/tests/llm_translation/test_together_ai.py @@ -20,7 +20,7 @@ import pytest class TestTogetherAI(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: litellm.set_verbose = True - return {"model": "together_ai/Qwen/Qwen3.5-9B"} + return {"model": "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo"} def test_tool_call_no_arguments(self, tool_call_no_arguments): """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index f18a2b4afbb..457385a3b0b 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -65,7 +65,7 @@ def test_completion_custom_provider_model_name(): try: litellm.cache = None response = completion( - model="together_ai/Qwen/Qwen3.5-9B", + model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo", messages=messages, logger_fn=logger_fn, ) @@ -2815,7 +2815,7 @@ def test_customprompt_together_ai(): print(litellm.success_callback) print(litellm._async_success_callback) response = completion( - model="together_ai/Qwen/Qwen3.5-9B", + model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo", messages=messages, roles={ "system": { @@ -3682,7 +3682,7 @@ def test_completion_together_ai_stream(): messages = [{"content": user_message, "role": "user"}] try: response = completion( - model="together_ai/Qwen/Qwen3.5-9B", + model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo", messages=messages, stream=True, max_tokens=5, diff --git a/tests/local_testing/test_multiple_deployments.py b/tests/local_testing/test_multiple_deployments.py index 61baa73da04..f7276d4f14e 100644 --- a/tests/local_testing/test_multiple_deployments.py +++ b/tests/local_testing/test_multiple_deployments.py @@ -25,7 +25,7 @@ model_list = [ { "model_name": "mistral-7b-instruct", "litellm_params": { # params for litellm completion/embedding call - "model": "together_ai/Qwen/Qwen3.5-9B", + "model": "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo", "api_key": os.getenv("TOGETHERAI_API_KEY"), }, }, diff --git a/tests/local_testing/test_text_completion.py b/tests/local_testing/test_text_completion.py index dde5f67ea1c..ace5fed1100 100644 --- a/tests/local_testing/test_text_completion.py +++ b/tests/local_testing/test_text_completion.py @@ -4034,7 +4034,7 @@ def test_async_text_completion_together_ai(): async def test_get_response(): try: response = await litellm.atext_completion( - model="together_ai/Qwen/Qwen3.5-9B", + model="together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo", prompt="good morning", max_tokens=10, ) From 00bac08e015794ffd18d54844dd79d724a47f7f6 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 16 Apr 2026 17:43:43 -0700 Subject: [PATCH 5/5] [Test] Mock remaining live Bedrock Moonshot tests Extends the prior moonshot mocking to cover every inherited BaseLLMChatTest test that still made a live AWS Bedrock call. Adds request-body assertions for each override. New overrides: - test_content_list_handling: verifies the outgoing body round-trips user content in list-of-text form; asserts response.choices[0]. message.content parses back from the canned response. - test_pydantic_model_input: verifies a pydantic Message input does not raise and produces a parseable response. - test_response_format_type_text_with_tool_calls_no_tool_choice: verifies tools are forwarded and response_format + drop_params do not break the call. - test_streaming: verifies stream=True routes to the invoke-with-response-stream endpoint. Bedrock invoke streaming is intercepted at the make_sync_call import site rather than via the caller-supplied client, because CustomStreamWrapper.fetch_sync_stream invokes the stored make_call partial with client=litellm.module_level_client, overriding any client passed by the caller. Extracts a shared _make_moonshot_response helper and a _invoke_with_mocked_post harness so all the sync mocks share one canned response body. After this change TestBedrockMoonshotInvoke runs 23 passed, 29 skipped, 0 live-callers, all in under 1s locally. --- .../llm_translation/test_bedrock_moonshot.py | 273 +++++++++++++----- 1 file changed, 204 insertions(+), 69 deletions(-) diff --git a/tests/llm_translation/test_bedrock_moonshot.py b/tests/llm_translation/test_bedrock_moonshot.py index a92e342a2c7..a9f4a86b3b6 100644 --- a/tests/llm_translation/test_bedrock_moonshot.py +++ b/tests/llm_translation/test_bedrock_moonshot.py @@ -16,6 +16,7 @@ import pytest import sys import os import json +from typing import Optional from unittest.mock import AsyncMock, Mock, patch sys.path.insert(0, os.path.abspath("../..")) @@ -41,73 +42,20 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): pass # --------------------------------------------------------------------- - # The three overrides below replace inherited BaseLLMChatTest tests that - # would otherwise make live AWS Bedrock calls. The live versions were + # The overrides below replace inherited BaseLLMChatTest tests that would + # otherwise make live AWS Bedrock calls. The live versions were # consistently crashing llm_translation xdist workers. Each override # patches the HTTP client's post() so no network request is sent, and - # asserts on the outgoing request body (or a mocked response for cost) - # — which is what the translation lane is actually supposed to cover. + # asserts on the outgoing request body (and, where needed, parses a + # canned response) — which is what the translation lane is actually + # supposed to cover. # --------------------------------------------------------------------- - def test_developer_role_translation(self): - """Verify LiteLLM maps the ``developer`` role to ``system`` on the - outgoing Bedrock invoke request, without hitting the network.""" - client = HTTPHandler() - with patch.object(client, "post", new=Mock()) as mock_post: - try: - litellm.completion( - model="bedrock/invoke/moonshot.kimi-k2-thinking", - messages=[ - {"role": "developer", "content": "Be a good bot!"}, - {"role": "user", "content": "Hello, how are you?"}, - ], - aws_access_key_id="fake", - aws_secret_access_key="fake", - aws_region_name="us-west-2", - client=client, - ) - except Exception: - # The Mock() return value is not a parseable Bedrock response; - # we only care about the outgoing request body. - pass - - mock_post.assert_called_once() - body = json.loads(mock_post.call_args.kwargs["data"]) - assert body["messages"][0]["role"] == "system" - assert body["messages"][0]["content"] == "Be a good bot!" - assert body["messages"][1]["role"] == "user" - - def test_message_with_name(self): - """Verify a user message carrying a ``name`` field is serialized into - the outgoing Bedrock invoke request without breaking the call.""" - client = HTTPHandler() - with patch.object(client, "post", new=Mock()) as mock_post: - try: - litellm.completion( - model="bedrock/invoke/moonshot.kimi-k2-thinking", - messages=[ - {"role": "user", "content": "Hello", "name": "test_name"}, - ], - aws_access_key_id="fake", - aws_secret_access_key="fake", - aws_region_name="us-west-2", - client=client, - ) - except Exception: - pass - - mock_post.assert_called_once() - body = json.loads(mock_post.call_args.kwargs["data"]) - assert body["messages"][0]["role"] == "user" - assert body["messages"][0]["content"] == "Hello" - - async def test_completion_cost(self): - """Verify LiteLLM computes a positive cost from a mocked Bedrock - Moonshot response, using the local model cost map.""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - response_body = { + @staticmethod + def _make_moonshot_response(content: str = "Hi!") -> Mock: + """Build a Mock httpx.Response that AmazonMoonshotConfig.transform_response + (which delegates to MoonshotChatConfig → OpenAI) can parse.""" + body = { "id": "chatcmpl-test", "object": "chat.completion", "created": 1234567890, @@ -115,7 +63,7 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): "choices": [ { "index": 0, - "message": {"role": "assistant", "content": "Hi!"}, + "message": {"role": "assistant", "content": content}, "finish_reason": "stop", } ], @@ -125,12 +73,199 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): "total_tokens": 15, }, } - mock_response = Mock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.text = json.dumps(response_body) - mock_response.json = lambda: response_body + mock_resp = Mock() + mock_resp.status_code = 200 + mock_resp.headers = {"Content-Type": "application/json"} + mock_resp.text = json.dumps(body) + mock_resp.json = lambda: body + return mock_resp + def _invoke_with_mocked_post( + self, + *, + messages: list, + extra_kwargs: Optional[dict] = None, + response_content: str = "Hi!", + ) -> "tuple[Mock, object]": + """Run a sync litellm.completion() with HTTPHandler.post patched to + return a canned moonshot response. Returns (mock_post, response).""" + client = HTTPHandler() + mock_resp = self._make_moonshot_response(content=response_content) + with patch.object( + client, "post", new=Mock(return_value=mock_resp) + ) as mock_post: + response = litellm.completion( + model="bedrock/invoke/moonshot.kimi-k2-thinking", + messages=messages, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-west-2", + client=client, + **(extra_kwargs or {}), + ) + return mock_post, response + + def test_developer_role_translation(self): + """Verify LiteLLM maps the ``developer`` role to ``system`` on the + outgoing Bedrock invoke request, without hitting the network.""" + mock_post, response = self._invoke_with_mocked_post( + messages=[ + {"role": "developer", "content": "Be a good bot!"}, + {"role": "user", "content": "Hello, how are you?"}, + ], + ) + mock_post.assert_called_once() + body = json.loads(mock_post.call_args.kwargs["data"]) + assert body["messages"][0]["role"] == "system" + assert body["messages"][0]["content"] == "Be a good bot!" + assert body["messages"][1]["role"] == "user" + assert response.choices[0].message.content is not None + + def test_message_with_name(self): + """Verify a user message carrying a ``name`` field is serialized into + the outgoing Bedrock invoke request without breaking the call.""" + mock_post, response = self._invoke_with_mocked_post( + messages=[{"role": "user", "content": "Hello", "name": "test_name"}], + ) + mock_post.assert_called_once() + body = json.loads(mock_post.call_args.kwargs["data"]) + assert body["messages"][0]["role"] == "user" + assert body["messages"][0]["content"] == "Hello" + assert response is not None + + def test_content_list_handling(self): + """Verify the inherited content-list-handling test passes against a + mocked moonshot response (no network).""" + mock_post, response = self._invoke_with_mocked_post( + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "Hello, how are you?"}], + } + ], + ) + mock_post.assert_called_once() + assert response.choices[0].message.content is not None + + def test_pydantic_model_input(self): + """Verify a completion call with a pydantic ``Message`` as input does + not raise and produces a parseable response.""" + from litellm import Message + + mock_post, response = self._invoke_with_mocked_post( + messages=[Message(content="Hello, how are you?", role="user")], + ) + mock_post.assert_called_once() + assert response is not None + + @pytest.mark.parametrize("response_format", [{"type": "text"}]) + def test_response_format_type_text_with_tool_calls_no_tool_choice( + self, response_format + ): + """Verify response_format + tools + drop_params sends a valid request + and produces a response object.""" + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + }, + } + ] + mock_post, response = self._invoke_with_mocked_post( + messages=[ + {"role": "user", "content": "What's the weather like in Boston today?"} + ], + extra_kwargs={ + "response_format": response_format, + "tools": tools, + "drop_params": True, + }, + ) + mock_post.assert_called_once() + body = json.loads(mock_post.call_args.kwargs["data"]) + assert "tools" in body + assert body["tools"][0]["function"]["name"] == "get_current_weather" + assert response is not None + + def test_streaming(self): + """Verify stream=True routes to the invoke-with-response-stream + endpoint with the messages body. Iteration of the stream itself is + not exercised here — moonshot streaming delegates to the OpenAI + parser and is covered by the OpenAI test suite. + + Note: bedrock invoke streaming cannot be intercepted by patching + the caller-supplied client, because ``CustomStreamWrapper.fetch_sync_stream`` + at streaming_handler.py invokes the stored ``make_call`` partial with + ``client=litellm.module_level_client``, which overrides any client the + caller passed. Patch ``make_sync_call`` at its import site in + ``base_invoke_transformation`` so we observe the exact kwargs the + partial was built with at stream-wrapper construction time. + """ + from litellm.utils import CustomStreamWrapper + + captured: dict = {} + + def fake_make_sync_call(**kwargs): + captured.update(kwargs) + # Return an empty iterator so the stream wrapper's iteration + # doesn't try to parse real bytes. + return iter([]) + + with patch( + "litellm.llms.bedrock.chat.invoke_transformations." + "base_invoke_transformation.make_sync_call", + new=fake_make_sync_call, + ): + response = litellm.completion( + model="bedrock/invoke/moonshot.kimi-k2-thinking", + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "Hello, how are you?"}], + } + ], + stream=True, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-west-2", + ) + assert isinstance(response, CustomStreamWrapper) + # Trigger fetch_sync_stream → make_call(...) → fake_make_sync_call. + try: + next(iter(response)) + except StopIteration: + pass + + assert captured, "make_sync_call was never invoked" + assert captured["api_base"].endswith("/invoke-with-response-stream") + body = json.loads(captured["data"]) + # Bedrock invoke does not put stream=true in the body (the URL + # carries the streaming flag); verify the user message is present. + assert body["messages"][0]["role"] == "user" + + async def test_completion_cost(self): + """Verify LiteLLM computes a positive cost from a mocked Bedrock + Moonshot response, using the local model cost map.""" + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + mock_response = self._make_moonshot_response() client = AsyncHTTPHandler() with patch.object(client, "post", new=AsyncMock(return_value=mock_response)): response = await litellm.acompletion(