diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 6d39f272f3e..7af1d341300 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -969,10 +969,10 @@ def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> b def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: """ - Mantle reports context overflow as a validation_error carrying the token counts, or as - OpenAI's context_length_exceeded code, either in a 400 body or in a streamed error event - with no error type. Clients such as Claude Code only treat "prompt is too long" as - overflow (litellm/litellm#36546). + Mantle reports context overflow as a validation_error carrying the token counts, as + OpenAI's context_length_exceeded code, or as the served model's own overflow text, either + in a 400 body or in a streamed error event with no error type. Clients such as Claude Code + only treat "prompt is too long" as overflow (litellm/litellm#36546). """ match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str) if match is not None: @@ -980,9 +980,17 @@ def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum" if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(error_str) is not None: return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE + if ExceptionCheckers.is_error_str_context_window_exceeded(error_str): + return _with_prompt_is_too_long(error_str) return None +def _with_prompt_is_too_long(error_str: str) -> str: + if "prompt is too long" in error_str: + return error_str + return f"prompt is too long: {error_str}" + + def _map_bedrock_exception( *, model: str, @@ -1009,9 +1017,10 @@ def _map_bedrock_exception( or "prompt is too long" in error_str or "prompt: length: 1.." in error_str or "Too many input tokens" in error_str + or ExceptionCheckers.is_error_str_context_window_exceeded(error_str) ): raise ContextWindowExceededError( - message=f"BedrockException: Context Window Error - {error_str}", + message=f"BedrockException: Context Window Error - {_with_prompt_is_too_long(error_str)}", model=model, llm_provider="bedrock", response=getattr(original_exception, "response", None), diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ad57685da04..a236f0218c5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -44336,7 +44336,7 @@ "input_cost_per_token_batches": 3e-07, "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock_converse", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, @@ -44357,7 +44357,7 @@ "input_cost_per_token_flex": 3e-07, "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock_converse", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, @@ -45632,7 +45632,7 @@ "input_cost_per_token_flex": 3e-08, "input_cost_per_token_priority": 1.1e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.4e-07, @@ -48109,7 +48109,7 @@ "input_cost_per_token_flex": 7e-08, "input_cost_per_token_priority": 2.6e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, @@ -48260,7 +48260,7 @@ "input_cost_per_token_flex": 2.5e-07, "input_cost_per_token_priority": 8.75e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, @@ -70383,7 +70383,7 @@ "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 1.25e-06, @@ -70404,7 +70404,7 @@ "output_cost_per_token": 8.8e-07, "output_cost_per_token_flex": 4.4e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 4.4e-07, @@ -70425,7 +70425,7 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 32000, + "max_input_tokens": 32768, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 3e-07, @@ -70448,7 +70448,7 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 3e-07, @@ -70490,7 +70490,7 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 6e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ad57685da04..a236f0218c5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -44336,7 +44336,7 @@ "input_cost_per_token_batches": 3e-07, "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock_converse", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, @@ -44357,7 +44357,7 @@ "input_cost_per_token_flex": 3e-07, "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock_converse", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, @@ -45632,7 +45632,7 @@ "input_cost_per_token_flex": 3e-08, "input_cost_per_token_priority": 1.1e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.4e-07, @@ -48109,7 +48109,7 @@ "input_cost_per_token_flex": 7e-08, "input_cost_per_token_priority": 2.6e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, @@ -48260,7 +48260,7 @@ "input_cost_per_token_flex": 2.5e-07, "input_cost_per_token_priority": 8.75e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, @@ -70383,7 +70383,7 @@ "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 1.25e-06, @@ -70404,7 +70404,7 @@ "output_cost_per_token": 8.8e-07, "output_cost_per_token_flex": 4.4e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 4.4e-07, @@ -70425,7 +70425,7 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 32000, + "max_input_tokens": 32768, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 3e-07, @@ -70448,7 +70448,7 @@ "output_cost_per_token": 6e-07, "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 3e-07, @@ -70490,7 +70490,7 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, + "max_input_tokens": 262144, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 6e-07, diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 32a99a37870..7ae0c88a82f 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1613,6 +1613,7 @@ PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW = ( "anthropic", "azure", "azure_ai", + "bedrock", "databricks", "deepseek", "fireworks_ai", @@ -1840,6 +1841,32 @@ def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_ assert "prompt is too long" in excinfo.value.message +def test_bedrock_mantle_served_model_overflow_text_maps_to_prompt_is_too_long(): + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + original_exception: Final = BaseLLMException( + status_code=400, + message=( + '{"error":{"code":"validation_error","message":"ErrorEvent { error: APIError { type: ' + '\\"BadRequestError\\", code: Some(400), message: \\"This model\'s maximum context length is 262144 ' + "tokens. However, you requested 16 output tokens and your prompt contains at least 262129 input tokens" + '\\", param: None } }","param":null,"type":"invalid_request_error"}}' + ), + ) + + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-next-80b-a3b-instruct", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.llm_provider == "bedrock_mantle" + assert "prompt is too long" in excinfo.value.message + assert "your prompt contains at least 262129 input tokens" in excinfo.value.message + + def test_bedrock_mantle_streamed_context_overflow_event_maps_to_context_window_exceeded(): from litellm.responses.streaming_iterator import map_stream_error_to_exception @@ -2053,6 +2080,46 @@ def test_bedrock_500_preserves_provider_response_headers(): assert exc_info.value.response.headers["x-amzn-requestid"] == "req-map-500" +def test_bedrock_converse_maximum_context_length_error_maps_to_context_window_exceeded(): + original_exception: Final = BedrockError( + status_code=400, + message=( + '{"message":"The model returned the following errors: ErrorEvent { error: APIError { type: ' + '\\"BadRequestError\\", code: Some(400), message: \\"This model\'s maximum context length is 262144 ' + "tokens. However, you requested 16 output tokens and your prompt contains at least 262129 input tokens" + '\\" } }"}' + ), + ) + + with pytest.raises(litellm.ContextWindowExceededError) as exc_info: + exception_type( + model="qwen.qwen3-coder-next", + original_exception=original_exception, + custom_llm_provider="bedrock", + ) + + assert exc_info.value.status_code == 400 + assert "maximum context length is 262144 tokens" in exc_info.value.message + assert "prompt is too long" in exc_info.value.message + + +def test_bedrock_overflow_that_already_says_prompt_is_too_long_is_not_repeated(): + original_exception: Final = BedrockError( + status_code=400, + message='{"message":"prompt is too long: 210000 tokens > 200000 maximum"}', + ) + + with pytest.raises(litellm.ContextWindowExceededError) as exc_info: + exception_type( + model="anthropic.claude-sonnet-4-5-20250929-v1:0", + original_exception=original_exception, + custom_llm_provider="bedrock", + ) + + assert exc_info.value.message.count("prompt is too long") == 1 + assert "210000 tokens > 200000 maximum" in exc_info.value.message + + @pytest.mark.parametrize( "custom_llm_provider, status_code, provider_message, expected_exception", [ diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index e195b970fd5..ee482bd8073 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -3,6 +3,8 @@ from __future__ import annotations import json import re from collections.abc import Iterator, Mapping +from itertools import groupby +from operator import itemgetter from pathlib import Path from types import MappingProxyType from typing import Final @@ -1024,3 +1026,70 @@ def test_bedrock_gpt_oss_rows_declare_vision_support_like_their_safeguard_siblin def test_bedrock_gpt_oss_vision_is_explicitly_disabled_for_the_router_gate() -> None: assert litellm.utils.is_vision_explicitly_disabled("bedrock/openai.gpt-oss-120b-1:0") is True assert litellm.utils.is_vision_explicitly_disabled("bedrock_mantle/openai.gpt-oss-20b") is True + + +class _BedrockWindowRow(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + litellm_provider: str + max_input_tokens: int | None = None + + +_BEDROCK_PROVIDERS: Final = frozenset({"bedrock", "bedrock_converse", "bedrock_mantle"}) +_ROWS_NAMING_A_MODEL_UNDER_ANOTHER_ID: Final = ( + ("moonshot.kimi-k2-thinking", "moonshotai.kimi-k2-thinking"), + ("bedrock_mantle/moonshotai.kimi-k2-thinking", "moonshotai.kimi-k2-thinking"), + ("bedrock_mantle/qwen.qwen3-next-80b-a3b-instruct", "qwen.qwen3-next-80b-a3b"), + ("bedrock_mantle/qwen.qwen3-235b-a22b-2507", "qwen.qwen3-235b-a22b-2507-v1:0"), + ("bedrock_mantle/qwen.qwen3-coder-30b-a3b-instruct", "qwen.qwen3-coder-30b-a3b-v1:0"), + ("bedrock_mantle/qwen.qwen3-32b", "qwen.qwen3-32b-v1:0"), +) +_BEDROCK_PREFIXED_ROW: Final = re.compile(r"bedrock/(?:[a-z0-9-]+/)?(?P[^/]+)") + + +def _bedrock_input_windows(path: Path) -> Mapping[str, int]: + prices: Final = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) + rows: Final = tuple( + (key, _BedrockWindowRow.model_validate(row)) + for key, row in prices.items() + if isinstance(row, dict) and row.get("litellm_provider") in _BEDROCK_PROVIDERS + ) + return MappingProxyType({key: row.max_input_tokens for key, row in rows if row.max_input_tokens is not None}) + + +def _widest_prefixed_window_per_model(windows: Mapping[str, int]) -> Mapping[str, int]: + prefixed: Final = sorted( + (match["model"], window) for key, window in windows.items() if (match := _BEDROCK_PREFIXED_ROW.fullmatch(key)) + ) + return MappingProxyType( + {model: max(window for _, window in group) for model, group in groupby(prefixed, key=itemgetter(0))} + ) + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_bare_bedrock_rows_admit_every_prompt_their_prefixed_rows_admit(path: Path) -> None: + windows: Final = _bedrock_input_windows(path) + widest: Final = _widest_prefixed_window_per_model(windows) + bare_with_prefixed_rows: Final = tuple(key for key in windows if "/" not in key and key in widest) + assert bare_with_prefixed_rows, f"No bare Bedrock row with a bedrock/ counterpart in {path}" + narrower: Final = tuple( + f"{key}: {windows[key]} < {widest[key]}" for key in bare_with_prefixed_rows if windows[key] < widest[key] + ) + assert narrower == () + + +def _widest_window_of_model(windows: Mapping[str, int], model: str) -> int: + return max( + window + for key, window in windows.items() + if key == model or ((match := _BEDROCK_PREFIXED_ROW.fullmatch(key)) is not None and match["model"] == model) + ) + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +@pytest.mark.parametrize(("row", "model"), _ROWS_NAMING_A_MODEL_UNDER_ANOTHER_ID) +def test_bedrock_rows_under_another_model_id_admit_every_prompt_that_model_admits( + path: Path, row: str, model: str +) -> None: + windows: Final = _bedrock_input_windows(path) + assert windows[row] >= _widest_window_of_model(windows, model)