From c8e4b28319aae9c6ef40473198cdb3c535cd8b2d Mon Sep 17 00:00:00 2001 From: Jairo Suarez Date: Mon, 10 Aug 2026 10:15:53 +0200 Subject: [PATCH] fix(bedrock_mantle): correct grok-4.3 and gemma-4 token limits Four bedrock_mantle entries advertise token limits that do not match what the service enforces. Verified by probing the live endpoint (bedrock-mantle us-east-2) and reading the caps it reports back: model field current actual xai.grok-4.3 input 131072 1048576 xai.grok-4.3 output 16384 1048576 google.gemma-4-31b in 256000 262144 google.gemma-4-26b-a4b in 256000 262144 google.gemma-4-e2b in 128000 131072 grok-4.3 is the significant one: the input limit is understated 8x. As with the GPT-5.6 entries, the non-Bedrock counterpart already carries the larger value (xai/grok-4.3 reads 1000000), which is what suggested the bedrock_mantle numbers were stale rather than deliberate. Evidence, input: "prompt tokens (1300031) exceed customer model maximum (1048576)" and a real request with 394937 prompt tokens (3x the advertised max) completed successfully on xai.grok-4.3, correctly retrieving a fact planted at 85% depth. Evidence, output: max_tokens=1048576 accepted; 1048577 -> "'max_tokens' (1048577) exceeds model maximum (1048576)" The gemma-4 models behave the same way, so their max_output_tokens and max_tokens move with the input value (they were already equal). The understated values cause premature context-window errors and early auto-compaction, and clamp max_tokens far below what the model allows. Left alone deliberately: openai.gpt-oss-* keep 131072 (they are not served on the Responses or /openai/v1 chat routes, so the cap could not be probed the same way) and the openai.gpt-5.x entries, which are handled separately. --- ...odel_prices_and_context_window_backup.json | 24 +++++++++---------- model_prices_and_context_window.json | 24 +++++++++---------- .../test_bedrock_mantle_transformation.py | 23 +++++++++++++++--- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d954e33da9c..e108b7d853d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45394,9 +45394,9 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 4e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -45413,9 +45413,9 @@ "input_cost_per_token": 1.3e-07, "output_cost_per_token": 4e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -45432,9 +45432,9 @@ "input_cost_per_token": 4e-08, "output_cost_per_token": 8e-08, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -45453,9 +45453,9 @@ "output_cost_per_token": 2.5e-06, "cache_read_input_token_cost": 2e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 131072, - "max_output_tokens": 16384, - "max_tokens": 16384, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8982b4f2565..bcb60ad471f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45515,9 +45515,9 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 4e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -45534,9 +45534,9 @@ "input_cost_per_token": 1.3e-07, "output_cost_per_token": 4e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -45553,9 +45553,9 @@ "input_cost_per_token": 4e-08, "output_cost_per_token": 8e-08, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -45574,9 +45574,9 @@ "output_cost_per_token": 2.5e-06, "cache_read_input_token_cost": 2e-07, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 131072, - "max_output_tokens": 16384, - "max_tokens": 16384, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 275fb460b9f..480492dda26 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -641,9 +641,9 @@ class TestBedrockMantlePricing: @pytest.mark.parametrize( "model_id,input_cost,output_cost,max_tokens", [ - ("google.gemma-4-31b", 1.4e-07, 4e-07, 256000), - ("google.gemma-4-26b-a4b", 1.3e-07, 4e-07, 256000), - ("google.gemma-4-e2b", 4e-08, 8e-08, 128000), + ("google.gemma-4-31b", 1.4e-07, 4e-07, 262144), + ("google.gemma-4-26b-a4b", 1.3e-07, 4e-07, 262144), + ("google.gemma-4-e2b", 4e-08, 8e-08, 131072), ], ) def test_gemma_4_bedrock_mantle_model_metadata( @@ -685,3 +685,20 @@ def test_gemma_4_models_register_under_bedrock_mantle(local_cost_map, model_id): resolved_model, provider, _, _ = litellm.get_llm_provider(full_model_name) assert provider == "bedrock_mantle" assert resolved_model == model_id + + +@pytest.mark.parametrize( + "model_id, expected_tokens", + [ + ("xai.grok-4.3", 1048576), + ("google.gemma-4-31b", 262144), + ("google.gemma-4-26b-a4b", 262144), + ("google.gemma-4-e2b", 131072), + ], +) +def test_context_windows_match_bedrock_limits( + local_cost_map, model_id, expected_tokens +): + info = litellm.get_model_info(model=model_id, custom_llm_provider="bedrock_mantle") + assert info["max_input_tokens"] == expected_tokens + assert info["max_output_tokens"] == expected_tokens