fix(bedrock_mantle): correct grok-4.3 and gemma-4 token limits

Four bedrock_mantle entries advertise token limits that do not match what
the service enforces. Verified by probing the live endpoint
(bedrock-mantle us-east-2) and reading the caps it reports back:

  model                  field  current    actual
  xai.grok-4.3           input   131072   1048576
  xai.grok-4.3          output    16384   1048576
  google.gemma-4-31b        in   256000    262144
  google.gemma-4-26b-a4b    in   256000    262144
  google.gemma-4-e2b        in   128000    131072

grok-4.3 is the significant one: the input limit is understated 8x. As
with the GPT-5.6 entries, the non-Bedrock counterpart already carries the
larger value (xai/grok-4.3 reads 1000000), which is what suggested the
bedrock_mantle numbers were stale rather than deliberate.

Evidence, input:
  "prompt tokens (1300031) exceed customer model maximum (1048576)"
and a real request with 394937 prompt tokens (3x the advertised max)
completed successfully on xai.grok-4.3, correctly retrieving a fact
planted at 85% depth.

Evidence, output:
  max_tokens=1048576 accepted; 1048577 -> "'max_tokens' (1048577) exceeds
  model maximum (1048576)"
The gemma-4 models behave the same way, so their max_output_tokens and
max_tokens move with the input value (they were already equal).

The understated values cause premature context-window errors and early
auto-compaction, and clamp max_tokens far below what the model allows.

Left alone deliberately: openai.gpt-oss-* keep 131072 (they are not
served on the Responses or /openai/v1 chat routes, so the cap could not
be probed the same way) and the openai.gpt-5.x entries, which are handled
separately.
This commit is contained in:
Jairo Suarez 2026-08-10 10:15:53 +02:00
parent 97a59c8c90
commit c8e4b28319
3 changed files with 44 additions and 27 deletions

View file

@ -45394,9 +45394,9 @@
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 256000,
"max_output_tokens": 256000,
"max_tokens": 256000,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"use_openai_responses_path": true,
"supported_endpoints": [
@ -45413,9 +45413,9 @@
"input_cost_per_token": 1.3e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 256000,
"max_output_tokens": 256000,
"max_tokens": 256000,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"use_openai_responses_path": true,
"supported_endpoints": [
@ -45432,9 +45432,9 @@
"input_cost_per_token": 4e-08,
"output_cost_per_token": 8e-08,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"use_openai_responses_path": true,
"supported_endpoints": [
@ -45453,9 +45453,9 @@
"output_cost_per_token": 2.5e-06,
"cache_read_input_token_cost": 2e-07,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 131072,
"max_output_tokens": 16384,
"max_tokens": 16384,
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
"max_tokens": 1048576,
"mode": "chat",
"supported_endpoints": [
"/v1/chat/completions",

View file

@ -45515,9 +45515,9 @@
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 256000,
"max_output_tokens": 256000,
"max_tokens": 256000,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"use_openai_responses_path": true,
"supported_endpoints": [
@ -45534,9 +45534,9 @@
"input_cost_per_token": 1.3e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 256000,
"max_output_tokens": 256000,
"max_tokens": 256000,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"use_openai_responses_path": true,
"supported_endpoints": [
@ -45553,9 +45553,9 @@
"input_cost_per_token": 4e-08,
"output_cost_per_token": 8e-08,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"use_openai_responses_path": true,
"supported_endpoints": [
@ -45574,9 +45574,9 @@
"output_cost_per_token": 2.5e-06,
"cache_read_input_token_cost": 2e-07,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 131072,
"max_output_tokens": 16384,
"max_tokens": 16384,
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
"max_tokens": 1048576,
"mode": "chat",
"supported_endpoints": [
"/v1/chat/completions",

View file

@ -641,9 +641,9 @@ class TestBedrockMantlePricing:
@pytest.mark.parametrize(
"model_id,input_cost,output_cost,max_tokens",
[
("google.gemma-4-31b", 1.4e-07, 4e-07, 256000),
("google.gemma-4-26b-a4b", 1.3e-07, 4e-07, 256000),
("google.gemma-4-e2b", 4e-08, 8e-08, 128000),
("google.gemma-4-31b", 1.4e-07, 4e-07, 262144),
("google.gemma-4-26b-a4b", 1.3e-07, 4e-07, 262144),
("google.gemma-4-e2b", 4e-08, 8e-08, 131072),
],
)
def test_gemma_4_bedrock_mantle_model_metadata(
@ -685,3 +685,20 @@ def test_gemma_4_models_register_under_bedrock_mantle(local_cost_map, model_id):
resolved_model, provider, _, _ = litellm.get_llm_provider(full_model_name)
assert provider == "bedrock_mantle"
assert resolved_model == model_id
@pytest.mark.parametrize(
"model_id, expected_tokens",
[
("xai.grok-4.3", 1048576),
("google.gemma-4-31b", 262144),
("google.gemma-4-26b-a4b", 262144),
("google.gemma-4-e2b", 131072),
],
)
def test_context_windows_match_bedrock_limits(
local_cost_map, model_id, expected_tokens
):
info = litellm.get_model_info(model=model_id, custom_llm_provider="bedrock_mantle")
assert info["max_input_tokens"] == expected_tokens
assert info["max_output_tokens"] == expected_tokens