mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
fix(bedrock_mantle): correct grok-4.3 and gemma-4 token limits
Four bedrock_mantle entries advertise token limits that do not match what the service enforces. Verified by probing the live endpoint (bedrock-mantle us-east-2) and reading the caps it reports back: model field current actual xai.grok-4.3 input 131072 1048576 xai.grok-4.3 output 16384 1048576 google.gemma-4-31b in 256000 262144 google.gemma-4-26b-a4b in 256000 262144 google.gemma-4-e2b in 128000 131072 grok-4.3 is the significant one: the input limit is understated 8x. As with the GPT-5.6 entries, the non-Bedrock counterpart already carries the larger value (xai/grok-4.3 reads 1000000), which is what suggested the bedrock_mantle numbers were stale rather than deliberate. Evidence, input: "prompt tokens (1300031) exceed customer model maximum (1048576)" and a real request with 394937 prompt tokens (3x the advertised max) completed successfully on xai.grok-4.3, correctly retrieving a fact planted at 85% depth. Evidence, output: max_tokens=1048576 accepted; 1048577 -> "'max_tokens' (1048577) exceeds model maximum (1048576)" The gemma-4 models behave the same way, so their max_output_tokens and max_tokens move with the input value (they were already equal). The understated values cause premature context-window errors and early auto-compaction, and clamp max_tokens far below what the model allows. Left alone deliberately: openai.gpt-oss-* keep 131072 (they are not served on the Responses or /openai/v1 chat routes, so the cap could not be probed the same way) and the openai.gpt-5.x entries, which are handled separately.
This commit is contained in:
parent
97a59c8c90
commit
c8e4b28319
3 changed files with 44 additions and 27 deletions
|
|
@ -45394,9 +45394,9 @@
|
|||
"input_cost_per_token": 1.4e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 256000,
|
||||
"max_output_tokens": 256000,
|
||||
"max_tokens": 256000,
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"use_openai_responses_path": true,
|
||||
"supported_endpoints": [
|
||||
|
|
@ -45413,9 +45413,9 @@
|
|||
"input_cost_per_token": 1.3e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 256000,
|
||||
"max_output_tokens": 256000,
|
||||
"max_tokens": 256000,
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"use_openai_responses_path": true,
|
||||
"supported_endpoints": [
|
||||
|
|
@ -45432,9 +45432,9 @@
|
|||
"input_cost_per_token": 4e-08,
|
||||
"output_cost_per_token": 8e-08,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"use_openai_responses_path": true,
|
||||
"supported_endpoints": [
|
||||
|
|
@ -45453,9 +45453,9 @@
|
|||
"output_cost_per_token": 2.5e-06,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 16384,
|
||||
"max_tokens": 16384,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"max_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
|
|||
|
|
@ -45515,9 +45515,9 @@
|
|||
"input_cost_per_token": 1.4e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 256000,
|
||||
"max_output_tokens": 256000,
|
||||
"max_tokens": 256000,
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"use_openai_responses_path": true,
|
||||
"supported_endpoints": [
|
||||
|
|
@ -45534,9 +45534,9 @@
|
|||
"input_cost_per_token": 1.3e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 256000,
|
||||
"max_output_tokens": 256000,
|
||||
"max_tokens": 256000,
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"max_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"use_openai_responses_path": true,
|
||||
"supported_endpoints": [
|
||||
|
|
@ -45553,9 +45553,9 @@
|
|||
"input_cost_per_token": 4e-08,
|
||||
"output_cost_per_token": 8e-08,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"use_openai_responses_path": true,
|
||||
"supported_endpoints": [
|
||||
|
|
@ -45574,9 +45574,9 @@
|
|||
"output_cost_per_token": 2.5e-06,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 16384,
|
||||
"max_tokens": 16384,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"max_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
|
|||
|
|
@ -641,9 +641,9 @@ class TestBedrockMantlePricing:
|
|||
@pytest.mark.parametrize(
|
||||
"model_id,input_cost,output_cost,max_tokens",
|
||||
[
|
||||
("google.gemma-4-31b", 1.4e-07, 4e-07, 256000),
|
||||
("google.gemma-4-26b-a4b", 1.3e-07, 4e-07, 256000),
|
||||
("google.gemma-4-e2b", 4e-08, 8e-08, 128000),
|
||||
("google.gemma-4-31b", 1.4e-07, 4e-07, 262144),
|
||||
("google.gemma-4-26b-a4b", 1.3e-07, 4e-07, 262144),
|
||||
("google.gemma-4-e2b", 4e-08, 8e-08, 131072),
|
||||
],
|
||||
)
|
||||
def test_gemma_4_bedrock_mantle_model_metadata(
|
||||
|
|
@ -685,3 +685,20 @@ def test_gemma_4_models_register_under_bedrock_mantle(local_cost_map, model_id):
|
|||
resolved_model, provider, _, _ = litellm.get_llm_provider(full_model_name)
|
||||
assert provider == "bedrock_mantle"
|
||||
assert resolved_model == model_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id, expected_tokens",
|
||||
[
|
||||
("xai.grok-4.3", 1048576),
|
||||
("google.gemma-4-31b", 262144),
|
||||
("google.gemma-4-26b-a4b", 262144),
|
||||
("google.gemma-4-e2b", 131072),
|
||||
],
|
||||
)
|
||||
def test_context_windows_match_bedrock_limits(
|
||||
local_cost_map, model_id, expected_tokens
|
||||
):
|
||||
info = litellm.get_model_info(model=model_id, custom_llm_provider="bedrock_mantle")
|
||||
assert info["max_input_tokens"] == expected_tokens
|
||||
assert info["max_output_tokens"] == expected_tokens
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue