scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-04 02:31:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 69

Merge pull request #24633 from Sameerlite/litellm_fix_opus_tests

Browse source 
Litellm fix opus tests
This commit is contained in:
yuneng-jiang 2026-03-26 22:06:36 -07:00 • committed by GitHub
commit 958cadebf3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 71 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
litellm/model_prices_and_context_window_backup.json
View file

@ -1104,7 +1104,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -1130,7 +1130,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -1156,7 +1156,7 @@
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -1182,7 +1182,7 @@
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -1208,7 +1208,7 @@
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -1791,7 +1791,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -8466,7 +8466,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -30352,7 +30352,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -37092,7 +37092,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",

18
model_prices_and_context_window.json
View file

@ -1104,7 +1104,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -1130,7 +1130,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -1156,7 +1156,7 @@
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -1182,7 +1182,7 @@
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -1208,7 +1208,7 @@
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -1791,7 +1791,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -8466,7 +8466,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -30352,7 +30352,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
@ -37092,7 +37092,7 @@
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",

98
tests/test_litellm/test_claude_opus_4_6_config.py
View file

@ -24,70 +24,82 @@ def test_claude_4_6_australia_region_uses_au_prefix_not_apac():
Related: The 'apac.' prefix is valid for Asia-Pacific (Singapore) region models,
but should not be used for Australia which has its own 'au.' prefix.
"""
json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
# Verify au.anthropic.claude-opus-4-6-v1 exists (correct)
assert "au.anthropic.claude-opus-4-6-v1" in model_data, \
"Missing Australia region model: au.anthropic.claude-opus-4-6-v1"
assert (
"au.anthropic.claude-opus-4-6-v1" in model_data
), "Missing Australia region model: au.anthropic.claude-opus-4-6-v1"
# Verify apac.anthropic.claude-opus-4-6-v1 does NOT exist (incorrect)
assert "apac.anthropic.claude-opus-4-6-v1" not in model_data, \
"Incorrect model entry exists: apac.anthropic.claude-opus-4-6-v1 should be au.anthropic.claude-opus-4-6-v1"
assert (
"apac.anthropic.claude-opus-4-6-v1" not in model_data
), "Incorrect model entry exists: apac.anthropic.claude-opus-4-6-v1 should be au.anthropic.claude-opus-4-6-v1"
# Verify au.anthropic.claude-sonnet-4-6 exists (correct)
assert "au.anthropic.claude-sonnet-4-6" in model_data, \
"Missing Australia region model: au.anthropic.claude-sonnet-4-6"
assert (
"au.anthropic.claude-sonnet-4-6" in model_data
), "Missing Australia region model: au.anthropic.claude-sonnet-4-6"
# Verify apac.anthropic.claude-sonnet-4-6 does NOT exist (incorrect)
assert "apac.anthropic.claude-sonnet-4-6" not in model_data, \
"Incorrect model entry exists: apac.anthropic.claude-sonnet-4-6 should be au.anthropic.claude-sonnet-4-6"
assert (
"apac.anthropic.claude-sonnet-4-6" not in model_data
), "Incorrect model entry exists: apac.anthropic.claude-sonnet-4-6 should be au.anthropic.claude-sonnet-4-6"
# Verify the au. model is registered in bedrock_converse_models
assert "au.anthropic.claude-opus-4-6-v1" in litellm.bedrock_converse_models, \
"au.anthropic.claude-opus-4-6-v1 not registered in bedrock_converse_models"
assert (
"au.anthropic.claude-opus-4-6-v1" in litellm.bedrock_converse_models
), "au.anthropic.claude-opus-4-6-v1 not registered in bedrock_converse_models"
# Verify apac. is NOT registered for this model
assert "apac.anthropic.claude-opus-4-6-v1" not in litellm.bedrock_converse_models, \
"apac.anthropic.claude-opus-4-6-v1 should not be in bedrock_converse_models"
assert (
"apac.anthropic.claude-opus-4-6-v1" not in litellm.bedrock_converse_models
), "apac.anthropic.claude-opus-4-6-v1 should not be in bedrock_converse_models"
# Verify the au. model is registered in bedrock_converse_models
assert "au.anthropic.claude-sonnet-4-6" in litellm.bedrock_converse_models, \
"au.anthropic.claude-sonnet-4-6 not registered in bedrock_converse_models"
assert (
"au.anthropic.claude-sonnet-4-6" in litellm.bedrock_converse_models
), "au.anthropic.claude-sonnet-4-6 not registered in bedrock_converse_models"
# Verify apac. is NOT registered for this model
assert "apac.anthropic.claude-sonnet-4-6" not in litellm.bedrock_converse_models, \
"apac.anthropic.claude-sonnet-4-6 should not be in bedrock_converse_models"
assert (
"apac.anthropic.claude-sonnet-4-6" not in litellm.bedrock_converse_models
), "apac.anthropic.claude-sonnet-4-6 should not be in bedrock_converse_models"
def test_opus_4_6_model_pricing_and_capabilities():
json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
expected_models = {
"claude-opus-4-6": {
"provider": "anthropic",
"has_long_context_pricing": True,
"has_long_context_pricing": False,
"tool_use_system_prompt_tokens": 346,
"max_input_tokens": 1000000,
},
"claude-opus-4-6-20260205": {
"provider": "anthropic",
"has_long_context_pricing": True,
"has_long_context_pricing": False,
"tool_use_system_prompt_tokens": 346,
"max_input_tokens": 1000000,
},
"anthropic.claude-opus-4-6-v1": {
"provider": "bedrock_converse",
"has_long_context_pricing": True,
"has_long_context_pricing": False,
"tool_use_system_prompt_tokens": 346,
"max_input_tokens": 1000000,
},
"vertex_ai/claude-opus-4-6": {
"provider": "vertex_ai-anthropic_models",
"has_long_context_pricing": True,
"has_long_context_pricing": False,
"tool_use_system_prompt_tokens": 346,
"max_input_tokens": 1000000,
},
@ -119,6 +131,11 @@ def test_opus_4_6_model_pricing_and_capabilities():
assert info["output_cost_per_token_above_200k_tokens"] == 3.75e-05
assert info["cache_creation_input_token_cost_above_200k_tokens"] == 1.25e-05
assert info["cache_read_input_token_cost_above_200k_tokens"] == 1e-06
else:
assert "input_cost_per_token_above_200k_tokens" not in info
assert "output_cost_per_token_above_200k_tokens" not in info
assert "cache_creation_input_token_cost_above_200k_tokens" not in info
assert "cache_read_input_token_cost_above_200k_tokens" not in info
assert info["supports_assistant_prefill"] is False
assert info["supports_function_calling"] is True
@ -126,11 +143,16 @@ def test_opus_4_6_model_pricing_and_capabilities():
assert info["supports_reasoning"] is True
assert info["supports_tool_choice"] is True
assert info["supports_vision"] is True
assert info["tool_use_system_prompt_tokens"] == config["tool_use_system_prompt_tokens"]
assert (
info["tool_use_system_prompt_tokens"]
== config["tool_use_system_prompt_tokens"]
)
def test_opus_4_6_bedrock_regional_model_pricing():
json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
@ -140,40 +162,24 @@ def test_opus_4_6_bedrock_regional_model_pricing():
"output_cost_per_token": 2.5e-05,
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token_above_200k_tokens": 1e-05,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
},
"us.anthropic.claude-opus-4-6-v1": {
"input_cost_per_token": 5.5e-06,
"output_cost_per_token": 2.75e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_read_input_token_cost": 5.5e-07,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
},
"eu.anthropic.claude-opus-4-6-v1": {
"input_cost_per_token": 5.5e-06,
"output_cost_per_token": 2.75e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_read_input_token_cost": 5.5e-07,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
},
"au.anthropic.claude-opus-4-6-v1": {
"input_cost_per_token": 5.5e-06,
"output_cost_per_token": 2.75e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_read_input_token_cost": 5.5e-07,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
},
}
@ -186,12 +192,18 @@ def test_opus_4_6_bedrock_regional_model_pricing():
assert info["max_tokens"] == 128000
assert info["supports_assistant_prefill"] is False
assert info["tool_use_system_prompt_tokens"] == 346
assert "input_cost_per_token_above_200k_tokens" not in info
assert "output_cost_per_token_above_200k_tokens" not in info
assert "cache_creation_input_token_cost_above_200k_tokens" not in info
assert "cache_read_input_token_cost_above_200k_tokens" not in info
for key, value in expected.items():
assert info[key] == value
def test_opus_4_6_alias_and_dated_metadata_match():
json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
json_path = os.path.join(
os.path.dirname(__file__), "../../model_prices_and_context_window.json"
)
with open(json_path) as f:
model_data = json.load(f)
@ -207,10 +219,6 @@ def test_opus_4_6_alias_and_dated_metadata_match():
"cache_creation_input_token_cost",
"cache_creation_input_token_cost_above_1hr",
"cache_read_input_token_cost",
"input_cost_per_token_above_200k_tokens",
"output_cost_per_token_above_200k_tokens",
"cache_creation_input_token_cost_above_200k_tokens",
"cache_read_input_token_cost_above_200k_tokens",
"supports_assistant_prefill",
"tool_use_system_prompt_tokens",
]