diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 21132db93cb..e15decc4b83 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -778,6 +778,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -865,6 +866,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -881,6 +883,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -896,6 +899,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -2103,6 +2107,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -2138,6 +2143,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -2176,6 +2182,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -8865,6 +8872,7 @@ }, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -9327,6 +9335,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -9343,6 +9352,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -9502,6 +9512,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -9518,6 +9529,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -14380,6 +14392,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -14435,6 +14448,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -14451,6 +14465,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -14466,6 +14481,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -32555,6 +32571,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -32610,6 +32627,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -32626,6 +32644,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -32641,6 +32660,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 73cefeb7c77..7122dbed42f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -778,6 +778,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -865,6 +866,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -881,6 +883,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -896,6 +899,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -2103,6 +2107,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -2138,6 +2143,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -2176,6 +2182,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -8865,6 +8872,7 @@ }, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -9327,6 +9335,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -9343,6 +9352,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -9502,6 +9512,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -9518,6 +9529,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -14380,6 +14392,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -14435,6 +14448,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -14451,6 +14465,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -14466,6 +14481,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -32732,6 +32748,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -32787,6 +32804,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, @@ -32803,6 +32821,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -32818,6 +32837,7 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, "supports_function_calling": true, "supports_pdf_input": true, "supports_response_schema": true, diff --git a/tests/test_litellm/test_claude_3_bedrock_assistant_prefill.py b/tests/test_litellm/test_claude_3_bedrock_assistant_prefill.py new file mode 100644 index 00000000000..304b0300da1 --- /dev/null +++ b/tests/test_litellm/test_claude_3_bedrock_assistant_prefill.py @@ -0,0 +1,99 @@ +""" +Regression test: the AWS Bedrock Anthropic Claude 3 family entries in the +model-cost maps were missing ``supports_assistant_prefill`` entirely. Because +the key was absent rather than ``true``, callers that gate on its truthiness +silently treated these models as not supporting trailing-assistant prefill, +even though every Claude 3 model supports it (the direct ``anthropic`` API +entries for ``claude-3-haiku``/``claude-3-opus`` already carry ``true``). + +These tests pin ``supports_assistant_prefill == true`` for the affected entries +in both the primary price map and the ``litellm/`` backup, and verify +``get_model_info`` surfaces it, so the field cannot silently drop back to absent. +""" + +import json +import os +import sys + +sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path + +import litellm + +CLAUDE_3_BEDROCK_MODELS = ( + "anthropic.claude-3-haiku-20240307-v1:0", + "anthropic.claude-3-sonnet-20240229-v1:0", + "anthropic.claude-3-opus-20240229-v1:0", + "anthropic.claude-3-5-sonnet-20240620-v1:0", + "us.anthropic.claude-3-haiku-20240307-v1:0", + "us.anthropic.claude-3-sonnet-20240229-v1:0", + "us.anthropic.claude-3-opus-20240229-v1:0", + "us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "eu.anthropic.claude-3-haiku-20240307-v1:0", + "eu.anthropic.claude-3-sonnet-20240229-v1:0", + "eu.anthropic.claude-3-opus-20240229-v1:0", + "eu.anthropic.claude-3-5-sonnet-20240620-v1:0", + "apac.anthropic.claude-3-haiku-20240307-v1:0", + "apac.anthropic.claude-3-sonnet-20240229-v1:0", + "apac.anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0", + "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0", + "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0", +) + + +def _load_json(path: str) -> dict: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _backup_path() -> str: + return os.path.join( + os.path.dirname(litellm.__file__), + "model_prices_and_context_window_backup.json", + ) + + +def _main_path() -> str: + # This test lives at ``tests/test_litellm/``; the primary price map sits at + # the repo root, two directories up. Resolve it relative to this file so the + # test works regardless of where ``litellm`` itself is installed. + return os.path.join( + os.path.dirname(__file__), + "..", + "..", + "model_prices_and_context_window.json", + ) + + +class TestClaude3BedrockAssistantPrefillData: + """Both JSON price maps must mark every Claude 3 Bedrock entry as supporting + assistant prefill.""" + + def test_backup_supports_assistant_prefill(self): + price_map = _load_json(_backup_path()) + for model in CLAUDE_3_BEDROCK_MODELS: + assert price_map[model]["supports_assistant_prefill"] is True, model + + def test_main_supports_assistant_prefill(self): + price_map = _load_json(_main_path()) + for model in CLAUDE_3_BEDROCK_MODELS: + assert price_map[model]["supports_assistant_prefill"] is True, model + + +class TestClaude3BedrockAssistantPrefillModelInfo: + """``get_model_info`` must report ``supports_assistant_prefill`` for these + models.""" + + def test_get_model_info_supports_assistant_prefill(self): + # Patch litellm.model_cost with the local backup so the assertion does + # not depend on the remote fetch hitting a not-yet-merged main branch. + original = litellm.model_cost + try: + litellm.model_cost = _load_json(_backup_path()) + for model in CLAUDE_3_BEDROCK_MODELS: + info = litellm.get_model_info(model) + assert info["supports_assistant_prefill"] is True, model + finally: + litellm.model_cost = original