From 6ee12cd9322f01d301bc1b20d506056f23ab2353 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 20 Oct 2026 15:59:08 -0700 Subject: [PATCH 1/6] feat(count_tokens): mark Bedrock models AWS cannot count as local in the cost map and skip the CountTokens call AWS CountTokens answers 400 "The provided model doesn't support counting tokens." for every non-Anthropic Bedrock model, so a count request for Nova, Llama, Mistral and the rest was doomed before it left the proxy. The cost map now carries token_count_model.litellm_provider "local" on those rows and the proxy answers the local estimate straight away, with litellm_count_source "local_estimate" in the response. A deployment's model_info.token_count_model still overrides the cost map in both directions. Resolves LIT-9763 --- ...odel_prices_and_context_window_backup.json | 1390 +++++++++++++---- .../common_utils/token_count_settings.py | 37 +- litellm/proxy/proxy_server.py | 26 +- litellm/types/token_count.py | 5 +- model_prices_and_context_window.json | 1390 +++++++++++++---- model_prices_and_context_window.schema.json | 29 +- .../test_bedrock_mantle_count_tokens_wire.py | 2 +- .../providers/test_count_tokens_model_wire.py | 73 +- .../common_utils/test_token_count_settings.py | 48 +- tests/unit/proxy/test_proxy_token_counter.py | 235 ++- 10 files changed, 2615 insertions(+), 620 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 15eba7f8af8..fafb8f4ec7c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -82,7 +82,10 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.25e-05, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "ai21.j2-ultra-v1": { "input_cost_per_token": 1.88e-05, @@ -92,7 +95,10 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.88e-05, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "ai21.jamba-1-5-large-v1:0": { "deprecation_date": "2026-11-26", @@ -103,7 +109,10 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 8e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md", + "token_count_model": { + "litellm_provider": "local" + } }, "ai21.jamba-1-5-mini-v1:0": { "deprecation_date": "2026-11-26", @@ -114,7 +123,10 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md", + "token_count_model": { + "litellm_provider": "local" + } }, "ai21.jamba-instruct-v1:0": { "input_cost_per_token": 5e-07, @@ -125,7 +137,10 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_system_messages": true + "supports_system_messages": true, + "token_count_model": { + "litellm_provider": "local" + } }, "aiml/dall-e-2": { "litellm_provider": "aiml", @@ -303,7 +318,10 @@ "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "supports_function_calling": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.writer.palmyra-x5-v1:0": { "deprecation_date": "2027-03-21", @@ -316,7 +334,10 @@ "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", "supports_function_calling": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "writer.palmyra-x4-v1:0": { "deprecation_date": "2027-03-09", @@ -329,7 +350,10 @@ "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "supports_function_calling": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "writer.palmyra-x5-v1:0": { "deprecation_date": "2027-03-21", @@ -342,7 +366,10 @@ "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", "supports_function_calling": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "writer.palmyra-vision-7b": { "input_cost_per_token": 1.5e-07, @@ -359,7 +386,10 @@ "output_cost_per_token_flex": 3e-07, "output_cost_per_token_priority": 1.05e-06, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-lite-v1:0": { "cache_read_input_token_cost": 1.5e-08, @@ -378,7 +408,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -402,7 +435,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 3.125e-07, @@ -427,7 +463,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "apac.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 9.9e-08, @@ -445,7 +484,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "apac.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 3.4375e-07, @@ -466,7 +508,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 1.0725e-07, @@ -484,7 +529,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 3.4375e-07, @@ -505,7 +553,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 8.25e-08, @@ -532,7 +583,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 3.4375e-07, @@ -557,7 +611,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", @@ -591,7 +648,10 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-premier-v1:0": { "cache_read_input_token_cost": 6.25e-07, @@ -613,7 +673,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, @@ -638,7 +701,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-2-sonic-v1:0": { "input_cost_per_audio_token": 3e-06, @@ -822,7 +888,10 @@ "litellm_provider": "bedrock", "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", @@ -831,7 +900,10 @@ "litellm_provider": "bedrock", "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", @@ -840,7 +912,10 @@ "litellm_provider": "bedrock", "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "global.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", @@ -849,7 +924,10 @@ "litellm_provider": "bedrock", "mode": "chat", "supports_video_input": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.titan-text-express-v1": { "input_cost_per_token": 2e-07, @@ -858,7 +936,10 @@ "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", - "output_cost_per_token": 6e-07 + "output_cost_per_token": 6e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.titan-text-lite-v1": { "input_cost_per_token": 1.5e-07, @@ -867,7 +948,10 @@ "max_output_tokens": 4000, "max_tokens": 4000, "mode": "chat", - "output_cost_per_token": 2e-07 + "output_cost_per_token": 2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.titan-text-premier-v1:0": { "input_cost_per_token": 5e-07, @@ -876,7 +960,10 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", - "output_cost_per_token": 1.5e-06 + "output_cost_per_token": 1.5e-06, + "token_count_model": { + "litellm_provider": "local" + } }, "anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -3461,7 +3548,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "apac.amazon.nova-micro-v1:0": { "cache_read_input_token_cost": 9.25e-09, @@ -3475,7 +3565,10 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "apac.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.1e-07, @@ -3494,7 +3587,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "apac.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -13460,7 +13556,10 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/*/1-month-commitment/cohere.command-text-v14": { "cost_per_second": 0.011, @@ -13470,7 +13569,10 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/*/6-month-commitment/cohere.command-light-text-v14": { "cost_per_second": 0.0011416, @@ -13480,7 +13582,10 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/*/6-month-commitment/cohere.command-text-v14": { "cost_per_second": 0.0066027, @@ -13490,7 +13595,10 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/guardrails": { "guardrail_cost_per_unit": { @@ -13612,7 +13720,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -13631,7 +13742,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -13650,7 +13764,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13665,7 +13782,10 @@ "output_cost_per_token_flex": 1.52e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -13683,7 +13803,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13701,7 +13824,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13716,7 +13842,10 @@ "output_cost_per_token_flex": 1.25e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, @@ -13736,7 +13865,10 @@ "supports_video_input": true, "supports_vision": true, "supports_audio_input": false, - "supports_response_schema": true + "supports_response_schema": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -13747,7 +13879,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -13758,7 +13893,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -13777,7 +13915,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -13796,7 +13937,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -13815,7 +13959,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13830,7 +13977,10 @@ "output_cost_per_token_flex": 1.47e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -13848,7 +13998,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13866,7 +14019,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { "input_cost_per_token": 3.1e-07, @@ -13885,7 +14041,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.24e-06, - "output_cost_per_token_flex": 6.2e-07 + "output_cost_per_token_flex": 6.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -13904,7 +14063,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -13923,7 +14085,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -13942,7 +14107,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -13960,7 +14128,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13978,7 +14149,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -13989,7 +14163,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.03e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ca-central-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14000,7 +14177,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.9e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-north-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -14019,7 +14199,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14038,7 +14221,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-north-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14057,7 +14243,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -14075,7 +14264,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.01635, @@ -14181,7 +14373,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-central-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14200,7 +14395,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14218,7 +14416,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14229,7 +14430,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.78e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14240,7 +14444,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.5e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14259,7 +14466,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14278,7 +14488,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14296,7 +14509,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14307,7 +14523,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.55e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14318,7 +14537,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.8e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14337,7 +14559,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 4.7e-07, @@ -14356,7 +14581,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.86e-06, - "output_cost_per_token_flex": 9.3e-07 + "output_cost_per_token_flex": 9.3e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 2.3e-07, @@ -14375,7 +14603,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, @@ -14393,7 +14624,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", @@ -14405,7 +14639,10 @@ "mode": "chat", "output_cost_per_token": 2.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-3/mistral.mistral-large-2402-v1:0": { "input_cost_per_token": 5.2e-06, @@ -14416,7 +14653,10 @@ "mode": "chat", "output_cost_per_token": 1.56e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", - "supports_function_calling": true + "supports_function_calling": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", @@ -14428,7 +14668,10 @@ "mode": "chat", "output_cost_per_token": 9.1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14447,7 +14690,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14466,7 +14712,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14484,7 +14733,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -14513,7 +14765,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.88e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14524,7 +14779,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.01e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -14543,7 +14801,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14562,7 +14823,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14581,7 +14845,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14596,7 +14863,10 @@ "output_cost_per_token_flex": 1.52e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -14614,7 +14884,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14632,7 +14905,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.011, @@ -14731,7 +15007,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14742,7 +15021,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", @@ -14754,7 +15036,10 @@ "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/mistral.mistral-large-2402-v1:0": { "input_cost_per_token": 4e-06, @@ -14765,7 +15050,10 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", - "supports_function_calling": true + "supports_function_calling": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", @@ -14777,7 +15065,10 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, @@ -14796,7 +15087,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14815,7 +15109,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, @@ -14833,7 +15130,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14848,7 +15148,10 @@ "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, @@ -14866,7 +15169,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -14884,7 +15190,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, @@ -14903,7 +15212,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14922,7 +15234,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, @@ -14941,7 +15256,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.2e-06, - "output_cost_per_token_flex": 6e-07 + "output_cost_per_token_flex": 6e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14956,7 +15274,10 @@ "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, @@ -14974,7 +15295,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -14992,7 +15316,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.4e-07, @@ -15008,7 +15335,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -15035,7 +15365,10 @@ "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", - "output_cost_per_token": 1.7e-06 + "output_cost_per_token": 1.7e-06, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/amazon.titan-text-lite-v1": { "input_cost_per_token": 3e-07, @@ -15044,7 +15377,10 @@ "max_output_tokens": 4000, "max_tokens": 4000, "mode": "chat", - "output_cost_per_token": 4e-07 + "output_cost_per_token": 4e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/amazon.titan-text-premier-v1:0": { "input_cost_per_token": 5e-07, @@ -15053,7 +15389,10 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", - "output_cost_per_token": 1.5e-06 + "output_cost_per_token": 1.5e-06, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -15117,7 +15456,10 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/meta.llama3-8b-instruct-v1:0": { "input_cost_per_token": 3e-07, @@ -15127,7 +15469,10 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.65e-06, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", @@ -15159,7 +15504,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.nova-micro-v1:0": { "cache_read_input_token_cost": 1.05e-08, @@ -15173,7 +15521,10 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.4e-07, @@ -15189,7 +15540,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -15216,7 +15570,10 @@ "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", - "output_cost_per_token": 1.6e-06 + "output_cost_per_token": 1.6e-06, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.titan-text-lite-v1": { "input_cost_per_token": 3e-07, @@ -15225,7 +15582,10 @@ "max_output_tokens": 4000, "max_tokens": 4000, "mode": "chat", - "output_cost_per_token": 4e-07 + "output_cost_per_token": 4e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.titan-text-premier-v1:0": { "input_cost_per_token": 5e-07, @@ -15234,7 +15594,10 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", - "output_cost_per_token": 1.5e-06 + "output_cost_per_token": 1.5e-06, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -15298,7 +15661,10 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": { "input_cost_per_token": 3e-07, @@ -15308,7 +15674,10 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6e-07, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -15319,7 +15688,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -15330,7 +15702,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.011, @@ -15430,7 +15805,10 @@ "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/mistral.mistral-large-2402-v1:0": { "input_cost_per_token": 4e-06, @@ -15441,7 +15819,10 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", - "supports_function_calling": true + "supports_function_calling": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", @@ -15453,7 +15834,10 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, @@ -15472,7 +15856,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -15491,7 +15878,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, @@ -15509,7 +15899,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -15524,7 +15917,10 @@ "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, @@ -15542,7 +15938,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -15560,7 +15959,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -16941,7 +17343,10 @@ "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "cohere.command-text-v14": { "input_cost_per_token": 1e-06, @@ -16952,7 +17357,10 @@ "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "cohere.embed-english-v3": { "input_cost_per_token": 1e-07, @@ -24271,7 +24679,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "token_count_model": { + "litellm_provider": "local" + } }, "deepseek.v3.2": { "input_cost_per_token": 6.2e-07, @@ -24294,7 +24705,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "dolphin": { "input_cost_per_token": 5e-07, @@ -24949,7 +25363,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.amazon.nova-micro-v1:0": { "cache_read_input_token_cost": 1.15e-08, @@ -24963,7 +25380,10 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.625e-07, @@ -24983,7 +25403,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { "input_cost_per_token": 8e-07, @@ -25254,7 +25677,10 @@ "output_cost_per_token": 1.3e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -25267,7 +25693,10 @@ "output_cost_per_token": 1.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.mistral.pixtral-large-2502-v1:0": { "input_cost_per_token": 2e-06, @@ -25279,7 +25708,10 @@ "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "fal_ai/bria/text-to-image/3.2": { "litellm_provider": "fal_ai", @@ -34349,7 +34781,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "google.gemma-3-27b-it": { "deprecation_date": "2027-03-30", @@ -34371,7 +34806,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "google.gemma-3-4b-it": { "deprecation_date": "2027-03-30", @@ -34392,7 +34830,10 @@ "supports_audio_input": false, "supports_function_calling": true, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "google_pse/search": { "input_cost_per_query": 0.005, @@ -34533,7 +34974,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "gpt-3.5-turbo": { "deprecation_date": "2026-10-23", @@ -41087,7 +41531,10 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama2-70b-chat-v1": { "input_cost_per_token": 1.95e-06, @@ -41097,7 +41544,10 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.56e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-1-405b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -41110,7 +41560,10 @@ "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-1-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41125,7 +41578,10 @@ "output_cost_per_token_batches": 3.6e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-1-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41140,7 +41596,10 @@ "output_cost_per_token_batches": 1.1e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-2-11b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -41154,7 +41613,10 @@ "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, "supports_tool_choice": false, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -41167,7 +41629,10 @@ "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -41180,7 +41645,10 @@ "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-2-90b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -41194,7 +41662,10 @@ "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, "supports_tool_choice": false, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41209,7 +41680,10 @@ "output_cost_per_token_batches": 3.6e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41220,7 +41694,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41231,7 +41708,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama4-maverick-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41254,7 +41734,10 @@ "code" ], "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama4-scout-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41277,7 +41760,10 @@ "code" ], "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta/muse-spark-1.1": { "cache_read_input_token_cost": 1.5e-07, @@ -41630,7 +42116,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -41653,7 +42142,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, @@ -41675,7 +42167,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, @@ -41875,7 +42370,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.magistral-small-2509": { "input_cost_per_token": 5e-07, @@ -41897,7 +42395,10 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.ministral-3-14b-instruct": { "input_cost_per_token": 2e-07, @@ -41919,7 +42420,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.ministral-3-3b-instruct": { "input_cost_per_token": 1e-07, @@ -41941,7 +42445,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.ministral-3-8b-instruct": { "input_cost_per_token": 1.5e-07, @@ -41963,7 +42470,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", @@ -41975,7 +42485,10 @@ "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mistral-large-2402-v1:0": { "input_cost_per_token": 4e-06, @@ -41986,7 +42499,10 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", - "supports_function_calling": true + "supports_function_calling": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mistral-large-2407-v1:0": { "input_cost_per_token": 2e-06, @@ -41998,7 +42514,10 @@ "output_cost_per_token": 6e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mistral-large-3-675b-instruct": { "input_cost_per_token": 5e-07, @@ -42020,7 +42539,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mistral-small-2402-v1:0": { "input_cost_per_token": 1e-06, @@ -42033,7 +42555,10 @@ "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-small.md", - "supports_function_calling": true + "supports_function_calling": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", @@ -42045,7 +42570,10 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", @@ -42065,7 +42593,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": true, "supports_system_messages": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", @@ -42085,7 +42616,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": true, "supports_system_messages": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, @@ -42910,7 +43444,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, @@ -42932,7 +43469,10 @@ "supports_vision": true, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, - "supports_response_schema": true + "supports_response_schema": true, + "token_count_model": { + "litellm_provider": "local" + } }, "moonshot/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, @@ -44039,7 +44579,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", @@ -44061,7 +44604,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 6e-08, @@ -44084,7 +44630,10 @@ "supports_audio_input": false, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.5e-07, @@ -44107,7 +44656,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "o1": { "cache_read_input_token_cost": 7.5e-06, @@ -45608,7 +46160,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "openai.gpt-oss-20b-1:0": { "supported_endpoints": [ @@ -45633,7 +46188,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "openai.gpt-oss-safeguard-120b": { "input_cost_per_token": 1.5e-07, @@ -45655,7 +46213,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "openai.gpt-oss-safeguard-20b": { "input_cost_per_token": 7e-08, @@ -45677,7 +46238,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, @@ -46536,7 +47100,10 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_native_structured_output": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-480b-a35b-instruct.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-480b-a35b-instruct.html", + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-235b-a22b-2507-v1:0": { "input_cost_per_token": 2.2e-07, @@ -46554,7 +47121,10 @@ "supports_native_structured_output": true, "input_cost_per_token_batches": 1.1e-07, "output_cost_per_token_batches": 4.4e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-235b-a22b-2507.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-235b-a22b-2507.html", + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-coder-30b-a3b-v1:0": { "deprecation_date": "2027-03-30", @@ -46573,7 +47143,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-32b-v1:0": { "input_cost_per_token": 1.5e-07, @@ -46593,7 +47166,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.5e-07, @@ -46609,7 +47185,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -46625,7 +47204,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -46641,7 +47223,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.545e-07, @@ -46657,7 +47242,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -46673,7 +47261,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 2.3e-07, @@ -46689,7 +47280,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -46705,7 +47299,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-vl-235b-a22b": { "input_cost_per_token": 5.3e-07, @@ -46725,7 +47322,10 @@ "supports_system_messages": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_response_schema": false + "supports_response_schema": false, + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -46747,7 +47347,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "reducto/parse-legacy": { "litellm_provider": "reducto", @@ -48777,7 +49380,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.amazon.nova-micro-v1:0": { "cache_read_input_token_cost": 8.75e-09, @@ -48794,7 +49400,10 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, @@ -48819,7 +49428,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -49263,7 +49875,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", @@ -49279,7 +49894,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_function_calling": true, - "supports_response_schema": true + "supports_response_schema": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", @@ -49295,7 +49913,10 @@ "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -49312,7 +49933,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -49331,7 +49955,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.openai.gpt-oss-120b-1:0": { "input_cost_per_token": 1.8e-07, @@ -49350,7 +49977,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.xai.grok-4.6": { "supports_regex_lookaround": false, @@ -49371,7 +50001,10 @@ "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -49591,7 +50224,10 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", "supports_function_calling": false, "supports_reasoning": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.deepseek.v3.2": { "input_cost_per_token": 6.2e-07, @@ -49614,7 +50250,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -49632,7 +50271,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-1-405b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -49645,7 +50287,10 @@ "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-1-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -49660,7 +50305,10 @@ "output_cost_per_token_batches": 3.6e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-1-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -49675,7 +50323,10 @@ "output_cost_per_token_batches": 1.1e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-2-11b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -49689,7 +50340,10 @@ "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, "supports_tool_choice": false, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -49702,7 +50356,10 @@ "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -49715,7 +50372,10 @@ "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-2-90b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -49729,7 +50389,10 @@ "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, "supports_tool_choice": false, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -49744,7 +50407,10 @@ "output_cost_per_token_batches": 3.6e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama4-maverick-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -49767,7 +50433,10 @@ "code" ], "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama4-scout-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -49790,7 +50459,10 @@ "code" ], "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.mistral.pixtral-large-2502-v1:0": { "input_cost_per_token": 2e-06, @@ -49802,7 +50474,10 @@ "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "v0/v0-1.0-md": { "input_cost_per_token": 3e-06, @@ -54466,7 +55141,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "zai.glm-5": { "input_cost_per_token": 1e-06, @@ -54488,7 +55166,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "zai.glm-4.7-flash": { "deprecation_date": "2027-03-30", @@ -54512,7 +55193,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "zai/glm-5": { "cache_creation_input_token_cost": 0, @@ -60208,7 +60892,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-5.6-sol": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60245,7 +60932,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-5.6-terra": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60282,7 +60972,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-5.6-terra": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60319,7 +61012,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-5.6-luna": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60356,7 +61052,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-5.4": { "input_cost_per_token": 2.75e-06, @@ -60388,7 +61087,10 @@ "supports_sampling_params": false, "supported_endpoints": [ "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-5.4": { "input_cost_per_token": 2.5e-06, @@ -60420,7 +61122,10 @@ "supports_sampling_params": false, "supported_endpoints": [ "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, @@ -60452,7 +61157,10 @@ "supports_sampling_params": false, "supported_endpoints": [ "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-5.5": { "input_cost_per_token": 5e-06, @@ -60484,7 +61192,10 @@ "supports_sampling_params": false, "supported_endpoints": [ "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-5.6-luna": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60521,7 +61232,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock_mantle/openai.gpt-6-astra": { "input_cost_per_token": 1.1e-05, @@ -60693,7 +61407,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-6-sol": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60732,7 +61449,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-6-luna": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60771,7 +61491,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-6-astra": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60818,7 +61541,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "openai.gpt-6-sol": { "input_cost_per_token": 2e-06, @@ -60852,7 +61578,10 @@ "supports_xhigh_reasoning_effort": true, "supports_vision": true, "supports_sampling_params": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-6-sol": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60891,7 +61620,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "openai.gpt-6-luna": { "input_cost_per_token": 1e-07, @@ -60925,7 +61657,10 @@ "supports_xhigh_reasoning_effort": true, "supports_vision": true, "supports_sampling_params": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-6-luna": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60964,7 +61699,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, @@ -61317,7 +62055,10 @@ "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "global.xai.grok-4.6": { "supports_regex_lookaround": false, @@ -61345,7 +62086,10 @@ "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "volcengine/doubao-seed-2-0-pro-260215": { "litellm_provider": "volcengine", @@ -61637,7 +62381,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/zai.glm-5": { "input_cost_per_token": 1e-06, @@ -61655,7 +62402,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, @@ -67746,7 +68496,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", @@ -67762,7 +68515,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_function_calling": true, - "supports_response_schema": true + "supports_response_schema": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", @@ -67778,7 +68534,10 @@ "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -67795,7 +68554,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -67814,7 +68576,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { "input_cost_per_token": 1.8e-07, @@ -67833,7 +68598,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -68043,7 +68811,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", @@ -68059,7 +68830,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_function_calling": true, - "supports_response_schema": true + "supports_response_schema": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", @@ -68075,7 +68849,10 @@ "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -68092,7 +68869,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -68111,7 +68891,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { "input_cost_per_token": 1.8e-07, @@ -68130,7 +68913,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -72293,7 +73079,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "global.moonshotai.kimi-k3": { "supports_regex_lookaround": false, @@ -72325,7 +73114,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.moonshotai.kimi-k3": { "supports_regex_lookaround": false, @@ -72357,7 +73149,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "xiaomi_mimo/mimo-v2.6-pro": { "cache_read_input_token_cost": 3.6e-09, @@ -73832,7 +74627,10 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", "supports_function_calling": false, "supports_reasoning": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.pixtral-large-2502-v1:0": { "input_cost_per_token": 2e-06, @@ -73844,7 +74642,10 @@ "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "azure_ai/MAI-Cyber-1-Flash": { "cache_read_input_token_cost": 6e-08, @@ -74581,7 +75382,10 @@ "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.xai.grok-4.7": { "supports_regex_lookaround": false, @@ -74607,7 +75411,10 @@ "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-Fast": { "cache_read_input_token_cost": 1.4e-08, @@ -74756,7 +75563,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_sampling_params": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock_mantle/openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.75e-06, @@ -74850,7 +75660,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_sampling_params": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "litellm_provider": "local" + } }, "vertex_ai/gemini-3.8-flash-tts": { "input_cost_per_token": 5e-07, @@ -74954,7 +75767,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.zai.glm-5.3": { "cache_creation_input_token_cost": 2.31e-06, @@ -74987,7 +75803,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-2-5-sonic": { "input_cost_per_audio_token": 3e-06, @@ -75211,7 +76030,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens_batches": 2.5e-07, @@ -75993,7 +76815,10 @@ "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "global.twelvelabs.pegasus-1-5-v1:0": { "input_cost_per_video_per_second": 0.00049, @@ -76004,7 +76829,10 @@ "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "twelvelabs.pegasus-1-5-v1:0": { "input_cost_per_video_per_second": 0.00049, @@ -76015,7 +76843,10 @@ "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "azure_ai/Microsoft-Decision-1": { "input_cost_per_token": 4.2e-08, @@ -76549,7 +77380,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "vertex_ai/deepseek-ai/deepseek-v4.1-flash-maas": { "cache_read_input_token_cost": 3e-08, diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index 22b35bd1001..3fa6c6489bd 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -9,7 +9,12 @@ import litellm from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import ProxyException -from litellm.types.token_count import TokenCountProvider, TokenCountSourceFields, TokenCountTokenizer +from litellm.types.token_count import ( + TokenCountEndpoint, + TokenCountProvider, + TokenCountSourceFields, + TokenCountTokenizer, +) from litellm.types.utils import TokenCountResponse CANNOT_COUNT_MODEL_STATUSES: Final = frozenset({400, 404}) @@ -46,13 +51,18 @@ def _with_hint(message: str, hint: str) -> str: class TokenCountModel: model: str region: str | None - litellm_provider: TokenCountProvider | None = None + litellm_provider: TokenCountEndpoint | None = None + + +@dataclass(frozen=True, slots=True) +class LocalTokenCount: + pass @dataclass(frozen=True, slots=True) class TokenCountSettings: framing: int | None - count_model: TokenCountModel | None + count_model: TokenCountModel | LocalTokenCount | None tokenizer: TokenCountTokenizer | None = None @@ -107,20 +117,21 @@ def _count_provider(value: object) -> TokenCountProvider | None: return "bedrock" case "bedrock_mantle": return "bedrock_mantle" + case "local": + return "local" case _: return None -def _count_model(value: Mapping[str, object]) -> TokenCountModel | None: +def _count_model(value: Mapping[str, object]) -> TokenCountModel | LocalTokenCount | None: + provider: Final = _count_provider(value.get("litellm_provider")) + if provider == "local": + return LocalTokenCount() model: Final = _non_blank(value.get("model")) return ( None if model is None - else TokenCountModel( - model=model, - region=_non_blank(value.get("region")), - litellm_provider=_count_provider(value.get("litellm_provider")), - ) + else TokenCountModel(model=model, region=_non_blank(value.get("region")), litellm_provider=provider) ) @@ -129,9 +140,9 @@ def _geography(region: str) -> str: def _within_deployment_geography( - count_model: TokenCountModel | None, deployment_region: Callable[[], str] -) -> TokenCountModel | None: - if count_model is None or count_model.region is None: + count_model: TokenCountModel | LocalTokenCount | None, deployment_region: Callable[[], str] +) -> TokenCountModel | LocalTokenCount | None: + if not isinstance(count_model, TokenCountModel) or count_model.region is None: return count_model model: Final = count_model.model.removeprefix("bedrock/").removeprefix("bedrock_mantle/") if model.startswith("global."): @@ -143,7 +154,7 @@ def _within_deployment_geography( def _resolved_count_model( configured: object, default: object, deployment_region: Callable[[], str] -) -> TokenCountModel | None: +) -> TokenCountModel | LocalTokenCount | None: configured_mapping: Final = as_str_mapping(configured) if configured_mapping is not None: return _count_model(configured_mapping) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 967cae27990..8bfd2881133 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -160,6 +160,7 @@ from litellm.proxy.common_utils.token_count_settings import ( TOKENIZER_LOAD_MAX_RUNNING, TOKENIZER_LOAD_REUSE_SECONDS, TOKENIZER_LOAD_WAIT_SECONDS, + LocalTokenCount, ProviderCountFailure, TokenCountSettings, cost_map_entry, @@ -14782,6 +14783,8 @@ async def _provider_token_count( Returns None when the count should be estimated locally, or the provider's failure when the framed local estimate stands in for it and that failure is the answer if the estimate cannot read the request.""" count_model: Final = settings.count_model + if isinstance(count_model, LocalTokenCount): + return None counting_model: Final = ( model_to_use if count_model is None @@ -14833,13 +14836,21 @@ def _provider_error_after_failed_estimate( provider_failure: ProviderCountFailure, error: BaseException ) -> ProxyException: verbose_proxy_logger.warning( - "Local token estimate failed after the provider's count error %s, answering that error: %s", + "Local token estimate failed, answering count error %s instead: %s", provider_failure.status_code, type(error).__name__, ) return provider_failure.to_proxy_exception() +def _local_estimate_failure(model: str) -> ProviderCountFailure: + return ProviderCountFailure( + status_code=400, + message=f"LiteLLM counts tokens for {model} locally and could not estimate this request", + rejected_before_provider_call=True, + ) + + async def _estimate_or_provider_error(estimate: Awaitable[int], provider_failure: ProviderCountFailure) -> int: try: return await estimate @@ -15259,6 +15270,7 @@ async def count_request_tokens( provider_counter, _model, custom_llm_provider = _get_provider_token_counter(deployment, model_to_use) if _model is not None: model_to_use = _model + counting_provider: Final = None if isinstance(settings.count_model, LocalTokenCount) else provider_counter sent_params: Final[Mapping[str, object]] = {**_router_default_litellm_params(llm_router), **request_litellm_params} count_deployment: Final = _deployment_with_request_drop_params(deployment, sent_params.get("drop_params")) @@ -15280,7 +15292,7 @@ async def count_request_tokens( ) chat_count: Final = ( None - if routed_request is None or provider_counter is None or isinstance(provider_counter, OpenAITokenCounter) + if routed_request is None or counting_provider is None or isinstance(counting_provider, OpenAITokenCounter) else await chat_count_request( model_to_use, custom_llm_provider, @@ -15293,10 +15305,10 @@ async def count_request_tokens( raise ProxyException(message=chat_count.message, type="invalid_request_error", param=None, code=400) ladder: Final = ( None - if provider_counter is None + if counting_provider is None else await _provider_token_count( count_with=_native_responses_count( - provider_counter, + counting_provider, custom_llm_provider, routed_request, routed_deployment, @@ -15304,7 +15316,7 @@ async def count_request_tokens( ) or partial( _try_provider_token_count, - provider_counter, + counting_provider, custom_llm_provider, messages=( [dict(message) for message in chat_count.messages] @@ -15384,7 +15396,9 @@ async def count_request_tokens( messages=counted_messages, tools=counted_tools, custom_tokenizer=custom_tokenizer, - provider_failure=ladder, + provider_failure=( + _local_estimate_failure(model_to_use) if isinstance(settings.count_model, LocalTokenCount) else ladder + ), ) template: Final = ( 0 if kimi_k3_messages is None else kimi_k3_template_tokens(len(kimi_k3_messages), bool(counted_tools)) diff --git a/litellm/types/token_count.py b/litellm/types/token_count.py index 90ccefea177..d0c6dcedf5f 100644 --- a/litellm/types/token_count.py +++ b/litellm/types/token_count.py @@ -4,11 +4,12 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict TokenCountSource: TypeAlias = Literal["provider", "token_count_model", "local_estimate"] TokenCountTokenizer: TypeAlias = Literal["kimi_k3"] -TokenCountProvider: TypeAlias = Literal["bedrock", "bedrock_mantle"] +TokenCountEndpoint: TypeAlias = Literal["bedrock", "bedrock_mantle"] +TokenCountProvider: TypeAlias = TokenCountEndpoint | Literal["local"] class TokenCountModelInfo(TypedDict): - model: ReadOnly[str] + model: ReadOnly[NotRequired[str]] region: ReadOnly[NotRequired[str]] litellm_provider: ReadOnly[NotRequired[TokenCountProvider]] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 15eba7f8af8..fafb8f4ec7c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -82,7 +82,10 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.25e-05, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "ai21.j2-ultra-v1": { "input_cost_per_token": 1.88e-05, @@ -92,7 +95,10 @@ "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.88e-05, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "ai21.jamba-1-5-large-v1:0": { "deprecation_date": "2026-11-26", @@ -103,7 +109,10 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 8e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md", + "token_count_model": { + "litellm_provider": "local" + } }, "ai21.jamba-1-5-mini-v1:0": { "deprecation_date": "2026-11-26", @@ -114,7 +123,10 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md", + "token_count_model": { + "litellm_provider": "local" + } }, "ai21.jamba-instruct-v1:0": { "input_cost_per_token": 5e-07, @@ -125,7 +137,10 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_system_messages": true + "supports_system_messages": true, + "token_count_model": { + "litellm_provider": "local" + } }, "aiml/dall-e-2": { "litellm_provider": "aiml", @@ -303,7 +318,10 @@ "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "supports_function_calling": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.writer.palmyra-x5-v1:0": { "deprecation_date": "2027-03-21", @@ -316,7 +334,10 @@ "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", "supports_function_calling": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "writer.palmyra-x4-v1:0": { "deprecation_date": "2027-03-09", @@ -329,7 +350,10 @@ "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", "supports_function_calling": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "writer.palmyra-x5-v1:0": { "deprecation_date": "2027-03-21", @@ -342,7 +366,10 @@ "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", "supports_function_calling": true, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "writer.palmyra-vision-7b": { "input_cost_per_token": 1.5e-07, @@ -359,7 +386,10 @@ "output_cost_per_token_flex": 3e-07, "output_cost_per_token_priority": 1.05e-06, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-lite-v1:0": { "cache_read_input_token_cost": 1.5e-08, @@ -378,7 +408,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -402,7 +435,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 3.125e-07, @@ -427,7 +463,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "apac.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 9.9e-08, @@ -445,7 +484,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "apac.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 3.4375e-07, @@ -466,7 +508,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 1.0725e-07, @@ -484,7 +529,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 3.4375e-07, @@ -505,7 +553,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 8.25e-08, @@ -532,7 +583,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 3.4375e-07, @@ -557,7 +611,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", @@ -591,7 +648,10 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-premier-v1:0": { "cache_read_input_token_cost": 6.25e-07, @@ -613,7 +673,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, @@ -638,7 +701,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-2-sonic-v1:0": { "input_cost_per_audio_token": 3e-06, @@ -822,7 +888,10 @@ "litellm_provider": "bedrock", "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", @@ -831,7 +900,10 @@ "litellm_provider": "bedrock", "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", @@ -840,7 +912,10 @@ "litellm_provider": "bedrock", "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "global.twelvelabs.pegasus-1-2-v1:0": { "deprecation_date": "2027-03-22", @@ -849,7 +924,10 @@ "litellm_provider": "bedrock", "mode": "chat", "supports_video_input": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-2.md", + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.titan-text-express-v1": { "input_cost_per_token": 2e-07, @@ -858,7 +936,10 @@ "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", - "output_cost_per_token": 6e-07 + "output_cost_per_token": 6e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.titan-text-lite-v1": { "input_cost_per_token": 1.5e-07, @@ -867,7 +948,10 @@ "max_output_tokens": 4000, "max_tokens": 4000, "mode": "chat", - "output_cost_per_token": 2e-07 + "output_cost_per_token": 2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.titan-text-premier-v1:0": { "input_cost_per_token": 5e-07, @@ -876,7 +960,10 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", - "output_cost_per_token": 1.5e-06 + "output_cost_per_token": 1.5e-06, + "token_count_model": { + "litellm_provider": "local" + } }, "anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -3461,7 +3548,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "apac.amazon.nova-micro-v1:0": { "cache_read_input_token_cost": 9.25e-09, @@ -3475,7 +3565,10 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "apac.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.1e-07, @@ -3494,7 +3587,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "apac.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -13460,7 +13556,10 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/*/1-month-commitment/cohere.command-text-v14": { "cost_per_second": 0.011, @@ -13470,7 +13569,10 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/*/6-month-commitment/cohere.command-light-text-v14": { "cost_per_second": 0.0011416, @@ -13480,7 +13582,10 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/*/6-month-commitment/cohere.command-text-v14": { "cost_per_second": 0.0066027, @@ -13490,7 +13595,10 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/guardrails": { "guardrail_cost_per_unit": { @@ -13612,7 +13720,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -13631,7 +13742,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -13650,7 +13764,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13665,7 +13782,10 @@ "output_cost_per_token_flex": 1.52e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -13683,7 +13803,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13701,7 +13824,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13716,7 +13842,10 @@ "output_cost_per_token_flex": 1.25e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, @@ -13736,7 +13865,10 @@ "supports_video_input": true, "supports_vision": true, "supports_audio_input": false, - "supports_response_schema": true + "supports_response_schema": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -13747,7 +13879,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -13758,7 +13893,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -13777,7 +13915,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -13796,7 +13937,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -13815,7 +13959,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13830,7 +13977,10 @@ "output_cost_per_token_flex": 1.47e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -13848,7 +13998,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13866,7 +14019,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { "input_cost_per_token": 3.1e-07, @@ -13885,7 +14041,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.24e-06, - "output_cost_per_token_flex": 6.2e-07 + "output_cost_per_token_flex": 6.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -13904,7 +14063,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -13923,7 +14085,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -13942,7 +14107,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -13960,7 +14128,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13978,7 +14149,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -13989,7 +14163,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.03e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ca-central-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14000,7 +14177,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.9e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-north-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -14019,7 +14199,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14038,7 +14221,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-north-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14057,7 +14243,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -14075,7 +14264,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.01635, @@ -14181,7 +14373,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-central-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14200,7 +14395,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14218,7 +14416,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14229,7 +14430,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.78e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14240,7 +14444,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.5e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14259,7 +14466,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14278,7 +14488,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14296,7 +14509,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14307,7 +14523,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.55e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14318,7 +14537,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.8e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14337,7 +14559,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 4.7e-07, @@ -14356,7 +14581,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.86e-06, - "output_cost_per_token_flex": 9.3e-07 + "output_cost_per_token_flex": 9.3e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 2.3e-07, @@ -14375,7 +14603,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, @@ -14393,7 +14624,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", @@ -14405,7 +14639,10 @@ "mode": "chat", "output_cost_per_token": 2.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-3/mistral.mistral-large-2402-v1:0": { "input_cost_per_token": 5.2e-06, @@ -14416,7 +14653,10 @@ "mode": "chat", "output_cost_per_token": 1.56e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", - "supports_function_calling": true + "supports_function_calling": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", @@ -14428,7 +14668,10 @@ "mode": "chat", "output_cost_per_token": 9.1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14447,7 +14690,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14466,7 +14712,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14484,7 +14733,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -14513,7 +14765,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.88e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14524,7 +14779,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.01e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -14543,7 +14801,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14562,7 +14823,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14581,7 +14845,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_flex": 7.2e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14596,7 +14863,10 @@ "output_cost_per_token_flex": 1.52e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -14614,7 +14884,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14632,7 +14905,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.011, @@ -14731,7 +15007,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14742,7 +15021,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", @@ -14754,7 +15036,10 @@ "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/mistral.mistral-large-2402-v1:0": { "input_cost_per_token": 4e-06, @@ -14765,7 +15050,10 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", - "supports_function_calling": true + "supports_function_calling": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", @@ -14777,7 +15065,10 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, @@ -14796,7 +15087,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14815,7 +15109,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, @@ -14833,7 +15130,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14848,7 +15148,10 @@ "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, @@ -14866,7 +15169,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -14884,7 +15190,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, @@ -14903,7 +15212,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14922,7 +15234,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, @@ -14941,7 +15256,10 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.2e-06, - "output_cost_per_token_flex": 6e-07 + "output_cost_per_token_flex": 6e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14956,7 +15274,10 @@ "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, @@ -14974,7 +15295,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-east-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -14992,7 +15316,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.4e-07, @@ -15008,7 +15335,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -15035,7 +15365,10 @@ "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", - "output_cost_per_token": 1.7e-06 + "output_cost_per_token": 1.7e-06, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/amazon.titan-text-lite-v1": { "input_cost_per_token": 3e-07, @@ -15044,7 +15377,10 @@ "max_output_tokens": 4000, "max_tokens": 4000, "mode": "chat", - "output_cost_per_token": 4e-07 + "output_cost_per_token": 4e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/amazon.titan-text-premier-v1:0": { "input_cost_per_token": 5e-07, @@ -15053,7 +15389,10 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", - "output_cost_per_token": 1.5e-06 + "output_cost_per_token": 1.5e-06, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -15117,7 +15456,10 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/meta.llama3-8b-instruct-v1:0": { "input_cost_per_token": 3e-07, @@ -15127,7 +15469,10 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.65e-06, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", @@ -15159,7 +15504,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.nova-micro-v1:0": { "cache_read_input_token_cost": 1.05e-08, @@ -15173,7 +15521,10 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.4e-07, @@ -15189,7 +15540,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -15216,7 +15570,10 @@ "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", - "output_cost_per_token": 1.6e-06 + "output_cost_per_token": 1.6e-06, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.titan-text-lite-v1": { "input_cost_per_token": 3e-07, @@ -15225,7 +15582,10 @@ "max_output_tokens": 4000, "max_tokens": 4000, "mode": "chat", - "output_cost_per_token": 4e-07 + "output_cost_per_token": 4e-07, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/amazon.titan-text-premier-v1:0": { "input_cost_per_token": 5e-07, @@ -15234,7 +15594,10 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", - "output_cost_per_token": 1.5e-06 + "output_cost_per_token": 1.5e-06, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -15298,7 +15661,10 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": { "input_cost_per_token": 3e-07, @@ -15308,7 +15674,10 @@ "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6e-07, - "supports_pdf_input": true + "supports_pdf_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -15319,7 +15688,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-1/meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -15330,7 +15702,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.011, @@ -15430,7 +15805,10 @@ "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/mistral.mistral-large-2402-v1:0": { "input_cost_per_token": 4e-06, @@ -15441,7 +15819,10 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", - "supports_function_calling": true + "supports_function_calling": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", @@ -15453,7 +15834,10 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, @@ -15472,7 +15856,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -15491,7 +15878,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, @@ -15509,7 +15899,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -15524,7 +15917,10 @@ "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, @@ -15542,7 +15938,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -15560,7 +15959,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -16941,7 +17343,10 @@ "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "cohere.command-text-v14": { "input_cost_per_token": 1e-06, @@ -16952,7 +17357,10 @@ "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "cohere.embed-english-v3": { "input_cost_per_token": 1e-07, @@ -24271,7 +24679,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "token_count_model": { + "litellm_provider": "local" + } }, "deepseek.v3.2": { "input_cost_per_token": 6.2e-07, @@ -24294,7 +24705,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "dolphin": { "input_cost_per_token": 5e-07, @@ -24949,7 +25363,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.amazon.nova-micro-v1:0": { "cache_read_input_token_cost": 1.15e-08, @@ -24963,7 +25380,10 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.625e-07, @@ -24983,7 +25403,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { "input_cost_per_token": 8e-07, @@ -25254,7 +25677,10 @@ "output_cost_per_token": 1.3e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -25267,7 +25693,10 @@ "output_cost_per_token": 1.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.mistral.pixtral-large-2502-v1:0": { "input_cost_per_token": 2e-06, @@ -25279,7 +25708,10 @@ "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "fal_ai/bria/text-to-image/3.2": { "litellm_provider": "fal_ai", @@ -34349,7 +34781,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "google.gemma-3-27b-it": { "deprecation_date": "2027-03-30", @@ -34371,7 +34806,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "google.gemma-3-4b-it": { "deprecation_date": "2027-03-30", @@ -34392,7 +34830,10 @@ "supports_audio_input": false, "supports_function_calling": true, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "google_pse/search": { "input_cost_per_query": 0.005, @@ -34533,7 +34974,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "gpt-3.5-turbo": { "deprecation_date": "2026-10-23", @@ -41087,7 +41531,10 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama2-70b-chat-v1": { "input_cost_per_token": 1.95e-06, @@ -41097,7 +41544,10 @@ "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.56e-06, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-1-405b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -41110,7 +41560,10 @@ "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-1-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41125,7 +41578,10 @@ "output_cost_per_token_batches": 3.6e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-1-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41140,7 +41596,10 @@ "output_cost_per_token_batches": 1.1e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-2-11b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -41154,7 +41613,10 @@ "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, "supports_tool_choice": false, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -41167,7 +41629,10 @@ "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -41180,7 +41645,10 @@ "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-2-90b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -41194,7 +41662,10 @@ "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, "supports_tool_choice": false, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41209,7 +41680,10 @@ "output_cost_per_token_batches": 3.6e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41220,7 +41694,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama3-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41231,7 +41708,10 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md", + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama4-maverick-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41254,7 +41734,10 @@ "code" ], "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta.llama4-scout-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -41277,7 +41760,10 @@ "code" ], "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "meta/muse-spark-1.1": { "cache_read_input_token_cost": 1.5e-07, @@ -41630,7 +42116,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -41653,7 +42142,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, @@ -41675,7 +42167,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, @@ -41875,7 +42370,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.magistral-small-2509": { "input_cost_per_token": 5e-07, @@ -41897,7 +42395,10 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.ministral-3-14b-instruct": { "input_cost_per_token": 2e-07, @@ -41919,7 +42420,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.ministral-3-3b-instruct": { "input_cost_per_token": 1e-07, @@ -41941,7 +42445,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.ministral-3-8b-instruct": { "input_cost_per_token": 1.5e-07, @@ -41963,7 +42470,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", @@ -41975,7 +42485,10 @@ "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mistral-large-2402-v1:0": { "input_cost_per_token": 4e-06, @@ -41986,7 +42499,10 @@ "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", - "supports_function_calling": true + "supports_function_calling": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mistral-large-2407-v1:0": { "input_cost_per_token": 2e-06, @@ -41998,7 +42514,10 @@ "output_cost_per_token": 6e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mistral-large-3-675b-instruct": { "input_cost_per_token": 5e-07, @@ -42020,7 +42539,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mistral-small-2402-v1:0": { "input_cost_per_token": 1e-06, @@ -42033,7 +42555,10 @@ "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-small.md", - "supports_function_calling": true + "supports_function_calling": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.mixtral-8x7b-instruct-v0:1": { "deprecation_date": "2027-03-30", @@ -42045,7 +42570,10 @@ "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", @@ -42065,7 +42593,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": true, "supports_system_messages": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", @@ -42085,7 +42616,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": true, "supports_system_messages": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, @@ -42910,7 +43444,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, @@ -42932,7 +43469,10 @@ "supports_vision": true, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, - "supports_response_schema": true + "supports_response_schema": true, + "token_count_model": { + "litellm_provider": "local" + } }, "moonshot/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, @@ -44039,7 +44579,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", @@ -44061,7 +44604,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 6e-08, @@ -44084,7 +44630,10 @@ "supports_audio_input": false, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.5e-07, @@ -44107,7 +44656,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "o1": { "cache_read_input_token_cost": 7.5e-06, @@ -45608,7 +46160,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "openai.gpt-oss-20b-1:0": { "supported_endpoints": [ @@ -45633,7 +46188,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "openai.gpt-oss-safeguard-120b": { "input_cost_per_token": 1.5e-07, @@ -45655,7 +46213,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "openai.gpt-oss-safeguard-20b": { "input_cost_per_token": 7e-08, @@ -45677,7 +46238,10 @@ "supports_function_calling": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, @@ -46536,7 +47100,10 @@ "supports_reasoning": true, "supports_tool_choice": true, "supports_native_structured_output": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-480b-a35b-instruct.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-480b-a35b-instruct.html", + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-235b-a22b-2507-v1:0": { "input_cost_per_token": 2.2e-07, @@ -46554,7 +47121,10 @@ "supports_native_structured_output": true, "input_cost_per_token_batches": 1.1e-07, "output_cost_per_token_batches": 4.4e-07, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-235b-a22b-2507.html" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-235b-a22b-2507.html", + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-coder-30b-a3b-v1:0": { "deprecation_date": "2027-03-30", @@ -46573,7 +47143,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-32b-v1:0": { "input_cost_per_token": 1.5e-07, @@ -46593,7 +47166,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_native_structured_output": true + "supports_native_structured_output": true, + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.5e-07, @@ -46609,7 +47185,10 @@ "supports_system_messages": true, "supports_native_structured_output": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -46625,7 +47204,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -46641,7 +47223,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.545e-07, @@ -46657,7 +47242,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -46673,7 +47261,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 2.3e-07, @@ -46689,7 +47280,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, @@ -46705,7 +47299,10 @@ "supports_system_messages": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-vl-235b-a22b": { "input_cost_per_token": 5.3e-07, @@ -46725,7 +47322,10 @@ "supports_system_messages": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_response_schema": false + "supports_response_schema": false, + "token_count_model": { + "litellm_provider": "local" + } }, "qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -46747,7 +47347,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "reducto/parse-legacy": { "litellm_provider": "reducto", @@ -48777,7 +49380,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.amazon.nova-micro-v1:0": { "cache_read_input_token_cost": 8.75e-09, @@ -48794,7 +49400,10 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, @@ -48819,7 +49428,10 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_vision": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -49263,7 +49875,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", @@ -49279,7 +49894,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_function_calling": true, - "supports_response_schema": true + "supports_response_schema": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", @@ -49295,7 +49913,10 @@ "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -49312,7 +49933,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -49331,7 +49955,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.openai.gpt-oss-120b-1:0": { "input_cost_per_token": 1.8e-07, @@ -49350,7 +49977,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us-gov.xai.grok-4.6": { "supports_regex_lookaround": false, @@ -49371,7 +50001,10 @@ "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -49591,7 +50224,10 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", "supports_function_calling": false, "supports_reasoning": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.deepseek.v3.2": { "input_cost_per_token": 6.2e-07, @@ -49614,7 +50250,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "eu.deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -49632,7 +50271,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-1-405b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -49645,7 +50287,10 @@ "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-1-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -49660,7 +50305,10 @@ "output_cost_per_token_batches": 3.6e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-1-8b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -49675,7 +50323,10 @@ "output_cost_per_token_batches": 1.1e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-2-11b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -49689,7 +50340,10 @@ "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, "supports_tool_choice": false, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-2-1b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -49702,7 +50356,10 @@ "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-2-3b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -49715,7 +50372,10 @@ "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-2-90b-instruct-v1:0": { "deprecation_date": "2026-07-07", @@ -49729,7 +50389,10 @@ "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", "supports_function_calling": true, "supports_tool_choice": false, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama3-3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -49744,7 +50407,10 @@ "output_cost_per_token_batches": 3.6e-07, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama4-maverick-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -49767,7 +50433,10 @@ "code" ], "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.meta.llama4-scout-17b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -49790,7 +50459,10 @@ "code" ], "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.mistral.pixtral-large-2502-v1:0": { "input_cost_per_token": 2e-06, @@ -49802,7 +50474,10 @@ "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "v0/v0-1.0-md": { "input_cost_per_token": 3e-06, @@ -54466,7 +55141,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "zai.glm-5": { "input_cost_per_token": 1e-06, @@ -54488,7 +55166,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "zai.glm-4.7-flash": { "deprecation_date": "2027-03-30", @@ -54512,7 +55193,10 @@ "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json", "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "zai/glm-5": { "cache_creation_input_token_cost": 0, @@ -60208,7 +60892,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-5.6-sol": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60245,7 +60932,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-5.6-terra": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60282,7 +60972,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-5.6-terra": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60319,7 +61012,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-5.6-luna": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60356,7 +61052,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-5.4": { "input_cost_per_token": 2.75e-06, @@ -60388,7 +61087,10 @@ "supports_sampling_params": false, "supported_endpoints": [ "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-5.4": { "input_cost_per_token": 2.5e-06, @@ -60420,7 +61122,10 @@ "supports_sampling_params": false, "supported_endpoints": [ "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, @@ -60452,7 +61157,10 @@ "supports_sampling_params": false, "supported_endpoints": [ "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-5.5": { "input_cost_per_token": 5e-06, @@ -60484,7 +61192,10 @@ "supports_sampling_params": false, "supported_endpoints": [ "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-5.6-luna": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60521,7 +61232,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock_mantle/openai.gpt-6-astra": { "input_cost_per_token": 1.1e-05, @@ -60693,7 +61407,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-6-sol": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60732,7 +61449,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "us.openai.gpt-6-luna": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60771,7 +61491,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-6-astra": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60818,7 +61541,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "openai.gpt-6-sol": { "input_cost_per_token": 2e-06, @@ -60852,7 +61578,10 @@ "supports_xhigh_reasoning_effort": true, "supports_vision": true, "supports_sampling_params": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-6-sol": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60891,7 +61620,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "openai.gpt-6-luna": { "input_cost_per_token": 1e-07, @@ -60925,7 +61657,10 @@ "supports_xhigh_reasoning_effort": true, "supports_vision": true, "supports_sampling_params": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "global.openai.gpt-6-luna": { "supports_bedrock_runtime_chat_completions_response_format": true, @@ -60964,7 +61699,10 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" - ] + ], + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, @@ -61317,7 +62055,10 @@ "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "global.xai.grok-4.6": { "supports_regex_lookaround": false, @@ -61345,7 +62086,10 @@ "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "volcengine/doubao-seed-2-0-pro-260215": { "litellm_provider": "volcengine", @@ -61637,7 +62381,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-west-2/zai.glm-5": { "input_cost_per_token": 1e-06, @@ -61655,7 +62402,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://aws.amazon.com/bedrock/pricing/", + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, @@ -67746,7 +68496,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", @@ -67762,7 +68515,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_function_calling": true, - "supports_response_schema": true + "supports_response_schema": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", @@ -67778,7 +68534,10 @@ "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -67795,7 +68554,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -67814,7 +68576,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { "input_cost_per_token": 1.8e-07, @@ -67833,7 +68598,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-west-1/anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -68043,7 +68811,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", @@ -68059,7 +68830,10 @@ "supports_vision": true, "supports_audio_input": false, "supports_function_calling": true, - "supports_response_schema": true + "supports_response_schema": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", @@ -68075,7 +68849,10 @@ "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.8e-07, @@ -68092,7 +68869,10 @@ "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { "input_cost_per_token": 8.4e-08, @@ -68111,7 +68891,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { "input_cost_per_token": 1.8e-07, @@ -68130,7 +68913,10 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-5": { "bedrock_converse_supports_strict_tools": false, @@ -72293,7 +73079,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "global.moonshotai.kimi-k3": { "supports_regex_lookaround": false, @@ -72325,7 +73114,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.moonshotai.kimi-k3": { "supports_regex_lookaround": false, @@ -72357,7 +73149,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "xiaomi_mimo/mimo-v2.6-pro": { "cache_read_input_token_cost": 3.6e-09, @@ -73832,7 +74627,10 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", "supports_function_calling": false, "supports_reasoning": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "mistral.pixtral-large-2502-v1:0": { "input_cost_per_token": 2e-06, @@ -73844,7 +74642,10 @@ "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, - "supports_tool_choice": false + "supports_tool_choice": false, + "token_count_model": { + "litellm_provider": "local" + } }, "azure_ai/MAI-Cyber-1-Flash": { "cache_read_input_token_cost": 6e-08, @@ -74581,7 +75382,10 @@ "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "us.xai.grok-4.7": { "supports_regex_lookaround": false, @@ -74607,7 +75411,10 @@ "supports_prompt_caching": false, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-Fast": { "cache_read_input_token_cost": 1.4e-08, @@ -74756,7 +75563,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_sampling_params": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "litellm_provider": "local" + } }, "bedrock_mantle/openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.75e-06, @@ -74850,7 +75660,10 @@ "supports_tool_choice": true, "supports_vision": true, "supports_sampling_params": false, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "token_count_model": { + "litellm_provider": "local" + } }, "vertex_ai/gemini-3.8-flash-tts": { "input_cost_per_token": 5e-07, @@ -74954,7 +75767,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "us.zai.glm-5.3": { "cache_creation_input_token_cost": 2.31e-06, @@ -74987,7 +75803,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_model": { + "litellm_provider": "local" + } }, "amazon.nova-2-5-sonic": { "input_cost_per_audio_token": 3e-06, @@ -75211,7 +76030,10 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens_batches": 2.5e-07, @@ -75993,7 +76815,10 @@ "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "global.twelvelabs.pegasus-1-5-v1:0": { "input_cost_per_video_per_second": 0.00049, @@ -76004,7 +76829,10 @@ "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "twelvelabs.pegasus-1-5-v1:0": { "input_cost_per_video_per_second": 0.00049, @@ -76015,7 +76843,10 @@ "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", - "supports_video_input": true + "supports_video_input": true, + "token_count_model": { + "litellm_provider": "local" + } }, "azure_ai/Microsoft-Decision-1": { "input_cost_per_token": 4.2e-08, @@ -76549,7 +77380,10 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "token_count_model": { + "litellm_provider": "local" + } }, "vertex_ai/deepseek-ai/deepseek-v4.1-flash-maas": { "cache_read_input_token_cost": 3e-08, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 016d40ea874..313c8e49622 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -1248,9 +1248,6 @@ }, "token_count_model": { "type": "object", - "required": [ - "model" - ], "additionalProperties": false, "properties": { "model": { @@ -1265,11 +1262,33 @@ "type": "string", "enum": [ "bedrock", - "bedrock_mantle" + "bedrock_mantle", + "local" ] } }, - "description": "Model that counts tokens for this one, its counting provider, and the AWS region its count API runs in." + "if": { + "required": [ + "litellm_provider" + ], + "properties": { + "litellm_provider": { + "const": "local" + } + } + }, + "then": { + "properties": { + "model": false, + "region": false + } + }, + "else": { + "required": [ + "model" + ] + }, + "description": "Model that counts tokens for this one, its counting provider, and the AWS region its count API runs in. The provider \"local\" marks a model no provider API can count, so litellm estimates its requests locally without calling the provider and takes no model or region." }, "token_count_tokenizer": { "type": "string", diff --git a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py index a4542f1f604..1cfdcf09675 100644 --- a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py @@ -1729,7 +1729,7 @@ def test_nova_count_sends_the_tool_specs_converse_inference_sends(counting_proxy path, body, count_field = _NOVA_COUNTS[route] gateway: Final = counting_proxy.gateway with wire_server(_counting_every_model) as runtime, gateway.scenario() as scenario: - model: Final = _metadata_deployment(scenario, runtime.url, _NOVA) + model: Final = _fallback_deployment(scenario, runtime.url, _NOVA) generated: Final = gateway.request( "POST", "/v1/chat/completions", diff --git a/tests/integration/providers/test_count_tokens_model_wire.py b/tests/integration/providers/test_count_tokens_model_wire.py index b6ace7ddef3..ea713e7a07b 100644 --- a/tests/integration/providers/test_count_tokens_model_wire.py +++ b/tests/integration/providers/test_count_tokens_model_wire.py @@ -105,6 +105,9 @@ _TOOL_TURNS: Final[list[JsonValue]] = [ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "Sunny"}]}, ] _LOCAL_SOURCE: Final = {"litellm_count_source": "local_estimate"} +_COUNT_ITSELF: Final[dict[str, JsonValue]] = {"token_count_model": {}} +_COUNT_LOCALLY: Final[dict[str, JsonValue]] = {"token_count_model": {"litellm_provider": "local"}} +_CANNOT_ESTIMATE: Final = "locally and could not estimate this request" def _prompt(status: int, count_model_status: int | None = None) -> str: @@ -279,9 +282,9 @@ def _messages_estimate(gateway: Gateway, model: str) -> tuple[int, JsonValue]: return counted.input_tokens, (counted.model_extra or {}).get("litellm_count_source") -def _utils_estimate(gateway: Gateway, model: str) -> tuple[int, JsonValue]: +def _utils_estimate(gateway: Gateway, model: str, path: str = "/utils/token_counter") -> tuple[int, JsonValue]: response: Final = gateway.request( - "POST", "/utils/token_counter", {"model": model, "messages": [{"role": "user", "content": _prompt(400)}]} + "POST", path, {"model": model, "messages": [{"role": "user", "content": _prompt(400)}]} ) assert response.status_code == 200, response.text body: Final = _JSON_OBJECT.validate_json(response.content) @@ -290,6 +293,10 @@ def _utils_estimate(gateway: Gateway, model: str) -> tuple[int, JsonValue]: return total, body.get("count_source") +def _utils_endpoint_estimate(gateway: Gateway, model: str) -> tuple[int, JsonValue]: + return _utils_estimate(gateway, model, "/utils/token_counter?call_endpoint=true") + + def _shipped_framing(model: str) -> JsonValue: return _COST_MAP.validate_json(_COST_MAP_PATH.read_bytes())[model].get("token_count_framing") @@ -355,9 +362,11 @@ def test_framing_is_added_to_the_local_estimate_when_the_runtime_cannot_count( gateway.scenario() as scenario, ): framed: Final = _count( - gateway, _deployment(scenario, runtime_port, _NOVA, {"token_count_framing": _FRAMING}), 400 + gateway, _deployment(scenario, runtime_port, _NOVA, {**_COUNT_ITSELF, "token_count_framing": _FRAMING}), 400 + ) + unframed: Final = _count( + gateway, _deployment(scenario, runtime_port, _NOVA, {**_COUNT_ITSELF, "token_count_framing": 0}), 400 ) - unframed: Final = _count(gateway, _deployment(scenario, runtime_port, _NOVA, {"token_count_framing": 0}), 400) assert (framed.model_extra, unframed.model_extra) == (_LOCAL_SOURCE, _LOCAL_SOURCE), (framed, unframed) assert framed.input_tokens - unframed.input_tokens == _FRAMING, (framed, unframed) assert _runtime_models(runtime) == (_NOVA, _NOVA) @@ -373,8 +382,8 @@ def test_framing_on_one_deployment_leaves_a_sibling_of_the_same_model_raising_th wire_server(_mantle, port=mantle_port) as mantle, gateway.scenario() as scenario, ): - framed: Final = _deployment(scenario, runtime_port, _NOVA, {"token_count_framing": _FRAMING}) - sibling: Final = _deployment(scenario, runtime_port, _NOVA) + framed: Final = _deployment(scenario, runtime_port, _NOVA, {**_COUNT_ITSELF, "token_count_framing": _FRAMING}) + sibling: Final = _deployment(scenario, runtime_port, _NOVA, _COUNT_ITSELF) assert _count(gateway, framed, 400).model_extra == _LOCAL_SOURCE with pytest.raises(anthropic.BadRequestError, match=re.escape(_UNSUPPORTED)): _count(gateway, sibling, 400) @@ -421,16 +430,13 @@ def test_inference_profile_arn_counts_with_the_token_count_model_of_its_base_mod @pytest.mark.parametrize( - ("estimate", "runtime_calls"), - [(_messages_estimate, (_KIMI_BASE, _KIMI_BASE)), (_utils_estimate, ())], - ids=["messages_count_tokens", "utils_token_counter"], + "estimate", [_messages_estimate, _utils_estimate], ids=["messages_count_tokens", "utils_token_counter"] ) def test_shipped_framing_is_added_to_the_estimate_and_a_configured_zero_turns_it_off( counting_proxy: OwnedProxy, runtime_port: int, mantle_port: int, estimate: Callable[[Gateway, str], tuple[int, JsonValue]], - runtime_calls: tuple[str, ...], ) -> None: gateway: Final = counting_proxy.gateway with ( @@ -444,7 +450,30 @@ def test_shipped_framing_is_added_to_the_estimate_and_a_configured_zero_turns_it framing: Final = _shipped_framing(_KIMI) assert isinstance(framing, int) and framing > 0, framing assert shipped[0] - unframed[0] == framing, (shipped, unframed) - assert _runtime_models(runtime) == runtime_calls + assert _runtime_models(runtime) == () + assert _mantle_models(mantle) == () + + +@pytest.mark.parametrize( + "estimate", [_messages_estimate, _utils_endpoint_estimate], ids=["messages_count_tokens", "utils_token_counter"] +) +def test_local_marker_counts_locally_without_asking_the_runtime_or_mantle( + counting_proxy: OwnedProxy, + runtime_port: int, + mantle_port: int, + estimate: Callable[[Gateway, str], tuple[int, JsonValue]], +) -> None: + gateway: Final = counting_proxy.gateway + with ( + wire_server(_runtime, port=runtime_port) as runtime, + wire_server(_mantle, port=mantle_port) as mantle, + gateway.scenario() as scenario, + ): + shipped: Final = estimate(gateway, _deployment(scenario, runtime_port, _NOVA)) + configured: Final = estimate(gateway, _deployment(scenario, runtime_port, _SONNET, _COUNT_LOCALLY)) + assert shipped[0] > 0 and configured[0] > 0, (shipped, configured) + assert (shipped[1], configured[1]) == ("local_estimate", "local_estimate"), (shipped, configured) + assert _runtime_models(runtime) == () assert _mantle_models(mantle) == () @@ -613,8 +642,20 @@ def _count_answer(response: httpx.Response, source_key: str) -> tuple[int, str]: @pytest.mark.parametrize("endpoint", sorted(_COUNT_ENDPOINTS)) @pytest.mark.parametrize("unreadable", sorted(_UNREADABLE_BODIES)) +@pytest.mark.parametrize( + ("model_info", "runtime_calls", "rejection"), + [(None, (), _CANNOT_ESTIMATE), (_COUNT_ITSELF, (_KIMI_BASE,), _UNSUPPORTED)], + ids=["shipped_local_marker", "counting_itself_on_the_runtime"], +) def test_shipped_framing_never_answers_a_500_for_a_body_the_local_count_cannot_read( - counting_proxy: OwnedProxy, runtime_port: int, mantle_port: int, unreadable: str, endpoint: str + counting_proxy: OwnedProxy, + runtime_port: int, + mantle_port: int, + unreadable: str, + endpoint: str, + model_info: dict[str, JsonValue] | None, + runtime_calls: tuple[str, ...], + rejection: str, ) -> None: gateway: Final = counting_proxy.gateway path, source_key = _COUNT_ENDPOINTS[endpoint] @@ -624,14 +665,16 @@ def test_shipped_framing_never_answers_a_500_for_a_body_the_local_count_cannot_r gateway.scenario() as scenario, ): response: Final = gateway.request( - "POST", path, {"model": _deployment(scenario, runtime_port, _KIMI), **_UNREADABLE_BODIES[unreadable]} + "POST", + path, + {"model": _deployment(scenario, runtime_port, _KIMI, model_info), **_UNREADABLE_BODIES[unreadable]}, ) status, answer = _count_answer(response, source_key) - assert (status, answer) == (200, "local_estimate") or (status == 400 and _UNSUPPORTED in answer), ( + assert (status, answer) == (200, "local_estimate") or (status == 400 and rejection in answer), ( response.status_code, response.text, ) - assert _runtime_models(runtime) == (_KIMI_BASE,) + assert _runtime_models(runtime) == runtime_calls assert _mantle_models(mantle) == () diff --git a/tests/unit/proxy/common_utils/test_token_count_settings.py b/tests/unit/proxy/common_utils/test_token_count_settings.py index d7314f1a6d3..6c1cb77d9b7 100644 --- a/tests/unit/proxy/common_utils/test_token_count_settings.py +++ b/tests/unit/proxy/common_utils/test_token_count_settings.py @@ -6,6 +6,7 @@ import pytest from litellm.llms.bedrock.common_utils import get_bedrock_base_model from litellm.proxy.common_utils.token_count_settings import ( + LocalTokenCount, ProviderCountFailure, TokenCountModel, TokenCountSettings, @@ -19,6 +20,7 @@ _DEFAULT: Final = { "token_count_framing": 97, "token_count_model": {"model": "anthropic.claude-opus-5-5", "region": "us-east-1"}, } +_LOCAL_DEFAULT: Final = {"token_count_framing": 97, "token_count_model": {"litellm_provider": "local"}} _COST_MAP: Final = Path(__file__).parents[4] / "model_prices_and_context_window.json" @@ -68,6 +70,15 @@ def _in_us_west_2() -> str: {"token_count_model": {"model": " ", "region": "us-east-1"}}, TokenCountSettings(framing=97, count_model=None), ), + ( + {"token_count_model": {"litellm_provider": "local"}}, + TokenCountSettings(framing=97, count_model=LocalTokenCount()), + ), + ( + {"token_count_model": {"litellm_provider": "local", "model": "custom-model", "region": "us-east-1"}}, + TokenCountSettings(framing=97, count_model=LocalTokenCount()), + ), + ({"token_count_model": {"litellm_provider": "bedrock"}}, TokenCountSettings(framing=97, count_model=None)), ], ) def test_config_overrides_the_cost_map_default_key_by_key( @@ -79,6 +90,37 @@ def test_config_overrides_the_cost_map_default_key_by_key( ) +@pytest.mark.parametrize( + ("configured", "expected"), + [ + ({}, LocalTokenCount()), + ({"token_count_framing": 0}, LocalTokenCount()), + ( + {"token_count_model": {"model": "custom-model", "litellm_provider": "bedrock"}}, + TokenCountModel(model="custom-model", region=None, litellm_provider="bedrock"), + ), + ({"token_count_model": {"model": "custom-model"}}, TokenCountModel(model="custom-model", region=None)), + ({"token_count_model": {}}, None), + ], +) +def test_shipped_local_marker_counts_locally_until_the_config_routes_the_count_to_a_provider( + configured: dict[str, object], expected: TokenCountModel | LocalTokenCount | None +) -> None: + settings: Final = resolve_token_count_settings( + configured=configured, default=_LOCAL_DEFAULT, deployment_region=lambda: "eu-central-1" + ) + assert (settings.count_model, settings.framing) == (expected, 0 if "token_count_framing" in configured else 97) + + +def test_region_on_a_shipped_local_marker_never_routes_the_count_anywhere() -> None: + settings: Final = resolve_token_count_settings( + configured={}, + default={"token_count_model": {"litellm_provider": "local", "region": "us-east-1"}}, + deployment_region=lambda: "eu-central-1", + ) + assert settings.count_model == LocalTokenCount() + + @pytest.mark.parametrize( ("deployment_region", "configured", "expected_region"), [ @@ -180,10 +222,14 @@ def test_every_shipped_token_count_model_resolves_to_a_mapped_aws_model() -> Non if isinstance(entry, dict) and "token_count_model" in entry } assert count_models + assert any(count_model.get("litellm_provider") == "local" for count_model in count_models.values()) for name, count_model in count_models.items(): + assert cost_map[name]["litellm_provider"] in ("bedrock", "bedrock_converse") + if count_model.get("litellm_provider") == "local": + assert count_model == {"litellm_provider": "local"}, name + continue model: Final = count_model["model"] assert get_bedrock_base_model(model) in aws_models - assert cost_map[name]["litellm_provider"] in ("bedrock", "bedrock_converse") assert "region" not in count_model or count_model["region"].strip() diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index fa0c9eda02d..aeddb1ab628 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -1644,6 +1644,14 @@ def _provider_reply( ) +def _kimi_counting_itself(**model_info: object) -> dict[str, object]: + return { + "model_name": "kimi", + "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}, + "model_info": {"token_count_model": {}, **model_info}, + } + + _COUNT_REQUEST = TokenCountRequest( model="sonnet-5-5", messages=[{"role": "user", "content": "hello"}], @@ -1994,7 +2002,7 @@ async def test_framed_estimate_that_cannot_read_the_body_answers_the_provider_er monkeypatch.setattr( litellm.proxy.proxy_server, "llm_router", - Router(model_list=[{"model_name": "kimi", "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}}]), + Router(model_list=[_kimi_counting_itself()]), ) monkeypatch.setattr( litellm.proxy.proxy_server, "_get_provider_token_counter", lambda deployment, model: (counter, model, "bedrock") @@ -2068,11 +2076,7 @@ async def test_framed_deployment_whose_custom_tokenizer_cannot_load_answers_the_ "llm_router", Router( model_list=[ - { - "model_name": "kimi", - "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}, - "model_info": {"custom_tokenizer": {"identifier": "moonshotai/kimi-k3", "revision": "main"}}, - } + _kimi_counting_itself(custom_tokenizer={"identifier": "moonshotai/kimi-k3", "revision": "main"}) ] ), ) @@ -2111,11 +2115,10 @@ def _framed_kimis_with_custom_tokenizers( }, *( { + **_kimi_counting_itself( + custom_tokenizer={"identifier": identifier, "revision": revision, "auth_token": None} + ), "model_name": name, - "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}, - "model_info": { - "custom_tokenizer": {"identifier": identifier, "revision": revision, "auth_token": None} - }, } for name, (identifier, revision) in tokenizers.items() ), @@ -2373,7 +2376,7 @@ def _framed_kimi(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( litellm.proxy.proxy_server, "llm_router", - Router(model_list=[{"model_name": "kimi", "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}}]), + Router(model_list=[_kimi_counting_itself()]), ) monkeypatch.setattr( litellm.proxy.proxy_server, "_get_provider_token_counter", lambda deployment, model: (counter, model, "bedrock") @@ -2467,7 +2470,7 @@ async def test_swallowed_estimate_error_is_logged_by_type_without_the_request_co monkeypatch.setattr( litellm.proxy.proxy_server, "llm_router", - Router(model_list=[{"model_name": "kimi", "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}}]), + Router(model_list=[_kimi_counting_itself()]), ) monkeypatch.setattr( litellm.proxy.proxy_server, "_get_provider_token_counter", lambda deployment, model: (counter, model, "bedrock") @@ -2744,15 +2747,7 @@ async def test_uncountable_model_keeps_the_provider_error_when_local_counting_is monkeypatch.setattr( litellm.proxy.proxy_server, "llm_router", - Router( - model_list=[ - { - "model_name": "kimi", - "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}, - "model_info": {"token_count_framing": 78}, - } - ] - ), + Router(model_list=[_kimi_counting_itself(token_count_framing=78)]), ) monkeypatch.setattr( litellm.proxy.proxy_server, "_get_provider_token_counter", lambda deployment, model: (counter, model, "bedrock") @@ -3892,3 +3887,201 @@ async def test_responses_count_on_claude_counts_the_turns_previous_response_id_r ] assert [sorted(message) for message in messages if isinstance(message, dict)] == [["content", "role"]] * 3 spend_logs.assert_awaited_once_with("chatcmpl-stored", ("chatcmpl-stored", "resp_client_form")) + + +_LOCAL_MARKED: Final = "litellm-test.local-marked-chat-v1:0" +_BEDROCK_ROUTED: Final = "litellm-test.bedrock-routed-chat-v1:0" +_LOCAL_MARKED_ENTRY: Final = MappingProxyType( + { + "litellm_provider": "bedrock_converse", + "mode": "chat", + "max_tokens": 4096, + "max_input_tokens": 4096, + "max_output_tokens": 4096, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "token_count_model": {"litellm_provider": "local"}, + } +) +_BEDROCK_ROUTED_ENTRY: Final = MappingProxyType( + {**_LOCAL_MARKED_ENTRY, "token_count_model": {"model": _BEDROCK_ROUTED, "litellm_provider": "bedrock"}} +) +_HELLO: Final = [{"role": "user", "content": "Hello, how many tokens is this?"}] +_CANNOT_ESTIMATE: Final = f"LiteLLM counts tokens for {_LOCAL_MARKED} locally and could not estimate this request" +_WEB_SEARCH_REPLAY: Final = [ + {"role": "user", "content": "What is the newest LiteLLM release?"}, + { + "role": "assistant", + "content": [ + {"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "LiteLLM"}}, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_01", + "content": [ + { + "type": "web_search_result", + "url": "https://docs.litellm.ai/release_notes", + "title": "Release notes", + "encrypted_content": "", + } + ], + }, + {"type": "text", "text": "The newest release is in the release notes."}, + ], + }, + {"role": "user", "content": "Thanks."}, +] +_BOOLEAN_SUBSCHEMA_TOOL: Final = {"name": "search", "input_schema": {"type": "object", "properties": {"query": True}}} + + +def _bedrock_count_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(url__regex=r"^https://bedrock-count-tokens\.test/model/.+/count-tokens$").mock( + return_value=httpx.Response(200, json={"inputTokens": 999}) + ) + + +def _uncountable_deployment( + monkeypatch: pytest.MonkeyPatch, + model: str, + entry: Mapping[str, object], + model_info: Mapping[str, object] | None = None, +) -> None: + monkeypatch.setitem(litellm.model_cost, model, dict(entry)) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setattr(litellm.proxy.proxy_server, "proxy_config", litellm.proxy.proxy_server.ProxyConfig()) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "uncountable", + "litellm_params": { + "model": f"bedrock/{model}", + "api_base": "https://bedrock-count-tokens.test", + "aws_access_key_id": "AKIATESTACCESSKEY", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-east-1", + }, + "model_info": dict(model_info or {}), + } + ] + ), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "entry", "model_info", "expected"), + [ + pytest.param(_LOCAL_MARKED, _LOCAL_MARKED_ENTRY, {}, ("local_estimate", 0), id="shipped_local_marker"), + pytest.param( + _LOCAL_MARKED, + _LOCAL_MARKED_ENTRY, + {"token_count_model": {"model": _LOCAL_MARKED, "litellm_provider": "bedrock"}}, + ("token_count_model", 1), + id="config_routes_a_local_marked_model_to_bedrock", + ), + pytest.param( + _LOCAL_MARKED, + _LOCAL_MARKED_ENTRY, + {"token_count_model": {}}, + ("provider", 1), + id="config_makes_a_local_marked_model_count_itself", + ), + pytest.param(_BEDROCK_ROUTED, _BEDROCK_ROUTED_ENTRY, {}, ("token_count_model", 1), id="shipped_bedrock_route"), + pytest.param( + _BEDROCK_ROUTED, + _BEDROCK_ROUTED_ENTRY, + {"token_count_model": {"litellm_provider": "local"}}, + ("local_estimate", 0), + id="config_marks_a_bedrock_routed_model_local", + ), + ], +) +async def test_config_flips_the_cost_maps_local_marker_in_both_directions( + monkeypatch: pytest.MonkeyPatch, + model: str, + entry: Mapping[str, object], + model_info: dict[str, object], + expected: tuple[str, int], +): + _uncountable_deployment(monkeypatch, model, entry, model_info) + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = _bedrock_count_route(respx_mock) + counted: Final = await count_request_tokens( + request=TokenCountRequest(model="uncountable", messages=_HELLO), call_endpoint=True + ) + + assert (counted.count_source, route.call_count) == expected + assert counted.total_tokens == (999 if route.called else litellm.token_counter(model=model, messages=_HELLO)) + + +@pytest.mark.asyncio +async def test_messages_count_tokens_of_a_local_marked_model_says_the_count_is_a_local_estimate( + monkeypatch: pytest.MonkeyPatch, +): + _uncountable_deployment(monkeypatch, _LOCAL_MARKED, _LOCAL_MARKED_ENTRY) + body: Final = { + "model": "uncountable", + "system": "Be terse.", + "messages": _HELLO, + "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}], + } + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = _bedrock_count_route(respx_mock) + response: Final = await anthropic_count_tokens( + _json_request(body, "claude-cli/2.1.0 (external, cli)"), UserAPIKeyAuth() + ) + + input_tokens: Final = response["input_tokens"] + assert (response["litellm_count_source"], route.called) == ("local_estimate", False) + assert isinstance(input_tokens, int) and input_tokens > 0 + + +@pytest.mark.asyncio +async def test_local_marked_model_answers_503_when_local_counting_is_disabled(monkeypatch: pytest.MonkeyPatch): + _uncountable_deployment(monkeypatch, _LOCAL_MARKED, _LOCAL_MARKED_ENTRY) + monkeypatch.setattr(litellm, "disable_token_counter", True) + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = _bedrock_count_route(respx_mock) + with pytest.raises(ProxyException) as raised: + await count_request_tokens( + request=TokenCountRequest(model="uncountable", messages=_HELLO), call_endpoint=True + ) + + assert (raised.value.code, raised.value.type, route.called) == ("503", "token_counting_disabled", False) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "unreadable_body", + [ + pytest.param({"messages": _WEB_SEARCH_REPLAY}, id="web_search_replay"), + pytest.param({"messages": _HELLO, "tools": [_BOOLEAN_SUBSCHEMA_TOOL]}, id="boolean_subschema_tool"), + ], +) +async def test_local_marked_model_answers_a_400_not_a_500_for_a_body_the_estimate_cannot_read( + monkeypatch: pytest.MonkeyPatch, unreadable_body: dict[str, list[dict[str, object]]] +): + _uncountable_deployment(monkeypatch, _LOCAL_MARKED, _LOCAL_MARKED_ENTRY) + + with respx.mock(assert_all_called=False) as respx_mock: + route: Final = _bedrock_count_route(respx_mock) + with pytest.raises(ProxyException) as raised: + await count_request_tokens( + request=TokenCountRequest(model="uncountable", **unreadable_body), + call_endpoint=True, + messages_format="anthropic", + ) + + assert (raised.value.code, raised.value.type, raised.value.message, route.called) == ( + "400", + "token_counting_error", + _CANNOT_ESTIMATE, + False, + ) From 1eca810397ad7f41c1606c888ba1d10d7ea35777 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 2 Nov 2026 16:36:44 -0800 Subject: [PATCH 2/6] feat(count_tokens): let an admin name the tokenizer a deployment counts with and skip the provider call --- litellm/llms/bedrock/count_tokens/kimi_k3.py | 4 +- .../common_utils/token_count_settings.py | 25 +- litellm/proxy/proxy_server.py | 183 +++++++++- litellm/types/token_count.py | 2 + litellm/types/utils.py | 3 +- litellm/utils.py | 21 +- .../common_utils/test_token_count_settings.py | 37 +- tests/unit/proxy/test_proxy_token_counter.py | 333 +++++++++++++++++- 8 files changed, 593 insertions(+), 15 deletions(-) diff --git a/litellm/llms/bedrock/count_tokens/kimi_k3.py b/litellm/llms/bedrock/count_tokens/kimi_k3.py index 5eb46944f9a..28c3d1ebe20 100644 --- a/litellm/llms/bedrock/count_tokens/kimi_k3.py +++ b/litellm/llms/bedrock/count_tokens/kimi_k3.py @@ -422,7 +422,7 @@ def _rendered(request: _ConverseRequest) -> _Segments: ) -def _text_tokens(encoding: tiktoken.Encoding, text: str) -> int: +def kimi_k3_text_tokens(encoding: tiktoken.Encoding, text: str) -> int: spans: Final = (match.span() for match in _CHUNK.finditer(text)) return sum(len(encoding.encode_ordinary(text[start:end])) for start, end in spans) @@ -481,7 +481,7 @@ def kimi_k3_tokens( if request is None: return None return sum( - 1 if isinstance(segment, _Control) else _text_tokens(encoding, segment) for segment in _rendered(request) + 1 if isinstance(segment, _Control) else kimi_k3_text_tokens(encoding, segment) for segment in _rendered(request) ) diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index 4e3059cfe26..cb9c0ea9334 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -3,13 +3,13 @@ from dataclasses import dataclass, replace from types import MappingProxyType from typing import Final, Literal -from pydantic import BaseModel, ConfigDict, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import ProxyException -from litellm.types.token_count import TokenCountProvider, TokenCountSourceFields, TokenCountTokenizer +from litellm.types.token_count import LocalTokenizer, TokenCountProvider, TokenCountSourceFields, TokenCountTokenizer from litellm.types.utils import TokenCountResponse CANNOT_COUNT_MODEL_STATUSES: Final = frozenset({400, 404}) @@ -51,11 +51,17 @@ class TokenCountModel: configured: bool = False +@dataclass(frozen=True, slots=True) +class UnknownLocalTokenizer: + value: str + + @dataclass(frozen=True, slots=True) class TokenCountSettings: framing: int | None count_model: TokenCountModel | None tokenizer: TokenCountTokenizer | None = None + local_tokenizer: LocalTokenizer | UnknownLocalTokenizer | None = None tool_framing: int | None = None responses_framing: int | None = None responses_tool_framing: int | None = None @@ -197,6 +203,18 @@ def _tokenizer(value: object) -> TokenCountTokenizer | None: return None +_LOCAL_TOKENIZER: Final[TypeAdapter[LocalTokenizer]] = TypeAdapter(LocalTokenizer) + + +def _local_tokenizer(value: object) -> LocalTokenizer | UnknownLocalTokenizer | None: + if value is None or (isinstance(value, str) and not value.strip()): + return None + try: + return _LOCAL_TOKENIZER.validate_python(value, strict=True) + except ValidationError: + return UnknownLocalTokenizer(value=str(value)) + + def resolve_token_count_settings( configured: Mapping[str, object], default: Mapping[str, object], deployment_region: Callable[[], str] ) -> TokenCountSettings: @@ -206,6 +224,7 @@ def resolve_token_count_settings( configured.get("token_count_model"), default.get("token_count_model"), deployment_region ), tokenizer=_tokenizer(default.get("token_count_tokenizer")), + local_tokenizer=_local_tokenizer(configured.get("token_count_tokenizer")), tool_framing=_configured_framing(configured, default, "token_count_tool_framing"), responses_framing=_configured_framing(configured, default, "token_count_responses_framing"), responses_tool_framing=_configured_framing( @@ -224,5 +243,7 @@ def count_source_fields(response: TokenCountResponse) -> TokenCountSourceFields: return {} case "token_count_model": return {"litellm_count_source": "token_count_model", "litellm_count_model": response.model_used} + case "local_estimate" if response.count_tokenizer is not None: + return {"litellm_count_source": "local_estimate", "litellm_count_tokenizer": response.count_tokenizer} case source: return {"litellm_count_source": source} diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 868ca383e02..59e9a93e5f2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -58,6 +58,7 @@ from typing import ( import anyio import anyio.lowlevel +import tiktoken import websockets import websockets.exceptions from pydantic import BaseModel, ConfigDict, Json, JsonValue, TypeAdapter, ValidationError, with_config @@ -125,6 +126,7 @@ from litellm.llms.bedrock.count_tokens.kimi_k3 import ( KimiK3Encodings, kimi_k3_anthropic_tokens, kimi_k3_template_tokens, + kimi_k3_text_tokens, kimi_k3_tokens, ) from litellm.proxy._types import ( @@ -182,6 +184,7 @@ from litellm.proxy.common_utils.token_count_settings import ( TOKENIZER_LOAD_WAIT_SECONDS, ProviderCountFailure, TokenCountSettings, + UnknownLocalTokenizer, cost_map_entry, resolve_token_count_settings, ) @@ -205,6 +208,7 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import ( ) from litellm.rust_bridge.catalog import Target from litellm.types.caching import RedisPipelineIncrementOperation +from litellm.types.token_count import LocalTokenizer from litellm.types.utils import ( PRICING_OVERRIDES_KEY, CallTypes, @@ -14965,6 +14969,7 @@ from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import VerificationTokenRepository from litellm.responses.litellm_completion_transformation.count_request import ( ChatCountBridgeError, + ChatCountRequest, chat_count_request, with_provider_file_ids, ) @@ -15247,7 +15252,7 @@ def _provider_error_after_failed_estimate( provider_failure: ProviderCountFailure, error: BaseException ) -> ProxyException: verbose_proxy_logger.warning( - "Local token estimate failed after the provider's count error %s, answering that error: %s", + "Local token estimate failed, answering count error %s instead: %s", provider_failure.status_code, type(error).__name__, ) @@ -15450,6 +15455,17 @@ async def _kimi_k3_count( encoding: Final = await _KIMI_K3_ENCODINGS.encoding(TOKENIZER_LOAD_WAIT_SECONDS) if encoding is None: return None + return await _kimi_k3_encoded_count(encoding, request, messages_format, model, messages, tools) + + +async def _kimi_k3_encoded_count( + encoding: tiktoken.Encoding, + request: TokenCountRequest, + messages_format: TokenCountMessagesFormat, + model: str, + messages: Sequence[AllMessageValues], + tools: Sequence[ChatCompletionToolParam] | None, +) -> int | None: match messages_format: case "anthropic": return await offload_token_count(kimi_k3_anthropic_tokens)( @@ -15459,6 +15475,132 @@ async def _kimi_k3_count( return await offload_token_count(kimi_k3_tokens)(encoding, model, messages, tools) +def _local_tokenizer_failure(tokenizer: str, model: str, reason: str) -> ProviderCountFailure: + return ProviderCountFailure( + status_code=400, + message=f"{model} counts tokens with the {tokenizer} tokenizer set in token_count_tokenizer, which {reason}", + rejected_before_provider_call=True, + ) + + +async def _kimi_k3_local_tokens( + prompt: str | None, + count_messages: Callable[[tiktoken.Encoding], Awaitable[int | None]], + unreadable: ProviderCountFailure, + unavailable: ProviderCountFailure, +) -> int: + encoding: Final = await _KIMI_K3_ENCODINGS.encoding(TOKENIZER_LOAD_WAIT_SECONDS) + if encoding is None: + raise unavailable.to_proxy_exception() + counted: Final = await _estimate_or_provider_error( + count_messages(encoding) if prompt is None else offload_token_count(kimi_k3_text_tokens)(encoding, prompt), + unreadable, + ) + if counted is None: + raise unreadable.to_proxy_exception() + return counted + + +async def _local_tokenizer_tokens( + tokenizer: LocalTokenizer, + request_model: str, + template_model: str, + prompt: str | None, + messages: Sequence[AllMessageValues] | None, + tools: list[ChatCompletionToolParam] | None, + kimi_k3_count: Callable[[tiktoken.Encoding], Awaitable[int | None]], +) -> _LocalTokenEstimate: + unreadable: Final = _local_tokenizer_failure(tokenizer, request_model, "could not count this request") + unavailable: Final = _local_tokenizer_failure(tokenizer, request_model, "could not be loaded") + if tokenizer == "kimi_k3": + return _LocalTokenEstimate( + total_tokens=await _kimi_k3_local_tokens(prompt, kimi_k3_count, unreadable, unavailable), + tokenizer_type="huggingface_tokenizer", + ) + selected: Final = await asyncify(litellm.utils.select_named_tokenizer)(tokenizer) + if selected is None: + raise unavailable.to_proxy_exception() + total_tokens: Final = await _estimate_or_provider_error( + offload_token_count(litellm.token_counter)( + model=template_model, text=prompt, messages=messages, tools=tools, custom_tokenizer=selected + ), + unreadable, + ) + return _LocalTokenEstimate(total_tokens=total_tokens, tokenizer_type=selected["type"]) + + +async def _count_with_local_tokenizer( + tokenizer: LocalTokenizer, + request: TokenCountRequest, + messages_format: TokenCountMessagesFormat, + model: str, + template_model: str, + prompt: str | None, + messages: Sequence[AllMessageValues] | None, + tools: list[ChatCompletionToolParam] | None, + bridged: ChatCountRequest | None, + settings: TokenCountSettings, + responses_api: bool, + contents_estimable: bool, + stored_turns_unseen: bool, +) -> TokenCountResponse: + """Count with the tokenizer the deployment names in token_count_tokenizer, never asking the provider. + + A Responses request bridged to chat is counted as the chat body inference sends, previous_response_id turns + included.""" + if stored_turns_unseen: + raise _local_tokenizer_failure( + tokenizer, request.model, "cannot see the earlier turns previous_response_id points to on the provider" + ).to_proxy_exception() + if not contents_estimable: + raise _local_tokenizer_failure(tokenizer, request.model, "cannot read these contents").to_proxy_exception() + counted_messages: Final = ( + messages + if bridged is None + else cast( # cast-ok: the chat body Responses inference sends, chat-shaped dicts token_counter normalizes + Sequence[AllMessageValues], [dict(message) for message in bridged.messages] + ) + ) + counted_tools: Final = ( + tools + if bridged is None + else cast( # cast-ok: the chat tools Responses inference sends + list[ChatCompletionToolParam] | None, [dict(tool) for tool in bridged.tools or ()] or None + ) + ) + estimate: Final = await _local_tokenizer_tokens( + tokenizer, + request.model, + template_model, + prompt, + counted_messages, + counted_tools, + kimi_k3_count=partial( + _kimi_k3_encoded_count, + request=request, + messages_format=messages_format if bridged is None else "openai", + model=model, + messages=counted_messages or (), + tools=counted_tools, + ), + ) + framing: Final = ( + 0 + if counted_messages is None + else settings.hidden_prompt_tokens( + carries_tools=sends_function_tools(counted_tools), responses_api=responses_api + ) + ) + return TokenCountResponse( + total_tokens=estimate.total_tokens + framing, + request_model=request.model, + model_used=model, + tokenizer_type=estimate.tokenizer_type, + count_source="local_estimate", + count_tokenizer=tokenizer, + ) + + class _TokenCountDeployment(TypedDict, total=False): litellm_params: ReadOnly[Mapping[str, object]] model_info: ReadOnly[Mapping[str, object]] @@ -15863,6 +16005,13 @@ async def _count_request_tokens( default=await asyncify(cost_map_entry)(*_token_count_settings_model(deployment_view, request.model)), deployment_region=partial(_deployment_aws_region, deployment_view), ) + local_tokenizer: Final = settings.local_tokenizer + if isinstance(local_tokenizer, UnknownLocalTokenizer): + raise _rejected_view( + 400, + f"{request.model} sets token_count_tokenizer to {local_tokenizer.value!r}, " + f"which is not one of {', '.join(get_args(LocalTokenizer))}", + ) # Try provider-specific token counting first - only for non-direct requests (from provider endpoints) provider_counter: BaseTokenCounter | None = None @@ -15926,12 +16075,16 @@ async def _count_request_tokens( routed_request: Final = ( None if responses_request is None else with_provider_file_ids(responses_request, routed_deployment) ) + counting_provider: Final = provider_counter if local_tokenizer is None else None chat_count: Final = ( None - if routed_request is None or provider_counter is None or isinstance(provider_counter, OpenAITokenCounter) + if routed_request is None + or (local_tokenizer is None and (provider_counter is None or isinstance(provider_counter, OpenAITokenCounter))) else await chat_count_request( model_to_use, - custom_llm_provider, + custom_llm_provider + if local_tokenizer is None + else custom_llm_provider or _deployment_provider(deployment_view, request.model, routed_request), _deployment_api_base(deployment_view, routed_request), routed_request, _count_drop_policy(routed_deployment, routed_request), @@ -15941,10 +16094,10 @@ async def _count_request_tokens( raise ProxyException(message=chat_count.message, type="invalid_request_error", param=None, code=400) ladder: Final = ( None - if provider_counter is None + if counting_provider is None else await _provider_token_count( count_with=_native_responses_count( - provider_counter, + counting_provider, custom_llm_provider, routed_request, routed_deployment, @@ -15952,7 +16105,7 @@ async def _count_request_tokens( ) or partial( _try_provider_token_count, - provider_counter, + counting_provider, custom_llm_provider, messages=( [dict(message) for message in chat_count.messages] @@ -16055,6 +16208,24 @@ async def _count_request_tokens( carries_tools=sends_function_tools(counted_tools), responses_api=native_responses ) ) + if local_tokenizer is not None: + return await _count_with_local_tokenizer( + local_tokenizer, + request + if counted_view is None + else request.model_copy(update={"messages": local_messages, "system": local_system}), + messages_format, + model_to_use, + template_model, + prompt=prompt if counted_messages is None else None, + messages=counted_messages, + tools=counted_tools, + bridged=chat_count, + settings=settings, + responses_api=native_responses, + contents_estimable=contents_estimable, + stored_turns_unseen=chat_count is None and bool(_responses_routing_kwargs(routed_request)), + ) kimi_k3_messages: Final = counted_messages if settings.tokenizer == "kimi_k3" else None exact: Final = ( await _kimi_k3_count( diff --git a/litellm/types/token_count.py b/litellm/types/token_count.py index 90ccefea177..f89bcbfaa35 100644 --- a/litellm/types/token_count.py +++ b/litellm/types/token_count.py @@ -4,6 +4,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict TokenCountSource: TypeAlias = Literal["provider", "token_count_model", "local_estimate"] TokenCountTokenizer: TypeAlias = Literal["kimi_k3"] +LocalTokenizer: TypeAlias = Literal["openai", "llama3", "deepseek", "cohere", "kimi_k3"] TokenCountProvider: TypeAlias = Literal["bedrock", "bedrock_mantle"] @@ -16,3 +17,4 @@ class TokenCountModelInfo(TypedDict): class TokenCountSourceFields(TypedDict, total=False): litellm_count_source: ReadOnly[TokenCountSource] litellm_count_model: ReadOnly[str] + litellm_count_tokenizer: ReadOnly[LocalTokenizer] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index eeb0956a06b..bd181748437 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -67,7 +67,7 @@ from litellm.types.llms.base import ( LiteLLMPydanticObjectBase, ) from litellm.types.mcp import MCPServerCostInfo -from litellm.types.token_count import TokenCountModelInfo, TokenCountSource, TokenCountTokenizer +from litellm.types.token_count import LocalTokenizer, TokenCountModelInfo, TokenCountSource, TokenCountTokenizer from litellm.types.workload_identity import ( ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS, @@ -4812,6 +4812,7 @@ class TokenCountResponse(LiteLLMPydanticObjectBase): model_used: str tokenizer_type: str count_source: TokenCountSource | None = None + count_tokenizer: LocalTokenizer | None = None original_response: dict | None = None """ Original Response from upstream API call - if an API call was made for token counting diff --git a/litellm/utils.py b/litellm/utils.py index 55423a42260..561f8094729 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2910,7 +2910,26 @@ def _return_huggingface_tokenizer( model: str, outcomes: _HuggingFaceTokenizerLoadOutcomes = _HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES ) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: kind: Final = huggingface_tokenizer_kind(model) - if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()): + return None if kind is None else _huggingface_tokenizer_of_kind(kind, model, outcomes) + + +def select_named_tokenizer( + name: Literal["openai"] | HuggingFaceTokenizerKind, + outcomes: _HuggingFaceTokenizerLoadOutcomes = _HUGGINGFACE_TOKENIZER_LOAD_OUTCOMES, +) -> SelectTokenizerResponse | None: + """The tokenizer `name` stands for, never another one: None when it cannot be loaded.""" + if name == "openai": + return _return_openai_tokenizer(name) + if litellm.disable_hf_tokenizer_download is True and name not in _BUNDLED_TOKENIZER_LOAD_LOCKS: + return None + loaded: Final = _huggingface_tokenizer_of_kind(name, name, outcomes) + return None if loaded is None or isinstance(loaded, _HuggingFaceTokenizerLoadFailure) else loaded + + +def _huggingface_tokenizer_of_kind( + kind: HuggingFaceTokenizerKind, model: str, outcomes: _HuggingFaceTokenizerLoadOutcomes +) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: + if kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode(): return None key: Final = (kind, _huggingface_tokenizer_backend()) with _BUNDLED_TOKENIZER_LOAD_LOCKS.get(kind, nullcontext()): diff --git a/tests/unit/proxy/common_utils/test_token_count_settings.py b/tests/unit/proxy/common_utils/test_token_count_settings.py index 68910f52b02..274eb051d6c 100644 --- a/tests/unit/proxy/common_utils/test_token_count_settings.py +++ b/tests/unit/proxy/common_utils/test_token_count_settings.py @@ -9,6 +9,7 @@ from litellm.proxy.common_utils.token_count_settings import ( ProviderCountFailure, TokenCountModel, TokenCountSettings, + UnknownLocalTokenizer, count_source_fields, resolve_token_count_settings, ) @@ -249,13 +250,40 @@ def test_only_a_tokenizer_litellm_ships_is_taken_from_the_cost_map(value: object assert settings.tokenizer == expected -def _counted(source: str | None, model_used: str = "us.anthropic.claude-opus-5-5") -> TokenCountResponse: +@pytest.mark.parametrize( + ("configured", "expected"), + [ + ({"token_count_tokenizer": "openai"}, "openai"), + ({"token_count_tokenizer": "deepseek"}, "deepseek"), + ({"token_count_tokenizer": "kimi_k3"}, "kimi_k3"), + ({"token_count_tokenizer": "local"}, UnknownLocalTokenizer(value="local")), + ({"token_count_tokenizer": "OpenAI"}, UnknownLocalTokenizer(value="OpenAI")), + ({"token_count_tokenizer": 5}, UnknownLocalTokenizer(value="5")), + ({"token_count_tokenizer": " "}, None), + ({"token_count_tokenizer": None}, None), + ({}, None), + ], +) +def test_only_the_deployment_names_the_tokenizer_that_counts_without_the_provider( + configured: dict[str, object], expected: object +) -> None: + settings: Final = resolve_token_count_settings( + configured=configured, default={"token_count_tokenizer": "kimi_k3"}, deployment_region=_in_us_west_2 + ) + + assert (settings.local_tokenizer, settings.tokenizer) == (expected, "kimi_k3") + + +def _counted( + source: str | None, model_used: str = "us.anthropic.claude-opus-5-5", count_tokenizer: str | None = None +) -> TokenCountResponse: return TokenCountResponse( total_tokens=7, request_model="sonnet", model_used=model_used, tokenizer_type="bedrock_api", count_source=source, + count_tokenizer=count_tokenizer, ) @@ -277,6 +305,13 @@ def test_count_source_fields_name_the_counting_model_only_when_token_count_model assert count_source_fields(_counted(source)) == expected +def test_count_source_fields_name_the_tokenizer_a_local_estimate_was_counted_with() -> None: + assert count_source_fields(_counted("local_estimate", count_tokenizer="llama3")) == { + "litellm_count_source": "local_estimate", + "litellm_count_tokenizer": "llama3", + } + + def test_every_shipped_token_count_model_resolves_to_a_mapped_aws_model() -> None: cost_map: Final = json.loads(_COST_MAP.read_text()) aws_models: Final = frozenset( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 2f85d1c7516..8c4f9b26302 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -4331,13 +4331,14 @@ async def test_messages_count_tokens_on_a_responses_api_bridge_rejects_an_invali class _CountTokensPeer: - def __init__(self, reply: Mapping[str, JsonValue]) -> None: + def __init__(self, reply: Mapping[str, JsonValue], status_code: int = 200) -> None: self._reply: Final = reply + self._status_code: Final = status_code self.requests: tuple[httpx.Request, ...] = () def __call__(self, request: httpx.Request) -> httpx.Response: self.requests = (*self.requests, request) - return httpx.Response(200, json=dict(self._reply), request=request) + return httpx.Response(self._status_code, json=dict(self._reply), request=request) class _BedrockInvokeModel(BaseModel): @@ -5359,3 +5360,331 @@ def test_utils_token_counter_line_stays_one_line_when_the_model_carries_a_newlin assert line["request_model"] == forged_model assert "\n" not in message assert message.startswith(f'token count - litellm_call_id="count log" request_model={json.dumps(forged_model)} ') + + +_NOVA_PRO: Final = "us.amazon.nova-pro-v1:0" +_AWS_CANNOT_COUNT: Final = "The provided model doesn't support counting tokens." +_MIXED_SCRIPT_TURNS: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20} +] +_DEEPSEEK_TOKENIZER_JSON: Final = ( + Path(litellm.__file__).parent / "litellm_core_utils/tokenizers/deepseek_tokenizer.json" +) + + +def _expected_local_count(model: str, tokenizer: str) -> int: + custom_tokenizer: Final = ( + litellm.utils.create_tokenizer(_DEEPSEEK_TOKENIZER_JSON.read_text(encoding="utf-8")) + if tokenizer == "deepseek" + else None + ) + return litellm.token_counter(model=model, messages=_MIXED_SCRIPT_TURNS, custom_tokenizer=custom_tokenizer) + + +def _bedrock_router(model: str, model_info: Mapping[str, JsonValue]) -> Router: + return Router( + model_list=[ + { + "model_name": "bedrock-agent", + "litellm_params": { + "model": f"bedrock/{model}", + "aws_access_key_id": "AKIATESTACCESSKEY", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-east-1", + }, + "model_info": dict(model_info), + } + ] + ) + + +async def _bedrock_counts( + monkeypatch: pytest.MonkeyPatch, model: str, model_info: Mapping[str, JsonValue], body: Mapping[str, JsonValue] +) -> tuple[object, TokenCountResponse | ProxyException, tuple[httpx.Request, ...]]: + from litellm.llms.bedrock.count_tokens import handler as bedrock_count_tokens_handler + + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + peer: Final = _CountTokensPeer({"message": _AWS_CANNOT_COUNT}, status_code=400) + router: Final = _bedrock_router(model, model_info) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + with patch.object( + bedrock_count_tokens_handler, + "get_async_httpx_client", + return_value=AsyncHTTPHandler(transport=httpx.MockTransport(peer)), + ): + messages_reply: Final = await _messages_endpoint_count(router, {**body, "model": "bedrock-agent"}) + try: + utils_reply: TokenCountResponse | ProxyException = await token_counter( + request=TokenCountRequest.model_validate({**body, "model": "bedrock-agent"}), call_endpoint=True + ) + except ProxyException as rejected: + utils_reply = rejected # rebind-ok: the utils endpoint answers its rejection as a raised ProxyException + return messages_reply, utils_reply, peer.requests + + +def _rejection(reply: object) -> tuple[int, str]: + assert isinstance(reply, JSONResponse), reply + return reply.status_code, json.loads(bytes(reply.body))["error"]["message"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tokenizer", ["openai", "deepseek"]) +async def test_a_named_token_count_tokenizer_counts_with_exactly_that_tokenizer_and_never_asks_the_provider( + monkeypatch: pytest.MonkeyPatch, tokenizer: str +) -> None: + counts: Final = {name: _expected_local_count(_NOVA_PRO, name) for name in ("openai", "deepseek")} + assert counts["openai"] != counts["deepseek"], counts + + messages_reply, utils_reply, sent = await _bedrock_counts( + monkeypatch, _NOVA_PRO, {"token_count_tokenizer": tokenizer}, {"messages": _MIXED_SCRIPT_TURNS} + ) + + assert sent == () + assert messages_reply == { + "input_tokens": counts[tokenizer], + "litellm_count_source": "local_estimate", + "litellm_count_tokenizer": tokenizer, + } + assert isinstance(utils_reply, TokenCountResponse), utils_reply + assert (utils_reply.total_tokens, utils_reply.count_source, utils_reply.count_tokenizer) == ( + counts[tokenizer], + "local_estimate", + tokenizer, + ) + + +@pytest.mark.asyncio +async def test_without_a_named_tokenizer_a_model_the_provider_cannot_count_answers_the_providers_400( + monkeypatch: pytest.MonkeyPatch, +) -> None: + messages_reply, utils_reply, sent = await _bedrock_counts( + monkeypatch, _NOVA_PRO, {}, {"messages": _MIXED_SCRIPT_TURNS} + ) + + status, message = _rejection(messages_reply) + assert (status, _AWS_CANNOT_COUNT in message) == (400, True), message + assert isinstance(utils_reply, ProxyException), utils_reply + assert (utils_reply.code, _AWS_CANNOT_COUNT in utils_reply.message) == ("400", True), utils_reply.message + assert [request.url.path.endswith("/count-tokens") for request in sent] == [True, True] + + +@pytest.mark.asyncio +async def test_the_deployments_named_tokenizer_beats_the_tokenizer_the_cost_map_ships( + monkeypatch: pytest.MonkeyPatch, +) -> None: + kimi: Final = "us.moonshotai.kimi-k3" + assert cost_map_entry(kimi).get("token_count_tokenizer") == "kimi_k3" + + messages_reply, _, sent = await _bedrock_counts( + monkeypatch, kimi, {"token_count_tokenizer": "deepseek"}, {"messages": _MIXED_SCRIPT_TURNS} + ) + + assert sent == () + assert messages_reply == { + "input_tokens": _expected_local_count(kimi, "deepseek") + + (cost_map_entry(kimi).get("token_count_framing") or 0), + "litellm_count_source": "local_estimate", + "litellm_count_tokenizer": "deepseek", + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_info", "reason"), + [ + ({"token_count_tokenizer": "local"}, "sets token_count_tokenizer to 'local', which is not one of openai, "), + ( + {"token_count_tokenizer": "kimi_k3"}, + "counts tokens with the kimi_k3 tokenizer set in token_count_tokenizer, which could not be loaded", + ), + ], + ids=["unknown_name", "tokenizer_that_cannot_load"], +) +async def test_a_named_tokenizer_litellm_cannot_count_with_answers_400_without_asking_the_provider( + monkeypatch: pytest.MonkeyPatch, model_info: dict[str, JsonValue], reason: str +) -> None: + messages_reply, utils_reply, sent = await _bedrock_counts( + monkeypatch, _NOVA_PRO, model_info, {"messages": _MIXED_SCRIPT_TURNS} + ) + + status, message = _rejection(messages_reply) + assert (status, reason in message) == (400, True), message + assert isinstance(utils_reply, ProxyException), utils_reply + assert (utils_reply.code, reason in utils_reply.message) == ("400", True), utils_reply.message + assert sent == () + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "unreadable_body", + [ + pytest.param( + { + "messages": [ + {"role": "user", "content": "What is new?"}, + { + "role": "assistant", + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_01", + "name": "web_search", + "input": {"query": "new"}, + }, + {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_01", "content": []}, + ], + }, + ] + }, + id="web_search_blocks", + ), + pytest.param( + { + "messages": [{"role": "user", "content": "Describe this"}], + "contents": [ + { + "role": "user", + "parts": [ + {"text": "Describe this"}, + {"fileData": {"mimeType": "image/png", "fileUri": "gs://b/c.png"}}, + ], + } + ], + }, + id="google_contents_with_media_the_translation_dropped", + ), + ], +) +async def test_a_named_tokenizer_that_cannot_read_the_request_answers_400_never_500( + monkeypatch: pytest.MonkeyPatch, unreadable_body: dict[str, JsonValue] +) -> None: + _, utils_reply, sent = await _bedrock_counts( + monkeypatch, _NOVA_PRO, {"token_count_tokenizer": "openai"}, unreadable_body + ) + + assert isinstance(utils_reply, ProxyException), utils_reply + assert utils_reply.code == "400", utils_reply.message + assert "counts tokens with the openai tokenizer set in token_count_tokenizer, which " in utils_reply.message + assert sent == () + + +_STORED_1998_TURN: Final[dict[str, JsonValue]] = { + "request_id": "chatcmpl-stored", + "call_type": "aresponses", + "session_id": "session-1", + "proxy_server_request": {"input": "Who won in 1998?", "model": "bedrock-agent"}, + "response": { + "id": "chatcmpl-stored", + "object": "chat.completion", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "The Bulls."}, "finish_reason": "stop"}], + }, + "status": "success", +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("litellm_params", "served_model"), + [ + ( + { + "model": f"bedrock/{_NOVA_PRO}", + "aws_access_key_id": "AKIATESTACCESSKEY", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-east-1", + }, + _NOVA_PRO, + ), + ({"model": "mistral/mistral-small-latest", "api_key": "sk-test"}, "mistral-small-latest"), + ], + ids=["provider_with_a_count_api", "provider_without_a_count_api"], +) +async def test_a_named_tokenizer_counts_the_turns_previous_response_id_replays( + monkeypatch: pytest.MonkeyPatch, litellm_params: dict[str, JsonValue], served_model: str +) -> None: + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "bedrock-agent", + "litellm_params": litellm_params, + "model_info": {"token_count_tokenizer": "openai"}, + } + ] + ), + ) + inline_turns: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": "Who won in 1998?"}, + {"role": "assistant", "content": "The Bulls."}, + {"role": "user", "content": "And in 1999?"}, + ] + + with patch.object( + ResponsesSessionHandler, + "get_all_spend_logs_for_previous_response_id", + new_callable=AsyncMock, + return_value=[_STORED_1998_TURN], + ): + replayed: Final = await count_request_tokens( + request=TokenCountRequest(model="bedrock-agent", messages=[{"role": "user", "content": "And in 1999?"}]), + call_endpoint=True, + responses_request={ + "model": "bedrock-agent", + "input": "And in 1999?", + "previous_response_id": "chatcmpl-stored", + }, + ) + inline: Final = await count_request_tokens( + request=TokenCountRequest(model="bedrock-agent", messages=inline_turns), + call_endpoint=True, + responses_request={"model": "bedrock-agent", "input": inline_turns}, + ) + + assert (replayed.count_source, replayed.count_tokenizer) == ("local_estimate", "openai") + assert ( + replayed.total_tokens == inline.total_tokens == litellm.token_counter(model=served_model, messages=inline_turns) + ) + + +@pytest.mark.asyncio +async def test_a_named_tokenizer_answers_400_for_previous_response_id_turns_only_the_provider_stores( + monkeypatch: pytest.MonkeyPatch, +) -> None: + peer: Final = _CountTokensPeer({"input_tokens": 99}) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "gpt-counted", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + "api_base": "https://count-tokens.test/v1", + }, + "model_info": {"token_count_tokenizer": "openai"}, + } + ] + ), + ) + + with ( + patch.object( + openai_count_tokens_http, + "get_async_httpx_client", + return_value=AsyncHTTPHandler(transport=httpx.MockTransport(peer)), + ), + pytest.raises(ProxyException) as rejected, + ): + await count_request_tokens( + request=TokenCountRequest(model="gpt-counted", messages=[{"role": "user", "content": "again"}]), + call_endpoint=True, + responses_request={"model": "gpt-counted", "input": "again", "previous_response_id": "resp_original"}, + ) + + assert rejected.value.code == "400", rejected.value.message + assert "cannot see the earlier turns previous_response_id points to" in rejected.value.message + assert peer.requests == () From c2c152f0607189e2ab95bdeaccb3e4a3612486e8 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 2 Nov 2026 16:40:50 -0800 Subject: [PATCH 3/6] refactor(count_tokens): share the tokenizer count call and keep estimate types exact --- litellm/proxy/proxy_server.py | 43 ++++++++++++------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 7 +++ 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 59e9a93e5f2..439978e1b0b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15259,7 +15259,12 @@ def _provider_error_after_failed_estimate( return provider_failure.to_proxy_exception() -async def _estimate_or_provider_error(estimate: Awaitable[int], provider_failure: ProviderCountFailure) -> int: +_EstimateT = TypeVar("_EstimateT", int, int | None) + + +async def _estimate_or_provider_error( + estimate: Awaitable[_EstimateT], provider_failure: ProviderCountFailure +) -> _EstimateT: try: return await estimate except BaseException as e: @@ -15413,6 +15418,22 @@ async def _tokenizer_or_provider_error( raise _provider_error_after_failed_estimate(provider_failure, tokenizer) from tokenizer +def _tokenizer_count( + template_model: str, + prompt: str | None, + messages: Sequence[AllMessageValues] | None, + tools: list[ChatCompletionToolParam] | None, + tokenizer: SelectTokenizerResponse, +) -> Awaitable[int]: + return offload_token_count(litellm.token_counter)( + model=template_model, + text=prompt, + messages=messages, + tools=tools, + custom_tokenizer=tokenizer, + ) + + async def _local_token_estimate( model: str, template_model: str, @@ -15427,13 +15448,7 @@ async def _local_token_estimate( if provider_failure is None else await _tokenizer_or_provider_error(model, custom_tokenizer, provider_failure) ) - count: Final = offload_token_count(litellm.token_counter)( - model=template_model, - text=prompt, - messages=messages, - tools=tools, - custom_tokenizer=tokenizer, - ) + count: Final = _tokenizer_count(template_model, prompt, messages, tools, tokenizer) total_tokens: Final = await ( count if provider_failure is None else _estimate_or_provider_error(count, provider_failure) ) @@ -15492,10 +15507,9 @@ async def _kimi_k3_local_tokens( encoding: Final = await _KIMI_K3_ENCODINGS.encoding(TOKENIZER_LOAD_WAIT_SECONDS) if encoding is None: raise unavailable.to_proxy_exception() - counted: Final = await _estimate_or_provider_error( - count_messages(encoding) if prompt is None else offload_token_count(kimi_k3_text_tokens)(encoding, prompt), - unreadable, - ) + if prompt is not None: + return await _estimate_or_provider_error(offload_token_count(kimi_k3_text_tokens)(encoding, prompt), unreadable) + counted: Final = await _estimate_or_provider_error(count_messages(encoding), unreadable) if counted is None: raise unreadable.to_proxy_exception() return counted @@ -15521,10 +15535,7 @@ async def _local_tokenizer_tokens( if selected is None: raise unavailable.to_proxy_exception() total_tokens: Final = await _estimate_or_provider_error( - offload_token_count(litellm.token_counter)( - model=template_model, text=prompt, messages=messages, tools=tools, custom_tokenizer=selected - ), - unreadable, + _tokenizer_count(template_model, prompt, messages, tools, selected), unreadable ) return _LocalTokenEstimate(total_tokens=total_tokens, tokenizer_type=selected["type"]) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3aa7c888e98..ad41cc10a2a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -47732,6 +47732,11 @@ export interface components { * @enum {string} */ litellm_count_source?: "provider" | "token_count_model" | "local_estimate"; + /** + * Litellm Count Tokenizer + * @enum {string} + */ + litellm_count_tokenizer?: "openai" | "llama3" | "deepseek" | "cohere" | "kimi_k3"; /** Prompttokensdetails */ promptTokensDetails: components["schemas"]["PromptTokensDetails"][]; /** Totaltokens */ @@ -47762,6 +47767,8 @@ export interface components { TokenCountResponse: { /** Count Source */ count_source?: ("provider" | "token_count_model" | "local_estimate") | null; + /** Count Tokenizer */ + count_tokenizer?: ("openai" | "llama3" | "deepseek" | "cohere" | "kimi_k3") | null; /** * Error * @default false From 9f0042e2072f35f49acb3128df33fbd50d77e3be Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 2 Nov 2026 16:47:42 -0800 Subject: [PATCH 4/6] refactor(count_tokens): keep the count handler as simple as before and its annotations read-only --- litellm/proxy/proxy_server.py | 62 +++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 439978e1b0b..821ae5958dc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15259,12 +15259,7 @@ def _provider_error_after_failed_estimate( return provider_failure.to_proxy_exception() -_EstimateT = TypeVar("_EstimateT", int, int | None) - - -async def _estimate_or_provider_error( - estimate: Awaitable[_EstimateT], provider_failure: ProviderCountFailure -) -> _EstimateT: +async def _estimate_or_provider_error(estimate: Awaitable[int], provider_failure: ProviderCountFailure) -> int: try: return await estimate except BaseException as e: @@ -15422,14 +15417,14 @@ def _tokenizer_count( template_model: str, prompt: str | None, messages: Sequence[AllMessageValues] | None, - tools: list[ChatCompletionToolParam] | None, + tools: Sequence[ChatCompletionToolParam] | None, tokenizer: SelectTokenizerResponse, ) -> Awaitable[int]: return offload_token_count(litellm.token_counter)( model=template_model, text=prompt, messages=messages, - tools=tools, + tools=None if tools is None else list(tools), custom_tokenizer=tokenizer, ) @@ -15490,6 +15485,17 @@ async def _kimi_k3_encoded_count( return await offload_token_count(kimi_k3_tokens)(encoding, model, messages, tools) +def _known_local_tokenizer(settings: TokenCountSettings, request_model: str) -> LocalTokenizer | None: + named: Final = settings.local_tokenizer + if isinstance(named, UnknownLocalTokenizer): + raise _rejected_view( + 400, + f"{request_model} sets token_count_tokenizer to {named.value!r}, " + f"which is not one of {', '.join(get_args(LocalTokenizer))}", + ) + return named + + def _local_tokenizer_failure(tokenizer: str, model: str, reason: str) -> ProviderCountFailure: return ProviderCountFailure( status_code=400, @@ -15507,11 +15513,18 @@ async def _kimi_k3_local_tokens( encoding: Final = await _KIMI_K3_ENCODINGS.encoding(TOKENIZER_LOAD_WAIT_SECONDS) if encoding is None: raise unavailable.to_proxy_exception() - if prompt is not None: - return await _estimate_or_provider_error(offload_token_count(kimi_k3_text_tokens)(encoding, prompt), unreadable) - counted: Final = await _estimate_or_provider_error(count_messages(encoding), unreadable) + return await _estimate_or_provider_error( + offload_token_count(kimi_k3_text_tokens)(encoding, prompt) + if prompt is not None + else _counted_or_rejected(count_messages(encoding), unreadable), + unreadable, + ) + + +async def _counted_or_rejected(count: Awaitable[int | None], rejection: ProviderCountFailure) -> int: + counted: Final = await count if counted is None: - raise unreadable.to_proxy_exception() + raise rejection.to_proxy_exception() return counted @@ -15521,7 +15534,7 @@ async def _local_tokenizer_tokens( template_model: str, prompt: str | None, messages: Sequence[AllMessageValues] | None, - tools: list[ChatCompletionToolParam] | None, + tools: Sequence[ChatCompletionToolParam] | None, kimi_k3_count: Callable[[tiktoken.Encoding], Awaitable[int | None]], ) -> _LocalTokenEstimate: unreadable: Final = _local_tokenizer_failure(tokenizer, request_model, "could not count this request") @@ -15548,7 +15561,7 @@ async def _count_with_local_tokenizer( template_model: str, prompt: str | None, messages: Sequence[AllMessageValues] | None, - tools: list[ChatCompletionToolParam] | None, + tools: Sequence[ChatCompletionToolParam] | None, bridged: ChatCountRequest | None, settings: TokenCountSettings, responses_api: bool, @@ -15576,7 +15589,7 @@ async def _count_with_local_tokenizer( tools if bridged is None else cast( # cast-ok: the chat tools Responses inference sends - list[ChatCompletionToolParam] | None, [dict(tool) for tool in bridged.tools or ()] or None + Sequence[ChatCompletionToolParam] | None, [dict(tool) for tool in bridged.tools or ()] or None ) ) estimate: Final = await _local_tokenizer_tokens( @@ -16016,13 +16029,7 @@ async def _count_request_tokens( default=await asyncify(cost_map_entry)(*_token_count_settings_model(deployment_view, request.model)), deployment_region=partial(_deployment_aws_region, deployment_view), ) - local_tokenizer: Final = settings.local_tokenizer - if isinstance(local_tokenizer, UnknownLocalTokenizer): - raise _rejected_view( - 400, - f"{request.model} sets token_count_tokenizer to {local_tokenizer.value!r}, " - f"which is not one of {', '.join(get_args(LocalTokenizer))}", - ) + local_tokenizer: Final = _known_local_tokenizer(settings, request.model) # Try provider-specific token counting first - only for non-direct requests (from provider endpoints) provider_counter: BaseTokenCounter | None = None @@ -16179,12 +16186,11 @@ async def _count_request_tokens( local_system: Final = system if counted_view is None else counted_view.system # Default LiteLLM token counting - custom_tokenizer: CustomHuggingfaceTokenizer | None = None - if model_info is not None: - custom_tokenizer = cast( - CustomHuggingfaceTokenizer | None, - model_info.get("custom_tokenizer", None), - ) + custom_tokenizer: Final = ( + None + if model_info is None + else cast(CustomHuggingfaceTokenizer | None, model_info.get("custom_tokenizer", None)) + ) system_message: Final = _system_message(local_system) prompt_turn: Final = prompt_only and not (system_message is None and not tools) typed_messages: Final = cast( # cast-ok: request messages are raw chat-shaped dicts that token_counter normalizes From 3e0c4f6e5dcffcc9255f5df8048c8d651bb8adcf Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 2 Nov 2026 17:08:50 -0800 Subject: [PATCH 5/6] refactor(count_tokens): return local tokenizer failures as values and raise them at one boundary --- litellm/proxy/proxy_server.py | 194 +++++++++++-------- tests/unit/proxy/test_proxy_token_counter.py | 50 +++++ 2 files changed, 162 insertions(+), 82 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 821ae5958dc..d3eed7ea26f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15485,15 +15485,13 @@ async def _kimi_k3_encoded_count( return await offload_token_count(kimi_k3_tokens)(encoding, model, messages, tools) -def _known_local_tokenizer(settings: TokenCountSettings, request_model: str) -> LocalTokenizer | None: - named: Final = settings.local_tokenizer - if isinstance(named, UnknownLocalTokenizer): - raise _rejected_view( - 400, - f"{request_model} sets token_count_tokenizer to {named.value!r}, " - f"which is not one of {', '.join(get_args(LocalTokenizer))}", - ) - return named +def _unknown_tokenizer_failure(named: UnknownLocalTokenizer, request_model: str) -> ProviderCountFailure: + return ProviderCountFailure( + status_code=400, + message=f"{request_model} sets token_count_tokenizer to {named.value!r}, " + f"which is not one of {', '.join(get_args(LocalTokenizer))}", + rejected_before_provider_call=True, + ) def _local_tokenizer_failure(tokenizer: str, model: str, reason: str) -> ProviderCountFailure: @@ -15504,30 +15502,34 @@ def _local_tokenizer_failure(tokenizer: str, model: str, reason: str) -> Provide ) +async def _count_or_failure( + count: Awaitable[int | None], unreadable: ProviderCountFailure +) -> int | ProviderCountFailure: + try: + counted: Final = await count + except BaseException as e: + if not _answers_provider_error(e): + raise + verbose_proxy_logger.warning("Local token count failed, answering 400 instead: %s", type(e).__name__) + return unreadable + return unreadable if counted is None else counted + + async def _kimi_k3_local_tokens( prompt: str | None, count_messages: Callable[[tiktoken.Encoding], Awaitable[int | None]], unreadable: ProviderCountFailure, unavailable: ProviderCountFailure, -) -> int: +) -> int | ProviderCountFailure: encoding: Final = await _KIMI_K3_ENCODINGS.encoding(TOKENIZER_LOAD_WAIT_SECONDS) if encoding is None: - raise unavailable.to_proxy_exception() - return await _estimate_or_provider_error( - offload_token_count(kimi_k3_text_tokens)(encoding, prompt) - if prompt is not None - else _counted_or_rejected(count_messages(encoding), unreadable), + return unavailable + return await _count_or_failure( + offload_token_count(kimi_k3_text_tokens)(encoding, prompt) if prompt is not None else count_messages(encoding), unreadable, ) -async def _counted_or_rejected(count: Awaitable[int | None], rejection: ProviderCountFailure) -> int: - counted: Final = await count - if counted is None: - raise rejection.to_proxy_exception() - return counted - - async def _local_tokenizer_tokens( tokenizer: LocalTokenizer, request_model: str, @@ -15536,57 +15538,64 @@ async def _local_tokenizer_tokens( messages: Sequence[AllMessageValues] | None, tools: Sequence[ChatCompletionToolParam] | None, kimi_k3_count: Callable[[tiktoken.Encoding], Awaitable[int | None]], -) -> _LocalTokenEstimate: +) -> _LocalTokenEstimate | ProviderCountFailure: unreadable: Final = _local_tokenizer_failure(tokenizer, request_model, "could not count this request") unavailable: Final = _local_tokenizer_failure(tokenizer, request_model, "could not be loaded") if tokenizer == "kimi_k3": - return _LocalTokenEstimate( - total_tokens=await _kimi_k3_local_tokens(prompt, kimi_k3_count, unreadable, unavailable), - tokenizer_type="huggingface_tokenizer", + kimi_k3: Final = await _kimi_k3_local_tokens(prompt, kimi_k3_count, unreadable, unavailable) + return ( + kimi_k3 + if isinstance(kimi_k3, ProviderCountFailure) + else _LocalTokenEstimate(total_tokens=kimi_k3, tokenizer_type="huggingface_tokenizer") ) selected: Final = await asyncify(litellm.utils.select_named_tokenizer)(tokenizer) if selected is None: - raise unavailable.to_proxy_exception() - total_tokens: Final = await _estimate_or_provider_error( + return unavailable + total_tokens: Final = await _count_or_failure( _tokenizer_count(template_model, prompt, messages, tools, selected), unreadable ) - return _LocalTokenEstimate(total_tokens=total_tokens, tokenizer_type=selected["type"]) + return ( + total_tokens + if isinstance(total_tokens, ProviderCountFailure) + else _LocalTokenEstimate(total_tokens=total_tokens, tokenizer_type=selected["type"]) + ) -async def _count_with_local_tokenizer( - tokenizer: LocalTokenizer, - request: TokenCountRequest, - messages_format: TokenCountMessagesFormat, - model: str, - template_model: str, - prompt: str | None, - messages: Sequence[AllMessageValues] | None, - tools: Sequence[ChatCompletionToolParam] | None, - bridged: ChatCountRequest | None, - settings: TokenCountSettings, - responses_api: bool, - contents_estimable: bool, - stored_turns_unseen: bool, -) -> TokenCountResponse: - """Count with the tokenizer the deployment names in token_count_tokenizer, never asking the provider. +@dataclass(frozen=True, slots=True) +class _LocalTokenizerRequest: + request: TokenCountRequest + messages_format: TokenCountMessagesFormat + model: str + template_model: str + prompt: str | None + messages: Sequence[AllMessageValues] | None + tools: Sequence[ChatCompletionToolParam] | None + bridged: ChatCountRequest | None + responses_api: bool + contents_estimable: bool + stored_turns_unseen: bool - A Responses request bridged to chat is counted as the chat body inference sends, previous_response_id turns - included.""" - if stored_turns_unseen: - raise _local_tokenizer_failure( + +async def _local_tokenizer_count( + tokenizer: LocalTokenizer, counted: _LocalTokenizerRequest, settings: TokenCountSettings +) -> TokenCountResponse | ProviderCountFailure: + request: Final = counted.request + if counted.stored_turns_unseen: + return _local_tokenizer_failure( tokenizer, request.model, "cannot see the earlier turns previous_response_id points to on the provider" - ).to_proxy_exception() - if not contents_estimable: - raise _local_tokenizer_failure(tokenizer, request.model, "cannot read these contents").to_proxy_exception() - counted_messages: Final = ( - messages + ) + if not counted.contents_estimable: + return _local_tokenizer_failure(tokenizer, request.model, "cannot read these contents") + bridged: Final = counted.bridged + messages: Final = ( + counted.messages if bridged is None else cast( # cast-ok: the chat body Responses inference sends, chat-shaped dicts token_counter normalizes Sequence[AllMessageValues], [dict(message) for message in bridged.messages] ) ) - counted_tools: Final = ( - tools + tools: Final = ( + counted.tools if bridged is None else cast( # cast-ok: the chat tools Responses inference sends Sequence[ChatCompletionToolParam] | None, [dict(tool) for tool in bridged.tools or ()] or None @@ -15595,36 +15604,55 @@ async def _count_with_local_tokenizer( estimate: Final = await _local_tokenizer_tokens( tokenizer, request.model, - template_model, - prompt, - counted_messages, - counted_tools, + counted.template_model, + counted.prompt, + messages, + tools, kimi_k3_count=partial( _kimi_k3_encoded_count, request=request, - messages_format=messages_format if bridged is None else "openai", - model=model, - messages=counted_messages or (), - tools=counted_tools, + messages_format=counted.messages_format if bridged is None else "openai", + model=counted.model, + messages=messages or (), + tools=tools, ), ) + if isinstance(estimate, ProviderCountFailure): + return estimate framing: Final = ( 0 - if counted_messages is None + if messages is None else settings.hidden_prompt_tokens( - carries_tools=sends_function_tools(counted_tools), responses_api=responses_api + carries_tools=sends_function_tools(tools), responses_api=counted.responses_api ) ) return TokenCountResponse( total_tokens=estimate.total_tokens + framing, request_model=request.model, - model_used=model, + model_used=counted.model, tokenizer_type=estimate.tokenizer_type, count_source="local_estimate", count_tokenizer=tokenizer, ) +async def _count_with_local_tokenizer( + named: LocalTokenizer | UnknownLocalTokenizer, counted: _LocalTokenizerRequest, settings: TokenCountSettings +) -> TokenCountResponse: + """Count with the tokenizer the deployment names in token_count_tokenizer, never asking the provider. + + A Responses request bridged to chat is counted as the chat body inference sends, previous_response_id turns + included.""" + outcome: Final = ( + _unknown_tokenizer_failure(named, counted.request.model) + if isinstance(named, UnknownLocalTokenizer) + else await _local_tokenizer_count(named, counted, settings) + ) + if isinstance(outcome, ProviderCountFailure): + raise outcome.to_proxy_exception() + return outcome + + class _TokenCountDeployment(TypedDict, total=False): litellm_params: ReadOnly[Mapping[str, object]] model_info: ReadOnly[Mapping[str, object]] @@ -16029,7 +16057,7 @@ async def _count_request_tokens( default=await asyncify(cost_map_entry)(*_token_count_settings_model(deployment_view, request.model)), deployment_region=partial(_deployment_aws_region, deployment_view), ) - local_tokenizer: Final = _known_local_tokenizer(settings, request.model) + local_tokenizer: Final = settings.local_tokenizer # Try provider-specific token counting first - only for non-direct requests (from provider endpoints) provider_counter: BaseTokenCounter | None = None @@ -16228,20 +16256,22 @@ async def _count_request_tokens( if local_tokenizer is not None: return await _count_with_local_tokenizer( local_tokenizer, - request - if counted_view is None - else request.model_copy(update={"messages": local_messages, "system": local_system}), - messages_format, - model_to_use, - template_model, - prompt=prompt if counted_messages is None else None, - messages=counted_messages, - tools=counted_tools, - bridged=chat_count, - settings=settings, - responses_api=native_responses, - contents_estimable=contents_estimable, - stored_turns_unseen=chat_count is None and bool(_responses_routing_kwargs(routed_request)), + _LocalTokenizerRequest( + request=request + if counted_view is None + else request.model_copy(update={"messages": local_messages, "system": local_system}), + messages_format=messages_format, + model=model_to_use, + template_model=template_model, + prompt=prompt if counted_messages is None else None, + messages=counted_messages, + tools=counted_tools, + bridged=chat_count, + responses_api=native_responses, + contents_estimable=contents_estimable, + stored_turns_unseen=chat_count is None and bool(_responses_routing_kwargs(routed_request)), + ), + settings, ) kimi_k3_messages: Final = counted_messages if settings.tokenizer == "kimi_k3" else None exact: Final = ( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 8c4f9b26302..66f10a5ee34 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -5488,6 +5488,56 @@ async def test_the_deployments_named_tokenizer_beats_the_tokenizer_the_cost_map_ } +@pytest.mark.asyncio +async def test_a_named_tokenizer_count_adds_the_framing_the_deployment_sets_for_requests_with_tools( + monkeypatch: pytest.MonkeyPatch, +) -> None: + model_info: Final = {"token_count_tokenizer": "openai", "token_count_framing": 5, "token_count_tool_framing": 40} + + _, utils_reply, sent = await _bedrock_counts( + monkeypatch, _NOVA_PRO, model_info, {"messages": _MIXED_SCRIPT_TURNS, "tools": [_KIMI_WEATHER]} + ) + + assert sent == () + assert isinstance(utils_reply, TokenCountResponse), utils_reply + assert utils_reply.total_tokens == 40 + litellm.token_counter( + model=_NOVA_PRO, messages=_MIXED_SCRIPT_TURNS, tools=[_KIMI_WEATHER] + ) + + +@pytest.mark.asyncio +async def test_a_named_kimi_k3_tokenizer_counts_text_with_kimi_k3_and_answers_400_for_a_body_it_cannot_render( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + encoding: Final = _available_kimi_k3(monkeypatch, tmp_path) + image_turn: Final[dict[str, JsonValue]] = { + "role": "user", + "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0K"}}], + } + + counted, _, counted_sent = await _bedrock_counts( + monkeypatch, _NOVA_PRO, {"token_count_tokenizer": "kimi_k3"}, {"messages": _MIXED_SCRIPT_TURNS} + ) + rejected, _, rejected_sent = await _bedrock_counts( + monkeypatch, _NOVA_PRO, {"token_count_tokenizer": "kimi_k3"}, {"messages": [image_turn]} + ) + + assert counted == { + "input_tokens": kimi_k3_tokens(encoding, _NOVA_PRO, _MIXED_SCRIPT_TURNS, None), + "litellm_count_source": "local_estimate", + "litellm_count_tokenizer": "kimi_k3", + } + status, message = _rejection(rejected) + assert ( + status, + "kimi_k3 tokenizer set in token_count_tokenizer, which could not count this request" in message, + ) == ( + 400, + True, + ), message + assert counted_sent == rejected_sent == () + + @pytest.mark.asyncio @pytest.mark.parametrize( ("model_info", "reason"), From ee402443f3c9f3cade306fbda5db2371aac33cef Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 2 Nov 2026 17:22:59 -0800 Subject: [PATCH 6/6] fix(count_tokens): leave count_tokenizer out of responses that no named tokenizer counted --- litellm/types/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index bd181748437..6b5c3b0a813 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4812,7 +4812,7 @@ class TokenCountResponse(LiteLLMPydanticObjectBase): model_used: str tokenizer_type: str count_source: TokenCountSource | None = None - count_tokenizer: LocalTokenizer | None = None + count_tokenizer: LocalTokenizer | None = Field(default=None, exclude_if=lambda tokenizer: tokenizer is None) original_response: dict | None = None """ Original Response from upstream API call - if an API call was made for token counting