From d2630bd98472eb9056f7bdaaa816b625fb36baa9 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 22:01:30 +0000 Subject: [PATCH 1/6] fix(token_counter): count OpenAI models newer than tiktoken's table with o200k_base Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- AGENTS.md | 1 + litellm/litellm_core_utils/token_counter.py | 8 ++++---- .../litellm_core_utils/test_token_counter.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f4b1de57807..485ef0e8b4c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,6 +105,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega - No file sprawl: deliberate file and folder structure - Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions - API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration +- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated Follow conventional commits for commit names and PR titles diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 8aded69aed2..bc87c20019e 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -742,17 +742,17 @@ def openai_tokenizer_encoding(model: str) -> "Encoding": def openai_tokenizer_encoding_name(model: str) -> str: - """The tiktoken encoding name for `model`, without loading the encoding.""" - from litellm.utils import print_verbose + """The tiktoken encoding name for `model`, without loading the encoding. + A listed OpenAI or Azure model that tiktoken does not know yet is a newer release than tiktoken's table, so it + gets o200k_base, the encoding every OpenAI model since gpt-4o uses.""" model_to_use: Final = _fix_model_name(model) if "gpt-4o" in model_to_use: return "o200k_base" try: return tiktoken.encoding_name_for_model(model_to_use) except KeyError: - print_verbose("Warning: model not found. Using cl100k_base encoding.") - return "cl100k_base" + return "o200k_base" def uses_legacy_message_accounting(model: str) -> bool: diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 309cbd33131..2fd4272feb8 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2211,3 +2211,22 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk ) == 200_000 ) + + +_MULTILINGUAL_SAMPLE: Final = "日本語のテキストとहिन्दी पाठ और русский текст. " * 20 + + +@pytest.mark.parametrize("model", ["gpt-5.1", "gpt-5.5", "gpt-6.1-sol", "azure/gpt-41"]) +def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200k(model: str): + # OpenAI's /v1/responses/input_tokens returned the same count for gpt-5 (o200k_base) and gpt-5.1 through + # gpt-6.1-sol on this sample, checked 2026-10-20 + expected: Final = len(tiktoken.get_encoding("o200k_base").encode(_MULTILINGUAL_SAMPLE)) + + assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE)) + assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == expected + + +def test_token_counter_keeps_cl100k_for_models_that_are_not_openai(): + assert litellm.token_counter(model="groq/qwen3-32b", text=_MULTILINGUAL_SAMPLE) == len( + tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE) + ) From 9b818dd831ce4f02cd628542c91aae11b175f234 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 22:09:41 +0000 Subject: [PATCH 2/6] fix(token_counter): keep tiktoken's encoding for suffixed variants of models it knows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 13 ++++++++++--- tests/unit/litellm_core_utils/test_token_counter.py | 5 +++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index bc87c20019e..8c195803d7e 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -12,6 +12,7 @@ import anyio import anyio.lowlevel import httpx import tiktoken +from tiktoken.model import MODEL_TO_ENCODING from typing_extensions import ParamSpec, TypeVar import litellm @@ -744,15 +745,21 @@ def openai_tokenizer_encoding(model: str) -> "Encoding": def openai_tokenizer_encoding_name(model: str) -> str: """The tiktoken encoding name for `model`, without loading the encoding. - A listed OpenAI or Azure model that tiktoken does not know yet is a newer release than tiktoken's table, so it - gets o200k_base, the encoding every OpenAI model since gpt-4o uses.""" + A listed OpenAI or Azure model that tiktoken does not know is either a suffixed variant of a model it does + know (`text-embedding-ada-002-v2`) or a newer release than its table, which gets o200k_base, the encoding every + OpenAI model since gpt-4o uses.""" model_to_use: Final = _fix_model_name(model) if "gpt-4o" in model_to_use: return "o200k_base" try: return tiktoken.encoding_name_for_model(model_to_use) except KeyError: - return "o200k_base" + return _suffixed_variant_encoding_name(model_to_use) or "o200k_base" + + +def _suffixed_variant_encoding_name(model: str) -> str | None: + known: Final = [name for name in MODEL_TO_ENCODING if model.startswith(f"{name}-")] + return MODEL_TO_ENCODING[max(known, key=len)] if known else None def uses_legacy_message_accounting(model: str) -> bool: diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 2fd4272feb8..076df472ecd 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2226,7 +2226,8 @@ def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200 assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == expected -def test_token_counter_keeps_cl100k_for_models_that_are_not_openai(): - assert litellm.token_counter(model="groq/qwen3-32b", text=_MULTILINGUAL_SAMPLE) == len( +@pytest.mark.parametrize("model", ["groq/qwen3-32b", "text-embedding-ada-002-v2"]) +def test_token_counter_keeps_cl100k_for_non_openai_models_and_variants_of_cl100k_models(model: str): + assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == len( tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE) ) From bff90869311f7d5a78a9cb98eac5f4567c5a262a Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 22:20:05 +0000 Subject: [PATCH 3/6] test(token_counter): move the o200k provenance note into the assert message Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/litellm_core_utils/test_token_counter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 076df472ecd..9b5df69e266 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2218,12 +2218,13 @@ _MULTILINGUAL_SAMPLE: Final = "日本語のテキストとहिन्दी प @pytest.mark.parametrize("model", ["gpt-5.1", "gpt-5.5", "gpt-6.1-sol", "azure/gpt-41"]) def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200k(model: str): - # OpenAI's /v1/responses/input_tokens returned the same count for gpt-5 (o200k_base) and gpt-5.1 through - # gpt-6.1-sol on this sample, checked 2026-10-20 expected: Final = len(tiktoken.get_encoding("o200k_base").encode(_MULTILINGUAL_SAMPLE)) assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE)) - assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == expected + assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == expected, ( + f"{model} should count with o200k_base. OpenAI's /v1/responses/input_tokens gave gpt-5 and gpt-5.1 through " + "gpt-6.1-sol the same count on this sample, checked 2026-10-20" + ) @pytest.mark.parametrize("model", ["groq/qwen3-32b", "text-embedding-ada-002-v2"]) From 970dd994813c4c46da9032436dda021b5545dc52 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 22:33:45 +0000 Subject: [PATCH 4/6] fix(token_counter): strip openai/ and azure/ prefixes before picking the encoding Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 8 +++++++- tests/unit/litellm_core_utils/test_token_counter.py | 6 ++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 8c195803d7e..15624d34c3c 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -748,7 +748,7 @@ def openai_tokenizer_encoding_name(model: str) -> str: A listed OpenAI or Azure model that tiktoken does not know is either a suffixed variant of a model it does know (`text-embedding-ada-002-v2`) or a newer release than its table, which gets o200k_base, the encoding every OpenAI model since gpt-4o uses.""" - model_to_use: Final = _fix_model_name(model) + model_to_use: Final = _fix_model_name(_without_openai_provider_prefix(model)) if "gpt-4o" in model_to_use: return "o200k_base" try: @@ -757,6 +757,12 @@ def openai_tokenizer_encoding_name(model: str) -> str: return _suffixed_variant_encoding_name(model_to_use) or "o200k_base" +def _without_openai_provider_prefix(model: str) -> str: + provider, separator, name = model.partition("/") + is_listed_openai_model: Final = provider in ("openai", "azure") and name in litellm.open_ai_chat_completion_models + return name if separator and is_listed_openai_model else model + + def _suffixed_variant_encoding_name(model: str) -> str | None: known: Final = [name for name in MODEL_TO_ENCODING if model.startswith(f"{name}-")] return MODEL_TO_ENCODING[max(known, key=len)] if known else None diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 9b5df69e266..9a160cc74ad 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2216,7 +2216,9 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk _MULTILINGUAL_SAMPLE: Final = "日本語のテキストとहिन्दी पाठ और русский текст. " * 20 -@pytest.mark.parametrize("model", ["gpt-5.1", "gpt-5.5", "gpt-6.1-sol", "azure/gpt-41"]) +@pytest.mark.parametrize( + "model", ["gpt-5.1", "gpt-5.5", "gpt-6.1-sol", "azure/gpt-41", "azure/gpt-5.1", "openai/gpt-5.1", "openai/gpt-4o"] +) def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200k(model: str): expected: Final = len(tiktoken.get_encoding("o200k_base").encode(_MULTILINGUAL_SAMPLE)) @@ -2227,7 +2229,7 @@ def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200 ) -@pytest.mark.parametrize("model", ["groq/qwen3-32b", "text-embedding-ada-002-v2"]) +@pytest.mark.parametrize("model", ["groq/qwen3-32b", "text-embedding-ada-002-v2", "azure/gpt-4", "azure/my-deployment"]) def test_token_counter_keeps_cl100k_for_non_openai_models_and_variants_of_cl100k_models(model: str): assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == len( tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE) From 571228cf32a0daf4af3139e787caa59114be4a3a Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 22:41:25 +0000 Subject: [PATCH 5/6] test(rust_bridge): azure/gpt-4o now resolves to o200k_base Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/rust_bridge/test_token_counter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/rust_bridge/test_token_counter.py b/tests/unit/rust_bridge/test_token_counter.py index 3af7127a9b3..59bbb0065bd 100644 --- a/tests/unit/rust_bridge/test_token_counter.py +++ b/tests/unit/rust_bridge/test_token_counter.py @@ -143,7 +143,7 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None) ("gemini/gemini-2.5-pro", "cl100k_base"), ("mistral/mistral-large-latest", "cl100k_base"), ("my-router-alias", "cl100k_base"), - ("azure/gpt-4o", "cl100k_base"), + ("azure/gpt-4o", "o200k_base"), ("command-r-plus", "cl100k_base"), ("gpt-4o", "o200k_base"), ("gpt-4o-mini", "o200k_base"), From 1d3e17d4304d8f95f4d3873b144e96144623717d Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 22:53:31 +0000 Subject: [PATCH 6/6] fix(token_counter): check the provider prefix through _fix_model_name to keep the type gate flat Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 15624d34c3c..39cfdadfa5e 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -759,7 +759,7 @@ def openai_tokenizer_encoding_name(model: str) -> str: def _without_openai_provider_prefix(model: str) -> str: provider, separator, name = model.partition("/") - is_listed_openai_model: Final = provider in ("openai", "azure") and name in litellm.open_ai_chat_completion_models + is_listed_openai_model: Final = provider in ("openai", "azure") and _fix_model_name(name) == name return name if separator and is_listed_openai_model else model