From d2630bd98472eb9056f7bdaaa816b625fb36baa9 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 22:01:30 +0000 Subject: [PATCH] fix(token_counter): count OpenAI models newer than tiktoken's table with o200k_base Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- AGENTS.md | 1 + litellm/litellm_core_utils/token_counter.py | 8 ++++---- .../litellm_core_utils/test_token_counter.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f4b1de57807..485ef0e8b4c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,6 +105,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega - No file sprawl: deliberate file and folder structure - Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions - API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration +- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated Follow conventional commits for commit names and PR titles diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 8aded69aed2..bc87c20019e 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -742,17 +742,17 @@ def openai_tokenizer_encoding(model: str) -> "Encoding": def openai_tokenizer_encoding_name(model: str) -> str: - """The tiktoken encoding name for `model`, without loading the encoding.""" - from litellm.utils import print_verbose + """The tiktoken encoding name for `model`, without loading the encoding. + A listed OpenAI or Azure model that tiktoken does not know yet is a newer release than tiktoken's table, so it + gets o200k_base, the encoding every OpenAI model since gpt-4o uses.""" model_to_use: Final = _fix_model_name(model) if "gpt-4o" in model_to_use: return "o200k_base" try: return tiktoken.encoding_name_for_model(model_to_use) except KeyError: - print_verbose("Warning: model not found. Using cl100k_base encoding.") - return "cl100k_base" + return "o200k_base" def uses_legacy_message_accounting(model: str) -> bool: diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 309cbd33131..2fd4272feb8 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2211,3 +2211,22 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk ) == 200_000 ) + + +_MULTILINGUAL_SAMPLE: Final = "日本語のテキストとहिन्दी पाठ और русский текст. " * 20 + + +@pytest.mark.parametrize("model", ["gpt-5.1", "gpt-5.5", "gpt-6.1-sol", "azure/gpt-41"]) +def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200k(model: str): + # OpenAI's /v1/responses/input_tokens returned the same count for gpt-5 (o200k_base) and gpt-5.1 through + # gpt-6.1-sol on this sample, checked 2026-10-20 + expected: Final = len(tiktoken.get_encoding("o200k_base").encode(_MULTILINGUAL_SAMPLE)) + + assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE)) + assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == expected + + +def test_token_counter_keeps_cl100k_for_models_that_are_not_openai(): + assert litellm.token_counter(model="groq/qwen3-32b", text=_MULTILINGUAL_SAMPLE) == len( + tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE) + )