fix(token_counter): count OpenAI models newer than tiktoken's table with o200k_base

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-10-20 22:01:30 +00:00
parent 70ece3501f
commit d2630bd984
3 changed files with 24 additions and 4 deletions

View file

@ -105,6 +105,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
- No file sprawl: deliberate file and folder structure
- Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated
Follow conventional commits for commit names and PR titles

View file

@ -742,17 +742,17 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
def openai_tokenizer_encoding_name(model: str) -> str:
"""The tiktoken encoding name for `model`, without loading the encoding."""
from litellm.utils import print_verbose
"""The tiktoken encoding name for `model`, without loading the encoding.
A listed OpenAI or Azure model that tiktoken does not know yet is a newer release than tiktoken's table, so it
gets o200k_base, the encoding every OpenAI model since gpt-4o uses."""
model_to_use: Final = _fix_model_name(model)
if "gpt-4o" in model_to_use:
return "o200k_base"
try:
return tiktoken.encoding_name_for_model(model_to_use)
except KeyError:
print_verbose("Warning: model not found. Using cl100k_base encoding.")
return "cl100k_base"
return "o200k_base"
def uses_legacy_message_accounting(model: str) -> bool:

View file

@ -2211,3 +2211,22 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk
)
== 200_000
)
_MULTILINGUAL_SAMPLE: Final = "日本語のテキストとहिन्दी पाठ और русский текст. " * 20
@pytest.mark.parametrize("model", ["gpt-5.1", "gpt-5.5", "gpt-6.1-sol", "azure/gpt-41"])
def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200k(model: str):
# OpenAI's /v1/responses/input_tokens returned the same count for gpt-5 (o200k_base) and gpt-5.1 through
# gpt-6.1-sol on this sample, checked 2026-10-20
expected: Final = len(tiktoken.get_encoding("o200k_base").encode(_MULTILINGUAL_SAMPLE))
assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE))
assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == expected
def test_token_counter_keeps_cl100k_for_models_that_are_not_openai():
assert litellm.token_counter(model="groq/qwen3-32b", text=_MULTILINGUAL_SAMPLE) == len(
tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE)
)