mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-21 10:15:47 +00:00
fix(token_counter): count OpenAI models newer than tiktoken's table with o200k_base
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
70ece3501f
commit
d2630bd984
3 changed files with 24 additions and 4 deletions
|
|
@ -105,6 +105,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
|
|||
- No file sprawl: deliberate file and folder structure
|
||||
- Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
|
||||
- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
|
||||
- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated
|
||||
|
||||
Follow conventional commits for commit names and PR titles
|
||||
|
||||
|
|
|
|||
|
|
@ -742,17 +742,17 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
|
|||
|
||||
|
||||
def openai_tokenizer_encoding_name(model: str) -> str:
|
||||
"""The tiktoken encoding name for `model`, without loading the encoding."""
|
||||
from litellm.utils import print_verbose
|
||||
"""The tiktoken encoding name for `model`, without loading the encoding.
|
||||
|
||||
A listed OpenAI or Azure model that tiktoken does not know yet is a newer release than tiktoken's table, so it
|
||||
gets o200k_base, the encoding every OpenAI model since gpt-4o uses."""
|
||||
model_to_use: Final = _fix_model_name(model)
|
||||
if "gpt-4o" in model_to_use:
|
||||
return "o200k_base"
|
||||
try:
|
||||
return tiktoken.encoding_name_for_model(model_to_use)
|
||||
except KeyError:
|
||||
print_verbose("Warning: model not found. Using cl100k_base encoding.")
|
||||
return "cl100k_base"
|
||||
return "o200k_base"
|
||||
|
||||
|
||||
def uses_legacy_message_accounting(model: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -2211,3 +2211,22 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk
|
|||
)
|
||||
== 200_000
|
||||
)
|
||||
|
||||
|
||||
_MULTILINGUAL_SAMPLE: Final = "日本語のテキストとहिन्दी पाठ और русский текст. " * 20
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5.1", "gpt-5.5", "gpt-6.1-sol", "azure/gpt-41"])
|
||||
def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200k(model: str):
|
||||
# OpenAI's /v1/responses/input_tokens returned the same count for gpt-5 (o200k_base) and gpt-5.1 through
|
||||
# gpt-6.1-sol on this sample, checked 2026-10-20
|
||||
expected: Final = len(tiktoken.get_encoding("o200k_base").encode(_MULTILINGUAL_SAMPLE))
|
||||
|
||||
assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE))
|
||||
assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == expected
|
||||
|
||||
|
||||
def test_token_counter_keeps_cl100k_for_models_that_are_not_openai():
|
||||
assert litellm.token_counter(model="groq/qwen3-32b", text=_MULTILINGUAL_SAMPLE) == len(
|
||||
tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE)
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue