This commit is contained in:
Mateo Wang 2026-10-27 02:23:02 +00:00 • committed by GitHub
commit cd26aaab3d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 43 additions and 6 deletions

View file

@ -105,6 +105,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
- No file sprawl: deliberate file and folder structure
- Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated
Follow conventional commits for commit names and PR titles

View file

@ -12,6 +12,7 @@ import anyio
import anyio.lowlevel
import httpx
import tiktoken
from tiktoken.model import MODEL_TO_ENCODING
from typing_extensions import ParamSpec, TypeVar
import litellm
@ -742,17 +743,29 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
def openai_tokenizer_encoding_name(model: str) -> str:
"""The tiktoken encoding name for `model`, without loading the encoding."""
from litellm.utils import print_verbose
"""The tiktoken encoding name for `model`, without loading the encoding.
model_to_use: Final = _fix_model_name(model)
A listed OpenAI or Azure model that tiktoken does not know is either a suffixed variant of a model it does
know (`text-embedding-ada-002-v2`) or a newer release than its table, which gets o200k_base, the encoding every
OpenAI model since gpt-4o uses."""
model_to_use: Final = _fix_model_name(_without_openai_provider_prefix(model))
if "gpt-4o" in model_to_use:
return "o200k_base"
try:
return tiktoken.encoding_name_for_model(model_to_use)
except KeyError:
print_verbose("Warning: model not found. Using cl100k_base encoding.")
return "cl100k_base"
return _suffixed_variant_encoding_name(model_to_use) or "o200k_base"
def _without_openai_provider_prefix(model: str) -> str:
provider, separator, name = model.partition("/")
is_listed_openai_model: Final = provider in ("openai", "azure") and _fix_model_name(name) == name
return name if separator and is_listed_openai_model else model
def _suffixed_variant_encoding_name(model: str) -> str | None:
known: Final = [name for name in MODEL_TO_ENCODING if model.startswith(f"{name}-")]
return MODEL_TO_ENCODING[max(known, key=len)] if known else None
def uses_legacy_message_accounting(model: str) -> bool:

View file

@ -2342,3 +2342,26 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk
)
== 200_000
)
_MULTILINGUAL_SAMPLE: Final = "日本語のテキストとहिन्दी पाठ और русский текст. " * 20
@pytest.mark.parametrize(
"model", ["gpt-5.1", "gpt-5.5", "gpt-6.1-sol", "azure/gpt-41", "azure/gpt-5.1", "openai/gpt-5.1", "openai/gpt-4o"]
)
def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200k(model: str):
expected: Final = len(tiktoken.get_encoding("o200k_base").encode(_MULTILINGUAL_SAMPLE))
assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE))
assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == expected, (
f"{model} should count with o200k_base. OpenAI's /v1/responses/input_tokens gave gpt-5 and gpt-5.1 through "
"gpt-6.1-sol the same count on this sample, checked 2026-10-20"
)
@pytest.mark.parametrize("model", ["groq/qwen3-32b", "text-embedding-ada-002-v2", "azure/gpt-4", "azure/my-deployment"])
def test_token_counter_keeps_cl100k_for_non_openai_models_and_variants_of_cl100k_models(model: str):
assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == len(
tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE)
)

View file

@ -143,7 +143,7 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None)
("gemini/gemini-2.5-pro", "cl100k_base"),
("mistral/mistral-large-latest", "cl100k_base"),
("my-router-alias", "cl100k_base"),
("azure/gpt-4o", "cl100k_base"),
("azure/gpt-4o", "o200k_base"),
("command-r-plus", "cl100k_base"),
("gpt-4o", "o200k_base"),
("gpt-4o-mini", "o200k_base"),