diff --git a/AGENTS.md b/AGENTS.md index f4b1de57807..485ef0e8b4c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,6 +105,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega - No file sprawl: deliberate file and folder structure - Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions - API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration +- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated Follow conventional commits for commit names and PR titles diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 90e79d4d853..573b39fee52 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -12,6 +12,7 @@ import anyio import anyio.lowlevel import httpx import tiktoken +from tiktoken.model import MODEL_TO_ENCODING from typing_extensions import ParamSpec, TypeVar import litellm @@ -742,17 +743,29 @@ def openai_tokenizer_encoding(model: str) -> "Encoding": def openai_tokenizer_encoding_name(model: str) -> str: - """The tiktoken encoding name for `model`, without loading the encoding.""" - from litellm.utils import print_verbose + """The tiktoken encoding name for `model`, without loading the encoding. - model_to_use: Final = _fix_model_name(model) + A listed OpenAI or Azure model that tiktoken does not know is either a suffixed variant of a model it does + know (`text-embedding-ada-002-v2`) or a newer release than its table, which gets o200k_base, the encoding every + OpenAI model since gpt-4o uses.""" + model_to_use: Final = _fix_model_name(_without_openai_provider_prefix(model)) if "gpt-4o" in model_to_use: return "o200k_base" try: return tiktoken.encoding_name_for_model(model_to_use) except KeyError: - print_verbose("Warning: model not found. Using cl100k_base encoding.") - return "cl100k_base" + return _suffixed_variant_encoding_name(model_to_use) or "o200k_base" + + +def _without_openai_provider_prefix(model: str) -> str: + provider, separator, name = model.partition("/") + is_listed_openai_model: Final = provider in ("openai", "azure") and _fix_model_name(name) == name + return name if separator and is_listed_openai_model else model + + +def _suffixed_variant_encoding_name(model: str) -> str | None: + known: Final = [name for name in MODEL_TO_ENCODING if model.startswith(f"{name}-")] + return MODEL_TO_ENCODING[max(known, key=len)] if known else None def uses_legacy_message_accounting(model: str) -> bool: diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 88af656ad2e..84fd7f43daa 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2342,3 +2342,26 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk ) == 200_000 ) + + +_MULTILINGUAL_SAMPLE: Final = "日本語のテキストとहिन्दी पाठ और русский текст. " * 20 + + +@pytest.mark.parametrize( + "model", ["gpt-5.1", "gpt-5.5", "gpt-6.1-sol", "azure/gpt-41", "azure/gpt-5.1", "openai/gpt-5.1", "openai/gpt-4o"] +) +def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200k(model: str): + expected: Final = len(tiktoken.get_encoding("o200k_base").encode(_MULTILINGUAL_SAMPLE)) + + assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE)) + assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == expected, ( + f"{model} should count with o200k_base. OpenAI's /v1/responses/input_tokens gave gpt-5 and gpt-5.1 through " + "gpt-6.1-sol the same count on this sample, checked 2026-10-20" + ) + + +@pytest.mark.parametrize("model", ["groq/qwen3-32b", "text-embedding-ada-002-v2", "azure/gpt-4", "azure/my-deployment"]) +def test_token_counter_keeps_cl100k_for_non_openai_models_and_variants_of_cl100k_models(model: str): + assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == len( + tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE) + ) diff --git a/tests/unit/rust_bridge/test_token_counter.py b/tests/unit/rust_bridge/test_token_counter.py index 3af7127a9b3..59bbb0065bd 100644 --- a/tests/unit/rust_bridge/test_token_counter.py +++ b/tests/unit/rust_bridge/test_token_counter.py @@ -143,7 +143,7 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None) ("gemini/gemini-2.5-pro", "cl100k_base"), ("mistral/mistral-large-latest", "cl100k_base"), ("my-router-alias", "cl100k_base"), - ("azure/gpt-4o", "cl100k_base"), + ("azure/gpt-4o", "o200k_base"), ("command-r-plus", "cl100k_base"), ("gpt-4o", "o200k_base"), ("gpt-4o-mini", "o200k_base"),