mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 1d3e17d430 into 3b8ad37296
This commit is contained in:
commit
cd26aaab3d
4 changed files with 43 additions and 6 deletions
|
|
@ -105,6 +105,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
|
|||
- No file sprawl: deliberate file and folder structure
|
||||
- Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
|
||||
- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
|
||||
- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated
|
||||
|
||||
Follow conventional commits for commit names and PR titles
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import anyio
|
|||
import anyio.lowlevel
|
||||
import httpx
|
||||
import tiktoken
|
||||
from tiktoken.model import MODEL_TO_ENCODING
|
||||
from typing_extensions import ParamSpec, TypeVar
|
||||
|
||||
import litellm
|
||||
|
|
@ -742,17 +743,29 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
|
|||
|
||||
|
||||
def openai_tokenizer_encoding_name(model: str) -> str:
|
||||
"""The tiktoken encoding name for `model`, without loading the encoding."""
|
||||
from litellm.utils import print_verbose
|
||||
"""The tiktoken encoding name for `model`, without loading the encoding.
|
||||
|
||||
model_to_use: Final = _fix_model_name(model)
|
||||
A listed OpenAI or Azure model that tiktoken does not know is either a suffixed variant of a model it does
|
||||
know (`text-embedding-ada-002-v2`) or a newer release than its table, which gets o200k_base, the encoding every
|
||||
OpenAI model since gpt-4o uses."""
|
||||
model_to_use: Final = _fix_model_name(_without_openai_provider_prefix(model))
|
||||
if "gpt-4o" in model_to_use:
|
||||
return "o200k_base"
|
||||
try:
|
||||
return tiktoken.encoding_name_for_model(model_to_use)
|
||||
except KeyError:
|
||||
print_verbose("Warning: model not found. Using cl100k_base encoding.")
|
||||
return "cl100k_base"
|
||||
return _suffixed_variant_encoding_name(model_to_use) or "o200k_base"
|
||||
|
||||
|
||||
def _without_openai_provider_prefix(model: str) -> str:
|
||||
provider, separator, name = model.partition("/")
|
||||
is_listed_openai_model: Final = provider in ("openai", "azure") and _fix_model_name(name) == name
|
||||
return name if separator and is_listed_openai_model else model
|
||||
|
||||
|
||||
def _suffixed_variant_encoding_name(model: str) -> str | None:
|
||||
known: Final = [name for name in MODEL_TO_ENCODING if model.startswith(f"{name}-")]
|
||||
return MODEL_TO_ENCODING[max(known, key=len)] if known else None
|
||||
|
||||
|
||||
def uses_legacy_message_accounting(model: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -2342,3 +2342,26 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk
|
|||
)
|
||||
== 200_000
|
||||
)
|
||||
|
||||
|
||||
_MULTILINGUAL_SAMPLE: Final = "日本語のテキストとहिन्दी पाठ और русский текст. " * 20
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["gpt-5.1", "gpt-5.5", "gpt-6.1-sol", "azure/gpt-41", "azure/gpt-5.1", "openai/gpt-5.1", "openai/gpt-4o"]
|
||||
)
|
||||
def test_token_counter_counts_openai_models_newer_than_tiktokens_table_with_o200k(model: str):
|
||||
expected: Final = len(tiktoken.get_encoding("o200k_base").encode(_MULTILINGUAL_SAMPLE))
|
||||
|
||||
assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE))
|
||||
assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == expected, (
|
||||
f"{model} should count with o200k_base. OpenAI's /v1/responses/input_tokens gave gpt-5 and gpt-5.1 through "
|
||||
"gpt-6.1-sol the same count on this sample, checked 2026-10-20"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["groq/qwen3-32b", "text-embedding-ada-002-v2", "azure/gpt-4", "azure/my-deployment"])
|
||||
def test_token_counter_keeps_cl100k_for_non_openai_models_and_variants_of_cl100k_models(model: str):
|
||||
assert litellm.token_counter(model=model, text=_MULTILINGUAL_SAMPLE) == len(
|
||||
tiktoken.get_encoding("cl100k_base").encode(_MULTILINGUAL_SAMPLE)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -143,7 +143,7 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None)
|
|||
("gemini/gemini-2.5-pro", "cl100k_base"),
|
||||
("mistral/mistral-large-latest", "cl100k_base"),
|
||||
("my-router-alias", "cl100k_base"),
|
||||
("azure/gpt-4o", "cl100k_base"),
|
||||
("azure/gpt-4o", "o200k_base"),
|
||||
("command-r-plus", "cl100k_base"),
|
||||
("gpt-4o", "o200k_base"),
|
||||
("gpt-4o-mini", "o200k_base"),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue