scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-25 01:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

refactor(tokenizer): route Python tokenization through the Rust extension

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Yujong Lee 2026-09-20 23:12:48 +00:00
parent b086dd17ae
commit 8ddec7249b
110 changed files with 343 additions and 609 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.github/scripts/verify_linux_native_wheel.py vendored
View file

@ -205,7 +205,7 @@ def main(
native_module: Final = load_native_module(native_path)
native_module_loads: Final = native_module is not None
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
native_size_limit: Final = 25_000_000
native_size_limit: Final = 35_000_000
native_size_within_limit: Final = native_member.file_size <= native_size_limit
validations: Final = (
(f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
@ -222,7 +222,7 @@ def main(
("Python extension entry point is present", extension_entry_point_present),
("Native module loads", native_module_loads),
("Production module omits the panic test hook", panic_test_hook_absent),
("Native extension does not exceed 25 MB", native_size_within_limit),
("Native extension does not exceed 35 MB", native_size_within_limit),
("Wheel contents are valid", not unexpected_members),
)
@ -267,7 +267,7 @@ def main(
),
(
not native_size_within_limit,
f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB",
f"native extension exceeds 35 MB: {native_member.file_size / 1_000_000:.2f} MB",
),
(bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
)

74
litellm-rust/crates/token-counter-huggingface/src/lib.rs
View file

@ -2,15 +2,19 @@
mod error;
use std::collections::HashSet;
pub use error::Error;
pub struct HuggingFaceTokenizer(Box<tokenizers::Tokenizer>);
pub struct HuggingFaceTokenizer {
tokenizer: Box<tokenizers::Tokenizer>,
special_token_ids: HashSet<u32>,
}
impl HuggingFaceTokenizer {
pub fn from_json(json: &str) -> Result<Self, Error> {
json.parse::<tokenizers::Tokenizer>()
.map(Box::new)
.map(Self)
.map(Self::new)
.map_err(Error::Load)
}
@ -27,28 +31,47 @@ impl HuggingFaceTokenizer {
..Default::default()
}),
)
.map(Box::new)
.map(Self)
.map(Self::new)
.map_err(Error::Download)
}
fn new(tokenizer: tokenizers::Tokenizer) -> Self {
let special_token_ids: HashSet<u32> = tokenizer
.get_added_tokens_decoder()
.into_iter()
.filter_map(|(id, token)| token.special.then_some(id))
.collect();
Self {
tokenizer: Box::new(tokenizer),
special_token_ids,
}
}
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
self.0
self.tokenizer
.encode_fast(text, true)
.map(|encoding| encoding.len())
.map_err(Error::Encode)
}
pub fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
self.0
self.tokenizer
.encode_fast(text, true)
.map(|encoding| encoding.get_ids().to_vec())
.map_err(Error::Encode)
}
pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
self.0
.decode(ids, skip_special_tokens)
if !skip_special_tokens {
return self.tokenizer.decode(ids, false).map_err(Error::Decode);
}
let filtered_ids: Vec<u32> = ids
.iter()
.copied()
.filter(|id| !self.special_token_ids.contains(id))
.collect();
self.tokenizer
.decode(&filtered_ids, true)
.map_err(Error::Decode)
}
@ -73,4 +96,37 @@ mod tests {
assert!(tokenizer.decode(&ids, false).unwrap().contains("<SOS>"));
assert_eq!(tokenizer.decode(&ids, true).unwrap(), "hello");
}
#[test]
fn decode_filters_special_added_tokens() {
let json = r#"{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 1,
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {"type": "Whitespace"},
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {"<unk>": 0, "<s>": 1, "hello": 2},
"unk_token": "<unk>"
}
}"#;
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
assert!(!tokenizer.decode(&[1, 2], true).unwrap().contains("<s>"));
assert!(tokenizer.decode(&[1, 2], false).unwrap().contains("<s>"));
}
}

11
litellm/_lazy_imports.py
View file

@ -58,7 +58,8 @@ from ._lazy_imports_registry import (
if TYPE_CHECKING:
import httpx
from tiktoken import Encoding
from litellm.rust_bridge._native import Tokenizer
def get_litellm_globals() -> dict[str, object]:
@ -89,15 +90,15 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo
# These are special lazy loaders for things that are used internally
# They're separate from the main lazy import system because they have specific use cases
# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup
_default_encoding: "Encoding | None" = None
# Lazy loader for default encoding - avoids importing the native extension at startup
_default_encoding: "Tokenizer | None" = None
def _get_default_encoding() -> "Encoding":
def _get_default_encoding() -> "Tokenizer":
"""
Lazily load and cache the default OpenAI encoding.
This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken)
This avoids importing `litellm.litellm_core_utils.default_encoding`
at `litellm` import time. The encoding is cached after the first import.
This is used internally by utils.py functions that need the encoding but shouldn't

3
litellm/litellm_core_utils/README.md
View file

@ -6,8 +6,7 @@ Core files:
- `streaming_handler.py`: The core streaming logic + streaming related helper utils
- `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`.
- `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types.
- `default_encoding.py`: code for loading the default encoding (tiktoken)
- `default_encoding.py`: code for loading the default native tokenizer
- `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name.
- `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])

38
litellm/litellm_core_utils/default_encoding.py
View file

@ -1,4 +1,3 @@
import os
from pathlib import Path
from typing import Final
@ -29,39 +28,6 @@ def o200k_base_rank_file() -> str:
return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii")
# Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory
# unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR.
# This keeps tiktoken fully offline-capable by default (see #1071).
custom_cache_dir: Final = os.getenv("CUSTOM_TIKTOKEN_CACHE_DIR")
if custom_cache_dir:
# If the user opts into a custom cache dir, ensure it exists.
os.makedirs(custom_cache_dir, exist_ok=True)
cache_dir = custom_cache_dir
else:
cache_dir = filename
from litellm.rust_bridge._native import Tokenizer
os.environ["TIKTOKEN_CACHE_DIR"] = (
cache_dir # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071
)
import random
import time
import tiktoken
# Retry logic to handle race conditions when multiple processes try to create
# the tiktoken cache file simultaneously (common in parallel test execution on Windows)
_max_retries: Final = 5
_retry_delay: Final = 0.1 # Start with 100ms
for attempt in range(_max_retries):
try:
encoding = tiktoken.get_encoding("cl100k_base")
break
except (FileExistsError, OSError):
if attempt == _max_retries - 1:
# Last attempt, re-raise the exception
raise
# Exponential backoff with jitter to reduce collision probability
delay = _retry_delay * (2**attempt) + random.uniform(0, 0.1)
time.sleep(delay)
encoding: Final = Tokenizer.from_tiktoken("cl100k_base")

29
litellm/litellm_core_utils/token_counter.py
View file

@ -4,13 +4,12 @@ import base64
import io
import struct
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from functools import lru_cache
from typing import Final, Literal, cast
import anyio
import anyio.lowlevel
import httpx
import tiktoken
from tokenizers import Tokenizer
from typing_extensions import ParamSpec, TypeVar
import litellm
@ -32,6 +31,7 @@ from litellm.constants import (
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.url_utils import safe_get
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.rust_bridge._native import Tokenizer, tiktoken_encoding_for_model
from litellm.types.llms.anthropic import (
AnthropicContentParamSource,
AnthropicContentParamSourceFileId,
@ -623,14 +623,14 @@ def _get_exact_count_function(
tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"]
def count_tokens(text: str) -> int:
return len(tokenizer.encode_batch_fast([text])[0])
return tokenizer.count(text)
return count_tokens
elif tokenizer_json["type"] == "openai_tokenizer":
encoding: Final = openai_tokenizer_encoding(model)
def encode_length(text: str) -> int:
return len(encoding.encode(text, disallowed_special=()))
return encoding.count(text)
return _get_tiktoken_count_function(encode_length)
else:
@ -638,23 +638,28 @@ def _get_exact_count_function(
else:
def encode_length(text: str) -> int:
return len(_get_default_encoding().encode(text, disallowed_special=()))
return _get_default_encoding().count(text)
return _get_tiktoken_count_function(encode_length)
def openai_tokenizer_encoding(model: str) -> tiktoken.Encoding:
"""The tiktoken encoding `token_counter` uses for a model on the `openai_tokenizer` path."""
@lru_cache(maxsize=8)
def _native_tokenizer_for_encoding(name: str) -> Tokenizer:
return Tokenizer.from_tiktoken(name)
def openai_tokenizer_encoding(model: str) -> Tokenizer:
"""The native encoding `token_counter` uses for a model on the `openai_tokenizer` path."""
from litellm.utils import print_verbose
model_to_use: Final = _fix_model_name(model)
if "gpt-4o" in model_to_use:
return tiktoken.get_encoding("o200k_base")
try:
return tiktoken.encoding_for_model(model_to_use)
except KeyError:
return _native_tokenizer_for_encoding("o200k_base")
name: Final = tiktoken_encoding_for_model(model_to_use)
if name is None:
print_verbose("Warning: model not found. Using cl100k_base encoding.")
return tiktoken.get_encoding("cl100k_base")
return _native_tokenizer_for_encoding("cl100k_base")
return _native_tokenizer_for_encoding(name)
def uses_legacy_message_accounting(model: str) -> bool:

5
litellm/llms/a2a/chat/transformation.py
View file

@ -23,9 +23,8 @@ from ..common_utils import (
from .streaming_iterator import A2AModelResponseIterator
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
_REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = (
@ -292,7 +291,7 @@ class A2AConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/aiml/image_generation/transformation.py
View file

@ -14,9 +14,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -171,7 +170,7 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/aiohttp_openai/chat/transformation.py
View file

@ -16,9 +16,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -68,7 +67,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/amazon_nova/chat/transformation.py
View file

@ -17,7 +17,7 @@ from litellm.types.utils import ModelResponse
from ...openai_like.chat.transformation import OpenAILikeChatConfig
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class AmazonNovaChatConfig(OpenAILikeChatConfig):
@ -86,7 +86,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/anthropic/batches/transformation.py
View file

@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
from litellm.types.utils import LiteLLMBatch, LlmProviders, ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -290,7 +289,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/anthropic/chat/transformation.py
View file

@ -100,9 +100,8 @@ from ..common_utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -2686,7 +2685,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/anthropic/completion/transformation.py
View file

@ -33,7 +33,7 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class AnthropicTextError(BaseLLMException):
@ -185,7 +185,7 @@ class AnthropicTextConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/azure/chat/gpt_transformation.py
View file

@ -29,9 +29,8 @@ from ...base_llm.chat.transformation import BaseConfig
from ..common_utils import AzureOpenAIError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -304,7 +303,7 @@ class AzureOpenAIConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/azure_ai/agents/transformation.py
View file

@ -34,10 +34,9 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -297,7 +296,7 @@ class AzureAIAgentsConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/azure_ai/azure_model_router/transformation.py
View file

@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class AzureModelRouterConfig(AzureAIStudioConfig):
@ -59,7 +59,7 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/azure_ai/chat/transformation.py
View file

@ -30,7 +30,7 @@ from litellm.types.utils import ModelResponse, ProviderField
from litellm.utils import _add_path_to_api_base, supports_tool_choice
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class AzureFoundryErrorStrings(str, enum.Enum):
@ -305,7 +305,7 @@ class AzureAIStudioConfig(OpenAIConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/azure_ai/image_generation/mai_transformation.py
View file

@ -12,9 +12,10 @@ from litellm.types.utils import ImageResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
"""Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5)."""
@ -245,7 +246,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/base_llm/audio_transcription/transformation.py
View file

@ -12,9 +12,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -121,7 +120,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/base_llm/bridges/completion_transformation.py
View file

@ -7,11 +7,11 @@ from collections.abc import AsyncIterator, Iterator
from typing import TYPE_CHECKING, Union
if TYPE_CHECKING:
import tiktoken
from pydantic import BaseModel
from litellm import LiteLLMLoggingObj, ModelResponse
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.rust_bridge._native import Tokenizer
from litellm.types.llms.openai import AllMessageValues
@ -39,7 +39,7 @@ class CompletionTransformationBridge(ABC):
messages: list["AllMessageValues"],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> "ModelResponse":

5
litellm/llms/base_llm/chat/transformation.py
View file

@ -21,9 +21,8 @@ from litellm.types.llms.openai import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.rust_bridge._native import Tokenizer
from litellm.types.utils import ModelResponse
from ..base_utils import (
@ -344,7 +343,7 @@ class BaseConfig(ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> "ModelResponse":

5
litellm/llms/base_llm/completion/transformation.py
View file

@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -68,7 +67,7 @@ class BaseTextCompletionConfig(BaseConfig, ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/base_llm/embedding/transformation.py
View file

@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
from litellm.types.utils import EmbeddingResponse, ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -80,7 +79,7 @@ class BaseEmbeddingConfig(BaseConfig, ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/base_llm/files/transformation.py
View file

@ -21,10 +21,9 @@ from litellm.types.utils import LlmProviders, ModelResponse
from ..chat.transformation import BaseConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.router import Router as _Router
from litellm.rust_bridge._native import Tokenizer
from litellm.types.llms.openai import HttpxBinaryResponseContent
LiteLLMLoggingObj = _LiteLLMLoggingObj
@ -231,7 +230,7 @@ class BaseFilesConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/base_llm/image_generation/transformation.py
View file

@ -11,9 +11,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -93,7 +92,7 @@ class BaseImageGenerationConfig(ABC):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

9
litellm/llms/base_llm/image_variations/transformation.py
View file

@ -17,9 +17,8 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -82,7 +81,7 @@ class BaseImageVariationConfig(BaseConfig, ABC):
image: FileTypes,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
) -> ImageResponse:
pass
@ -98,7 +97,7 @@ class BaseImageVariationConfig(BaseConfig, ABC):
image: FileTypes,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
) -> ImageResponse:
pass
@ -125,7 +124,7 @@ class BaseImageVariationConfig(BaseConfig, ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/bedrock/chat/agentcore/transformation.py
View file

@ -40,10 +40,9 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -990,7 +989,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -99,7 +99,7 @@ from ..common_utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
# Computer use tool prefixes supported by Bedrock
BEDROCK_COMPUTER_USE_TOOLS: Final = [
@ -1920,7 +1920,7 @@ class AmazonConverseConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/bedrock/chat/invoke_agent/transformation.py
View file

@ -37,9 +37,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Message, ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -438,7 +437,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py
View file

@ -25,7 +25,7 @@ from litellm.types.utils import (
from .amazon_llama_transformation import AmazonLlamaConfig
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class AmazonDeepSeekR1Config(AmazonLlamaConfig):
@ -39,7 +39,7 @@ class AmazonDeepSeekR1Config(AmazonLlamaConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py
View file

@ -21,9 +21,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
from litellm.types.utils import ModelResponse
LiteLLMLoggingObj = _LiteLLMLoggingObj
@ -198,7 +197,7 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> "ModelResponse":

4
litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py
View file

@ -28,7 +28,7 @@ from ..converse_transformation import AmazonConverseConfig
from .base_invoke_transformation import AmazonInvokeConfig
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
_CachePointCarrier = TypeVar("_CachePointCarrier", SystemContentBlock, ContentBlock)
_INJECTION_POINTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
@ -128,7 +128,7 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py
View file

@ -21,7 +21,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class AmazonQwen2Config(AmazonQwen3Config):
@ -44,7 +44,7 @@ class AmazonQwen2Config(AmazonQwen3Config):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py
View file

@ -19,7 +19,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
@ -170,7 +170,7 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py
View file

@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, Usage
from litellm.utils import get_base64_str
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -190,7 +189,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -359,7 +358,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -34,9 +34,8 @@ from litellm.types.utils import ModelResponse, Usage
from litellm.utils import CustomStreamWrapper
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -288,7 +287,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/black_forest_labs/image_generation/transformation.py
View file

@ -29,9 +29,8 @@ from ..common_utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -258,7 +257,7 @@ class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/bytez/chat/transformation.py
View file

@ -23,9 +23,8 @@ from litellm.utils import CustomStreamWrapper, ModelResponse, Usage
from ..common_utils import API_BASE, BytezError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -187,7 +186,7 @@ class BytezChatConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/clarifai/chat/transformation.py
View file

@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -87,7 +86,7 @@ class ClarifaiConfig(OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/cohere/chat/transformation.py
View file

@ -15,9 +15,8 @@ from ..common_utils import ModelResponseIterator as CohereModelResponseIterator
from ..common_utils import validate_environment as cohere_validate_environment
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -227,7 +226,7 @@ class CohereChatConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/cohere/chat/v2_transformation.py
View file

@ -20,9 +20,8 @@ from ..common_utils import CohereError, CohereV2ModelResponseIterator
from ..common_utils import validate_environment as cohere_validate_environment
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -191,7 +190,7 @@ class CohereV2ChatConfig(OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

6
litellm/llms/cohere/embed/handler.py
View file

@ -20,7 +20,7 @@ from litellm.types.utils import EmbeddingResponse
from .v1_transformation import CohereEmbeddingConfig
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
def validate_environment(api_key, headers: dict):
@ -60,7 +60,7 @@ async def async_embedding(
api_base: str,
api_key: str | None,
headers: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
client: AsyncHTTPHandler | None = None,
):
## LOGGING
@ -122,7 +122,7 @@ def embedding(
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
headers: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
data: dict | CohereEmbeddingRequest | None = None,
complete_api_base: str | None = None,
api_key: str | None = None,

5
litellm/llms/cometapi/image_generation/transformation.py
View file

@ -13,9 +13,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -132,7 +131,7 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/compactifai/chat/transformation.py
View file

@ -17,9 +17,8 @@ from litellm.types.utils import ModelResponse
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -66,7 +65,7 @@ class CompactifAIChatConfig(OpenAIGPTConfig):
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/custom_httpx/aiohttp_handler.py
View file

@ -26,9 +26,8 @@ from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProv
from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -268,7 +267,7 @@ class BaseLLMAIOHTTPHandler:
messages: list,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
client: ClientSession | None = None,
):

6
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -191,7 +191,6 @@ def _rust_responses_websocket_enabled(
from .http_handler import get_shared_realtime_ssl_context
if TYPE_CHECKING:
import tiktoken
from aiohttp import ClientSession
from websockets.asyncio.client import ClientConnection
@ -201,6 +200,7 @@ if TYPE_CHECKING:
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.rust_bridge._native import Tokenizer
from litellm.types.llms.openai_evals import (
CancelEvalResponse,
CancelRunResponse,
@ -492,7 +492,7 @@ class BaseLLMHTTPHandler:
messages: list,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
client: AsyncHTTPHandler | None = None,
json_mode: bool = False,
@ -558,7 +558,7 @@ class BaseLLMHTTPHandler:
api_base: str | None,
custom_llm_provider: str,
model_response: ModelResponse,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
timeout: float | httpx.Timeout,

5
litellm/llms/dashscope/image_generation/transformation.py
View file

@ -38,9 +38,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -165,7 +164,7 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/databricks/chat/transformation.py
View file

@ -148,9 +148,8 @@ def _split_parallel_tool_calls(messages: list[AllMessageValues]) -> list[AllMess
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -648,7 +647,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/deprecated_providers/aleph_alpha.py
View file

@ -278,10 +278,7 @@ def completion(
## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
prompt_tokens: Final = len(encoding.encode(prompt))
completion_tokens: Final = len(
encoding.encode(
model_response["choices"][0]["message"]["content"],
disallowed_special=(),
)
encoding.encode(model_response["choices"][0]["message"]["content"])
)
model_response.created = int(time.time())

5
litellm/llms/fal_ai/image_generation/bria_transformation.py
View file

@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse
from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -187,7 +186,7 @@ class FalAIBriaConfig(FalAIBaseConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py
View file

@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse
from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -194,7 +193,7 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py
View file

@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse
from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -150,7 +149,7 @@ class FalAIIdeogramV3Config(FalAIBaseConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/fal_ai/image_generation/imagen4_transformation.py
View file

@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse
from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -182,7 +181,7 @@ class FalAIImagen4Config(FalAIBaseConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py
View file

@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse
from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -172,7 +171,7 @@ class FalAIRecraftV3Config(FalAIBaseConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py
View file

@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse
from .transformation import FalAIBaseConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -208,7 +207,7 @@ class FalAIStableDiffusionConfig(FalAIBaseConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/fal_ai/image_generation/transformation.py
View file

@ -13,9 +13,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -78,7 +77,7 @@ class FalAIBaseConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

4
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -46,7 +46,7 @@ from ..common_utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
def _map_reasoning_effort(value: object) -> object:
@ -708,7 +708,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/gemini/image_generation/transformation.py
View file

@ -24,9 +24,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -173,7 +172,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/gigachat/chat/transformation.py
View file

@ -26,9 +26,8 @@ from ..authenticator import get_access_token
from ..file_handler import upload_file_sync
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -408,7 +407,7 @@ class GigaChatConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: tiktoken.Encoding | None,
encoding: Tokenizer | None,
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/groq/chat/transformation.py
View file

@ -27,7 +27,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, ServerToolUs
from ...openai_like.chat.transformation import OpenAILikeChatConfig
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
GROQ_COMPOUND_MODELS: Final = frozenset({"compound", "compound-mini"})
@ -286,7 +286,7 @@ class GroqChatConfig(OpenAILikeChatConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/huggingface/embedding/handler.py
View file

@ -1,6 +1,5 @@
import json
import os
from collections.abc import Sequence
from typing import Final, Literal, Protocol, get_args
import httpx
@ -32,7 +31,7 @@ hf_tasks_embeddings: Final = (
class _SupportsTokenEncode(Protocol):
"""Token encoder handle. Only ``encode`` is ever called on it here."""
def encode(self, text: str, *, disallowed_special: tuple[str, ...]) -> Sequence[int]: ...
def encode(self, text: str) -> list[int]: ...
def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> str | None:
@ -214,7 +213,7 @@ class HuggingFaceEmbedding(BaseLLM):
model_response.model = model
input_tokens = 0
for text in input:
input_tokens += len(encoding.encode(text, disallowed_special=()))
input_tokens += len(encoding.encode(text))
setattr(
model_response,

5
litellm/llms/huggingface/embedding/transformation.py
View file

@ -25,9 +25,8 @@ from litellm.utils import token_counter
from ..common_utils import HuggingFaceError, hf_task_list, hf_tasks, output_parser
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -479,7 +478,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/langflow/chat/transformation.py
View file

@ -14,10 +14,9 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Message, ModelResponse, Usage
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.rust_bridge._native import Tokenizer
from litellm.utils import CustomStreamWrapper
LiteLLMLoggingObj = _LiteLLMLoggingObj
@ -225,7 +224,7 @@ class LangFlowConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/langgraph/chat/transformation.py
View file

@ -23,10 +23,9 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Message, ModelResponse, Usage
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.rust_bridge._native import Tokenizer
from litellm.utils import CustomStreamWrapper
LiteLLMLoggingObj = _LiteLLMLoggingObj
@ -415,7 +414,7 @@ class LangGraphConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/lemonade/chat/transformation.py
View file

@ -19,7 +19,7 @@ from litellm.types.utils import ModelResponse
from ...openai_like.chat.transformation import OpenAILikeChatConfig
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class LemonadeChatConfig(OpenAILikeChatConfig):
@ -231,7 +231,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/mistral/chat/transformation.py
View file

@ -32,7 +32,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream
from litellm.utils import convert_to_model_response_object, supports_reasoning
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: str) -> str:
@ -580,7 +580,7 @@ class MistralConfig(OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/nlp_cloud/chat/transformation.py
View file

@ -14,9 +14,8 @@ from litellm.utils import ModelResponse, Usage
from ..common_utils import NLPCloudError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -175,7 +174,7 @@ class NLPCloudConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/oci/chat/transformation.py
View file

@ -65,9 +65,8 @@ from litellm.types.utils import (
from litellm.utils import supports_reasoning
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -603,7 +602,7 @@ class OCIChatConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/ollama/chat/transformation.py
View file

@ -31,9 +31,8 @@ from litellm.types.utils import ModelResponse, ModelResponseStream
from ..common_utils import OllamaError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -321,7 +320,7 @@ class OllamaChatConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

7
litellm/llms/ollama/completion/transformation.py
View file

@ -35,9 +35,8 @@ from litellm.types.utils import (
from ..common_utils import OllamaError, OllamaModelInfo, _convert_image
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -252,7 +251,7 @@ class OllamaConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:
@ -332,7 +331,7 @@ class OllamaConfig(BaseConfig):
tokenizer: Final = encoding if encoding is not None else litellm.encoding
prompt_tokens: Final = response_json.get(
"prompt_eval_count",
len(tokenizer.encode(_prompt, disallowed_special=())),
len(tokenizer.encode(_prompt)),
)
completion_tokens: Final = response_json.get(
"eval_count", len(response_json.get("message", dict()).get("content", ""))

5
litellm/llms/oobabooga/chat/transformation.py
View file

@ -11,9 +11,8 @@ from litellm.types.utils import ModelResponse, Usage
from ..common_utils import OobaboogaError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -39,7 +38,7 @@ class OobaboogaConfig(OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -60,10 +60,9 @@ from litellm.utils import convert_to_model_response_object
from ..common_utils import OpenAIError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.rust_bridge._native import Tokenizer
from litellm.types.llms.openai import ChatCompletionToolParam
LiteLLMLoggingObj = _LiteLLMLoggingObj
@ -671,7 +670,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/openai/image_generation/dall_e_2_transformation.py
View file

@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
class DallE2ImageGenerationConfig(BaseImageGenerationConfig):
"""
@ -52,7 +53,7 @@ class DallE2ImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/openai/image_generation/dall_e_3_transformation.py
View file

@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
class DallE3ImageGenerationConfig(BaseImageGenerationConfig):
"""
@ -52,7 +53,7 @@ class DallE3ImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/openai/image_generation/gpt_transformation.py
View file

@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
class GPTImageGenerationConfig(BaseImageGenerationConfig):
"""
@ -61,7 +62,7 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

6
litellm/llms/openai/image_variations/transformation.py
View file

@ -12,7 +12,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig
from ..common_utils import OpenAIError
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class OpenAIImageVariationConfig(BaseImageVariationConfig):
@ -53,7 +53,7 @@ class OpenAIImageVariationConfig(BaseImageVariationConfig):
image: FileTypes,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
) -> ImageResponse:
return model_response
@ -68,7 +68,7 @@ class OpenAIImageVariationConfig(BaseImageVariationConfig):
image: FileTypes,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
) -> ImageResponse:
return model_response

5
litellm/llms/openai/openai.py
View file

@ -6,9 +6,10 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast
import httpx
if TYPE_CHECKING:
import tiktoken
from aiohttp import ClientSession
from litellm.rust_bridge._native import Tokenizer
import openai
from openai import AsyncOpenAI, OpenAI
from openai._base_client import make_request_options
@ -277,7 +278,7 @@ class OpenAIConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/openai_like/chat/transformation.py
View file

@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -131,7 +130,7 @@ class OpenAILikeChatConfig(OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/openrouter/chat/transformation.py
View file

@ -23,9 +23,8 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
from ..common_utils import OpenRouterException
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
class CacheControlSupportedModels(str, Enum):
@ -182,7 +181,7 @@ class OpenrouterConfig(OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/openrouter/image_generation/transformation.py
View file

@ -50,9 +50,8 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
else:
LiteLLMLoggingObj = Any
@ -319,7 +318,7 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

4
litellm/llms/perplexity/chat/transformation.py
View file

@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues, ChatCompletionAnnotation
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class PerplexityChatConfig(OpenAIGPTConfig):
@ -75,7 +75,7 @@ class PerplexityChatConfig(OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/petals/completion/transformation.py
View file

@ -14,7 +14,7 @@ from litellm.types.utils import ModelResponse
from ..common_utils import PetalsError
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class PetalsConfig(BaseConfig):
@ -112,7 +112,7 @@ class PetalsConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/predibase/chat/transformation.py
View file

@ -18,9 +18,8 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage
from ..common_utils import PredibaseError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -150,7 +149,7 @@ class PredibaseConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/recraft/image_generation/transformation.py
View file

@ -14,9 +14,8 @@ from litellm.types.llms.recraft import RecraftImageGenerationRequestParams
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -122,7 +121,7 @@ class RecraftImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/replicate/chat/transformation.py
View file

@ -19,9 +19,8 @@ from litellm.utils import token_counter
from ..common_utils import ReplicateError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -237,7 +236,7 @@ class ReplicateConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

7
litellm/llms/runwayml/image_generation/transformation.py
View file

@ -22,9 +22,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -308,7 +307,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:
@ -383,7 +382,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/sagemaker/completion/transformation.py
View file

@ -24,9 +24,8 @@ from litellm.utils import token_counter
from ..common_utils import SagemakerError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -198,7 +197,7 @@ class SagemakerConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/sap/chat/transformation.py
View file

@ -15,9 +15,8 @@ from litellm.types.utils import ModelResponse
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -383,7 +382,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/stability/image_generation/transformation.py
View file

@ -26,9 +26,8 @@ from litellm.types.llms.stability import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -207,7 +206,7 @@ class StabilityImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

6
litellm/llms/topaz/image_variations/transformation.py
View file

@ -23,7 +23,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig
from ..common_utils import TopazException, TopazModelInfo
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig):
@ -139,7 +139,7 @@ class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig):
image: FileTypes,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
) -> ImageResponse:
image_content: Final = await raw_response.read()
@ -158,7 +158,7 @@ class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig):
image: FileTypes,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
) -> ImageResponse:
image_content: Final = raw_response.content

8
litellm/llms/triton/completion/transformation.py
View file

@ -29,7 +29,7 @@ from litellm.types.utils import (
from ..common_utils import TritonError
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class TritonConfig(BaseConfig):
@ -95,7 +95,7 @@ class TritonConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:
@ -215,7 +215,7 @@ class TritonGenerateConfig(TritonConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:
@ -280,7 +280,7 @@ class TritonInferConfig(TritonConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/vertex_ai/agent_engine/transformation.py
View file

@ -29,10 +29,9 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Message, ModelResponse, Usage
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.rust_bridge._native import Tokenizer
from litellm.utils import CustomStreamWrapper
LiteLLMLoggingObj = _LiteLLMLoggingObj
@ -285,7 +284,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

5
litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py
View file

@ -24,9 +24,8 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -284,7 +283,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

5
litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py
View file

@ -20,9 +20,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -214,7 +213,7 @@ class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

4
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
View file

@ -14,7 +14,7 @@ from ....anthropic.chat.transformation import AnthropicConfig
from .output_params_utils import sanitize_vertex_anthropic_output_params
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class VertexAIError(Exception):
@ -197,7 +197,7 @@ class VertexAIAnthropicConfig(AnthropicConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

4
litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py
View file

@ -21,7 +21,7 @@ from litellm.types.utils import (
from ...common_utils import VertexAIError
if TYPE_CHECKING:
import tiktoken
from litellm.rust_bridge._native import Tokenizer
class VertexAILlama3Config(OpenAIGPTConfig):
@ -112,7 +112,7 @@ class VertexAILlama3Config(OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

7
litellm/llms/vertex_ai/vertex_gemma_models/transformation.py
View file

@ -24,9 +24,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
class VertexGemmaConfig(OpenAIGPTConfig):
@ -275,7 +274,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
litellm_params: dict,
client: HTTPHandler | httpx.Client | None = None,
timeout: float | httpx.Timeout | None = None,
encoding: "tiktoken.Encoding | None" = None,
encoding: "Tokenizer | None" = None,
):
"""Synchronous completion request"""
from litellm.utils import convert_to_model_response_object
@ -365,7 +364,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
litellm_params: dict,
client: AsyncHTTPHandler | httpx.AsyncClient | None = None,
timeout: float | httpx.Timeout | None = None,
encoding: "tiktoken.Encoding | None" = None,
encoding: "Tokenizer | None" = None,
):
"""Asynchronous completion request"""
from litellm.utils import convert_to_model_response_object

5
litellm/llms/watsonx/completion/transformation.py
View file

@ -21,9 +21,8 @@ from ..common_utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.rust_bridge._native import Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -280,7 +279,7 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

19
litellm/main.py
View file

@ -37,7 +37,6 @@ if TYPE_CHECKING:
import dotenv
import httpx
import openai
import tiktoken
from pydantic import BaseModel
from typing_extensions import overload
@ -114,6 +113,7 @@ from litellm.llms.vertex_ai.common_utils import (
get_vertex_ai_model_route,
)
from litellm.realtime_api.main import _realtime_health_check
from litellm.rust_bridge._native import Tokenizer
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.types.completion import (
_CompletionDispatchContext,
@ -7393,7 +7393,7 @@ def text_completion(
if isinstance(prompt, list):
import concurrent.futures
tokenizer: Final = tiktoken.encoding_for_model("text-davinci-003")
tokenizer: Final = Tokenizer.from_tiktoken("p50k_base")
## if it's a 2d list - each element in the list is a text_completion() request
if len(prompt) > 0 and isinstance(prompt[0], list):
responses: Final = [None for x in prompt] # init responses
@ -9147,7 +9147,7 @@ async def acount_tokens(
except Exception as e:
verbose_logger.debug("Provider token counting failed for model=%s, falling back to local: %s", model, e)
# Fallback to local tiktoken-based token counting
# Fallback to local token counting
fallback_messages = messages or []
if system and fallback_messages:
fallback_messages = [{"role": "system", "content": system}] + fallback_messages
@ -9166,16 +9166,16 @@ async def acount_tokens(
# Cache for encoding to avoid repeated __getattr__ calls
_encoding_cache: tiktoken.Encoding | None = None
_encoding_cache: Tokenizer | None = None
def _load_module_encoding() -> tiktoken.Encoding:
def _load_module_encoding() -> Tokenizer:
import sys
return sys.modules[__name__].encoding
def _get_encoding() -> tiktoken.Encoding:
def _get_encoding() -> Tokenizer:
"""Get encoding, loading it lazily if needed."""
global _encoding_cache
if _encoding_cache is None:
@ -9184,18 +9184,15 @@ def _get_encoding() -> tiktoken.Encoding:
return _encoding_cache
def _load_default_encoding() -> tiktoken.Encoding:
def _load_default_encoding() -> Tokenizer:
from litellm._lazy_imports import _get_default_encoding
return _get_default_encoding()
def __getattr__(name: str) -> tiktoken.Encoding:
def __getattr__(name: str) -> Tokenizer:
"""Lazy import handler for main module"""
if name == "encoding":
# Use _get_default_encoding which properly sets TIKTOKEN_CACHE_DIR
# before loading tiktoken, ensuring the local cache is used
# instead of downloading from the internet
_encoding: Final = _load_default_encoding()
# Cache it in the module's __dict__ for subsequent accesses
import sys

Some files were not shown because too many files have changed in this diff Show more