Merge remote-tracking branch 'origin/main' into litellm_responses-callback-cost-zero

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

# Conflicts:
#	tests/e2e/conftest.py
#	tests/e2e/e2e_config.py
#	tests/e2e/pytest.ini
This commit is contained in:
kerry 2026-09-22 07:37:54 +00:00
commit 3587090c5c
256 changed files with 10111 additions and 1235 deletions

View file

@ -15,6 +15,12 @@ description: >-
cache the same directory for different workloads, and a shared key would let
whichever ran first deny the others a save.
inputs:
profile:
description: "Cargo profile the build uses (dev or release)"
required: false
default: "dev"
runs:
using: composite
steps:
@ -25,6 +31,6 @@ runs:
~/.cargo/registry
~/.cargo/git
litellm-rust/target
key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }}
key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-maturin-dev-
${{ runner.os }}-maturin-${{ inputs.profile }}-

View file

@ -9,6 +9,7 @@ from pydantic import TypeAdapter, ValidationError
secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
MIN_MASKED_LENGTH: Final = 8
ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS"
def main() -> int:
@ -30,10 +31,15 @@ def main() -> int:
f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
)
return 1
for value in secrets.values():
if len(value) >= MIN_MASKED_LENGTH:
_ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
sys.stdout.flush()
if os.environ.get(ACTIONS_RUNNER_FLAG) == "true":
_ = sys.stdout.write(
"".join(
f"::add-mask::{value.replace('%', '%25')}\n"
for value in secrets.values()
if len(value) >= MIN_MASKED_LENGTH
)
)
sys.stdout.flush()
lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
try:
with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:

View file

@ -9,6 +9,7 @@ UNSUPPORTED: Final = re.compile(
r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
)
HARNESS: Final = re.compile(
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"

View file

@ -134,7 +134,16 @@ def main(
uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members)
native_path: Final = wheel.parent / "native" / Path(native_member.filename).name
native_path.parent.mkdir(parents=True, exist_ok=True)
native_path.write_bytes(archive.read(native_member))
native_bytes: Final = archive.read(native_member)
native_path.write_bytes(native_bytes)
duplicated_vocabularies: Final = tuple(
member.filename
for member in wheel_members
if member.filename.startswith("litellm/litellm_core_utils/tokenizers/")
and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name)
and member.file_size > 0
and archive.read(member) in native_bytes
)
wheel_metadata_tags_match: Final = (
len(wheel_metadata_tags) == len(expanded_filename_tags)
@ -205,7 +214,7 @@ def main(
native_module: Final = load_native_module(native_path)
native_module_loads: Final = native_module is not None
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
native_size_limit: Final = 40_000_000
native_size_limit: Final = 35_000_000
native_size_within_limit: Final = native_member.file_size <= native_size_limit
validations: Final = (
(f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
@ -223,6 +232,7 @@ def main(
("Native module loads", native_module_loads),
("Production module omits the panic test hook", panic_test_hook_absent),
(f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies),
("Wheel contents are valid", not unexpected_members),
)

View file

@ -13,6 +13,7 @@ on:
- ".github/workflows/codspeed.yml"
- ".github/actions/setup-uv-with-retries/**"
- ".github/actions/cache-cargo-build/**"
- ".github/scripts/uv_sync_with_retries.sh"
pull_request:
branches:
- main
@ -25,6 +26,7 @@ on:
- ".github/workflows/codspeed.yml"
- ".github/actions/setup-uv-with-retries/**"
- ".github/actions/cache-cargo-build/**"
- ".github/scripts/uv_sync_with_retries.sh"
# Allow CodSpeed to trigger backtest performance analysis
# in order to generate initial data
workflow_dispatch:
@ -59,19 +61,27 @@ jobs:
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
with:
profile: release
# Build the wheel and resolve every dependency outside the CodSpeed
# runner: the same maturin build took 42 minutes inside `codspeed run`
# versus under 3 minutes as a plain step (LIT-6183)
- name: Build environment
- name: Build the release wheel
run: uv build --wheel --out-dir dist
- name: Install the wheel into the benchmark environment
run: |
UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12
uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl
- name: Collect benchmarks
env:
PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
LITELLM_REQUIRE_INSTALLED_WHEEL: "1"
run: >
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
uv run --frozen --no-default-groups
--with pytest==8.3.5
--with pytest-codspeed==4.3.0
--with "mcp>=2.2.0,<3.0"
--with "a2a-sdk>=1.1.0,<2.0"
pytest
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
--import-mode=importlib
-p pytest_codspeed.plugin
tests/benchmarks/
--codspeed
@ -82,13 +92,9 @@ jobs:
with:
mode: simulation
run: >
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
uv run --frozen --no-default-groups
--with pytest==8.3.5
--with pytest-codspeed==4.3.0
--with "mcp>=2.2.0,<3.0"
--with "a2a-sdk>=1.1.0,<2.0"
pytest
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
--import-mode=importlib
-p pytest_codspeed.plugin
tests/benchmarks/
--codspeed

View file

@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra saml \
--python python3.13
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma

View file

@ -1,8 +1,11 @@
"""Guard the cost map on pull requests.
Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file,
and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named
litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models.
Every pull request whose diff against its merge base touches one of the three cost map files gets the file
checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
files and may only add or update models.
"""
from __future__ import annotations
@ -108,20 +111,37 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
)
def touches_cost_map(changed_files: Sequence[str]) -> bool:
return any(path in GUARDED_PATHS for path in changed_files)
def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
if bot:
return "bot contract enforced"
return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
if not bot and not touches_cost_map(changed_files):
return ()
head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
if isinstance(head_map, str):
return (head_map,)
return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
def _git(*args: str) -> str:
def _git(*args: str) -> str | None:
result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True)
return result.stdout if result.returncode == 0 else ""
return result.stdout if result.returncode == 0 else None
def snapshot(revision: str) -> Snapshot:
return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS))
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
def changed_files(base: str, head: str) -> tuple[str, ...] | None:
diff: Final = _git("diff", "--name-only", "--no-renames", base, head)
return None if diff is None else tuple(diff.splitlines())
def main(argv: Sequence[str]) -> int:
@ -131,9 +151,12 @@ def main(argv: Sequence[str]) -> int:
parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
args: Final = parser.parse_args(argv)
bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines())
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot)
contract: Final = "bot contract enforced" if bot else "human PR, file checks only"
changed: Final = changed_files(args.base, args.head)
if changed is None:
print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
return 1
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
contract: Final = contract_for(bot, changed)
if failures:
print(f"cost map guard failed ({contract}):")
print("\n".join(f"- {failure}" for failure in failures))

View file

@ -3253,6 +3253,7 @@ dependencies = [
name = "litellm-token-counter-huggingface"
version = "0.1.0"
dependencies = [
"serde_json",
"thiserror 2.0.19",
"tokenizers",
]
@ -3261,6 +3262,9 @@ dependencies = [
name = "litellm-token-counter-tiktoken"
version = "0.1.0"
dependencies = [
"base64 0.22.1",
"once_cell",
"rustc-hash",
"thiserror 2.0.19",
"tiktoken-rs",
]

View file

@ -89,7 +89,7 @@ veil = "0.3.0"
[profile.release]
opt-level = 3
lto = "thin"
lto = "fat"
codegen-units = 1
panic = "unwind"
debug = false

View file

@ -29,7 +29,7 @@ pyo3::create_exception!(
static FORK_GATE: ForkGate = ForkGate::new();
/// Whether this process has started the Tokio runtime.
/// Whether this process has entered process-bound native execution.
pub fn runtime_started() -> bool {
FORK_GATE.started(std::process::id())
}
@ -40,9 +40,8 @@ pub fn reserve_process_for_forking() -> Result<(), RuntimeAlreadyStarted> {
FORK_GATE.reserve(std::process::id())
}
/// The only door to the Tokio runtime: every route reaches it through this module, which is
/// what lets the gate speak for the whole extension. `clippy.toml` disallows going around it.
fn enter_runtime() -> PyResult<()> {
/// Claims process-bound native state before runtime startup or tokenizer execution.
pub fn enter_native() -> PyResult<()> {
FORK_GATE
.enter(std::process::id())
.map_err(|refused| match refused {
@ -60,7 +59,7 @@ fn enter_runtime() -> PyResult<()> {
#[expect(clippy::disallowed_methods, reason = "this is the gated door")]
fn runtime() -> PyResult<&'static Runtime> {
enter_runtime()?;
enter_native()?;
Ok(pyo3_async_runtimes::tokio::get_runtime())
}
@ -70,7 +69,7 @@ where
F: Future<Output = PyResult<T>> + Send + 'static,
T: for<'py> IntoPyObject<'py> + Send + 'static,
{
enter_runtime()?;
enter_native()?;
pyo3_async_runtimes::tokio::future_into_py(py, future)
}

View file

@ -20,7 +20,7 @@ pub use argument::lookup;
pub use callable::wrap_failure;
pub use driver::run_call;
pub use execution::{
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value,
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, enter_native, poll_async_value,
reserve_process_for_forking, run_async, run_async_value, run_sync, run_sync_value,
runtime_started,
};

View file

@ -10,7 +10,7 @@ name = "_native"
crate-type = ["cdylib"]
[features]
default = ["abi3", "fast"]
default = ["abi3", "fast", "huggingface", "tiktoken"]
abi3 = ["pyo3/abi3-py310"]
extension-module = ["pyo3/extension-module"]
panic-test = []

View file

@ -12,7 +12,7 @@ mod routes;
reason = "secret-manager foundations await rollout activation"
)]
mod secrets;
mod token_counter;
mod tokenizer;
#[pymodule(gil_used = true)]
mod _native {
@ -37,7 +37,12 @@ mod _native {
#[pymodule_export]
use crate::routes::responses::ResponsesWebSocketConnection;
#[pymodule_export]
use crate::token_counter::TokenCounter;
use crate::routes::token_counter::TokenCounter;
#[cfg(feature = "huggingface")]
#[pymodule_export]
use crate::tokenizer::HuggingFaceEncoding;
#[pymodule_export]
use crate::tokenizer::Tokenizer;
#[pymodule_export]
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
use pyo3::{prelude::*, types::PyModule};
@ -83,10 +88,13 @@ mod tests {
"achat_completions",
"ResponsesWebSocketConnection",
"TokenCounter",
"Tokenizer",
"gil_stats",
"process_state_started",
"reserve_process_for_forking",
];
#[cfg(feature = "huggingface")]
expected.push("HuggingFaceEncoding");
expected.sort_unstable();
let mut public_names: Vec<String> = native_module(py)

View file

@ -3,6 +3,7 @@ pub(crate) mod chat_completions;
pub(crate) mod messages;
pub(crate) mod ocr;
pub(crate) mod responses;
pub(crate) mod token_counter;
#[cfg(test)]
mod tests {

View file

@ -0,0 +1,87 @@
use std::sync::Arc;
use std::{num::NonZero, thread::available_parallelism};
use litellm_host_python::{enter_native, run_async};
use litellm_token_counter::{
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
};
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
prelude::*,
types::PyAny,
};
use tokio::sync::Semaphore;
use crate::errors::RustBridgeDeclined;
use crate::tokenizer::Tokenizer;
/// Counts the input tokens of a raw request body off the Python event loop with
/// the GIL released. Python owns which requests get here and what to do with
/// the count. At most one encode per core runs at a time; the rest wait in the
/// async task, where a cancelled Python awaiter drops them before any blocking
/// work is scheduled.
#[pyclass(frozen)]
pub(crate) struct TokenCounter {
inner: Arc<CoreTokenCounter>,
encode_slots: Arc<Semaphore>,
}
#[pymethods]
impl TokenCounter {
#[staticmethod]
#[pyo3(signature = (tokenizer, fast = false))]
fn from_tokenizer(py: Python<'_>, tokenizer: &Tokenizer, fast: bool) -> PyResult<Self> {
enter_native()?;
let inner = CoreTokenCounter::new(tokenizer.counter(py, fast));
Ok(Self {
inner: Arc::new(inner),
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
})
}
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
let counter = Arc::clone(&self.inner);
let encode_slots = Arc::clone(&self.encode_slots);
let body = body.to_vec();
run_async(
py,
async move {
let _slot = encode_slots
.acquire_owned()
.await
.map_err(|error| Error::Task(error.to_string()))?;
tokio::task::spawn_blocking(move || count_body(&counter, &body))
.await
.map_err(|error| Error::Task(error.to_string()))?
},
token_count_error_to_pyerr,
)
}
}
fn encode_parallelism() -> usize {
available_parallelism().map_or(1, NonZero::get)
}
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
let request = CountableRequest::parse(body)?;
counter.count_request(&request)
}
pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr {
let message = error.to_string();
match error {
Error::Load(_)
| Error::Ranks(_)
| Error::UnicodeClasses
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
Error::RequestParse(_)
| Error::MissingInput
| Error::FloatText
| Error::ContentBlock
| Error::ArrayItems
| Error::JsonSerialization(_)
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
Error::Encode(_) | Error::Decode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
}
}

View file

@ -1,158 +0,0 @@
use std::sync::Arc;
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
use std::{num::NonZero, thread::available_parallelism};
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
use litellm_host_python::release_gil;
use litellm_host_python::run_async;
use litellm_token_counter::{
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
};
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
prelude::*,
types::PyAny,
};
use tokio::sync::Semaphore;
use crate::errors::RustBridgeDeclined;
/// Counts the input tokens of a raw request body off the Python event loop with
/// the GIL released. Python owns which requests get here and what to do with
/// the count. At most one encode per core runs at a time; the rest wait in the
/// async task, where a cancelled Python awaiter drops them before any blocking
/// work is scheduled.
#[pyclass(frozen)]
pub(crate) struct TokenCounter {
inner: Arc<CoreTokenCounter>,
encode_slots: Arc<Semaphore>,
}
#[pymethods]
impl TokenCounter {
#[new]
fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
#[cfg(feature = "fast")]
{
Self::load(py, || CoreTokenCounter::from_json_fast(tokenizer_json))
}
#[cfg(all(not(feature = "fast"), feature = "huggingface"))]
{
Self::load(py, || CoreTokenCounter::from_json(tokenizer_json))
}
#[cfg(not(any(feature = "fast", feature = "huggingface")))]
{
let _ = (py, tokenizer_json);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the fast or huggingface feature",
))
}
}
#[staticmethod]
fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
#[cfg(feature = "fast")]
{
Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file))
}
#[cfg(not(feature = "fast"))]
{
let _ = (py, rank_file);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the fast feature",
))
}
}
#[staticmethod]
fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
#[cfg(feature = "fast")]
{
Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file))
}
#[cfg(not(feature = "fast"))]
{
let _ = (py, rank_file);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the fast feature",
))
}
}
#[staticmethod]
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
#[cfg(feature = "tiktoken")]
{
Self::load(py, || CoreTokenCounter::from_tiktoken(encoding))
}
#[cfg(not(feature = "tiktoken"))]
{
let _ = (py, encoding);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the tiktoken feature",
))
}
}
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
let counter = Arc::clone(&self.inner);
let encode_slots = Arc::clone(&self.encode_slots);
let body = body.to_vec();
run_async(
py,
async move {
let _slot = encode_slots
.acquire_owned()
.await
.map_err(|error| Error::Task(error.to_string()))?;
tokio::task::spawn_blocking(move || count_body(&counter, &body))
.await
.map_err(|error| Error::Task(error.to_string()))?
},
token_count_error_to_pyerr,
)
}
}
impl TokenCounter {
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
fn load(
py: Python<'_>,
load: impl FnOnce() -> Result<CoreTokenCounter, Error> + Send,
) -> PyResult<Self> {
let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?;
Ok(Self {
inner: Arc::new(inner),
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
})
}
}
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
fn encode_parallelism() -> usize {
available_parallelism().map_or(1, NonZero::get)
}
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
let request = CountableRequest::parse(body)?;
counter.count_request(&request)
}
fn token_count_error_to_pyerr(error: Error) -> PyErr {
let message = error.to_string();
match error {
Error::Load(_)
| Error::Ranks(_)
| Error::UnicodeClasses
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
Error::RequestParse(_)
| Error::MissingInput
| Error::FloatText
| Error::ContentBlock
| Error::ArrayItems
| Error::JsonSerialization(_)
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
Error::Encode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
}
}

View file

@ -0,0 +1,713 @@
//! The Python face of the text codecs: one `Tokenizer` class over the tiktoken and Hugging
//! Face backends, carrying the read-only surface of `tiktoken.Encoding` and
//! `tokenizers.Tokenizer` that `litellm/litellm_core_utils/tokenizer.py` wraps.
use std::borrow::Cow;
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
use std::collections::HashMap;
use std::sync::Arc;
#[cfg(feature = "fast")]
use std::sync::OnceLock;
use litellm_host_python::{enter_native, release_gil};
#[cfg(feature = "fast")]
use litellm_token_counter::fast::{FastCounter, FastTokenizer};
use litellm_token_counter::{Error, TextCodec};
use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString};
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
use pyo3::exceptions::PyValueError;
#[cfg(feature = "huggingface")]
use pyo3::{exceptions::PyIOError, types::PyDict};
#[cfg(feature = "tiktoken")]
use pyo3::{
exceptions::{PyKeyError, PyRuntimeError},
types::PyBytes,
};
#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))]
use crate::errors::RustBridgeDeclined;
use crate::routes::token_counter::token_count_error_to_pyerr;
#[cfg(feature = "huggingface")]
use litellm_token_counter::huggingface::{
EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, PaddingStrategy,
TruncationDirection, encoding_from_json, encoding_to_json,
};
#[cfg(feature = "tiktoken")]
use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary};
#[cfg(feature = "tiktoken")]
pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<TiktokenTokenizer> {
enter_native()?;
let resource: std::path::PathBuf =
PyModule::import(py, "litellm.litellm_core_utils.tokenizers")?
.getattr("__file__")?
.extract()?;
release_gil(py, || {
TiktokenTokenizer::from_cached_ranks(encoding, |file| {
std::fs::read_to_string(resource.with_file_name(file))
})
})
.map_err(|error| token_count_error_to_pyerr(error.into()))
}
pub(crate) enum Codec {
#[cfg(feature = "tiktoken")]
Tiktoken(TiktokenTokenizer),
#[cfg(feature = "huggingface")]
HuggingFace(HuggingFaceTokenizer),
}
impl Codec {
pub(crate) fn codec(&self) -> &dyn TextCodec {
match *self {
#[cfg(feature = "tiktoken")]
Self::Tiktoken(ref tokenizer) => tokenizer,
#[cfg(feature = "huggingface")]
Self::HuggingFace(ref tokenizer) => tokenizer,
}
}
#[cfg(feature = "fast")]
fn fast_counter(&self) -> Option<FastTokenizer> {
match *self {
#[cfg(feature = "tiktoken")]
Self::Tiktoken(ref tokenizer) => tokenizer.fast_counter(),
#[cfg(feature = "huggingface")]
Self::HuggingFace(ref tokenizer) => tokenizer.fast_counter(),
}
}
}
/// The loaded model is shared: `TokenCounter::from_tokenizer` counts with the same parse,
/// and the opt-in count-only counter is derived from it once, on first use.
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Tokenizer {
inner: Arc<Codec>,
#[cfg(feature = "fast")]
fast: OnceLock<Option<Arc<FastTokenizer>>>,
}
#[pymethods]
impl Tokenizer {
#[staticmethod]
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
#[cfg(feature = "tiktoken")]
{
let tokenizer = load_tiktoken(py, encoding)?;
Ok(Self::new(Codec::Tiktoken(tokenizer)))
}
#[cfg(not(feature = "tiktoken"))]
{
let _ = (py, encoding);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the tiktoken feature",
))
}
}
#[staticmethod]
fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
#[cfg(feature = "huggingface")]
{
enter_native()?;
let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json))
.map_err(|error| token_count_error_to_pyerr(error.into()))?;
Ok(Self::new(Codec::HuggingFace(tokenizer)))
}
#[cfg(not(feature = "huggingface"))]
{
let _ = (py, tokenizer_json);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the huggingface feature",
))
}
}
#[staticmethod]
#[pyo3(signature = (identifier, revision = "main", token = None))]
fn from_pretrained(
py: Python<'_>,
identifier: &str,
revision: &str,
token: Option<&str>,
) -> PyResult<Self> {
#[cfg(feature = "huggingface")]
{
enter_native()?;
let kwargs = PyDict::new(py);
kwargs.set_item("repo_id", identifier)?;
kwargs.set_item("filename", "tokenizer.json")?;
kwargs.set_item("revision", revision)?;
kwargs.set_item("token", token)?;
let path: String = PyModule::import(py, "huggingface_hub")?
.getattr("hf_hub_download")?
.call((), Some(&kwargs))?
.extract()?;
let json =
release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?;
Self::from_json(py, &json)
}
#[cfg(not(feature = "huggingface"))]
{
let _ = (py, identifier, revision, token);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the huggingface feature",
))
}
}
fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult<Vec<u32>> {
enter_native()?;
let text = self.text(text)?;
release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr)
}
#[pyo3(signature = (ids, skip_special_tokens = true))]
fn decode(&self, py: Python<'_>, ids: Vec<u32>, skip_special_tokens: bool) -> PyResult<String> {
enter_native()?;
release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens))
.map_err(token_count_error_to_pyerr)
}
#[pyo3(signature = (text, fast = false))]
fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>, fast: bool) -> PyResult<usize> {
enter_native()?;
let text = self.text(text)?;
let counter = self.counter(py, fast);
release_gil(py, || {
litellm_token_counter::Tokenizer::count_tokens(&counter, &text)
})
.map_err(token_count_error_to_pyerr)
}
#[getter]
fn name(&self) -> &str {
self.inner.codec().name()
}
// ---- tiktoken: the `tiktoken.Encoding` surface ------------------------------------------
#[cfg(feature = "tiktoken")]
fn encode_special(
&self,
py: Python<'_>,
text: &Bound<'_, PyString>,
allowed: Vec<String>,
) -> PyResult<Vec<u32>> {
enter_native()?;
let tokenizer = self.tiktoken()?;
let text = self.text(text)?;
release_gil(py, || tokenizer.encode_special(&text, &allowed))
.map_err(PyRuntimeError::new_err)
}
/// tiktoken's `encode_with_unstable`: `(stable_tokens, completions)`.
#[cfg(feature = "tiktoken")]
fn encode_with_unstable(
&self,
py: Python<'_>,
text: &Bound<'_, PyString>,
allowed: Vec<String>,
) -> PyResult<(Vec<u32>, Vec<Vec<u32>>)> {
enter_native()?;
let tokenizer = self.tiktoken()?;
let text = self.text(text)?;
Ok(release_gil(py, || {
tokenizer.encode_with_unstable(&text, &allowed)
}))
}
/// The special tokens by text: tiktoken's `_special_tokens`.
#[cfg(feature = "tiktoken")]
fn special_tokens(&self) -> PyResult<HashMap<String, u32>> {
Ok(self
.vocabulary()?
.special_tokens()
.map(|(token, rank)| (token.to_owned(), rank))
.collect())
}
#[cfg(feature = "tiktoken")]
fn max_token_value(&self) -> PyResult<u32> {
Ok(self.vocabulary()?.max_token_value())
}
#[cfg(feature = "tiktoken")]
fn is_special_token(&self, token: u32) -> PyResult<bool> {
Ok(self.vocabulary()?.is_special_token(token))
}
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
#[cfg(feature = "tiktoken")]
fn token_byte_values<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py, PyBytes>>> {
let vocabulary = self.vocabulary()?;
let values = release_gil(py, || vocabulary.token_byte_values());
Ok(values.iter().map(|value| PyBytes::new(py, value)).collect())
}
/// The token of one whole piece; `KeyError` when it is not in the vocabulary.
#[cfg(feature = "tiktoken")]
fn encode_single_token(&self, py: Python<'_>, piece: Vec<u8>) -> PyResult<u32> {
self.vocabulary()?
.encode_single_token(&piece)
.ok_or_else(|| PyKeyError::new_err(PyBytes::new(py, &piece).unbind()))
}
#[cfg(feature = "tiktoken")]
fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec<u32>) -> PyResult<Bound<'py, PyBytes>> {
enter_native()?;
let tokenizer = self.tiktoken()?;
let bytes =
release_gil(py, || tokenizer.decode_bytes(&ids)).map_err(PyKeyError::new_err)?;
Ok(PyBytes::new(py, &bytes))
}
// ---- Hugging Face: the `tokenizers.Tokenizer` surface -----------------------------------
#[cfg(feature = "huggingface")]
#[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))]
fn encode_huggingface(
&self,
py: Python<'_>,
sequence: Sequence,
pair: Option<Sequence>,
is_pretokenized: bool,
add_special_tokens: bool,
fast: bool,
) -> PyResult<HuggingFaceEncoding> {
enter_native()?;
let tokenizer = self.huggingface()?;
let sequence = sequence.input(is_pretokenized)?;
let input = match pair {
Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?),
None => EncodeInput::Single(sequence),
};
release_gil(py, || {
tokenizer.encode_result(input, add_special_tokens, fast)
})
.map(|inner| HuggingFaceEncoding { inner })
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))]
fn encode_batch_huggingface(
&self,
py: Python<'_>,
inputs: Vec<(Sequence, Option<Sequence>)>,
is_pretokenized: bool,
add_special_tokens: bool,
fast: bool,
) -> PyResult<Vec<HuggingFaceEncoding>> {
enter_native()?;
let tokenizer = self.huggingface()?;
let inputs = inputs
.into_iter()
.map(|(sequence, pair)| {
let sequence = sequence.input(is_pretokenized)?;
match pair {
Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)),
None => Ok(EncodeInput::Single(sequence)),
}
})
.collect::<PyResult<Vec<_>>>()?;
release_gil(py, || {
tokenizer.encode_batch_result(inputs, add_special_tokens, fast)
})
.map(|encodings| {
encodings
.into_iter()
.map(|inner| HuggingFaceEncoding { inner })
.collect()
})
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (pretty = false))]
fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult<String> {
enter_native()?;
let tokenizer = self.huggingface()?;
release_gil(py, || tokenizer.to_json(pretty))
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
}
#[cfg(feature = "huggingface")]
fn token_to_id(&self, token: &str) -> PyResult<Option<u32>> {
Ok(self.huggingface()?.token_to_id(token))
}
#[cfg(feature = "huggingface")]
fn id_to_token(&self, id: u32) -> PyResult<Option<String>> {
Ok(self.huggingface()?.id_to_token(id))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (with_added_tokens = true))]
fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult<HashMap<String, u32>> {
let tokenizer = self.huggingface()?;
Ok(release_gil(py, || tokenizer.vocab(with_added_tokens)))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (with_added_tokens = true))]
fn get_vocab_size(&self, with_added_tokens: bool) -> PyResult<usize> {
Ok(self.huggingface()?.vocab_size(with_added_tokens))
}
/// The added tokens by id as `(id, (content, single_word, lstrip, rstrip, normalized,
/// special))`, for Python to rebuild as `tokenizers.AddedToken`.
#[cfg(feature = "huggingface")]
fn added_tokens_decoder(&self) -> PyResult<Vec<(u32, AddedTokenFields)>> {
Ok(self
.huggingface()?
.added_tokens_decoder()
.into_iter()
.map(|(id, token)| {
(
id,
(
token.content,
token.single_word,
token.lstrip,
token.rstrip,
token.normalized,
token.special,
),
)
})
.collect())
}
/// The padding parameters as `tokenizers.Tokenizer.padding` reports them.
#[cfg(feature = "huggingface")]
fn padding<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
let Some(params) = self.huggingface()?.padding() else {
return Ok(None);
};
let padding = PyDict::new(py);
padding.set_item(
"length",
match params.strategy {
PaddingStrategy::BatchLongest => None,
PaddingStrategy::Fixed(length) => Some(length),
},
)?;
padding.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?;
padding.set_item("pad_id", params.pad_id)?;
padding.set_item("pad_type_id", params.pad_type_id)?;
padding.set_item("pad_token", &params.pad_token)?;
padding.set_item("direction", params.direction.as_ref())?;
Ok(Some(padding))
}
/// The truncation parameters as `tokenizers.Tokenizer.truncation` reports them.
#[cfg(feature = "huggingface")]
fn truncation<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
let Some(params) = self.huggingface()?.truncation() else {
return Ok(None);
};
let truncation = PyDict::new(py);
truncation.set_item("max_length", params.max_length)?;
truncation.set_item("stride", params.stride)?;
truncation.set_item("strategy", params.strategy.as_ref())?;
truncation.set_item("direction", params.direction.as_ref())?;
Ok(Some(truncation))
}
#[cfg(feature = "huggingface")]
fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult<usize> {
Ok(self.huggingface()?.num_special_tokens_to_add(is_pair))
}
#[cfg(feature = "huggingface")]
fn encode_special_tokens(&self) -> PyResult<bool> {
Ok(self.huggingface()?.encode_special_tokens())
}
}
#[cfg(feature = "huggingface")]
type AddedTokenFields = (String, bool, bool, bool, bool, bool);
impl Tokenizer {
fn new(inner: Codec) -> Self {
Self {
inner: Arc::new(inner),
#[cfg(feature = "fast")]
fast: OnceLock::new(),
}
}
pub(crate) fn counter(&self, py: Python<'_>, fast: bool) -> SharedCounter {
#[cfg(feature = "fast")]
if fast {
let counter = self.fast.get().unwrap_or_else(|| {
release_gil(py, || {
self.fast
.get_or_init(|| self.inner.fast_counter().map(Arc::new))
})
});
if let Some(counter) = counter {
return SharedCounter::Fast(Arc::clone(counter));
}
}
#[cfg(not(feature = "fast"))]
let _ = (py, fast);
SharedCounter::Codec(Arc::clone(&self.inner))
}
/// A Python `str` as UTF-8. tiktoken replaces lone surrogates the way its Python `encode`
/// does; `tokenizers` rejects them, so that backend keeps the encode error.
fn text<'a>(&self, text: &'a Bound<'_, PyString>) -> PyResult<Cow<'a, str>> {
match text.to_cow() {
Ok(text) => Ok(text),
Err(error) => match *self.inner {
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(_) if error.is_instance_of::<PyUnicodeEncodeError>(text.py()) => {
text.call_method1("encode", ("utf-16", "surrogatepass"))?
.call_method1("decode", ("utf-16", "replace"))?
.extract::<String>()
.map(Cow::Owned)
}
_ => Err(error),
},
}
}
#[cfg(feature = "tiktoken")]
fn tiktoken(&self) -> PyResult<&TiktokenTokenizer> {
match *self.inner {
Codec::Tiktoken(ref tokenizer) => Ok(tokenizer),
#[cfg(feature = "huggingface")]
Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")),
}
}
#[cfg(feature = "tiktoken")]
fn vocabulary(&self) -> PyResult<&Vocabulary> {
self.tiktoken()?.vocabulary().ok_or_else(|| {
PyRuntimeError::new_err("this encoding was built without its vocabulary")
})
}
#[cfg(feature = "huggingface")]
fn huggingface(&self) -> PyResult<&HuggingFaceTokenizer> {
match *self.inner {
Codec::HuggingFace(ref tokenizer) => Ok(tokenizer),
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")),
}
}
}
pub(crate) enum SharedCounter {
Codec(Arc<Codec>),
#[cfg(feature = "fast")]
Fast(Arc<FastTokenizer>),
}
impl litellm_token_counter::Tokenizer for SharedCounter {
fn count_tokens(&self, text: &str) -> Result<usize, Error> {
match self {
Self::Codec(codec) => codec.codec().count_tokens(text),
#[cfg(feature = "fast")]
Self::Fast(counter) => counter.count_tokens(text).map_err(Error::from),
}
}
}
#[cfg(feature = "huggingface")]
#[derive(FromPyObject)]
pub(crate) enum Sequence {
Text(String),
Words(Vec<String>),
}
#[cfg(feature = "huggingface")]
impl Sequence {
fn input(self, is_pretokenized: bool) -> PyResult<InputSequence<'static>> {
match (self, is_pretokenized) {
(Self::Text(text), false) => Ok(text.into()),
(Self::Words(words), true) => Ok(words.into()),
_ => Err(pyo3::exceptions::PyTypeError::new_err(
"input must match is_pretokenized",
)),
}
}
}
#[cfg(feature = "huggingface")]
fn direction<T>(value: &str, left: T, right: T, what: &str) -> PyResult<T> {
match value {
"left" => Ok(left),
"right" => Ok(right),
other => Err(PyValueError::new_err(format!(
"invalid {what} direction {other:?}: expected 'left' or 'right'"
))),
}
}
/// `tokenizers.Encoding`, mutable like the original: `pad`, `truncate` and `set_sequence_id`
/// change it in place.
#[cfg(feature = "huggingface")]
#[pyclass(module = "litellm.rust_bridge._native")]
pub(crate) struct HuggingFaceEncoding {
inner: Encoding,
}
#[cfg(feature = "huggingface")]
#[pymethods]
impl HuggingFaceEncoding {
#[new]
#[pyo3(signature = (json = None))]
fn new(json: Option<&str>) -> PyResult<Self> {
let inner = match json {
Some(json) => encoding_from_json(json)
.map_err(|error| PyValueError::new_err(error.to_string()))?,
None => Encoding::default(),
};
Ok(Self { inner })
}
#[staticmethod]
#[pyo3(signature = (encodings, growing_offsets = true))]
fn merge(encodings: Vec<PyRef<'_, Self>>, growing_offsets: bool) -> Self {
Self {
inner: Encoding::merge(
encodings.iter().map(|encoding| encoding.inner.clone()),
growing_offsets,
),
}
}
fn __reduce__<'py>(
&self,
py: Python<'py>,
) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> {
let json = encoding_to_json(&self.inner)
.map_err(|error| PyValueError::new_err(error.to_string()))?;
Ok((py.get_type::<Self>(), (json,)))
}
fn __repr__(&self) -> String {
format!(
"Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \
attention_mask, special_tokens_mask, overflowing])",
self.inner.len()
)
}
fn __len__(&self) -> usize {
self.inner.len()
}
#[getter]
fn ids(&self) -> Vec<u32> {
self.inner.get_ids().to_vec()
}
#[getter]
fn tokens(&self) -> Vec<String> {
self.inner.get_tokens().to_vec()
}
#[getter]
fn offsets(&self) -> Vec<(usize, usize)> {
self.inner.get_offsets().to_vec()
}
#[getter]
fn type_ids(&self) -> Vec<u32> {
self.inner.get_type_ids().to_vec()
}
#[getter]
fn attention_mask(&self) -> Vec<u32> {
self.inner.get_attention_mask().to_vec()
}
#[getter]
fn special_tokens_mask(&self) -> Vec<u32> {
self.inner.get_special_tokens_mask().to_vec()
}
#[getter]
fn word_ids(&self) -> Vec<Option<u32>> {
self.inner.get_word_ids().to_vec()
}
#[getter]
fn sequence_ids(&self) -> Vec<Option<usize>> {
self.inner.get_sequence_ids()
}
#[getter]
fn overflowing(&self) -> Vec<Self> {
self.inner
.get_overflowing()
.iter()
.cloned()
.map(|inner| Self { inner })
.collect()
}
#[getter]
fn n_sequences(&self) -> usize {
self.inner.n_sequences()
}
#[pyo3(signature = (word_index, sequence_index = 0))]
fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
self.inner.word_to_tokens(word_index, sequence_index)
}
#[pyo3(signature = (word_index, sequence_index = 0))]
fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
self.inner.word_to_chars(word_index, sequence_index)
}
fn token_to_sequence(&self, token_index: usize) -> Option<usize> {
self.inner.token_to_sequence(token_index)
}
fn token_to_chars(&self, token_index: usize) -> Option<(usize, usize)> {
self.inner
.token_to_chars(token_index)
.map(|(_, offsets)| offsets)
}
fn token_to_word(&self, token_index: usize) -> Option<u32> {
self.inner.token_to_word(token_index).map(|(_, word)| word)
}
#[pyo3(signature = (char_pos, sequence_index = 0))]
fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option<usize> {
self.inner.char_to_token(char_pos, sequence_index)
}
#[pyo3(signature = (char_pos, sequence_index = 0))]
fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option<u32> {
self.inner.char_to_word(char_pos, sequence_index)
}
fn set_sequence_id(&mut self, sequence_id: usize) {
self.inner.set_sequence_id(sequence_id);
}
#[pyo3(signature = (length, direction = "right", pad_id = 0, pad_type_id = 0, pad_token = "[PAD]"))]
fn pad(
&mut self,
length: usize,
direction: &str,
pad_id: u32,
pad_type_id: u32,
pad_token: &str,
) -> PyResult<()> {
let direction = self::direction(
direction,
PaddingDirection::Left,
PaddingDirection::Right,
"padding",
)?;
self.inner
.pad(length, pad_id, pad_type_id, pad_token, direction);
Ok(())
}
#[pyo3(signature = (max_length, stride = 0, direction = "right"))]
fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> {
let direction = self::direction(
direction,
TruncationDirection::Left,
TruncationDirection::Right,
"truncation",
)?;
self.inner.truncate(max_length, stride, direction);
Ok(())
}
}

View file

@ -8,6 +8,8 @@ mod scanner;
mod tiktoken;
mod unicode_classes;
use std::sync::Arc;
use byte_level::ByteLevelCounter;
use scanner::{SplitPattern, TiktokenCounter};
@ -15,22 +17,29 @@ pub use error::Error;
enum Encoder {
HuggingFace {
tokenizer: Box<tokenizers::Tokenizer>,
tokenizer: Arc<tokenizers::Tokenizer>,
byte_level: Option<ByteLevelCounter>,
},
Tiktoken(TiktokenCounter),
}
/// A count-only tokenizer. Its model tables are immutable, so one built from an already
/// loaded model (`from_shared`, `from_*_pairs`) adds only the count-specific tables.
pub struct FastTokenizer(Encoder);
impl FastTokenizer {
pub fn from_json(json: &str) -> Result<Self, Error> {
let tokenizer = json.parse::<tokenizers::Tokenizer>().map_err(Error::Load)?;
Ok(Self::from_shared(Arc::new(tokenizer)))
}
/// Counts with a Hugging Face model another codec already holds; nothing is re-parsed.
pub fn from_shared(tokenizer: Arc<tokenizers::Tokenizer>) -> Self {
let byte_level = ByteLevelCounter::detect(&tokenizer);
Ok(Self(Encoder::HuggingFace {
tokenizer: Box::new(tokenizer),
Self(Encoder::HuggingFace {
tokenizer,
byte_level,
}))
})
}
pub fn from_cl100k_ranks(ranks: &str) -> Result<Self, Error> {
@ -41,12 +50,36 @@ impl FastTokenizer {
Self::from_ranks(SplitPattern::O200k, ranks)
}
/// `cl100k_base` from ranks another loader already parsed.
pub fn from_cl100k_pairs<'a>(
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
Self::from_pairs(SplitPattern::Cl100k, pairs)
}
/// `o200k_base` (and `o200k_harmony`, whose ordinary tokens are the same) from ranks
/// another loader already parsed.
pub fn from_o200k_pairs<'a>(
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
Self::from_pairs(SplitPattern::O200k, pairs)
}
fn from_ranks(split: SplitPattern, ranks: &str) -> Result<Self, Error> {
TiktokenCounter::from_ranks(split, ranks)
.map(Encoder::Tiktoken)
.map(Self)
}
fn from_pairs<'a>(
split: SplitPattern,
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
TiktokenCounter::from_pairs(split, pairs)
.map(Encoder::Tiktoken)
.map(Self)
}
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
match &self.0 {
Encoder::Tiktoken(counter) => Ok(counter.count(text)),

View file

@ -39,8 +39,19 @@ pub(super) struct TiktokenCounter {
impl TiktokenCounter {
pub(super) fn from_ranks(split: SplitPattern, rank_file: &str) -> Result<Self, Error> {
Self::new(split, MergeRanks::parse(rank_file)?)
}
pub(super) fn from_pairs<'a>(
split: SplitPattern,
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
Self::new(split, MergeRanks::from_pairs(pairs)?)
}
fn new(split: SplitPattern, ranks: MergeRanks) -> Result<Self, Error> {
Ok(Self {
ranks: MergeRanks::parse(rank_file)?,
ranks,
piece_len: split.piece_len(),
unicode_classes: UnicodeClasses::get().ok_or(Error::UnicodeClasses)?,
})

View file

@ -22,11 +22,25 @@ pub(super) struct MergeRanks(FxHashMap<Box<[u8]>, Rank>);
impl MergeRanks {
pub(super) fn parse(text: &str) -> Result<Self, Error> {
let ranks = text
.lines()
.filter(|line| !line.is_empty())
.map(parse_line)
.collect::<Result<FxHashMap<_, _>, _>>()?;
Self::from_entries(text.lines().filter(|line| !line.is_empty()).map(parse_line))
}
/// The same table from ranks another loader already parsed.
pub(super) fn from_pairs<'a>(
pairs: impl IntoIterator<Item = (&'a [u8], Rank)>,
) -> Result<Self, Error> {
Self::from_entries(pairs.into_iter().map(|(bytes, rank)| {
if rank == NO_RANK {
return Err(Error::Ranks(format!("rank {rank} is reserved")));
}
Ok((Box::from(bytes), rank))
}))
}
fn from_entries(
entries: impl Iterator<Item = Result<(Box<[u8]>, Rank), Error>>,
) -> Result<Self, Error> {
let ranks = entries.collect::<Result<FxHashMap<_, _>, _>>()?;
if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) {
return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token")));
}

View file

@ -6,5 +6,6 @@ license.workspace = true
repository.workspace = true
[dependencies]
serde_json.workspace = true
thiserror.workspace = true
tokenizers.workspace = true

View file

@ -6,4 +6,6 @@ pub enum Error {
Load(#[source] tokenizers::Error),
#[error("tokenization failed: {0}")]
Encode(#[source] tokenizers::Error),
#[error("token decoding failed: {0}")]
Decode(#[source] tokenizers::Error),
}

View file

@ -2,22 +2,243 @@
mod error;
pub use error::Error;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
pub struct HuggingFaceTokenizer(Box<tokenizers::Tokenizer>);
pub use error::Error;
use tokenizers::PostProcessor;
pub use tokenizers::{
AddedToken, EncodeInput, Encoding, InputSequence, PaddingDirection, PaddingParams,
PaddingStrategy, TruncationDirection, TruncationParams,
};
pub fn encoding_from_json(json: &str) -> Result<Encoding, Error> {
serde_json::from_str(json).map_err(|error| Error::Load(error.into()))
}
pub fn encoding_to_json(encoding: &Encoding) -> Result<String, Error> {
serde_json::to_string(encoding).map_err(|error| Error::Load(error.into()))
}
pub struct HuggingFaceTokenizer {
tokenizer: Arc<tokenizers::Tokenizer>,
special_token_ids: HashSet<u32>,
}
impl HuggingFaceTokenizer {
pub fn from_json(json: &str) -> Result<Self, Error> {
json.parse::<tokenizers::Tokenizer>()
.map(Box::new)
.map(Self)
.map(Self::new)
.map_err(Error::Load)
}
fn new(tokenizer: tokenizers::Tokenizer) -> Self {
let special_token_ids: HashSet<u32> = tokenizer
.get_added_tokens_decoder()
.into_iter()
.filter_map(|(id, token)| token.special.then_some(id))
.collect();
Self {
tokenizer: Arc::new(tokenizer),
special_token_ids,
}
}
/// The parsed model, for a count-only counter to share instead of parsing it again.
pub fn shared(&self) -> Arc<tokenizers::Tokenizer> {
Arc::clone(&self.tokenizer)
}
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
self.0
self.tokenizer
.encode_fast(text, true)
.map(|encoding| encoding.len())
.map_err(Error::Encode)
}
pub fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
self.tokenizer
.encode_fast(text, true)
.map(|encoding| encoding.get_ids().to_vec())
.map_err(Error::Encode)
}
pub fn encode_result<'a>(
&self,
input: EncodeInput<'a>,
add_special_tokens: bool,
fast: bool,
) -> Result<Encoding, Error> {
if fast {
return self
.tokenizer
.encode_fast(input, add_special_tokens)
.map_err(Error::Encode);
}
self.tokenizer
.encode_char_offsets(input, add_special_tokens)
.map_err(Error::Encode)
}
pub fn encode_batch_result<'a>(
&self,
inputs: Vec<EncodeInput<'a>>,
add_special_tokens: bool,
fast: bool,
) -> Result<Vec<Encoding>, Error> {
if fast {
return self
.tokenizer
.encode_batch_fast(inputs, add_special_tokens)
.map_err(Error::Encode);
}
self.tokenizer
.encode_batch_char_offsets(inputs, add_special_tokens)
.map_err(Error::Encode)
}
pub fn to_json(&self, pretty: bool) -> Result<String, Error> {
self.tokenizer.to_string(pretty).map_err(Error::Load)
}
pub fn token_to_id(&self, token: &str) -> Option<u32> {
self.tokenizer.token_to_id(token)
}
pub fn id_to_token(&self, id: u32) -> Option<String> {
self.tokenizer.id_to_token(id)
}
pub fn vocab(&self, with_added_tokens: bool) -> HashMap<String, u32> {
self.tokenizer.get_vocab(with_added_tokens)
}
pub fn vocab_size(&self, with_added_tokens: bool) -> usize {
self.tokenizer.get_vocab_size(with_added_tokens)
}
/// The added tokens by id, in id order.
pub fn added_tokens_decoder(&self) -> Vec<(u32, AddedToken)> {
let mut added: Vec<(u32, AddedToken)> = self
.tokenizer
.get_added_tokens_decoder()
.into_iter()
.collect();
added.sort_unstable_by_key(|(id, _)| *id);
added
}
pub fn padding(&self) -> Option<&PaddingParams> {
self.tokenizer.get_padding()
}
pub fn truncation(&self) -> Option<&TruncationParams> {
self.tokenizer.get_truncation()
}
/// How many special tokens the post-processor adds to a single sequence or a pair.
pub fn num_special_tokens_to_add(&self, is_pair: bool) -> usize {
self.tokenizer
.get_post_processor()
.map_or(0, |processor| processor.added_tokens(is_pair))
}
pub fn encode_special_tokens(&self) -> bool {
self.tokenizer.get_encode_special_tokens()
}
pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
if !skip_special_tokens {
return self.tokenizer.decode(ids, false).map_err(Error::Decode);
}
let filtered_ids: Vec<u32> = ids
.iter()
.copied()
.filter(|id| !self.special_token_ids.contains(id))
.collect();
self.tokenizer
.decode(&filtered_ids, true)
.map_err(Error::Decode)
}
pub fn name(&self) -> &str {
"huggingface"
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn codecs_round_trip_and_skip_special_tokens() {
let json = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
));
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
let ids = tokenizer.encode("<SOS>hello<EOT>").unwrap();
assert!(tokenizer.decode(&ids, false).unwrap().contains("<SOS>"));
assert_eq!(tokenizer.decode(&ids, true).unwrap(), "hello");
}
#[test]
fn decode_filters_special_added_tokens() {
let json = r#"{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 1,
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {"type": "Whitespace"},
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {"<unk>": 0, "<s>": 1, "hello": 2},
"unk_token": "<unk>"
}
}"#;
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
assert!(!tokenizer.decode(&[1, 2], true).unwrap().contains("<s>"));
assert!(tokenizer.decode(&[1, 2], false).unwrap().contains("<s>"));
}
#[test]
fn vocabulary_lookups_mirror_the_tokenizers_api() {
let json = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
));
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
let ids = tokenizer.encode("hello").unwrap();
let token = tokenizer.id_to_token(ids[0]).unwrap();
assert_eq!(tokenizer.token_to_id(&token), Some(ids[0]));
assert_eq!(tokenizer.id_to_token(u32::MAX), None);
assert_eq!(tokenizer.vocab(true).len(), tokenizer.vocab_size(true));
assert!(tokenizer.vocab_size(true) >= tokenizer.vocab_size(false));
let added = tokenizer.added_tokens_decoder();
assert!(added.windows(2).all(|pair| pair[0].0 < pair[1].0));
assert!(added.iter().any(|(_, token)| token.special));
assert!(tokenizer.padding().is_none());
assert!(tokenizer.truncation().is_none());
assert!(!tokenizer.encode_special_tokens());
assert_eq!(
tokenizer.num_special_tokens_to_add(false),
tokenizer.encode("").unwrap().len()
);
}
}

View file

@ -6,5 +6,8 @@ license.workspace = true
repository.workspace = true
[dependencies]
base64.workspace = true
once_cell = "1.21.3"
rustc-hash = "2.1.3"
thiserror.workspace = true
tiktoken-rs.workspace = true

View file

@ -1,27 +1,125 @@
#![forbid(unsafe_code)]
mod error;
mod ranks;
use std::collections::HashSet;
pub use error::UnsupportedTokenizer;
pub use ranks::{LoadError, Vocabulary};
pub struct TiktokenTokenizer(&'static tiktoken_rs::CoreBPE);
pub struct TiktokenTokenizer {
encoder: &'static tiktoken_rs::CoreBPE,
/// Present for encodings built from a rank file; the embedded tiktoken-rs singletons
/// behind [`from_name`](Self::from_name) keep their ranks private.
vocabulary: Option<&'static Vocabulary>,
name: &'static str,
}
impl TiktokenTokenizer {
/// Builds `name` from its packaged rank file (read through `load`), once per process.
/// The tokenizer reports the requested name, so `gpt2` stays `gpt2` like tiktoken does.
pub fn from_cached_ranks(
name: &str,
load: impl FnOnce(&str) -> std::io::Result<String>,
) -> Result<Self, LoadError> {
let (loaded, name) = ranks::load(name, load)?;
Ok(Self {
encoder: &loaded.bpe,
vocabulary: Some(&loaded.vocabulary),
name,
})
}
/// The encodings tiktoken-rs embeds, for hosts without the packaged rank files.
pub fn from_name(name: &str) -> Result<Self, UnsupportedTokenizer> {
let tokenizer = match name {
"cl100k_base" => tiktoken_rs::cl100k_base_singleton(),
"o200k_base" => tiktoken_rs::o200k_base_singleton(),
"o200k_harmony" => tiktoken_rs::o200k_harmony_singleton(),
"p50k_base" => tiktoken_rs::p50k_base_singleton(),
"p50k_edit" => tiktoken_rs::p50k_edit_singleton(),
"r50k_base" | "gpt2" => tiktoken_rs::r50k_base_singleton(),
let (encoder, name) = match name {
"cl100k_base" => (tiktoken_rs::cl100k_base_singleton(), "cl100k_base"),
"o200k_base" => (tiktoken_rs::o200k_base_singleton(), "o200k_base"),
"o200k_harmony" => (tiktoken_rs::o200k_harmony_singleton(), "o200k_harmony"),
"p50k_base" => (tiktoken_rs::p50k_base_singleton(), "p50k_base"),
"p50k_edit" => (tiktoken_rs::p50k_edit_singleton(), "p50k_edit"),
"r50k_base" => (tiktoken_rs::r50k_base_singleton(), "r50k_base"),
"gpt2" => (tiktoken_rs::r50k_base_singleton(), "gpt2"),
_ => return Err(UnsupportedTokenizer(name.to_owned())),
};
Ok(Self(tokenizer))
Ok(Self {
encoder,
vocabulary: None,
name,
})
}
pub fn vocabulary(&self) -> Option<&Vocabulary> {
self.vocabulary
}
pub fn count_tokens(&self, text: &str) -> usize {
self.0.count_ordinary(text)
self.encoder.count_ordinary(text)
}
pub fn encode(&self, text: &str) -> Vec<u32> {
self.encoder.encode_ordinary(text)
}
pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result<Vec<u32>, String> {
let allowed = allowed.iter().map(String::as_str).collect();
self.encoder
.encode(text, &allowed)
.map(|(ids, _)| ids)
.map_err(|error| error.to_string())
}
pub fn special_tokens(&self) -> HashSet<String> {
self.encoder
.special_tokens()
.into_iter()
.map(str::to_owned)
.collect()
}
/// tiktoken's `encode_with_unstable`: the stable prefix of `text`'s tokens and every
/// token sequence the unstable tail could still become, sorted for a stable order.
pub fn encode_with_unstable(
&self,
text: &str,
allowed: &[String],
) -> (Vec<u32>, Vec<Vec<u32>>) {
let allowed = allowed.iter().map(String::as_str).collect();
let (stable, completions) = self.encoder._encode_unstable_native(text, &allowed);
let mut completions: Vec<Vec<u32>> = completions.into_iter().collect();
completions.sort_unstable();
(stable, completions)
}
pub fn decode_bytes(&self, ids: &[u32]) -> Result<Vec<u8>, String> {
self.encoder
.decode_bytes(ids)
.map_err(|error| error.to_string())
}
pub fn decode(&self, ids: &[u32]) -> Result<String, String> {
self.encoder
.decode_bytes(ids)
.map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
.map_err(|error| error.to_string())
}
pub fn name(&self) -> &str {
self.name
}
}
pub fn encoding_for_model(model: &str) -> Option<&'static str> {
match tiktoken_rs::tokenizer::get_tokenizer(model)? {
tiktoken_rs::tokenizer::Tokenizer::Cl100kBase => Some("cl100k_base"),
tiktoken_rs::tokenizer::Tokenizer::O200kBase => Some("o200k_base"),
tiktoken_rs::tokenizer::Tokenizer::O200kHarmony => Some("o200k_harmony"),
tiktoken_rs::tokenizer::Tokenizer::P50kBase => Some("p50k_base"),
tiktoken_rs::tokenizer::Tokenizer::P50kEdit => Some("p50k_edit"),
tiktoken_rs::tokenizer::Tokenizer::R50kBase | tiktoken_rs::tokenizer::Tokenizer::Gpt2 => {
Some("r50k_base")
}
}
}
@ -66,5 +164,75 @@ mod tests {
panic!("unknown encoding must be rejected");
};
assert_eq!(name, "unknown-encoding");
assert_eq!(TiktokenTokenizer::from_name("gpt2").unwrap().name(), "gpt2");
}
#[test]
fn codecs_round_trip_named_encodings() {
let encodings = [
"cl100k_base",
"o200k_base",
"o200k_harmony",
"p50k_base",
"p50k_edit",
"r50k_base",
"gpt2",
];
let texts = ["hello world", "café 漢字 مرحبا 🙂", "line one\nline two"];
for name in encodings {
let tokenizer = TiktokenTokenizer::from_name(name).unwrap();
for text in texts {
assert_eq!(
tokenizer.decode(&tokenizer.encode(text)).unwrap(),
text,
"{name}: {text:?}",
);
}
}
}
#[test]
fn decoding_token_prefixes_replaces_incomplete_utf8() {
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
let reference = tiktoken_rs::cl100k_base_singleton();
let ids = tokenizer.encode("🙂漢字");
for end in 1..=ids.len() {
let bytes = reference.decode_bytes(&ids[..end]).unwrap();
assert_eq!(
tokenizer.decode(&ids[..end]).unwrap(),
String::from_utf8_lossy(&bytes),
);
}
assert!(tokenizer.decode(&[u32::MAX]).is_err());
}
#[test]
fn unstable_encoding_prefixes_stay_consistent_with_full_encoding() {
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
let text = "hello fanta";
let (stable, completions) = tokenizer.encode_with_unstable(text, &[]);
assert!(
text.as_bytes()
.starts_with(&tokenizer.decode_bytes(&stable).unwrap())
);
assert!(!completions.is_empty());
for completion in &completions {
let mut ids = stable.clone();
ids.extend(completion);
assert!(
tokenizer
.decode_bytes(&ids)
.unwrap()
.starts_with(text.as_bytes())
);
}
assert!(completions.windows(2).all(|pair| pair[0] < pair[1]));
}
#[test]
fn encoding_for_model_maps_known_models() {
assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base"));
assert_eq!(encoding_for_model("text-davinci-003"), Some("p50k_base"));
assert_eq!(encoding_for_model("unknown-model"), None);
}
}

View file

@ -0,0 +1,340 @@
use base64::{Engine, engine::general_purpose::STANDARD};
use once_cell::sync::OnceCell;
use rustc_hash::FxHashMap;
use thiserror::Error;
use tiktoken_rs::{CoreBPE, O200K_BASE_PAT_STR, Rank};
use crate::UnsupportedTokenizer;
const CL100K: &str = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4";
const O200K: &str = "fb374d419588a4632f3f557e76b4b70aebbca790";
const P50K: &str = "ec7223a39ce59f226a68acc30dc1af2788490e15";
const LEGACY_PATTERN: &str =
r"'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s";
const CL100K_PATTERN: &str = r"'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s";
static CL100K_ENCODER: OnceCell<Loaded> = OnceCell::new();
static O200K_ENCODER: OnceCell<Loaded> = OnceCell::new();
static HARMONY_ENCODER: OnceCell<Loaded> = OnceCell::new();
static P50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
static EDIT_ENCODER: OnceCell<Loaded> = OnceCell::new();
static R50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
/// One encoding built from a rank file: the BPE engine plus the vocabulary it was built
/// from, kept because `CoreBPE` does not expose its ranks and tiktoken's Python API does
/// (`token_byte_values`, `encode_single_token`, `max_token_value`, `_special_tokens`).
pub(super) struct Loaded {
pub(super) bpe: CoreBPE,
pub(super) vocabulary: Vocabulary,
}
/// The byte-level vocabulary of a tiktoken encoding.
pub struct Vocabulary {
ranks: FxHashMap<Vec<u8>, Rank>,
special_tokens: FxHashMap<String, Rank>,
max_token_value: Rank,
}
impl Vocabulary {
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
pub fn token_byte_values(&self) -> Vec<Vec<u8>> {
let mut values: Vec<Vec<u8>> = self.ranks.keys().cloned().collect();
values.sort_unstable();
values
}
/// The rank of one whole token: a mergeable piece first, then a special token's text.
pub fn encode_single_token(&self, piece: &[u8]) -> Option<Rank> {
if let Some(rank) = self.ranks.get(piece) {
return Some(*rank);
}
std::str::from_utf8(piece)
.ok()
.and_then(|text| self.special_tokens.get(text).copied())
}
pub fn max_token_value(&self) -> Rank {
self.max_token_value
}
/// Every mergeable token with its rank, for building other tables from one parse.
pub fn ranks(&self) -> impl Iterator<Item = (&[u8], Rank)> + '_ {
self.ranks
.iter()
.map(|(bytes, rank)| (bytes.as_slice(), *rank))
}
/// The special tokens with their ranks, tiktoken's `_special_tokens`.
pub fn special_tokens(&self) -> impl Iterator<Item = (&str, Rank)> + '_ {
self.special_tokens
.iter()
.map(|(token, rank)| (token.as_str(), *rank))
}
pub fn is_special_token(&self, rank: Rank) -> bool {
self.special_tokens.values().any(|special| *special == rank)
}
}
#[derive(Debug, Error)]
pub enum LoadError {
#[error(transparent)]
Unsupported(#[from] UnsupportedTokenizer),
#[error("failed to load tiktoken ranks: {0}")]
Ranks(String),
}
/// Loads `name` once per process. The returned name is the one requested (`gpt2` stays
/// `gpt2`, as `tiktoken.get_encoding("gpt2").name` does), while `gpt2` and `r50k_base` share
/// one cached encoder.
pub(super) fn load(
name: &str,
load_file: impl FnOnce(&str) -> std::io::Result<String>,
) -> Result<(&'static Loaded, &'static str), LoadError> {
let (requested, canonical, file, cache) = match name {
"cl100k_base" => ("cl100k_base", "cl100k_base", CL100K, &CL100K_ENCODER),
"o200k_base" => ("o200k_base", "o200k_base", O200K, &O200K_ENCODER),
"o200k_harmony" => ("o200k_harmony", "o200k_harmony", O200K, &HARMONY_ENCODER),
"p50k_base" => ("p50k_base", "p50k_base", P50K, &P50K_ENCODER),
"p50k_edit" => ("p50k_edit", "p50k_edit", P50K, &EDIT_ENCODER),
"r50k_base" => ("r50k_base", "r50k_base", P50K, &R50K_ENCODER),
"gpt2" => ("gpt2", "r50k_base", P50K, &R50K_ENCODER),
_ => return Err(UnsupportedTokenizer(name.to_owned()).into()),
};
let loaded = cache.get_or_try_init(|| {
let ranks = load_file(file).map_err(|error| LoadError::Ranks(error.to_string()))?;
build(canonical, &ranks)
})?;
Ok((loaded, requested))
}
fn build(name: &str, ranks: &str) -> Result<Loaded, LoadError> {
let parsed = ranks
.lines()
.map(parse_rank)
.collect::<Result<Vec<_>, _>>()?;
let encoder: FxHashMap<_, _> = parsed
.into_iter()
.filter(|(_, rank)| name != "r50k_base" || *rank < 50256)
.collect();
if encoder
.values()
.collect::<std::collections::HashSet<_>>()
.len()
!= encoder.len()
|| (0..=u8::MAX).any(|byte| !encoder.contains_key(&[byte][..]))
{
return Err(LoadError::Ranks("invalid vocabulary ranks".into()));
}
let (pattern, specials): (&str, &[(&str, Rank)]) = match name {
"cl100k_base" => (
CL100K_PATTERN,
&[
("<|endoftext|>", 100257),
("<|fim_prefix|>", 100258),
("<|fim_middle|>", 100259),
("<|fim_suffix|>", 100260),
("<|endofprompt|>", 100276),
],
),
"o200k_base" => (
O200K_BASE_PAT_STR,
&[("<|endoftext|>", 199999), ("<|endofprompt|>", 200018)],
),
"o200k_harmony" => (
O200K_BASE_PAT_STR,
&[
("<|startoftext|>", 199998),
("<|endoftext|>", 199999),
("<|reserved_200000|>", 200000),
("<|reserved_200001|>", 200001),
("<|return|>", 200002),
("<|constrain|>", 200003),
("<|reserved_200004|>", 200004),
("<|channel|>", 200005),
("<|start|>", 200006),
("<|end|>", 200007),
("<|message|>", 200008),
("<|reserved_200009|>", 200009),
("<|reserved_200010|>", 200010),
("<|reserved_200011|>", 200011),
("<|call|>", 200012),
],
),
"p50k_edit" => (
LEGACY_PATTERN,
&[
("<|endoftext|>", 50256),
("<|fim_prefix|>", 50281),
("<|fim_middle|>", 50282),
("<|fim_suffix|>", 50283),
],
),
_ => (LEGACY_PATTERN, &[("<|endoftext|>", 50256)]),
};
let reserved = (200013..=201087)
.filter(|_| name == "o200k_harmony")
.map(|rank| (format!("<|reserved_{rank}|>"), rank));
let special_tokens: FxHashMap<String, Rank> = specials
.iter()
.map(|(token, rank)| ((*token).to_owned(), *rank))
.chain(reserved)
.collect();
let max_token_value = encoder
.values()
.chain(special_tokens.values())
.copied()
.max()
.ok_or_else(|| LoadError::Ranks("empty vocabulary".into()))?;
let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern)
.map_err(|error| LoadError::Ranks(error.to_string()))?;
Ok(Loaded {
bpe,
vocabulary: Vocabulary {
ranks: encoder,
special_tokens,
max_token_value,
},
})
}
fn parse_rank(line: &str) -> Result<(Vec<u8>, Rank), LoadError> {
let (token, rank) = line
.split_once(' ')
.ok_or_else(|| LoadError::Ranks("missing rank".into()))?;
let bytes = STANDARD
.decode(token)
.map_err(|error| LoadError::Ranks(error.to_string()))?;
let rank = rank
.parse()
.map_err(|error: std::num::ParseIntError| LoadError::Ranks(error.to_string()))?;
Ok((bytes, rank))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::TiktokenTokenizer;
fn read_packaged_ranks(file: &str) -> std::io::Result<String> {
std::fs::read_to_string(
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../../litellm/litellm_core_utils/tokenizers")
.join(file),
)
}
#[test]
fn packaged_encodings_match_embedded_encodings_and_reuse_successful_loads() {
for name in [
"cl100k_base",
"o200k_base",
"o200k_harmony",
"p50k_base",
"p50k_edit",
"r50k_base",
"gpt2",
] {
if name != "gpt2" {
assert!(
TiktokenTokenizer::from_cached_ranks(name, |_| {
Err(std::io::Error::other("unreadable vocabulary"))
})
.is_err()
);
}
let loads = std::sync::atomic::AtomicUsize::new(0);
let barrier = std::sync::Barrier::new(4);
let encoders = std::thread::scope(|scope| {
let tasks: Vec<_> = (0..4)
.map(|_| {
scope.spawn(|| {
barrier.wait();
TiktokenTokenizer::from_cached_ranks(name, |file| {
loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
read_packaged_ranks(file)
})
.unwrap()
})
})
.collect();
tasks
.into_iter()
.map(|task| task.join().unwrap())
.collect::<Vec<_>>()
});
assert_eq!(loads.into_inner(), usize::from(name != "gpt2"));
let actual = &encoders[0];
let expected = TiktokenTokenizer::from_name(name).unwrap();
assert_eq!(actual.special_tokens(), expected.special_tokens());
let specials: Vec<_> = expected.special_tokens().into_iter().collect();
let special_text = specials.join(" ");
assert_eq!(
actual.encode_special(&special_text, &specials).unwrap(),
expected.encode_special(&special_text, &specials).unwrap()
);
for text in [
"",
"café 漢字 ع 🙂",
"a\r\nb\t ",
" hello 123456789",
&special_text,
] {
let ids = expected.encode(text);
assert_eq!(actual.encode(text), ids, "{name}: {text:?}");
assert_eq!(actual.count_tokens(text), ids.len(), "{name}: {text:?}");
assert_eq!(
actual.decode_bytes(&ids).unwrap(),
expected.decode_bytes(&ids).unwrap()
);
}
let cached = TiktokenTokenizer::from_cached_ranks(name, |_| {
panic!("reloaded cached vocabulary")
})
.unwrap();
assert_eq!(cached.encode("cached"), expected.encode("cached"));
assert_eq!(cached.name(), name);
assert!(expected.vocabulary().is_none());
assert_vocabulary_lookups(name, actual);
}
}
/// The token-level lookups tiktoken's Python `Encoding` exposes, checked against the
/// encoder itself and against the known vocabulary sizes.
fn assert_vocabulary_lookups(name: &str, tokenizer: &TiktokenTokenizer) {
let max_token_value = match name {
"cl100k_base" => 100_276,
"o200k_base" => 200_018,
"o200k_harmony" => 201_087,
"p50k_base" => 50_280,
"p50k_edit" => 50_283,
"r50k_base" | "gpt2" => 50_256,
_ => unreachable!("{name}"),
};
let vocabulary = tokenizer.vocabulary().unwrap();
assert_eq!(vocabulary.max_token_value(), max_token_value, "{name}");
let values = vocabulary.token_byte_values();
assert!(values.windows(2).all(|pair| pair[0] < pair[1]), "{name}");
for piece in values.iter().step_by(997) {
let rank = vocabulary.encode_single_token(piece).unwrap();
assert_eq!(tokenizer.decode_bytes(&[rank]).unwrap(), *piece, "{name}");
assert!(!vocabulary.is_special_token(rank), "{name}");
}
for (token, rank) in vocabulary.special_tokens() {
assert_eq!(vocabulary.encode_single_token(token.as_bytes()), Some(rank));
assert!(vocabulary.is_special_token(rank), "{name}: {token}");
}
assert_eq!(vocabulary.encode_single_token(b"<|not-a-token|>"), None);
}
#[test]
fn malformed_ranks_return_errors_instead_of_panicking() {
for ranks in ["", "IQ==", "IQ== x", "!!! 1", "IQ== 1"] {
assert!(build("cl100k_base", ranks).is_err());
}
let repeated_rank = (0..=u8::MAX)
.map(|byte| format!("{} 0\n", STANDARD.encode([byte])))
.collect::<String>();
assert!(build("cl100k_base", &repeated_rank).is_err());
}
}

View file

@ -1,6 +1,10 @@
# Token counting
`Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface
`Tokenizer` is the text-counting interface. `TextCodec` adds encoding, decoding, and a name. `TokenCounter` applies LiteLLM request, message, and tool accounting using any `Tokenizer`
Counts follow the codec: tiktoken treats special-token spellings as ordinary text, while Hugging Face applies its added tokens, post-processing, padding, and truncation. `fast=True` preserves those semantics and requests acceleration where available. Unsupported configurations use the normal codec, including tiktoken encodings without a scanner and builds without the `fast` feature. Invalid input and process-guard errors still propagate. Runtime request counting currently uses the normal codec; the custom accelerator is retained for explicit use and testing
`FastCounter: TextCodec` exposes an optional accelerator over a loaded codec. `None` means callers should use that codec. The Python bridge caches this selection per immutable tokenizer, shares it with request counters, and initializes it with the GIL released. Hugging Face can also choose the full encoder per input when added tokens require it
The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation
@ -8,7 +12,9 @@ The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through t
The `tiktoken` feature provides `tiktoken::TiktokenTokenizer` through `tiktoken-rs`. Select an encoding with `TokenCounter::from_tiktoken`. The supported names are `cl100k_base`, `o200k_base`, `o200k_harmony`, `p50k_base`, `p50k_edit`, `r50k_base`, and `gpt2`
All three backends are enabled by default. The Python extension builds with `fast` only, which keeps the wheel at the size it had before the split. With `default-features = false`, callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
All three backends are enabled by default in this crate and the Python extension. With `default-features = false`, Rust callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
Python `tiktoken` and `tokenizers` remain runtime dependencies and the default implementations. The catalog independently selects the tokenizer and request-counting routes. Enabling Rust changes factory dispatch; existing tokenizer objects keep their backend. Native Hugging Face wrappers provide an immutable encoding and decoding API, while training and mutable configuration remain available through the Python backend
Budget checks, cost calculation, and the `max_tokens` adjustment policy belong to `litellm-core-utils`. The counter does not own prices, budgets, or request limits

View file

@ -32,6 +32,8 @@ pub enum Error {
JsonUtf8(#[source] FromUtf8Error),
#[error("tokenization failed: {0}")]
Encode(#[source] Box<dyn std::error::Error + Send + Sync>),
#[error("token decoding failed: {0}")]
Decode(String),
#[error("token counting task failed: {0}")]
Task(String),
}

View file

@ -1,7 +1,31 @@
use litellm_token_counter_fast::Error as BackendError;
pub use litellm_token_counter_fast::FastTokenizer;
use crate::{Error, TokenCounter, Tokenizer};
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
pub trait FastCounter: TextCodec {
fn fast_counter(&self) -> Option<FastTokenizer>;
}
#[cfg(feature = "huggingface")]
impl FastCounter for crate::huggingface::HuggingFaceTokenizer {
fn fast_counter(&self) -> Option<FastTokenizer> {
Some(FastTokenizer::from_shared(self.shared()))
}
}
#[cfg(feature = "tiktoken")]
impl FastCounter for crate::tiktoken::TiktokenTokenizer {
fn fast_counter(&self) -> Option<FastTokenizer> {
let vocabulary = self.vocabulary()?;
match self.name() {
"cl100k_base" => FastTokenizer::from_cl100k_pairs(vocabulary.ranks()),
"o200k_base" | "o200k_harmony" => FastTokenizer::from_o200k_pairs(vocabulary.ranks()),
_ => return None,
}
.ok()
}
}
impl TokenCounter {
pub fn from_json_fast(tokenizer_json: &str) -> Result<Self, Error> {
@ -39,3 +63,65 @@ impl From<BackendError> for Error {
}
}
}
#[cfg(all(test, feature = "huggingface", feature = "tiktoken"))]
mod tests {
use super::*;
use crate::huggingface::HuggingFaceTokenizer;
use crate::tiktoken::TiktokenTokenizer;
const TEXTS: [&str; 4] = [
"",
"hello world <|endoftext|>",
"café 漢字 ع 🙂 line\r\n indented 123456789",
"<SOS>system<EOT> a\u{301} fi",
];
fn packaged(file: &str) -> String {
std::fs::read_to_string(format!(
"{}/../../../litellm/litellm_core_utils/tokenizers/{file}",
env!("CARGO_MANIFEST_DIR")
))
.unwrap()
}
#[test]
fn fast_counters_derived_from_codecs_count_like_the_codecs() {
let huggingface =
HuggingFaceTokenizer::from_json(&packaged("anthropic_tokenizer.json")).unwrap();
let fast = huggingface.fast_counter().unwrap();
for text in TEXTS {
assert_eq!(
fast.count_tokens(text).unwrap(),
Tokenizer::count_tokens(&huggingface, text).unwrap(),
"{text:?}"
);
}
for name in ["cl100k_base", "o200k_base", "o200k_harmony"] {
let tiktoken =
TiktokenTokenizer::from_cached_ranks(name, |file| Ok(packaged(file))).unwrap();
let fast = tiktoken.fast_counter().unwrap();
for text in TEXTS {
assert_eq!(
fast.count_tokens(text).unwrap(),
tiktoken.count_tokens(text),
"{name}: {text:?}"
);
}
}
}
#[test]
fn encodings_without_a_fast_scanner_keep_the_codec() {
let tiktoken =
TiktokenTokenizer::from_cached_ranks("p50k_base", |file| Ok(packaged(file))).unwrap();
assert!(tiktoken.fast_counter().is_none());
assert!(
TiktokenTokenizer::from_name("cl100k_base")
.unwrap()
.fast_counter()
.is_none()
);
}
}

View file

@ -1,7 +1,11 @@
use litellm_token_counter_huggingface::Error as BackendError;
pub use litellm_token_counter_huggingface::HuggingFaceTokenizer;
pub use litellm_token_counter_huggingface::{
AddedToken, EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection,
PaddingParams, PaddingStrategy, TruncationDirection, TruncationParams, encoding_from_json,
encoding_to_json,
};
use crate::{Error, TokenCounter, Tokenizer};
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
impl TokenCounter {
pub fn from_json(tokenizer_json: &str) -> Result<Self, Error> {
@ -17,11 +21,26 @@ impl Tokenizer for HuggingFaceTokenizer {
}
}
impl TextCodec for HuggingFaceTokenizer {
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
HuggingFaceTokenizer::encode(self, text).map_err(Error::from)
}
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
HuggingFaceTokenizer::decode(self, ids, skip_special_tokens).map_err(Error::from)
}
fn name(&self) -> &str {
HuggingFaceTokenizer::name(self)
}
}
impl From<BackendError> for Error {
fn from(error: BackendError) -> Self {
match error {
BackendError::Load(source) => Self::Load(source),
BackendError::Encode(source) => Self::Encode(source),
BackendError::Decode(source) => Self::Decode(source.to_string()),
}
}
}

View file

@ -20,5 +20,5 @@ pub mod tiktoken;
pub use counter::{InputTokenCount, TokenCounter};
pub use error::Error;
pub use tokenizer::Tokenizer;
pub use tokenizer::{TextCodec, Tokenizer};
pub use types::CountableRequest;

View file

@ -1,7 +1,7 @@
pub use litellm_token_counter_tiktoken::TiktokenTokenizer;
use litellm_token_counter_tiktoken::UnsupportedTokenizer;
use litellm_token_counter_tiktoken::{LoadError, UnsupportedTokenizer};
pub use litellm_token_counter_tiktoken::{TiktokenTokenizer, Vocabulary, encoding_for_model};
use crate::{Error, TokenCounter, Tokenizer};
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
impl TokenCounter {
pub fn from_tiktoken(encoding: &str) -> Result<Self, Error> {
@ -17,8 +17,31 @@ impl Tokenizer for TiktokenTokenizer {
}
}
impl TextCodec for TiktokenTokenizer {
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
Ok(TiktokenTokenizer::encode(self, text))
}
fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result<String, Error> {
TiktokenTokenizer::decode(self, ids).map_err(|error| Error::Decode(error.to_string()))
}
fn name(&self) -> &str {
TiktokenTokenizer::name(self)
}
}
impl From<UnsupportedTokenizer> for Error {
fn from(error: UnsupportedTokenizer) -> Self {
Self::UnsupportedTokenizer(error.0)
}
}
impl From<LoadError> for Error {
fn from(error: LoadError) -> Self {
match error {
LoadError::Unsupported(error) => error.into(),
LoadError::Ranks(message) => Self::Ranks(message),
}
}
}

View file

@ -4,6 +4,12 @@ pub trait Tokenizer: Send + Sync {
fn count_tokens(&self, text: &str) -> Result<usize, Error>;
}
pub trait TextCodec: Tokenizer {
fn encode(&self, text: &str) -> Result<Vec<u32>, Error>;
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error>;
fn name(&self) -> &str;
}
#[cfg(test)]
mod tests {
use super::*;

View file

@ -58,7 +58,8 @@ from ._lazy_imports_registry import (
if TYPE_CHECKING:
import httpx
from tiktoken import Encoding
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
def get_litellm_globals() -> dict[str, object]:
@ -89,26 +90,11 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo
# These are special lazy loaders for things that are used internally
# They're separate from the main lazy import system because they have specific use cases
# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup
_default_encoding: "Encoding | None" = None
def _get_default_encoding() -> "Tokenizer":
from litellm.rust_bridge.tokenizer import get_encoding
def _get_default_encoding() -> "Encoding":
"""
Lazily load and cache the default OpenAI encoding.
This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken)
at `litellm` import time. The encoding is cached after the first import.
This is used internally by utils.py functions that need the encoding but shouldn't
trigger its import during module load.
"""
global _default_encoding
if _default_encoding is None:
from litellm.litellm_core_utils.default_encoding import encoding
_default_encoding = encoding
return _default_encoding
return get_encoding("cl100k_base")
# Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time

View file

@ -7,6 +7,7 @@
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": "compact-2026-01-12",
"compact-2026-09-04": "compact-2026-09-04",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",

View file

@ -115,6 +115,7 @@ from litellm.types.utils import (
LlmProviders,
LlmProvidersSet,
ModelInfo,
ModelInfoBase,
PromptTokensDetailsWrapper,
ServiceTier,
StandardBuiltInToolsParams,
@ -322,6 +323,48 @@ class OCRPricing(TypedDict, total=False):
annotation_cost_per_page: ReadOnly[float | None]
_WALL_CLOCK_PRICED_MODES: Final = frozenset({"chat", "completion", "embedding", "responses"})
def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool:
return (
(model_info.get("input_cost_per_token") or 0.0) > 0
or (model_info.get("output_cost_per_token") or 0.0) > 0
or model_info.get("tiered_pricing") is not None
)
def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool:
mode: Final = model_info.get("mode")
return mode is None or mode in _WALL_CLOCK_PRICED_MODES
def _per_second_pricing_cost(
model: str,
custom_llm_provider: str | None,
response_time_ms: float | None,
) -> tuple[float, float] | None:
try:
model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
return None
if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
return None
input_cost_per_second: Final = model_info.get("input_cost_per_second")
output_cost_per_second: Final = model_info.get("output_cost_per_second")
if input_cost_per_second is None and output_cost_per_second is None:
return None
seconds: Final = (response_time_ms or 0.0) / 1000
verbose_logger.debug(
"For model=%s - input_cost_per_second: %s; output_cost_per_second: %s; response time: %s",
model,
input_cost_per_second,
output_cost_per_second,
response_time_ms,
)
return (input_cost_per_second or 0.0) * seconds, (output_cost_per_second or 0.0) * seconds
def cost_per_token(
model: str = "",
prompt_tokens: int = 0,
@ -448,9 +491,6 @@ def cost_per_token(
if response_cost is not None:
return response_cost[0], response_cost[1]
# given
prompt_tokens_cost_usd_dollar: float = 0
completion_tokens_cost_usd_dollar: float = 0
model_cost_ref: Final = litellm.model_cost
# Only callers that explicitly pass `custom_llm_provider` get the
# dedup/prefix-join treatment. When provider is omitted, preserve legacy
@ -611,6 +651,14 @@ def cost_per_token(
number_of_queries=number_of_queries or 1,
optional_params=(getattr(response, "_hidden_params", None) if response else None),
)
elif (
per_second_cost := _per_second_pricing_cost(
model=model,
custom_llm_provider=custom_llm_provider,
response_time_ms=response_time_ms,
)
) is not None:
return per_second_cost
elif custom_llm_provider == "vertex_ai":
cost_router: Final = google_cost_router(
model=model_without_prefix,
@ -685,12 +733,7 @@ def cost_per_token(
)
else:
model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
if (
(model_info.get("input_cost_per_token") or 0.0) > 0
or (model_info.get("output_cost_per_token") or 0.0) > 0
or model_info.get("tiered_pricing") is not None
):
if _has_token_or_tiered_pricing(model_info):
return generic_cost_per_token(
model=model,
usage=usage_block,
@ -698,36 +741,8 @@ def cost_per_token(
service_tier=service_tier,
data_residency=data_residency,
)
input_cost_per_second: Final = model_info.get("input_cost_per_second")
if input_cost_per_second is not None and response_time_ms is not None:
verbose_logger.debug(
"For model=%s - input_cost_per_second: %s; response time: %s",
model,
input_cost_per_second,
response_time_ms,
)
## COST PER SECOND ##
prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000
output_cost_per_second: Final = model_info.get("output_cost_per_second")
if output_cost_per_second is not None and response_time_ms is not None:
verbose_logger.debug(
"For model=%s - output_cost_per_second: %s; response time: %s",
model,
output_cost_per_second,
response_time_ms,
)
## COST PER SECOND ##
completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000
verbose_logger.debug(
"Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s",
model,
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
)
return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar
verbose_logger.debug("No per-token, tiered, or per-second pricing for model=%s; cost is 0", model)
return 0.0, 0.0
def get_replicate_completion_pricing(completion_response: dict, total_time=0.0):
@ -1222,6 +1237,21 @@ def _split_responses_ws_logging_object_by_service_tier(
)
def _response_time_ms_for_cost(
completion_response: object,
litellm_logging_obj: LitellmLoggingObject | None,
total_time: float | None,
) -> float:
stamped: Final = getattr(completion_response, "_response_ms", None)
if isinstance(stamped, (int, float)):
return float(stamped)
if total_time:
return total_time
if litellm_logging_obj is not None:
return litellm_logging_obj.get_response_ms()
return 0.0
def completion_cost(
completion_response: object | None = None,
model: str | None = None,
@ -1443,8 +1473,6 @@ def completion_cost(
prompt_tokens_details = _usage.get("prompt_tokens_details") or {}
cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0)
total_time = getattr(completion_response, "_response_ms", 0)
hidden_params = getattr(completion_response, "_hidden_params", None)
if hidden_params is not None:
custom_llm_provider = hidden_params.get("custom_llm_provider", custom_llm_provider or None)
@ -1676,6 +1704,11 @@ def completion_cost(
)
return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj)
response_time_ms = _response_time_ms_for_cost(
completion_response=completion_response,
litellm_logging_obj=litellm_logging_obj,
total_time=total_time,
)
# Calculate cost based on prompt_tokens, completion_tokens
if (
"togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
@ -1686,7 +1719,7 @@ def completion_cost(
# see https://replicate.com/pricing
elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost:
# for unmapped replicate model, default to replicate's time tracking logic
return get_replicate_completion_pricing(completion_response, total_time)
return get_replicate_completion_pricing(completion_response, response_time_ms)
if model is None:
raise ValueError(
@ -1718,7 +1751,7 @@ def completion_cost(
prompt_tokens=prompt_tokens or 0,
completion_tokens=completion_tokens or 0,
custom_llm_provider=custom_llm_provider,
response_time_ms=total_time,
response_time_ms=response_time_ms,
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
custom_cost_per_token=custom_cost_per_token,

View file

@ -427,7 +427,7 @@ class LLMCallSpanData:
# plain ``.get`` — no repeated ``isinstance`` guards.
raw_response: Final = payload.get("response")
response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {})
choices_out: Final = _dicts(response.get("choices")) or _responses_choices(response) or _ocr_choices(response)
choices_out: Final = _output_choices(response)
# ``finish_reasons`` is metadata, not content, so derive it from
# ``choices_out`` before gating. The raw message/choice bodies are only
# retained when content capture is enabled (see ``capture_span_content``);
@ -752,20 +752,99 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return (choice,)
def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
markdowns: Final = tuple(
text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None
def _output_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
"""The response output as chat-shaped choices; images and binary bodies become size summaries, never bytes."""
return (
_completion_choices(response)
or _responses_choices(response)
or _ocr_choices(response)
or _transcription_choices(response)
or _moderation_choices(response)
or _image_choices(response)
or _binary_choices(response)
)
if not markdowns:
def _text_choice(content: str, finish_reason: str | None = None) -> _Choice:
message: Final[_AssistantMessage] = {"role": "assistant", "content": content, "refusal": None, "tool_calls": None}
return {"message": message, "finish_reason": finish_reason}
def _joined_choice(parts: tuple[str, ...]) -> tuple[_Choice, ...]:
return (_text_choice("\n\n".join(parts)),) if parts else ()
def _completion_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
return tuple(
_text_choice(text, as_str(choice.get("finish_reason")))
if "message" not in choice and isinstance(text := choice.get("text"), str)
else choice
for choice in _dicts(response.get("choices"))
)
def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return _joined_choice(
tuple(text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None)
)
def _transcription_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
text: Final = response.get("text")
return (_text_choice(text),) if isinstance(text, str) and text else ()
def _moderation_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return _joined_choice(
tuple(
_moderation_verdict(flagged, result.get("categories"))
for result in _dicts(response.get("results"))
if isinstance(flagged := result.get("flagged"), bool)
)
)
def _moderation_verdict(flagged: bool, categories: object) -> str:
if not flagged:
return "not flagged"
hits: Final = (
tuple(name for name, hit in cast(Mapping[str, object], categories).items() if hit is True)
if isinstance(categories, dict)
else ()
)
return f"flagged: {', '.join(hits)}" if hits else "flagged"
def _image_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return _joined_choice(
tuple(summary for item in _dicts(response.get("data")) if (summary := _image_summary(item)) is not None)
)
def _image_summary(item: Mapping[str, object]) -> str | None:
location: Final = _image_location(item)
if location is None:
return None
revised: Final = as_str(item.get("revised_prompt"))
return f"{revised}\n{location}" if revised else location
def _image_location(item: Mapping[str, object]) -> str | None:
url: Final = as_str(item.get("url"))
if url is not None:
return url
encoded: Final = item.get("b64_json")
if not isinstance(encoded, str):
return None
return f"b64_json image ({len(encoded) * 3 // 4 - encoded[-2:].count('=')} bytes)"
def _binary_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
size: Final = as_int(response.get("num_bytes"))
if size is None:
return ()
message: Final[_AssistantMessage] = {
"role": "assistant",
"content": "\n\n".join(markdowns),
"refusal": None,
"tool_calls": None,
}
choice: Final[_Choice] = {"message": message, "finish_reason": None}
return (choice,)
content_type: Final = as_str(response.get("content_type"))
return (_text_choice(f"{content_type} ({size} bytes)" if content_type else f"{size} bytes"),)
def _responses_parts_text(parts: tuple[Mapping[str, object], ...], part_type: str, field: str) -> str | None:

View file

@ -6,8 +6,9 @@ Core files:
- `streaming_handler.py`: The core streaming logic + streaming related helper utils
- `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`.
- `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types.
- `default_encoding.py`: code for loading the default encoding (tiktoken)
- `default_encoding.py`: code for loading the default Python tokenizer and bundled cache
- `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name.
- `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.

View file

@ -0,0 +1,180 @@
from __future__ import annotations
import os
import platform
import traceback
from dataclasses import dataclass
from pathlib import Path
from typing import Final, Literal
from urllib.parse import urlencode
import litellm
from litellm._version import version as litellm_version
from litellm.types.utils import LlmProviders
ISSUE_URL_BASE: Final = "https://github.com/BerriAI/litellm/issues/new"
MAX_URL_LENGTH: Final = 6000
MAX_FRAMES: Final = 12
DISABLE_ENV_VAR: Final = "LITELLM_DISABLE_BUG_REPORT_LINK"
NOTICE_PREFIX: Final = "This looks like a bug in LiteLLM rather than in your request."
KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders)
Surface = Literal["sdk", "proxy"]
@dataclass(frozen=True, slots=True)
class BugReport:
surface: Surface
exception_type: str
litellm_frames: tuple[str, ...]
litellm_version: str
python_version: str
call_type: str | None
custom_llm_provider: str | None
stream: bool | None
config_lines: tuple[str, ...]
def bug_report_enabled() -> bool:
return os.getenv(DISABLE_ENV_VAR, "").lower() != "true"
def should_report_bug(exc: object) -> bool:
return bug_report_enabled() and isinstance(exc, BaseException) and getattr(exc, "status_code", None) is None
def _format_frame(frame: traceback.FrameSummary, package_dir: Path, package_parent: Path) -> str | None:
frame_path: Final = Path(frame.filename).resolve()
try:
frame_path.relative_to(package_dir)
relative_path: Final = frame_path.relative_to(package_parent)
except ValueError:
return None
return f"{relative_path.as_posix()}:{frame.lineno} in {frame.name}"
def _get_litellm_frames(exc: BaseException) -> tuple[str, ...]:
if exc.__traceback__ is None:
return ()
package_dir: Final = Path(litellm.__file__).resolve().parent
package_parent: Final = package_dir.parent
return tuple(
frame_text
for frame in traceback.extract_tb(exc.__traceback__)
if (frame_text := _format_frame(frame, package_dir, package_parent)) is not None
)[-MAX_FRAMES:]
def allowlisted(value: object, allowed: frozenset[str]) -> str | None:
return value if isinstance(value, str) and value in allowed else None
def build_bug_report(
exc: BaseException,
*,
surface: Surface,
call_type: str | None = None,
custom_llm_provider: object = None,
stream: object = None,
config_lines: tuple[str, ...] = (),
) -> BugReport:
return BugReport(
surface=surface,
exception_type=type(exc).__name__,
litellm_frames=_get_litellm_frames(exc),
litellm_version=litellm_version,
python_version=platform.python_version(),
call_type=call_type,
custom_llm_provider=allowlisted(custom_llm_provider, KNOWN_PROVIDERS),
stream=stream if isinstance(stream, bool) else None,
config_lines=config_lines,
)
def _domain(report: BugReport) -> str:
if report.surface == "sdk":
return "Python SDK: the litellm package itself"
if report.custom_llm_provider is not None:
return "LLM translation: a specific provider's request or response"
return "Proxy core: startup, config, health checks, endpoints"
def _title(report: BugReport, frames: tuple[str, ...]) -> str:
location: Final = frames[-1].split(":", 1)[0] if frames else "litellm"
return f"[Bug]: {report.exception_type} in {location}"
def _description(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str:
frame_block: Final = "LiteLLM frames:\n```\n" + "\n".join(frames) + "\n```\n\n" if frames else ""
stream_line: Final = "" if report.stream is None else f"Stream: {str(report.stream).lower()}\n"
config_block: Final = (
"\nConfig (true/false flags and LiteLLM-defined values only):\n```\n" + "\n".join(config_lines) + "\n```\n"
if config_lines
else ""
)
return (
"Auto-generated by LiteLLM's bug report link. It carries no request data or error text. "
"Please describe what you were doing, and paste the error message from your log below "
"if it contains nothing sensitive.\n\n"
"```\n\n```\n\n"
f"Exception: `{report.exception_type}`\n\n"
f"{frame_block}"
f"Surface: {report.surface}\n"
f"Endpoint / call: {report.call_type or 'unknown'}\n"
f"Provider: {report.custom_llm_provider or 'unknown'}\n"
f"LiteLLM: {report.litellm_version}\n"
f"Python: {report.python_version}\n"
f"{stream_line}"
f"{config_block}"
)
def _issue_url(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str:
deployment: Final[tuple[tuple[str, str], ...]] = (
(("deployment", "pip / Python SDK"),)
if report.surface == "sdk"
else (("deployment", "Docker"),)
if os.path.exists("/.dockerenv")
else ()
)
fields: Final = (
("template", "bug_report.yml"),
("labels", "bug"),
("title", _title(report, frames)),
("version", report.litellm_version),
("domain", _domain(report)),
("description", _description(report, frames, config_lines)),
) + deployment
return f"{ISSUE_URL_BASE}?{urlencode(fields)}"
def bug_report_issue_url(report: BugReport) -> str:
frames: Final = report.litellm_frames
config_lines: Final = report.config_lines
candidates: Final = (
*((frames, config_lines[:count]) for count in range(len(config_lines), -1, -1)),
*((frames[index:], ()) for index in range(1, len(frames) + 1)),
)
return next(
(
url
for candidate_frames, candidate_config in candidates
if len(url := _issue_url(report, candidate_frames, candidate_config)) <= MAX_URL_LENGTH
),
_issue_url(report, (), ()),
)
def strip_bug_report_notice(message: str) -> str:
index: Final = message.find(NOTICE_PREFIX)
if index == -1:
return message
head: Final = message[:index]
return head.removesuffix("\n")
def bug_report_notice(report: BugReport) -> str:
return (
f"{NOTICE_PREFIX} File it with one click "
f"(prefilled, no request data or error text, review before submitting): {bug_report_issue_url(report)}"
)

View file

@ -1,5 +1,4 @@
import os
from pathlib import Path
from typing import Final
import litellm
@ -15,20 +14,6 @@ except (ImportError, AttributeError):
filename = pkg_resources.resource_filename(__name__, "litellm_core_utils/tokenizers")
CL100K_BASE_RANK_FILE: Final = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
O200K_BASE_RANK_FILE: Final = "fb374d419588a4632f3f557e76b4b70aebbca790"
def cl100k_base_rank_file() -> str:
"""The vendored tiktoken `cl100k_base` rank file (`base64(token) rank` lines)."""
return Path(filename, CL100K_BASE_RANK_FILE).read_text(encoding="ascii")
def o200k_base_rank_file() -> str:
"""The vendored tiktoken `o200k_base` rank file (`base64(token) rank` lines)."""
return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii")
# Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory
# unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR.
# This keeps tiktoken fully offline-capable by default (see #1071).

View file

@ -10,6 +10,11 @@ import httpx
import litellm
from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger
from litellm.litellm_core_utils.bug_report import (
bug_report_notice,
build_bug_report,
should_report_bug,
)
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.types.utils import LlmProviders
@ -2673,7 +2678,21 @@ def exception_type(
)
else:
raise APIConnectionError(
message=f"{original_exception}\n{_redact_string(traceback.format_exc())}",
message=(
f"{original_exception}\n{_redact_string(traceback.format_exc())}"
+ (
"\n"
+ bug_report_notice(
build_bug_report(
original_exception,
surface="sdk",
custom_llm_provider=custom_llm_provider,
)
)
if should_report_bug(original_exception)
else ""
)
),
llm_provider=custom_llm_provider,
model=model,
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request

View file

@ -1,10 +1,24 @@
import re
from collections.abc import Iterator, Mapping
from collections.abc import Generator, Iterator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Any, Final
from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams
_CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata")
_inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar(
"inherited_message_logging_disabled", default=False
)
@contextmanager
def inherit_message_logging_privacy(disabled: bool) -> Generator[None]:
token: Final = _inherited_message_logging_disabled.set(_inherited_message_logging_disabled.get() or disabled)
try:
yield
finally:
_inherited_message_logging_disabled.reset(token)
def iter_client_callback_metadata_dicts(
@ -143,7 +157,7 @@ def get_trusted_callback_params(kwargs: Mapping[str, Any] | None) -> tuple[tuple
def initialize_standard_callback_dynamic_params(
kwargs: dict | None = None,
kwargs: dict[str, object] | None = None,
) -> StandardCallbackDynamicParams:
"""
Initialize the standard callback dynamic params from the kwargs
@ -179,4 +193,10 @@ def initialize_standard_callback_dynamic_params(
if param in _trusted_overlay_callback_params:
standard_callback_dynamic_params[param] = trusted_value
if _inherited_message_logging_disabled.get():
private_params: Final[StandardCallbackDynamicParams] = {
**standard_callback_dynamic_params,
"turn_off_message_logging": True,
}
return private_params
return standard_callback_dynamic_params

View file

@ -496,6 +496,14 @@ def mask_api_base_credentials(api_base: str) -> str:
return api_base[:key_end] + "*" * 5 + api_base[-4:]
def _timestamp_seconds(moment: object) -> float | None:
if isinstance(moment, datetime.datetime):
return moment.timestamp()
if isinstance(moment, (int, float)):
return float(moment)
return None
class Logging(LiteLLMLoggingBaseClass):
global \
supabaseClient, \
@ -1634,10 +1642,12 @@ class Logging(LiteLLMLoggingBaseClass):
return response.mcp_tool_call_response
def get_response_ms(self) -> float:
return (
self.model_call_details.get("end_time", datetime.datetime.now())
- self.model_call_details.get("start_time", datetime.datetime.now())
).total_seconds() * 1000
now: Final = datetime.datetime.now()
start_seconds: Final = _timestamp_seconds(self.model_call_details.get("start_time", now))
end_seconds: Final = _timestamp_seconds(self.model_call_details.get("end_time", now))
if start_seconds is None or end_seconds is None:
return 0.0
return (end_seconds - start_seconds) * 1000
def set_cost_breakdown(
self,
@ -6277,6 +6287,8 @@ def _extract_response_obj_and_hidden_params(
hidden_params = getattr(init_response_obj, "_hidden_params", None)
elif isinstance(init_response_obj, dict):
response_obj = init_response_obj
elif isinstance(init_response_obj, HttpxBinaryResponseContent):
response_obj = dict(init_response_obj.logging_summary())
else:
response_obj = {}

View file

@ -251,6 +251,6 @@ def update_response_metadata(
return
metadata: Final = ResponseMetadata(result)
metadata.set_hidden_params(logging_obj, model, kwargs)
metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead)
metadata.set_hidden_params(logging_obj, model, kwargs)
metadata.apply()

View file

@ -245,6 +245,8 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
_redact_tool_calls_dict(choice["delta"])
elif choice.get("text") is not None:
choice["text"] = redacted_str
else:
_redact_choice_content(choice)

View file

@ -10,7 +10,6 @@ import anyio
import anyio.lowlevel
import httpx
import tiktoken
from tokenizers import Tokenizer
from typing_extensions import ParamSpec, TypeVar
import litellm
@ -30,8 +29,10 @@ from litellm.constants import (
TOKEN_COUNTER_MAX_EXACT_CHARS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding
from litellm.litellm_core_utils.url_utils import safe_get
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.rust_bridge.tokenizer import get_encoding
from litellm.types.llms.anthropic import (
AnthropicContentParamSource,
AnthropicContentParamSourceFileId,
@ -622,9 +623,11 @@ def _get_exact_count_function(
if model is not None or custom_tokenizer is not None:
tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model)
if tokenizer_json["type"] == "huggingface_tokenizer":
tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"]
tokenizer: Final[HuggingFace] = tokenizer_json["tokenizer"]
def count_tokens(text: str) -> int:
if isinstance(tokenizer, HuggingFaceTokenizer):
return tokenizer.count(text)
return len(tokenizer.encode_batch_fast([text])[0])
return count_tokens
@ -632,31 +635,43 @@ def _get_exact_count_function(
encoding: Final = openai_tokenizer_encoding(model)
def encode_length(text: str) -> int:
return len(encoding.encode(text, disallowed_special=()))
return _encoding_count(encoding, text)
return _get_tiktoken_count_function(encode_length)
else:
raise ValueError("Unsupported tokenizer type")
else:
default_encoding: Final = _get_default_encoding()
def encode_length(text: str) -> int:
return len(_get_default_encoding().encode(text, disallowed_special=()))
return _encoding_count(default_encoding, text)
return _get_tiktoken_count_function(encode_length)
def openai_tokenizer_encoding(model: str) -> tiktoken.Encoding:
"""The tiktoken encoding `token_counter` uses for a model on the `openai_tokenizer` path."""
def _encoding_count(encoding: Encoding, text: str) -> int:
if isinstance(encoding, OpenAIEncoding):
return encoding.count(text)
return len(encoding.encode(text, disallowed_special=()))
def openai_tokenizer_encoding(model: str) -> Encoding:
"""The encoding `token_counter` uses for a model on the `openai_tokenizer` path."""
return get_encoding(openai_tokenizer_encoding_name(model))
def openai_tokenizer_encoding_name(model: str) -> str:
"""The tiktoken encoding name for `model`, without loading the encoding."""
from litellm.utils import print_verbose
model_to_use: Final = _fix_model_name(model)
if "gpt-4o" in model_to_use:
return tiktoken.get_encoding("o200k_base")
return "o200k_base"
try:
return tiktoken.encoding_for_model(model_to_use)
return tiktoken.encoding_name_for_model(model_to_use)
except KeyError:
print_verbose("Warning: model not found. Using cl100k_base encoding.")
return tiktoken.get_encoding("cl100k_base")
return "cl100k_base"
def uses_legacy_message_accounting(model: str) -> bool:

View file

@ -0,0 +1,402 @@
"""Python faces of the Rust text codecs.
``OpenAIEncoding`` mirrors ``tiktoken.Encoding`` and ``HuggingFaceTokenizer`` mirrors
``tokenizers.Tokenizer``, so a caller holding ``litellm.encoding`` or the object returned by
``litellm.create_tokenizer`` sees the same read-only surface whichever backend the Rust catalog
selected. Both wrappers are immutable: ``tokenizers`` mutators (``enable_padding``,
``enable_truncation``, ``add_tokens``) stay on the Python tokenizer.
"""
from __future__ import annotations
from collections.abc import Callable, Collection, Mapping, Sequence, Set
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from functools import partial
from pathlib import Path
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable
import tiktoken
from tokenizers import AddedToken
from tokenizers import Tokenizer as PythonHuggingFaceTokenizer
if TYPE_CHECKING:
import numpy as np
import numpy.typing as npt
from litellm.rust_bridge._native import HuggingFaceEncoding
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
SpecialTokens: TypeAlias = Literal["all"] | Collection[str]
AllowedSpecial: TypeAlias = Literal["all"] | Set[str]
HuggingFaceInput: TypeAlias = str | list[str] | tuple[str, ...]
HuggingFaceBatchInput: TypeAlias = HuggingFaceInput | tuple[HuggingFaceInput, HuggingFaceInput] | list[HuggingFaceInput]
@dataclass(frozen=True, slots=True)
class OpenAIEncoding:
"""``tiktoken.Encoding`` over the Rust tiktoken codec."""
_native: NativeTokenizer
_special_tokens: Mapping[str, int]
@staticmethod
def wrap(native: NativeTokenizer) -> OpenAIEncoding:
return OpenAIEncoding(native, MappingProxyType(native.special_tokens()))
@staticmethod
def from_tiktoken(encoding: str) -> OpenAIEncoding:
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
return OpenAIEncoding.wrap(NativeTokenizer.from_tiktoken(encoding))
def __repr__(self) -> str:
return f"<Encoding {self.name!r}>"
@property
def name(self) -> str:
return self._native.name
@property
def max_token_value(self) -> int:
return self._native.max_token_value()
@property
def n_vocab(self) -> int:
"""For backwards compatibility. Prefer to use `enc.max_token_value + 1`."""
return self.max_token_value + 1
@property
def eot_token(self) -> int:
return self._special_tokens["<|endoftext|>"]
@property
def special_tokens_set(self) -> set[str]: # mutable-ok: [LIT001, LIT002] SDK return type
return set(self._special_tokens)
def is_special_token(self, token: int) -> bool:
return self._native.is_special_token(token)
# ---- encoding -------------------------------------------------------------------------
def encode_ordinary(self, text: str) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.encode(text)
def encode(
self,
text: str,
*,
allowed_special: AllowedSpecial = frozenset(),
disallowed_special: SpecialTokens = "all",
) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type
allowed: Final = self._allowed(text, allowed_special, disallowed_special)
if not allowed:
return self.encode_ordinary(text)
return self._native.encode_special(text, tuple(allowed))
def encode_to_numpy(
self,
text: str,
*,
allowed_special: AllowedSpecial = frozenset(),
disallowed_special: SpecialTokens = "all",
) -> npt.NDArray[np.uint32]:
import numpy
return numpy.asarray(
self.encode(text, allowed_special=allowed_special, disallowed_special=disallowed_special),
dtype=numpy.uint32,
)
def encode_ordinary_batch(
self, text: Sequence[str], *, num_threads: int = 8
) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(self.encode_ordinary, text)
)
def encode_batch(
self,
text: Sequence[str],
*,
num_threads: int = 8,
allowed_special: AllowedSpecial = frozenset(),
disallowed_special: SpecialTokens = "all",
) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
encode: Final = partial(self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special)
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(encode, text)
)
def encode_with_unstable(
self,
text: str,
*,
allowed_special: AllowedSpecial = frozenset(),
disallowed_special: SpecialTokens = "all",
) -> tuple[list[int], list[list[int]]]: # mutable-ok: [LIT001, LIT002] SDK return type
"""The stable tokens of `text` and every completion its unstable tail could become.
Completions come back sorted; tiktoken returns them in hash order."""
allowed: Final = self._allowed(text, allowed_special, disallowed_special)
return self._native.encode_with_unstable(text, tuple(allowed))
def encode_single_token(self, text_or_bytes: str | bytes) -> int:
"""The token of one whole piece, special tokens included. Raises `KeyError` otherwise."""
piece: Final = text_or_bytes.encode("utf-8") if isinstance(text_or_bytes, str) else text_or_bytes
return self._native.encode_single_token(piece)
def count(self, text: str, fast: bool = False) -> int:
"""Count ordinary text; `fast` accelerates supported encodings and otherwise counts normally."""
return self._native.count(text, fast)
# ---- decoding -------------------------------------------------------------------------
def decode_bytes(self, tokens: Sequence[int]) -> bytes:
return self._native.decode_bytes(tokens)
def decode(self, tokens: Sequence[int], errors: str = "replace") -> str:
return self.decode_bytes(tokens).decode("utf-8", errors=errors)
def decode_single_token_bytes(self, token: int) -> bytes:
return self.decode_bytes((token,))
def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
return [ # mutable-ok: [LIT002] SDK returns a list
self.decode_single_token_bytes(token) for token in tokens
]
def decode_with_offsets(
self, tokens: Sequence[int]
) -> tuple[str, list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
"""The decoded text and, per token, the index of the first character holding its bytes.
Like tiktoken, raises `UnicodeDecodeError` when the tokens do not decode to valid UTF-8."""
token_bytes: Final = self.decode_tokens_bytes(tokens)
text_len = 0
offsets: Final[list[int]] = [] # mutable-ok: [LIT001] local accumulator
for token in token_bytes:
offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0)))
text_len += sum(1 for c in token if not 0x80 <= c < 0xC0)
return b"".join(token_bytes).decode("utf-8", errors="strict"), offsets
def decode_batch(
self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8
) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(partial(self.decode, errors=errors), batch)
)
def decode_bytes_batch(
self, batch: Sequence[Sequence[int]], *, num_threads: int = 8
) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(self.decode_bytes, batch)
)
def token_byte_values(self) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.token_byte_values()
def __reduce__(self) -> tuple[Callable[[str], OpenAIEncoding], tuple[str]]:
return (OpenAIEncoding.from_tiktoken, (self.name,))
# ---- private --------------------------------------------------------------------------
def _allowed(self, text: str, allowed_special: AllowedSpecial, disallowed_special: SpecialTokens) -> frozenset[str]:
"""tiktoken's special-token policy: which specials `text` may encode, after rejecting
any it must not contain."""
allowed: Final = frozenset(self._special_tokens) if allowed_special == "all" else frozenset(allowed_special)
disallowed: Final = (
frozenset(self._special_tokens) - allowed if disallowed_special == "all" else frozenset(disallowed_special)
)
for token in disallowed:
if token in text:
raise ValueError(
f"Encountered text corresponding to disallowed special token {token!r}.\n"
"If you want this text to be encoded as a special token, "
f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n"
"If you want this text to be encoded as normal text, disable the check for this token "
f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n"
"To disable this check for all special tokens, pass `disallowed_special=()`.\n"
)
return allowed
@dataclass(frozen=True, slots=True)
class HuggingFaceTokenizer:
"""The read-only ``tokenizers.Tokenizer`` surface over the Rust Hugging Face codec."""
_native: NativeTokenizer
@staticmethod
def from_str(json: str) -> HuggingFaceTokenizer:
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
return HuggingFaceTokenizer(NativeTokenizer.from_json(json))
from_json = from_str
@staticmethod
def from_buffer(buffer: bytes) -> HuggingFaceTokenizer:
return HuggingFaceTokenizer.from_str(buffer.decode("utf-8"))
@staticmethod
def from_file(path: str) -> HuggingFaceTokenizer:
return HuggingFaceTokenizer.from_str(Path(path).read_text(encoding="utf-8"))
@staticmethod
def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFaceTokenizer:
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
return HuggingFaceTokenizer(NativeTokenizer.from_pretrained(identifier, revision=revision, token=token))
def to_str(self, pretty: bool = False) -> str:
return self._native.to_json(pretty)
def save(self, path: str, pretty: bool = True) -> None:
Path(path).write_text(self.to_str(pretty), encoding="utf-8")
@property
def name(self) -> str:
return self._native.name
# ---- vocabulary -----------------------------------------------------------------------
def token_to_id(self, token: str) -> int | None:
return self._native.token_to_id(token)
def id_to_token(self, id: int) -> str | None:
return self._native.id_to_token(id)
def get_vocab(
self, with_added_tokens: bool = True
) -> dict[str, int]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.get_vocab(with_added_tokens)
def get_vocab_size(self, with_added_tokens: bool = True) -> int:
return self._native.get_vocab_size(with_added_tokens)
def get_added_tokens_decoder(self) -> dict[int, AddedToken]: # mutable-ok: [LIT001, LIT002] SDK return type
return { # mutable-ok: [LIT002] SDK returns a dict
token_id: AddedToken(
content, single_word=single_word, lstrip=lstrip, rstrip=rstrip, normalized=normalized, special=special
)
for token_id, (
content,
single_word,
lstrip,
rstrip,
normalized,
special,
) in self._native.added_tokens_decoder()
}
def num_special_tokens_to_add(self, is_pair: bool) -> int:
return self._native.num_special_tokens_to_add(is_pair)
@property
def padding(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.padding()
@property
def truncation(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.truncation()
@property
def encode_special_tokens(self) -> bool:
return self._native.encode_special_tokens()
# ---- encoding and decoding ------------------------------------------------------------
def encode(
self,
sequence: HuggingFaceInput,
pair: HuggingFaceInput | None = None,
is_pretokenized: bool = False,
add_special_tokens: bool = True,
) -> HuggingFaceEncoding:
return self._native.encode_huggingface(sequence, pair, is_pretokenized, add_special_tokens)
def encode_batch(
self,
input: Sequence[HuggingFaceBatchInput],
is_pretokenized: bool = False,
add_special_tokens: bool = True,
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=False)
def encode_batch_fast(
self,
input: Sequence[HuggingFaceBatchInput],
is_pretokenized: bool = False,
add_special_tokens: bool = True,
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=True)
def _encode_batch(
self, input: Sequence[HuggingFaceBatchInput], is_pretokenized: bool, add_special_tokens: bool, fast: bool
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
sequences: Final = tuple(_batch_input(item, is_pretokenized) for item in input)
return self._native.encode_batch_huggingface(sequences, is_pretokenized, add_special_tokens, fast)
def count(self, text: str, fast: bool = False) -> int:
"""Count with this tokenizer's configuration; `fast` uses acceleration where supported."""
return self._native.count(text, fast)
def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str:
return self._native.decode(ids, skip_special_tokens=skip_special_tokens)
def decode_batch(
self, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True
) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type
return [ # mutable-ok: [LIT002] SDK returns a list
self.decode(ids, skip_special_tokens=skip_special_tokens) for ids in sequences
]
def __reduce__(self) -> tuple[Callable[[str], HuggingFaceTokenizer], tuple[str]]:
return (HuggingFaceTokenizer.from_str, (self.to_str(),))
def _batch_input(
item: HuggingFaceBatchInput, is_pretokenized: bool
) -> tuple[HuggingFaceInput, HuggingFaceInput | None]:
if isinstance(item, str):
return (item, None)
if is_pretokenized and all(isinstance(word, str) for word in item):
return (tuple(word for word in item if isinstance(word, str)), None)
if len(item) != 2:
raise TypeError("batch input must be a sequence or a pair of sequences")
return (item[0], item[1])
Encoding: TypeAlias = tiktoken.Encoding | OpenAIEncoding
HuggingFace: TypeAlias = PythonHuggingFaceTokenizer | HuggingFaceTokenizer
Tokenizer: TypeAlias = Encoding | HuggingFace
class _AddedToken(Protocol):
@property
def special(self) -> bool: ...
@runtime_checkable
class _AddedTokenDecoder(Protocol):
def get_added_tokens_decoder(self) -> Mapping[int, _AddedToken]: ...
def strip_special_tokens(tokenizer: object, tokens: Sequence[int]) -> Sequence[int]:
"""Drop the special added tokens before a Python `tokenizers` decode; the Rust codec's
`decode(skip_special_tokens=True)` already does this itself."""
if isinstance(tokenizer, HuggingFaceTokenizer) or not isinstance(tokenizer, _AddedTokenDecoder):
return tokens
try:
added: Final = tokenizer.get_added_tokens_decoder()
except Exception: # noqa: BLE001 # optional metadata failures historically fall back to decoding
return tokens
special_ids: Final = frozenset(token_id for token_id, token in added.items() if token.special)
return tuple(token for token in tokens if token not in special_ids)

View file

@ -23,9 +23,8 @@ from ..common_utils import (
from .streaming_iterator import A2AModelResponseIterator
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
_REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = (
@ -292,7 +291,7 @@ class A2AConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -14,9 +14,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -171,7 +170,7 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

View file

@ -16,9 +16,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -68,7 +67,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -17,7 +17,7 @@ from litellm.types.utils import ModelResponse
from ...openai_like.chat.transformation import OpenAILikeChatConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AmazonNovaChatConfig(OpenAILikeChatConfig):
@ -86,7 +86,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
from litellm.types.utils import LiteLLMBatch, LlmProviders, ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -290,7 +289,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -96,13 +96,13 @@ from ..common_utils import (
AnthropicModelInfo,
eager_input_streaming_flag,
process_anthropic_headers,
requires_native_compaction_beta,
strip_advisor_blocks_from_messages,
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -1771,7 +1771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
return tools
def _ensure_beta_header(self, headers: dict, beta_value: str) -> None:
def _ensure_beta_header(self, headers: dict[str, str], beta_value: str) -> None:
"""
Ensure a beta header value is present in the anthropic-beta header.
Merges with existing values instead of overriding them.
@ -1780,13 +1780,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers: Dictionary of headers to update
beta_value: The beta header value to add
"""
existing_beta: Final = headers.get("anthropic-beta")
if existing_beta is None:
headers["anthropic-beta"] = beta_value
return
existing_values: Final = [beta.strip() for beta in existing_beta.split(",")]
if beta_value not in existing_values:
headers["anthropic-beta"] = f"{existing_beta}, {beta_value}"
existing_values: Final = tuple(
beta.strip()
for key, value in headers.items()
if key.lower() == "anthropic-beta"
for beta in value.split(",")
if beta.strip()
)
for key in tuple(headers):
if key.lower() == "anthropic-beta":
headers.pop(key)
headers["anthropic-beta"] = ", ".join(dict.fromkeys((*existing_values, beta_value)))
def _ensure_context_management_beta_header(self, headers: dict, context_management: object) -> None:
"""
@ -1824,7 +1828,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
)
def update_headers_with_optional_anthropic_beta(self, headers: dict, optional_params: dict) -> dict:
def update_headers_with_optional_anthropic_beta(
self, headers: dict, optional_params: dict, messages: Sequence[object] = ()
) -> dict:
"""Update headers with optional anthropic beta."""
# Skip adding beta headers for Vertex requests
@ -1833,6 +1839,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if is_vertex_request:
return headers
if requires_native_compaction_beta(self._resolved_provider, optional_params, messages):
self._ensure_beta_header(headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
_tools: Final = optional_params.get("tools", [])
for tool in _tools:
if tool.get("type", None) and tool.get("type").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value):
@ -1929,8 +1938,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
custom_llm_provider=self._resolved_provider,
)
headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params)
# === Tool-name sanitization (single chokepoint) ===
# Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We
# sanitize *here* -- not in map_openai_params -- because:
@ -1977,6 +1984,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
message=f"{e}\nReceived Messages={messages}",
) # don't use verbose_logger.exception, if exception is raised
self.update_headers_with_optional_anthropic_beta(
headers=headers, optional_params=optional_params, messages=anthropic_messages
)
## Auto-strip advisor blocks from history if advisor tool is absent.
## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
_all_tools: Final = optional_params.get("tools") or []
@ -2688,7 +2699,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -79,6 +79,27 @@ _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
def requires_native_compaction_beta(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> bool:
return custom_llm_provider == "anthropic" and (
optional_params.get("compaction") is not None
or any(
isinstance(block, Mapping)
and block.get("type") == "compaction"
and isinstance(block.get("signature"), str)
and bool(block.get("signature"))
for message in messages
if isinstance(message, Mapping)
for content in (message.get("content"),)
if isinstance(content, (list, tuple))
for block in content
)
)
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.utils import supports_prompt_caching

View file

@ -0,0 +1,71 @@
from __future__ import annotations
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from pydantic import TypeAdapter
from litellm.llms.compaction import CompactionProtocol
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicCompaction
_MAPPING: Final = TypeAdapter(Mapping[str, object])
_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_HEADERS: Final = TypeAdapter(dict[str, str])
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
_CONFLICTS: Final = ("context_management", "response_format", "stop", "stop_sequences", "tool_choice")
def supports_native_compaction(params: Mapping[str, object]) -> bool:
from litellm.utils import get_model_info
if params.get("custom_llm_provider") not in (None, "anthropic", "openai"):
return False
model: Final = str(params.get("model", "")).removeprefix("openai/").removeprefix("anthropic/")
try:
return get_model_info(model=model, custom_llm_provider="anthropic").get("supports_anthropic_compaction") is True
except Exception:
return False
def compatible_defaults(payload: Mapping[str, object]) -> bool:
return all(payload.get(key) is None for key in _CONFLICTS)
def request_kwargs() -> Mapping[str, object]:
operation: Final[AnthropicCompaction] = {"type": "summarize"}
return MappingProxyType(
{
"compaction": operation,
"extra_headers": _HEADERS.validate_python(
MappingProxyType({"anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
),
}
)
def _native_blocks(protocol: CompactionProtocol, response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
if protocol == "messages":
return (
_OBJECTS.validate_python(response.get("content", ())) if response.get("stop_reason") == "compaction" else ()
)
choices: Final = _OBJECTS.validate_python(response.get("choices", ()))
choice: Final = choices[0] if len(choices) == 1 else _EMPTY
message: Final = _MAPPING.validate_python(choice.get("message", _EMPTY))
fields: Final = _MAPPING.validate_python(message.get("provider_specific_fields") or _EMPTY)
return _OBJECTS.validate_python(fields.get("compaction_blocks", ()))
def extract_summary(protocol: CompactionProtocol, response: Mapping[str, object]) -> str | None:
blocks: Final = _native_blocks(protocol, response)
block: Final = blocks[0] if len(blocks) == 1 else _EMPTY
content: Final = block.get("content")
return (
content
if block.get("type") == "compaction"
and isinstance(block.get("signature"), str)
and block.get("signature")
and isinstance(content, str)
and content.strip()
else None
)

View file

@ -33,7 +33,7 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AnthropicTextError(BaseLLMException):
@ -185,7 +185,7 @@ class AnthropicTextConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -2,8 +2,11 @@ import copy
import hashlib
import json
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.llms.anthropic.experimental_pass_through.utils import (
is_reasoning_auto_summary_enabled,
@ -16,6 +19,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
TOOL_NAME_HASH_LENGTH: Final = 8
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
def _optional_attr(source: object, name: str) -> object:
@ -36,6 +40,20 @@ def _thought_signature(provider_specific_fields: object) -> str | None:
return signature if isinstance(signature, str) else None
def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]:
fields: Final = _as_string_mapping(provider_specific_fields)
raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None
return (
tuple(
block
for raw_block in raw_blocks
if (block := _as_string_mapping(raw_block)) is not None and block.get("type") == "compaction"
)
if isinstance(raw_blocks, (list, tuple))
else ()
)
_ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset(
{"name", "type", "input_schema", "description", "cache_control", "strict"}
)
@ -1330,7 +1348,11 @@ class LiteLLMAnthropicMessagesAdapter:
tool_name_mapping: dict[str, str] | None = None,
) -> list[dict[str, Any]]:
new_content: Final[list[dict[str, Any]]] = []
for choice in choices:
for choice, compaction_blocks in (
(choice, _compaction_blocks(_optional_attr(choice.message, "provider_specific_fields")))
for choice in choices
):
new_content.extend(_COMPACTION_BLOCK.validate_python(block) for block in compaction_blocks)
# Handle thinking blocks first
if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
for thinking_block in choice.message.thinking_blocks:
@ -1365,7 +1387,7 @@ class LiteLLMAnthropicMessagesAdapter:
)
# Handle text content
if choice.message.content is not None:
if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks):
new_content.append(
AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
)
@ -1545,21 +1567,35 @@ class LiteLLMAnthropicMessagesAdapter:
openai_finish_reason=openai_finish_reason
)
anthropic_finish_reason: Final = (
"refusal"
"compaction"
if len(anthropic_content) == 1 and anthropic_content[0].get("type") == "compaction"
else "refusal"
if refusal_text is not None and translated_finish_reason != "max_tokens"
else translated_finish_reason
)
# extract usage
usage: Final[Usage] = getattr(response, "usage")
anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
if polyfill_result is not None and polyfill_result.iterations_usage is not None:
message_iteration: Final[UsageIteration] = {
"type": "message",
"input_tokens": anthropic_usage["input_tokens"],
"output_tokens": usage.completion_tokens or 0,
}
anthropic_usage["iterations"] = list(polyfill_result.iterations_usage) + [message_iteration]
message_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None
anthropic_usage: Final[AnthropicUsage] = (
TypeAdapter(AnthropicUsage).validate_python(
MappingProxyType(
{
**message_usage,
"iterations": (
*polyfill_iterations,
UsageIteration(
type="message",
input_tokens=message_usage.get("input_tokens", 0),
output_tokens=usage.completion_tokens or 0,
),
),
}
)
)
if polyfill_iterations is not None
else message_usage
)
translated_obj: Final = AnthropicMessagesResponse(
id=response.id,

View file

@ -580,7 +580,9 @@ def anthropic_messages_handler(
)
if anthropic_messages_provider_config is None:
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
if _should_route_to_responses_api(custom_llm_provider, original_model, model):
if kwargs.get("compaction") is None and _should_route_to_responses_api(
custom_llm_provider, original_model, model
):
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=max_tokens,
messages=messages,

View file

@ -24,6 +24,7 @@ from ...common_utils import (
AnthropicError,
AnthropicModelInfo,
optionally_handle_anthropic_oauth,
requires_native_compaction_beta,
strip_advisor_blocks_from_messages,
strip_encrypted_reasoning_blocks_from_anthropic_messages,
)
@ -74,6 +75,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"tool_choice",
"thinking",
"context_management",
*(("compaction",) if self._resolved_provider == "anthropic" else ()),
"output_format",
"inference_geo",
"speed",
@ -637,6 +639,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
)
beta_values.update(existing_beta)
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:

View file

@ -29,9 +29,8 @@ from ...base_llm.chat.transformation import BaseConfig
from ..common_utils import AzureOpenAIError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -304,7 +303,7 @@ class AzureOpenAIConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
LiteLLMLoggingObj = _LiteLLMLoggingObj
@ -297,7 +296,7 @@ class AzureAIAgentsConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AzureModelRouterConfig(AzureAIStudioConfig):
@ -59,7 +59,7 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -30,7 +30,7 @@ from litellm.types.utils import ModelResponse, ProviderField
from litellm.utils import _add_path_to_api_base, supports_tool_choice
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AzureFoundryErrorStrings(str, enum.Enum):
@ -305,7 +305,7 @@ class AzureAIStudioConfig(OpenAIConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -12,9 +12,10 @@ from litellm.types.utils import ImageResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
"""Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5)."""
@ -245,7 +246,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

View file

@ -12,9 +12,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -121,7 +120,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -7,10 +7,10 @@ from collections.abc import AsyncIterator, Iterator
from typing import TYPE_CHECKING, Union
if TYPE_CHECKING:
import tiktoken
from pydantic import BaseModel
from litellm import LiteLLMLoggingObj, ModelResponse
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.types.llms.openai import AllMessageValues
@ -39,7 +39,7 @@ class CompletionTransformationBridge(ABC):
messages: list["AllMessageValues"],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> "ModelResponse":

View file

@ -21,9 +21,8 @@ from litellm.types.llms.openai import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.types.utils import ModelResponse
from ..base_utils import (
@ -344,7 +343,7 @@ class BaseConfig(ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> "ModelResponse":

View file

@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -68,7 +67,7 @@ class BaseTextCompletionConfig(BaseConfig, ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
from litellm.types.utils import EmbeddingResponse, ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -80,7 +79,7 @@ class BaseEmbeddingConfig(BaseConfig, ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -21,9 +21,8 @@ from litellm.types.utils import LlmProviders, ModelResponse
from ..chat.transformation import BaseConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.router import Router as _Router
from litellm.types.llms.openai import HttpxBinaryResponseContent
@ -231,7 +230,7 @@ class BaseFilesConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -11,9 +11,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -93,7 +92,7 @@ class BaseImageGenerationConfig(ABC):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

View file

@ -17,9 +17,8 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -82,7 +81,7 @@ class BaseImageVariationConfig(BaseConfig, ABC):
image: FileTypes,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
) -> ImageResponse:
pass
@ -98,7 +97,7 @@ class BaseImageVariationConfig(BaseConfig, ABC):
image: FileTypes,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
) -> ImageResponse:
pass
@ -125,7 +124,7 @@ class BaseImageVariationConfig(BaseConfig, ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -40,9 +40,8 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
LiteLLMLoggingObj = _LiteLLMLoggingObj
@ -990,7 +989,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -99,7 +99,7 @@ from ..common_utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
# Computer use tool prefixes supported by Bedrock
BEDROCK_COMPUTER_USE_TOOLS: Final = [
@ -1920,7 +1920,7 @@ class AmazonConverseConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -37,9 +37,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Message, ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -438,7 +437,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -25,7 +25,7 @@ from litellm.types.utils import (
from .amazon_llama_transformation import AmazonLlamaConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AmazonDeepSeekR1Config(AmazonLlamaConfig):
@ -39,7 +39,7 @@ class AmazonDeepSeekR1Config(AmazonLlamaConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -21,9 +21,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.types.utils import ModelResponse
LiteLLMLoggingObj = _LiteLLMLoggingObj
@ -198,7 +197,7 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> "ModelResponse":

View file

@ -28,7 +28,7 @@ from ..converse_transformation import AmazonConverseConfig
from .base_invoke_transformation import AmazonInvokeConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
_CachePointCarrier = TypeVar("_CachePointCarrier", SystemContentBlock, ContentBlock)
_INJECTION_POINTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
@ -128,7 +128,7 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -21,7 +21,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AmazonQwen2Config(AmazonQwen3Config):
@ -44,7 +44,7 @@ class AmazonQwen2Config(AmazonQwen3Config):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -19,7 +19,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse, Usage
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
@ -170,7 +170,7 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, Usage
from litellm.utils import get_base64_str
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -190,7 +189,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -359,7 +358,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -34,9 +34,8 @@ from litellm.types.utils import ModelResponse, Usage
from litellm.utils import CustomStreamWrapper
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -288,7 +287,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -29,9 +29,8 @@ from ..common_utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -258,7 +257,7 @@ class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

View file

@ -1,7 +1,7 @@
"""
Brave Search API module.
"""
from litellm.llms.brave.search.transformation import BraveSearchConfig
__all__ = ["BraveSearchConfig"]
"""
Brave Search API module.
"""
from litellm.llms.brave.search.transformation import BraveSearchConfig
__all__ = ["BraveSearchConfig"]

View file

@ -23,9 +23,8 @@ from litellm.utils import CustomStreamWrapper, ModelResponse, Usage
from ..common_utils import API_BASE, BytezError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -187,7 +186,7 @@ class BytezChatConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -87,7 +86,7 @@ class ClarifaiConfig(OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -15,9 +15,8 @@ from ..common_utils import ModelResponseIterator as CohereModelResponseIterator
from ..common_utils import validate_environment as cohere_validate_environment
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -227,7 +226,7 @@ class CohereChatConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -20,9 +20,8 @@ from ..common_utils import CohereError, CohereV2ModelResponseIterator
from ..common_utils import validate_environment as cohere_validate_environment
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -191,7 +190,7 @@ class CohereV2ChatConfig(OpenAIGPTConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -20,7 +20,7 @@ from litellm.types.utils import EmbeddingResponse
from .v1_transformation import CohereEmbeddingConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
def validate_environment(api_key, headers: dict):
@ -60,7 +60,7 @@ async def async_embedding(
api_base: str,
api_key: str | None,
headers: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
client: AsyncHTTPHandler | None = None,
):
## LOGGING
@ -122,7 +122,7 @@ def embedding(
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
headers: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
data: dict | CohereEmbeddingRequest | None = None,
complete_api_base: str | None = None,
api_key: str | None = None,

View file

@ -13,9 +13,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -132,7 +131,7 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

View file

@ -17,9 +17,8 @@ from litellm.types.utils import ModelResponse
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -66,7 +65,7 @@ class CompactifAIChatConfig(OpenAIGPTConfig):
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -0,0 +1,48 @@
from __future__ import annotations
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias
from pydantic import TypeAdapter
from litellm.types.llms.openai import AllMessageValues
if TYPE_CHECKING:
from litellm.router import Router
CompactionProtocol: TypeAlias = Literal["chat", "messages"]
_MAPPING: Final = TypeAdapter(Mapping[str, object])
_MESSAGES: Final = TypeAdapter(list[AllMessageValues])
class NativeCompactionProvider(Protocol):
def supports_native_compaction(self, params: Mapping[str, object]) -> bool: ...
def compatible_defaults(self, payload: Mapping[str, object]) -> bool: ...
def request_kwargs(self) -> Mapping[str, object]: ...
def extract_summary(self, protocol: CompactionProtocol, response: Mapping[str, object]) -> str | None: ...
def get_native_compaction_provider(params: Mapping[str, object]) -> NativeCompactionProvider | None:
from litellm.llms.anthropic import compaction
return compaction if compaction.supports_native_compaction(params) else None
async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mapping[str, object]) -> Mapping[str, object]:
if protocol == "messages":
return _MAPPING.validate_python(
await router.aanthropic_messages(custom_llm_provider=None, client=None, **payload)
)
response: Final = await router.acompletion(
model=str(payload["model"]),
messages=_MESSAGES.validate_python(payload["messages"]),
stream=False,
**MappingProxyType(
{key: value for key, value in payload.items() if key not in ("model", "messages", "stream")}
),
)
return _MAPPING.validate_python(response.model_dump())

View file

@ -26,9 +26,8 @@ from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProv
from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -268,7 +267,7 @@ class BaseLLMAIOHTTPHandler:
messages: list,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
client: ClientSession | None = None,
):

View file

@ -0,0 +1,46 @@
from collections.abc import Generator
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from typing import Final
import httpx
from starlette.types import ASGIApp, Receive, Scope, Send
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared cache retains its legacy parameter mapping
)
from litellm.types.llms.custom_http import httpxSpecialProvider
@dataclass(frozen=True, slots=True)
class _ASGITarget:
app: ASGIApp
root_path: str
client: tuple[str, int] | None
_target: Final[ContextVar[_ASGITarget]] = ContextVar("httpx_asgi_target")
async def _dispatch(scope: Scope, receive: Receive, send: Send) -> None:
target: Final = _target.get()
await target.app({**scope, "root_path": target.root_path, "client": target.client}, receive, send)
_TRANSPORT: Final = httpx.ASGITransport(app=_dispatch, raise_app_exceptions=False)
@contextmanager
def get_async_asgi_client(
app: ASGIApp, root_path: str = "", client: tuple[str, int] | None = None
) -> Generator[httpx.AsyncClient]:
handler: Final = get_async_httpx_client(
llm_provider=httpxSpecialProvider.ASGI,
params={"transport": _TRANSPORT, "timeout": httpx.Timeout(None), "follow_redirects": False},
)
token: Final = _target.set(_ASGITarget(app, root_path, client))
try:
yield handler.client
finally:
_target.reset(token)

View file

@ -614,11 +614,15 @@ class AsyncHTTPHandler:
client_alias: str | None = None, # name for client in logs
ssl_verify: VerifyTypes | None = None,
shared_session: Optional["ClientSession"] = None,
transport: httpx.AsyncBaseTransport | None = None,
follow_redirects: bool = True,
):
self.timeout = timeout
self.event_hooks = event_hooks
self.ssl_verify = ssl_verify
self.shared_session = shared_session
self.transport = transport
self.follow_redirects = follow_redirects
self._owns_client = True
self._client = self.create_client(
timeout=timeout,
@ -651,6 +655,16 @@ class AsyncHTTPHandler:
ssl_verify: VerifyTypes | None = None,
shared_session: Optional["ClientSession"] = None,
) -> httpx.AsyncClient:
if self.transport is not None:
return httpx.AsyncClient(
transport=self.transport,
event_hooks=event_hooks,
timeout=timeout if timeout is not None else _DEFAULT_TIMEOUT,
headers=get_default_headers(),
cookies=blocked_cookie_jar(),
follow_redirects=self.follow_redirects,
trust_env=False,
)
# Get unified SSL configuration
ssl_config: Final = get_ssl_configuration(ssl_verify)
@ -680,7 +694,7 @@ class AsyncHTTPHandler:
cert=cert,
headers=default_headers,
cookies=blocked_cookie_jar(),
follow_redirects=True,
follow_redirects=self.follow_redirects,
http2=http2_enabled(),
)

View file

@ -192,12 +192,12 @@ def _rust_responses_websocket_enabled(
from .http_handler import get_shared_realtime_ssl_context
if TYPE_CHECKING:
import tiktoken
from aiohttp import ClientSession
from websockets.asyncio.client import ClientConnection
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
@ -493,7 +493,7 @@ class BaseLLMHTTPHandler:
messages: list,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
client: AsyncHTTPHandler | None = None,
json_mode: bool = False,
@ -559,7 +559,7 @@ class BaseLLMHTTPHandler:
api_base: str | None,
custom_llm_provider: str,
model_response: ModelResponse,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
timeout: float | httpx.Timeout,

View file

@ -38,9 +38,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -165,7 +164,7 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

View file

@ -149,9 +149,8 @@ def _split_parallel_tool_calls(messages: list[AllMessageValues]) -> list[AllMess
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -189,7 +188,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
return "databricks"
@classmethod
def get_config(cls):
def get_config(cls, *, model: str | None = None):
return super().get_config()
def get_required_params(self) -> list[ProviderField]:
@ -651,7 +650,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -277,12 +277,7 @@ def completion(
## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
prompt_tokens: Final = len(encoding.encode(prompt))
completion_tokens: Final = len(
encoding.encode(
model_response["choices"][0]["message"]["content"],
disallowed_special=(),
)
)
completion_tokens: Final = len(encoding.encode(model_response["choices"][0]["message"]["content"]))
model_response.created = int(time.time())
model_response.model = model

Some files were not shown because too many files have changed in this diff Show more