mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
feat(tokenizer): preserve Python defaults with opt-in Rust dispatch (#42174)
* ci: benchmark and gate an installed release wheel Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: simplify installed-wheel benchmark check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(rust): add native tokenizer codec Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(tokenizer): route Python tokenization through the Rust extension Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(lint): format tokenizer call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(packaging): restore runtime dependencies and native images Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tokenizer): preserve Python SDK behavior with Rust tokenizers * fix(tokenizer): restore compatibility paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(tokenizer): count custom tokenizers directly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tokenizer): preserve caller-supplied Python tokenizer counts * fix(tokenizer): reuse packaged vocabularies in the native wheel * refactor(rust_bridge): route token counting through the catalog as RUST_OPT_IN Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend_tracking): compare tokenizer groups by value Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(deps): re-resolve filelock under the <4.0 pin Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(llms): align transformation override signatures with base configs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * build(rust): use fat LTO to keep the native wheel under the 35 MB limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(tokenizer): preserve Python defaults with opt-in Rust dispatch * test(proxy): tolerate missing litellm.utils.Tokenizer when patching it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): patch the tokenizer dispatch function instead of the removed alias Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(tokenizer): give the Rust wrappers the tiktoken and tokenizers surface Callers of litellm.encoding and litellm.create_tokenizer must see the same read-only API whichever backend the catalog selects. - OpenAIEncoding mirrors tiktoken.Encoding: n_vocab, max_token_value, token_byte_values, encode_single_token, encode_with_unstable, encode_to_numpy, decode_with_offsets, is_special_token, repr; the Rust tiktoken crate keeps a Vocabulary beside each CoreBPE and reports the requested encoding name (gpt2 stays gpt2). - HuggingFaceTokenizer mirrors the read-only tokenizers.Tokenizer surface (token_to_id, id_to_token, get_vocab, get_vocab_size, get_added_tokens_decoder, num_special_tokens_to_add, padding, truncation, encode_special_tokens, from_buffer); HuggingFaceEncoding gains the char/word/token lookups, pad, truncate, set_sequence_id and merge. Mutators stay on the Python tokenizer. - from_json/from_pretrained claim the fork gate only when the huggingface feature is compiled in; the surrogate fallback matches on the Codec. - Tokenizer caching is keyed on the same catalog Context the dispatch runs on; rust_tokenizer reads the encoding name without loading an encoding; LITELLM_RUST parsing is cached. - Drop the unused tiktoken_encoding_for_model export and Error::Download. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * fix(tokenizer): close the exhaustive matches with assert_never CodeQL reads a `match` over a Literal with no default arm as an implicit `None` return. `assert_never` makes the exhaustiveness explicit for both the HuggingFace tokenizer loader and the Rust token-counter factory. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * feat(tokenizer): derive the fast counter from the shared tokenizer The count-only counter (`fast` feature) and the codec each parsed the same artifact: TokenCounter took the Anthropic JSON and the tiktoken rank files from Python while Tokenizer loaded them again. One parse now serves both. - FastTokenizer builds from a model another loader holds: `from_shared` takes the Arc<tokenizers::Tokenizer> the HF codec keeps, and `from_*_pairs` take the ranks the tiktoken vocabulary already parsed. - `FastCounter::fast_counter` in the core crate derives it from either codec; encodings the fast scanner does not reproduce are refused. - Native `Tokenizer.count(text, fast=False)` opts into that counter, built once per tokenizer on first use; `TokenCounter.from_tokenizer(tokenizer, fast=False)` replaces the JSON and rank-file constructors. - The Python route counts over the native tokenizers the codec path shares (`native_encoding`, `native_anthropic`) and no longer reads rank files; the packaged Anthropic tokenizer has one loader, `tokenizer_dispatch.anthropic`. - Public wrappers gain `count(text, fast=False)`. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Yujong Lee <yujong@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
a550b95d70
commit
0abd9267c1
159 changed files with 4151 additions and 954 deletions
10
.github/actions/cache-cargo-build/action.yml
vendored
10
.github/actions/cache-cargo-build/action.yml
vendored
|
|
@ -15,6 +15,12 @@ description: >-
|
|||
cache the same directory for different workloads, and a shared key would let
|
||||
whichever ran first deny the others a save.
|
||||
|
||||
inputs:
|
||||
profile:
|
||||
description: "Cargo profile the build uses (dev or release)"
|
||||
required: false
|
||||
default: "dev"
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
|
|
@ -25,6 +31,6 @@ runs:
|
|||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
litellm-rust/target
|
||||
key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }}
|
||||
key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maturin-dev-
|
||||
${{ runner.os }}-maturin-${{ inputs.profile }}-
|
||||
|
|
|
|||
14
.github/scripts/verify_linux_native_wheel.py
vendored
14
.github/scripts/verify_linux_native_wheel.py
vendored
|
|
@ -134,7 +134,16 @@ def main(
|
|||
uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members)
|
||||
native_path: Final = wheel.parent / "native" / Path(native_member.filename).name
|
||||
native_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
native_path.write_bytes(archive.read(native_member))
|
||||
native_bytes: Final = archive.read(native_member)
|
||||
native_path.write_bytes(native_bytes)
|
||||
duplicated_vocabularies: Final = tuple(
|
||||
member.filename
|
||||
for member in wheel_members
|
||||
if member.filename.startswith("litellm/litellm_core_utils/tokenizers/")
|
||||
and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name)
|
||||
and member.file_size > 0
|
||||
and archive.read(member) in native_bytes
|
||||
)
|
||||
|
||||
wheel_metadata_tags_match: Final = (
|
||||
len(wheel_metadata_tags) == len(expanded_filename_tags)
|
||||
|
|
@ -205,7 +214,7 @@ def main(
|
|||
native_module: Final = load_native_module(native_path)
|
||||
native_module_loads: Final = native_module is not None
|
||||
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
|
||||
native_size_limit: Final = 40_000_000
|
||||
native_size_limit: Final = 35_000_000
|
||||
native_size_within_limit: Final = native_member.file_size <= native_size_limit
|
||||
validations: Final = (
|
||||
(f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
|
||||
|
|
@ -223,6 +232,7 @@ def main(
|
|||
("Native module loads", native_module_loads),
|
||||
("Production module omits the panic test hook", panic_test_hook_absent),
|
||||
(f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
|
||||
("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies),
|
||||
("Wheel contents are valid", not unexpected_members),
|
||||
)
|
||||
|
||||
|
|
|
|||
36
.github/workflows/codspeed.yml
vendored
36
.github/workflows/codspeed.yml
vendored
|
|
@ -13,6 +13,7 @@ on:
|
|||
- ".github/workflows/codspeed.yml"
|
||||
- ".github/actions/setup-uv-with-retries/**"
|
||||
- ".github/actions/cache-cargo-build/**"
|
||||
- ".github/scripts/uv_sync_with_retries.sh"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
|
@ -25,6 +26,7 @@ on:
|
|||
- ".github/workflows/codspeed.yml"
|
||||
- ".github/actions/setup-uv-with-retries/**"
|
||||
- ".github/actions/cache-cargo-build/**"
|
||||
- ".github/scripts/uv_sync_with_retries.sh"
|
||||
# Allow CodSpeed to trigger backtest performance analysis
|
||||
# in order to generate initial data
|
||||
workflow_dispatch:
|
||||
|
|
@ -59,19 +61,27 @@ jobs:
|
|||
|
||||
- name: Cache the Rust build
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
with:
|
||||
profile: release
|
||||
|
||||
# Build the wheel and resolve every dependency outside the CodSpeed
|
||||
# runner: the same maturin build took 42 minutes inside `codspeed run`
|
||||
# versus under 3 minutes as a plain step (LIT-6183)
|
||||
- name: Build environment
|
||||
- name: Build the release wheel
|
||||
run: uv build --wheel --out-dir dist
|
||||
|
||||
- name: Install the wheel into the benchmark environment
|
||||
run: |
|
||||
UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12
|
||||
uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl
|
||||
|
||||
- name: Collect benchmarks
|
||||
env:
|
||||
PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
|
||||
LITELLM_REQUIRE_INSTALLED_WHEEL: "1"
|
||||
run: >
|
||||
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
|
||||
uv run --frozen --no-default-groups
|
||||
--with pytest==8.3.5
|
||||
--with pytest-codspeed==4.3.0
|
||||
--with "mcp>=2.2.0,<3.0"
|
||||
--with "a2a-sdk>=1.1.0,<2.0"
|
||||
pytest
|
||||
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
|
||||
--import-mode=importlib
|
||||
-p pytest_codspeed.plugin
|
||||
tests/benchmarks/
|
||||
--codspeed
|
||||
|
|
@ -82,13 +92,9 @@ jobs:
|
|||
with:
|
||||
mode: simulation
|
||||
run: >
|
||||
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
|
||||
uv run --frozen --no-default-groups
|
||||
--with pytest==8.3.5
|
||||
--with pytest-codspeed==4.3.0
|
||||
--with "mcp>=2.2.0,<3.0"
|
||||
--with "a2a-sdk>=1.1.0,<2.0"
|
||||
pytest
|
||||
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1
|
||||
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
|
||||
--import-mode=importlib
|
||||
-p pytest_codspeed.plugin
|
||||
tests/benchmarks/
|
||||
--codspeed
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
--extra saml \
|
||||
--python python3.13
|
||||
|
||||
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
prisma generate --schema=./schema.prisma
|
||||
|
|
|
|||
4
litellm-rust/Cargo.lock
generated
4
litellm-rust/Cargo.lock
generated
|
|
@ -3253,6 +3253,7 @@ dependencies = [
|
|||
name = "litellm-token-counter-huggingface"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"tokenizers",
|
||||
]
|
||||
|
|
@ -3261,6 +3262,9 @@ dependencies = [
|
|||
name = "litellm-token-counter-tiktoken"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"once_cell",
|
||||
"rustc-hash",
|
||||
"thiserror 2.0.19",
|
||||
"tiktoken-rs",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ veil = "0.3.0"
|
|||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
lto = "thin"
|
||||
lto = "fat"
|
||||
codegen-units = 1
|
||||
panic = "unwind"
|
||||
debug = false
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ pyo3::create_exception!(
|
|||
|
||||
static FORK_GATE: ForkGate = ForkGate::new();
|
||||
|
||||
/// Whether this process has started the Tokio runtime.
|
||||
/// Whether this process has entered process-bound native execution.
|
||||
pub fn runtime_started() -> bool {
|
||||
FORK_GATE.started(std::process::id())
|
||||
}
|
||||
|
|
@ -40,9 +40,8 @@ pub fn reserve_process_for_forking() -> Result<(), RuntimeAlreadyStarted> {
|
|||
FORK_GATE.reserve(std::process::id())
|
||||
}
|
||||
|
||||
/// The only door to the Tokio runtime: every route reaches it through this module, which is
|
||||
/// what lets the gate speak for the whole extension. `clippy.toml` disallows going around it.
|
||||
fn enter_runtime() -> PyResult<()> {
|
||||
/// Claims process-bound native state before runtime startup or tokenizer execution.
|
||||
pub fn enter_native() -> PyResult<()> {
|
||||
FORK_GATE
|
||||
.enter(std::process::id())
|
||||
.map_err(|refused| match refused {
|
||||
|
|
@ -60,7 +59,7 @@ fn enter_runtime() -> PyResult<()> {
|
|||
|
||||
#[expect(clippy::disallowed_methods, reason = "this is the gated door")]
|
||||
fn runtime() -> PyResult<&'static Runtime> {
|
||||
enter_runtime()?;
|
||||
enter_native()?;
|
||||
Ok(pyo3_async_runtimes::tokio::get_runtime())
|
||||
}
|
||||
|
||||
|
|
@ -70,7 +69,7 @@ where
|
|||
F: Future<Output = PyResult<T>> + Send + 'static,
|
||||
T: for<'py> IntoPyObject<'py> + Send + 'static,
|
||||
{
|
||||
enter_runtime()?;
|
||||
enter_native()?;
|
||||
pyo3_async_runtimes::tokio::future_into_py(py, future)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ pub use argument::lookup;
|
|||
pub use callable::wrap_failure;
|
||||
pub use driver::run_call;
|
||||
pub use execution::{
|
||||
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value,
|
||||
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, enter_native, poll_async_value,
|
||||
reserve_process_for_forking, run_async, run_async_value, run_sync, run_sync_value,
|
||||
runtime_started,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ name = "_native"
|
|||
crate-type = ["cdylib"]
|
||||
|
||||
[features]
|
||||
default = ["abi3", "fast"]
|
||||
default = ["abi3", "fast", "huggingface", "tiktoken"]
|
||||
abi3 = ["pyo3/abi3-py310"]
|
||||
extension-module = ["pyo3/extension-module"]
|
||||
panic-test = []
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ mod routes;
|
|||
reason = "secret-manager foundations await rollout activation"
|
||||
)]
|
||||
mod secrets;
|
||||
mod token_counter;
|
||||
mod tokenizer;
|
||||
|
||||
#[pymodule(gil_used = true)]
|
||||
mod _native {
|
||||
|
|
@ -37,7 +37,12 @@ mod _native {
|
|||
#[pymodule_export]
|
||||
use crate::routes::responses::ResponsesWebSocketConnection;
|
||||
#[pymodule_export]
|
||||
use crate::token_counter::TokenCounter;
|
||||
use crate::routes::token_counter::TokenCounter;
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pymodule_export]
|
||||
use crate::tokenizer::HuggingFaceEncoding;
|
||||
#[pymodule_export]
|
||||
use crate::tokenizer::Tokenizer;
|
||||
#[pymodule_export]
|
||||
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
|
||||
use pyo3::{prelude::*, types::PyModule};
|
||||
|
|
@ -83,10 +88,13 @@ mod tests {
|
|||
"achat_completions",
|
||||
"ResponsesWebSocketConnection",
|
||||
"TokenCounter",
|
||||
"Tokenizer",
|
||||
"gil_stats",
|
||||
"process_state_started",
|
||||
"reserve_process_for_forking",
|
||||
];
|
||||
#[cfg(feature = "huggingface")]
|
||||
expected.push("HuggingFaceEncoding");
|
||||
expected.sort_unstable();
|
||||
|
||||
let mut public_names: Vec<String> = native_module(py)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ pub(crate) mod chat_completions;
|
|||
pub(crate) mod messages;
|
||||
pub(crate) mod ocr;
|
||||
pub(crate) mod responses;
|
||||
pub(crate) mod token_counter;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,87 @@
|
|||
use std::sync::Arc;
|
||||
use std::{num::NonZero, thread::available_parallelism};
|
||||
|
||||
use litellm_host_python::{enter_native, run_async};
|
||||
use litellm_token_counter::{
|
||||
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
|
||||
};
|
||||
use pyo3::{
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
prelude::*,
|
||||
types::PyAny,
|
||||
};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
use crate::tokenizer::Tokenizer;
|
||||
|
||||
/// Counts the input tokens of a raw request body off the Python event loop with
|
||||
/// the GIL released. Python owns which requests get here and what to do with
|
||||
/// the count. At most one encode per core runs at a time; the rest wait in the
|
||||
/// async task, where a cancelled Python awaiter drops them before any blocking
|
||||
/// work is scheduled.
|
||||
#[pyclass(frozen)]
|
||||
pub(crate) struct TokenCounter {
|
||||
inner: Arc<CoreTokenCounter>,
|
||||
encode_slots: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl TokenCounter {
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (tokenizer, fast = false))]
|
||||
fn from_tokenizer(py: Python<'_>, tokenizer: &Tokenizer, fast: bool) -> PyResult<Self> {
|
||||
enter_native()?;
|
||||
let inner = CoreTokenCounter::new(tokenizer.counter(py, fast));
|
||||
Ok(Self {
|
||||
inner: Arc::new(inner),
|
||||
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
|
||||
})
|
||||
}
|
||||
|
||||
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
|
||||
let counter = Arc::clone(&self.inner);
|
||||
let encode_slots = Arc::clone(&self.encode_slots);
|
||||
let body = body.to_vec();
|
||||
run_async(
|
||||
py,
|
||||
async move {
|
||||
let _slot = encode_slots
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?;
|
||||
tokio::task::spawn_blocking(move || count_body(&counter, &body))
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?
|
||||
},
|
||||
token_count_error_to_pyerr,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_parallelism() -> usize {
|
||||
available_parallelism().map_or(1, NonZero::get)
|
||||
}
|
||||
|
||||
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
|
||||
let request = CountableRequest::parse(body)?;
|
||||
counter.count_request(&request)
|
||||
}
|
||||
|
||||
pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr {
|
||||
let message = error.to_string();
|
||||
match error {
|
||||
Error::Load(_)
|
||||
| Error::Ranks(_)
|
||||
| Error::UnicodeClasses
|
||||
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
|
||||
Error::RequestParse(_)
|
||||
| Error::MissingInput
|
||||
| Error::FloatText
|
||||
| Error::ContentBlock
|
||||
| Error::ArrayItems
|
||||
| Error::JsonSerialization(_)
|
||||
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
|
||||
Error::Encode(_) | Error::Decode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
|
||||
}
|
||||
}
|
||||
|
|
@ -1,158 +0,0 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
use std::{num::NonZero, thread::available_parallelism};
|
||||
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
use litellm_host_python::release_gil;
|
||||
use litellm_host_python::run_async;
|
||||
use litellm_token_counter::{
|
||||
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
|
||||
};
|
||||
use pyo3::{
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
prelude::*,
|
||||
types::PyAny,
|
||||
};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
|
||||
/// Counts the input tokens of a raw request body off the Python event loop with
|
||||
/// the GIL released. Python owns which requests get here and what to do with
|
||||
/// the count. At most one encode per core runs at a time; the rest wait in the
|
||||
/// async task, where a cancelled Python awaiter drops them before any blocking
|
||||
/// work is scheduled.
|
||||
#[pyclass(frozen)]
|
||||
pub(crate) struct TokenCounter {
|
||||
inner: Arc<CoreTokenCounter>,
|
||||
encode_slots: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl TokenCounter {
|
||||
#[new]
|
||||
fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "fast")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_json_fast(tokenizer_json))
|
||||
}
|
||||
#[cfg(all(not(feature = "fast"), feature = "huggingface"))]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_json(tokenizer_json))
|
||||
}
|
||||
#[cfg(not(any(feature = "fast", feature = "huggingface")))]
|
||||
{
|
||||
let _ = (py, tokenizer_json);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the fast or huggingface feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "fast")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file))
|
||||
}
|
||||
#[cfg(not(feature = "fast"))]
|
||||
{
|
||||
let _ = (py, rank_file);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the fast feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "fast")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file))
|
||||
}
|
||||
#[cfg(not(feature = "fast"))]
|
||||
{
|
||||
let _ = (py, rank_file);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the fast feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_tiktoken(encoding))
|
||||
}
|
||||
#[cfg(not(feature = "tiktoken"))]
|
||||
{
|
||||
let _ = (py, encoding);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the tiktoken feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
|
||||
let counter = Arc::clone(&self.inner);
|
||||
let encode_slots = Arc::clone(&self.encode_slots);
|
||||
let body = body.to_vec();
|
||||
run_async(
|
||||
py,
|
||||
async move {
|
||||
let _slot = encode_slots
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?;
|
||||
tokio::task::spawn_blocking(move || count_body(&counter, &body))
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?
|
||||
},
|
||||
token_count_error_to_pyerr,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenCounter {
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
fn load(
|
||||
py: Python<'_>,
|
||||
load: impl FnOnce() -> Result<CoreTokenCounter, Error> + Send,
|
||||
) -> PyResult<Self> {
|
||||
let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?;
|
||||
Ok(Self {
|
||||
inner: Arc::new(inner),
|
||||
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
fn encode_parallelism() -> usize {
|
||||
available_parallelism().map_or(1, NonZero::get)
|
||||
}
|
||||
|
||||
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
|
||||
let request = CountableRequest::parse(body)?;
|
||||
counter.count_request(&request)
|
||||
}
|
||||
|
||||
fn token_count_error_to_pyerr(error: Error) -> PyErr {
|
||||
let message = error.to_string();
|
||||
match error {
|
||||
Error::Load(_)
|
||||
| Error::Ranks(_)
|
||||
| Error::UnicodeClasses
|
||||
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
|
||||
Error::RequestParse(_)
|
||||
| Error::MissingInput
|
||||
| Error::FloatText
|
||||
| Error::ContentBlock
|
||||
| Error::ArrayItems
|
||||
| Error::JsonSerialization(_)
|
||||
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
|
||||
Error::Encode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
|
||||
}
|
||||
}
|
||||
713
litellm-rust/crates/python-bridge/src/tokenizer.rs
Normal file
713
litellm-rust/crates/python-bridge/src/tokenizer.rs
Normal file
|
|
@ -0,0 +1,713 @@
|
|||
//! The Python face of the text codecs: one `Tokenizer` class over the tiktoken and Hugging
|
||||
//! Face backends, carrying the read-only surface of `tiktoken.Encoding` and
|
||||
//! `tokenizers.Tokenizer` that `litellm/litellm_core_utils/tokenizer.py` wraps.
|
||||
use std::borrow::Cow;
|
||||
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
#[cfg(feature = "fast")]
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use litellm_host_python::{enter_native, release_gil};
|
||||
#[cfg(feature = "fast")]
|
||||
use litellm_token_counter::fast::{FastCounter, FastTokenizer};
|
||||
use litellm_token_counter::{Error, TextCodec};
|
||||
use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString};
|
||||
|
||||
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
|
||||
use pyo3::exceptions::PyValueError;
|
||||
#[cfg(feature = "huggingface")]
|
||||
use pyo3::{exceptions::PyIOError, types::PyDict};
|
||||
#[cfg(feature = "tiktoken")]
|
||||
use pyo3::{
|
||||
exceptions::{PyKeyError, PyRuntimeError},
|
||||
types::PyBytes,
|
||||
};
|
||||
|
||||
#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))]
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
use crate::routes::token_counter::token_count_error_to_pyerr;
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
use litellm_token_counter::huggingface::{
|
||||
EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, PaddingStrategy,
|
||||
TruncationDirection, encoding_from_json, encoding_to_json,
|
||||
};
|
||||
#[cfg(feature = "tiktoken")]
|
||||
use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary};
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<TiktokenTokenizer> {
|
||||
enter_native()?;
|
||||
let resource: std::path::PathBuf =
|
||||
PyModule::import(py, "litellm.litellm_core_utils.tokenizers")?
|
||||
.getattr("__file__")?
|
||||
.extract()?;
|
||||
release_gil(py, || {
|
||||
TiktokenTokenizer::from_cached_ranks(encoding, |file| {
|
||||
std::fs::read_to_string(resource.with_file_name(file))
|
||||
})
|
||||
})
|
||||
.map_err(|error| token_count_error_to_pyerr(error.into()))
|
||||
}
|
||||
|
||||
pub(crate) enum Codec {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Tiktoken(TiktokenTokenizer),
|
||||
#[cfg(feature = "huggingface")]
|
||||
HuggingFace(HuggingFaceTokenizer),
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
pub(crate) fn codec(&self) -> &dyn TextCodec {
|
||||
match *self {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Self::Tiktoken(ref tokenizer) => tokenizer,
|
||||
#[cfg(feature = "huggingface")]
|
||||
Self::HuggingFace(ref tokenizer) => tokenizer,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "fast")]
|
||||
fn fast_counter(&self) -> Option<FastTokenizer> {
|
||||
match *self {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Self::Tiktoken(ref tokenizer) => tokenizer.fast_counter(),
|
||||
#[cfg(feature = "huggingface")]
|
||||
Self::HuggingFace(ref tokenizer) => tokenizer.fast_counter(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The loaded model is shared: `TokenCounter::from_tokenizer` counts with the same parse,
|
||||
/// and the opt-in count-only counter is derived from it once, on first use.
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Tokenizer {
|
||||
inner: Arc<Codec>,
|
||||
#[cfg(feature = "fast")]
|
||||
fast: OnceLock<Option<Arc<FastTokenizer>>>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Tokenizer {
|
||||
#[staticmethod]
|
||||
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
{
|
||||
let tokenizer = load_tiktoken(py, encoding)?;
|
||||
Ok(Self::new(Codec::Tiktoken(tokenizer)))
|
||||
}
|
||||
#[cfg(not(feature = "tiktoken"))]
|
||||
{
|
||||
let _ = (py, encoding);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the tiktoken feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "huggingface")]
|
||||
{
|
||||
enter_native()?;
|
||||
let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json))
|
||||
.map_err(|error| token_count_error_to_pyerr(error.into()))?;
|
||||
Ok(Self::new(Codec::HuggingFace(tokenizer)))
|
||||
}
|
||||
#[cfg(not(feature = "huggingface"))]
|
||||
{
|
||||
let _ = (py, tokenizer_json);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the huggingface feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (identifier, revision = "main", token = None))]
|
||||
fn from_pretrained(
|
||||
py: Python<'_>,
|
||||
identifier: &str,
|
||||
revision: &str,
|
||||
token: Option<&str>,
|
||||
) -> PyResult<Self> {
|
||||
#[cfg(feature = "huggingface")]
|
||||
{
|
||||
enter_native()?;
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("repo_id", identifier)?;
|
||||
kwargs.set_item("filename", "tokenizer.json")?;
|
||||
kwargs.set_item("revision", revision)?;
|
||||
kwargs.set_item("token", token)?;
|
||||
let path: String = PyModule::import(py, "huggingface_hub")?
|
||||
.getattr("hf_hub_download")?
|
||||
.call((), Some(&kwargs))?
|
||||
.extract()?;
|
||||
let json =
|
||||
release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?;
|
||||
Self::from_json(py, &json)
|
||||
}
|
||||
#[cfg(not(feature = "huggingface"))]
|
||||
{
|
||||
let _ = (py, identifier, revision, token);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the huggingface feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult<Vec<u32>> {
|
||||
enter_native()?;
|
||||
let text = self.text(text)?;
|
||||
release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr)
|
||||
}
|
||||
|
||||
#[pyo3(signature = (ids, skip_special_tokens = true))]
|
||||
fn decode(&self, py: Python<'_>, ids: Vec<u32>, skip_special_tokens: bool) -> PyResult<String> {
|
||||
enter_native()?;
|
||||
release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens))
|
||||
.map_err(token_count_error_to_pyerr)
|
||||
}
|
||||
|
||||
#[pyo3(signature = (text, fast = false))]
|
||||
fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>, fast: bool) -> PyResult<usize> {
|
||||
enter_native()?;
|
||||
let text = self.text(text)?;
|
||||
let counter = self.counter(py, fast);
|
||||
release_gil(py, || {
|
||||
litellm_token_counter::Tokenizer::count_tokens(&counter, &text)
|
||||
})
|
||||
.map_err(token_count_error_to_pyerr)
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn name(&self) -> &str {
|
||||
self.inner.codec().name()
|
||||
}
|
||||
|
||||
// ---- tiktoken: the `tiktoken.Encoding` surface ------------------------------------------
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn encode_special(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
text: &Bound<'_, PyString>,
|
||||
allowed: Vec<String>,
|
||||
) -> PyResult<Vec<u32>> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.tiktoken()?;
|
||||
let text = self.text(text)?;
|
||||
release_gil(py, || tokenizer.encode_special(&text, &allowed))
|
||||
.map_err(PyRuntimeError::new_err)
|
||||
}
|
||||
|
||||
/// tiktoken's `encode_with_unstable`: `(stable_tokens, completions)`.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn encode_with_unstable(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
text: &Bound<'_, PyString>,
|
||||
allowed: Vec<String>,
|
||||
) -> PyResult<(Vec<u32>, Vec<Vec<u32>>)> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.tiktoken()?;
|
||||
let text = self.text(text)?;
|
||||
Ok(release_gil(py, || {
|
||||
tokenizer.encode_with_unstable(&text, &allowed)
|
||||
}))
|
||||
}
|
||||
|
||||
/// The special tokens by text: tiktoken's `_special_tokens`.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn special_tokens(&self) -> PyResult<HashMap<String, u32>> {
|
||||
Ok(self
|
||||
.vocabulary()?
|
||||
.special_tokens()
|
||||
.map(|(token, rank)| (token.to_owned(), rank))
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn max_token_value(&self) -> PyResult<u32> {
|
||||
Ok(self.vocabulary()?.max_token_value())
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn is_special_token(&self, token: u32) -> PyResult<bool> {
|
||||
Ok(self.vocabulary()?.is_special_token(token))
|
||||
}
|
||||
|
||||
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn token_byte_values<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py, PyBytes>>> {
|
||||
let vocabulary = self.vocabulary()?;
|
||||
let values = release_gil(py, || vocabulary.token_byte_values());
|
||||
Ok(values.iter().map(|value| PyBytes::new(py, value)).collect())
|
||||
}
|
||||
|
||||
/// The token of one whole piece; `KeyError` when it is not in the vocabulary.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn encode_single_token(&self, py: Python<'_>, piece: Vec<u8>) -> PyResult<u32> {
|
||||
self.vocabulary()?
|
||||
.encode_single_token(&piece)
|
||||
.ok_or_else(|| PyKeyError::new_err(PyBytes::new(py, &piece).unbind()))
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec<u32>) -> PyResult<Bound<'py, PyBytes>> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.tiktoken()?;
|
||||
let bytes =
|
||||
release_gil(py, || tokenizer.decode_bytes(&ids)).map_err(PyKeyError::new_err)?;
|
||||
Ok(PyBytes::new(py, &bytes))
|
||||
}
|
||||
|
||||
// ---- Hugging Face: the `tokenizers.Tokenizer` surface -----------------------------------
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))]
|
||||
fn encode_huggingface(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
sequence: Sequence,
|
||||
pair: Option<Sequence>,
|
||||
is_pretokenized: bool,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> PyResult<HuggingFaceEncoding> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.huggingface()?;
|
||||
let sequence = sequence.input(is_pretokenized)?;
|
||||
let input = match pair {
|
||||
Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?),
|
||||
None => EncodeInput::Single(sequence),
|
||||
};
|
||||
release_gil(py, || {
|
||||
tokenizer.encode_result(input, add_special_tokens, fast)
|
||||
})
|
||||
.map(|inner| HuggingFaceEncoding { inner })
|
||||
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))]
|
||||
fn encode_batch_huggingface(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
inputs: Vec<(Sequence, Option<Sequence>)>,
|
||||
is_pretokenized: bool,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> PyResult<Vec<HuggingFaceEncoding>> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.huggingface()?;
|
||||
let inputs = inputs
|
||||
.into_iter()
|
||||
.map(|(sequence, pair)| {
|
||||
let sequence = sequence.input(is_pretokenized)?;
|
||||
match pair {
|
||||
Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)),
|
||||
None => Ok(EncodeInput::Single(sequence)),
|
||||
}
|
||||
})
|
||||
.collect::<PyResult<Vec<_>>>()?;
|
||||
release_gil(py, || {
|
||||
tokenizer.encode_batch_result(inputs, add_special_tokens, fast)
|
||||
})
|
||||
.map(|encodings| {
|
||||
encodings
|
||||
.into_iter()
|
||||
.map(|inner| HuggingFaceEncoding { inner })
|
||||
.collect()
|
||||
})
|
||||
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (pretty = false))]
|
||||
fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult<String> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.huggingface()?;
|
||||
release_gil(py, || tokenizer.to_json(pretty))
|
||||
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn token_to_id(&self, token: &str) -> PyResult<Option<u32>> {
|
||||
Ok(self.huggingface()?.token_to_id(token))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn id_to_token(&self, id: u32) -> PyResult<Option<String>> {
|
||||
Ok(self.huggingface()?.id_to_token(id))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (with_added_tokens = true))]
|
||||
fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult<HashMap<String, u32>> {
|
||||
let tokenizer = self.huggingface()?;
|
||||
Ok(release_gil(py, || tokenizer.vocab(with_added_tokens)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (with_added_tokens = true))]
|
||||
fn get_vocab_size(&self, with_added_tokens: bool) -> PyResult<usize> {
|
||||
Ok(self.huggingface()?.vocab_size(with_added_tokens))
|
||||
}
|
||||
|
||||
/// The added tokens by id as `(id, (content, single_word, lstrip, rstrip, normalized,
|
||||
/// special))`, for Python to rebuild as `tokenizers.AddedToken`.
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn added_tokens_decoder(&self) -> PyResult<Vec<(u32, AddedTokenFields)>> {
|
||||
Ok(self
|
||||
.huggingface()?
|
||||
.added_tokens_decoder()
|
||||
.into_iter()
|
||||
.map(|(id, token)| {
|
||||
(
|
||||
id,
|
||||
(
|
||||
token.content,
|
||||
token.single_word,
|
||||
token.lstrip,
|
||||
token.rstrip,
|
||||
token.normalized,
|
||||
token.special,
|
||||
),
|
||||
)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// The padding parameters as `tokenizers.Tokenizer.padding` reports them.
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn padding<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
|
||||
let Some(params) = self.huggingface()?.padding() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let padding = PyDict::new(py);
|
||||
padding.set_item(
|
||||
"length",
|
||||
match params.strategy {
|
||||
PaddingStrategy::BatchLongest => None,
|
||||
PaddingStrategy::Fixed(length) => Some(length),
|
||||
},
|
||||
)?;
|
||||
padding.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?;
|
||||
padding.set_item("pad_id", params.pad_id)?;
|
||||
padding.set_item("pad_type_id", params.pad_type_id)?;
|
||||
padding.set_item("pad_token", ¶ms.pad_token)?;
|
||||
padding.set_item("direction", params.direction.as_ref())?;
|
||||
Ok(Some(padding))
|
||||
}
|
||||
|
||||
/// The truncation parameters as `tokenizers.Tokenizer.truncation` reports them.
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn truncation<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
|
||||
let Some(params) = self.huggingface()?.truncation() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let truncation = PyDict::new(py);
|
||||
truncation.set_item("max_length", params.max_length)?;
|
||||
truncation.set_item("stride", params.stride)?;
|
||||
truncation.set_item("strategy", params.strategy.as_ref())?;
|
||||
truncation.set_item("direction", params.direction.as_ref())?;
|
||||
Ok(Some(truncation))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult<usize> {
|
||||
Ok(self.huggingface()?.num_special_tokens_to_add(is_pair))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn encode_special_tokens(&self) -> PyResult<bool> {
|
||||
Ok(self.huggingface()?.encode_special_tokens())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
type AddedTokenFields = (String, bool, bool, bool, bool, bool);
|
||||
|
||||
impl Tokenizer {
|
||||
fn new(inner: Codec) -> Self {
|
||||
Self {
|
||||
inner: Arc::new(inner),
|
||||
#[cfg(feature = "fast")]
|
||||
fast: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn counter(&self, py: Python<'_>, fast: bool) -> SharedCounter {
|
||||
#[cfg(feature = "fast")]
|
||||
if fast {
|
||||
let counter = self.fast.get().unwrap_or_else(|| {
|
||||
release_gil(py, || {
|
||||
self.fast
|
||||
.get_or_init(|| self.inner.fast_counter().map(Arc::new))
|
||||
})
|
||||
});
|
||||
if let Some(counter) = counter {
|
||||
return SharedCounter::Fast(Arc::clone(counter));
|
||||
}
|
||||
}
|
||||
#[cfg(not(feature = "fast"))]
|
||||
let _ = (py, fast);
|
||||
SharedCounter::Codec(Arc::clone(&self.inner))
|
||||
}
|
||||
|
||||
/// A Python `str` as UTF-8. tiktoken replaces lone surrogates the way its Python `encode`
|
||||
/// does; `tokenizers` rejects them, so that backend keeps the encode error.
|
||||
fn text<'a>(&self, text: &'a Bound<'_, PyString>) -> PyResult<Cow<'a, str>> {
|
||||
match text.to_cow() {
|
||||
Ok(text) => Ok(text),
|
||||
Err(error) => match *self.inner {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Codec::Tiktoken(_) if error.is_instance_of::<PyUnicodeEncodeError>(text.py()) => {
|
||||
text.call_method1("encode", ("utf-16", "surrogatepass"))?
|
||||
.call_method1("decode", ("utf-16", "replace"))?
|
||||
.extract::<String>()
|
||||
.map(Cow::Owned)
|
||||
}
|
||||
_ => Err(error),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn tiktoken(&self) -> PyResult<&TiktokenTokenizer> {
|
||||
match *self.inner {
|
||||
Codec::Tiktoken(ref tokenizer) => Ok(tokenizer),
|
||||
#[cfg(feature = "huggingface")]
|
||||
Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn vocabulary(&self) -> PyResult<&Vocabulary> {
|
||||
self.tiktoken()?.vocabulary().ok_or_else(|| {
|
||||
PyRuntimeError::new_err("this encoding was built without its vocabulary")
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn huggingface(&self) -> PyResult<&HuggingFaceTokenizer> {
|
||||
match *self.inner {
|
||||
Codec::HuggingFace(ref tokenizer) => Ok(tokenizer),
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) enum SharedCounter {
|
||||
Codec(Arc<Codec>),
|
||||
#[cfg(feature = "fast")]
|
||||
Fast(Arc<FastTokenizer>),
|
||||
}
|
||||
|
||||
impl litellm_token_counter::Tokenizer for SharedCounter {
|
||||
fn count_tokens(&self, text: &str) -> Result<usize, Error> {
|
||||
match self {
|
||||
Self::Codec(codec) => codec.codec().count_tokens(text),
|
||||
#[cfg(feature = "fast")]
|
||||
Self::Fast(counter) => counter.count_tokens(text).map_err(Error::from),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[derive(FromPyObject)]
|
||||
pub(crate) enum Sequence {
|
||||
Text(String),
|
||||
Words(Vec<String>),
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
impl Sequence {
|
||||
fn input(self, is_pretokenized: bool) -> PyResult<InputSequence<'static>> {
|
||||
match (self, is_pretokenized) {
|
||||
(Self::Text(text), false) => Ok(text.into()),
|
||||
(Self::Words(words), true) => Ok(words.into()),
|
||||
_ => Err(pyo3::exceptions::PyTypeError::new_err(
|
||||
"input must match is_pretokenized",
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn direction<T>(value: &str, left: T, right: T, what: &str) -> PyResult<T> {
|
||||
match value {
|
||||
"left" => Ok(left),
|
||||
"right" => Ok(right),
|
||||
other => Err(PyValueError::new_err(format!(
|
||||
"invalid {what} direction {other:?}: expected 'left' or 'right'"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// `tokenizers.Encoding`, mutable like the original: `pad`, `truncate` and `set_sequence_id`
|
||||
/// change it in place.
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyclass(module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct HuggingFaceEncoding {
|
||||
inner: Encoding,
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pymethods]
|
||||
impl HuggingFaceEncoding {
|
||||
#[new]
|
||||
#[pyo3(signature = (json = None))]
|
||||
fn new(json: Option<&str>) -> PyResult<Self> {
|
||||
let inner = match json {
|
||||
Some(json) => encoding_from_json(json)
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))?,
|
||||
None => Encoding::default(),
|
||||
};
|
||||
Ok(Self { inner })
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (encodings, growing_offsets = true))]
|
||||
fn merge(encodings: Vec<PyRef<'_, Self>>, growing_offsets: bool) -> Self {
|
||||
Self {
|
||||
inner: Encoding::merge(
|
||||
encodings.iter().map(|encoding| encoding.inner.clone()),
|
||||
growing_offsets,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn __reduce__<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> {
|
||||
let json = encoding_to_json(&self.inner)
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))?;
|
||||
Ok((py.get_type::<Self>(), (json,)))
|
||||
}
|
||||
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \
|
||||
attention_mask, special_tokens_mask, overflowing])",
|
||||
self.inner.len()
|
||||
)
|
||||
}
|
||||
|
||||
fn __len__(&self) -> usize {
|
||||
self.inner.len()
|
||||
}
|
||||
#[getter]
|
||||
fn ids(&self) -> Vec<u32> {
|
||||
self.inner.get_ids().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn tokens(&self) -> Vec<String> {
|
||||
self.inner.get_tokens().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn offsets(&self) -> Vec<(usize, usize)> {
|
||||
self.inner.get_offsets().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn type_ids(&self) -> Vec<u32> {
|
||||
self.inner.get_type_ids().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn attention_mask(&self) -> Vec<u32> {
|
||||
self.inner.get_attention_mask().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn special_tokens_mask(&self) -> Vec<u32> {
|
||||
self.inner.get_special_tokens_mask().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn word_ids(&self) -> Vec<Option<u32>> {
|
||||
self.inner.get_word_ids().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn sequence_ids(&self) -> Vec<Option<usize>> {
|
||||
self.inner.get_sequence_ids()
|
||||
}
|
||||
#[getter]
|
||||
fn overflowing(&self) -> Vec<Self> {
|
||||
self.inner
|
||||
.get_overflowing()
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|inner| Self { inner })
|
||||
.collect()
|
||||
}
|
||||
#[getter]
|
||||
fn n_sequences(&self) -> usize {
|
||||
self.inner.n_sequences()
|
||||
}
|
||||
|
||||
#[pyo3(signature = (word_index, sequence_index = 0))]
|
||||
fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
|
||||
self.inner.word_to_tokens(word_index, sequence_index)
|
||||
}
|
||||
#[pyo3(signature = (word_index, sequence_index = 0))]
|
||||
fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
|
||||
self.inner.word_to_chars(word_index, sequence_index)
|
||||
}
|
||||
fn token_to_sequence(&self, token_index: usize) -> Option<usize> {
|
||||
self.inner.token_to_sequence(token_index)
|
||||
}
|
||||
fn token_to_chars(&self, token_index: usize) -> Option<(usize, usize)> {
|
||||
self.inner
|
||||
.token_to_chars(token_index)
|
||||
.map(|(_, offsets)| offsets)
|
||||
}
|
||||
fn token_to_word(&self, token_index: usize) -> Option<u32> {
|
||||
self.inner.token_to_word(token_index).map(|(_, word)| word)
|
||||
}
|
||||
#[pyo3(signature = (char_pos, sequence_index = 0))]
|
||||
fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option<usize> {
|
||||
self.inner.char_to_token(char_pos, sequence_index)
|
||||
}
|
||||
#[pyo3(signature = (char_pos, sequence_index = 0))]
|
||||
fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option<u32> {
|
||||
self.inner.char_to_word(char_pos, sequence_index)
|
||||
}
|
||||
|
||||
fn set_sequence_id(&mut self, sequence_id: usize) {
|
||||
self.inner.set_sequence_id(sequence_id);
|
||||
}
|
||||
|
||||
#[pyo3(signature = (length, direction = "right", pad_id = 0, pad_type_id = 0, pad_token = "[PAD]"))]
|
||||
fn pad(
|
||||
&mut self,
|
||||
length: usize,
|
||||
direction: &str,
|
||||
pad_id: u32,
|
||||
pad_type_id: u32,
|
||||
pad_token: &str,
|
||||
) -> PyResult<()> {
|
||||
let direction = self::direction(
|
||||
direction,
|
||||
PaddingDirection::Left,
|
||||
PaddingDirection::Right,
|
||||
"padding",
|
||||
)?;
|
||||
self.inner
|
||||
.pad(length, pad_id, pad_type_id, pad_token, direction);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[pyo3(signature = (max_length, stride = 0, direction = "right"))]
|
||||
fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> {
|
||||
let direction = self::direction(
|
||||
direction,
|
||||
TruncationDirection::Left,
|
||||
TruncationDirection::Right,
|
||||
"truncation",
|
||||
)?;
|
||||
self.inner.truncate(max_length, stride, direction);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -8,6 +8,8 @@ mod scanner;
|
|||
mod tiktoken;
|
||||
mod unicode_classes;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use byte_level::ByteLevelCounter;
|
||||
use scanner::{SplitPattern, TiktokenCounter};
|
||||
|
||||
|
|
@ -15,22 +17,29 @@ pub use error::Error;
|
|||
|
||||
enum Encoder {
|
||||
HuggingFace {
|
||||
tokenizer: Box<tokenizers::Tokenizer>,
|
||||
tokenizer: Arc<tokenizers::Tokenizer>,
|
||||
byte_level: Option<ByteLevelCounter>,
|
||||
},
|
||||
Tiktoken(TiktokenCounter),
|
||||
}
|
||||
|
||||
/// A count-only tokenizer. Its model tables are immutable, so one built from an already
|
||||
/// loaded model (`from_shared`, `from_*_pairs`) adds only the count-specific tables.
|
||||
pub struct FastTokenizer(Encoder);
|
||||
|
||||
impl FastTokenizer {
|
||||
pub fn from_json(json: &str) -> Result<Self, Error> {
|
||||
let tokenizer = json.parse::<tokenizers::Tokenizer>().map_err(Error::Load)?;
|
||||
Ok(Self::from_shared(Arc::new(tokenizer)))
|
||||
}
|
||||
|
||||
/// Counts with a Hugging Face model another codec already holds; nothing is re-parsed.
|
||||
pub fn from_shared(tokenizer: Arc<tokenizers::Tokenizer>) -> Self {
|
||||
let byte_level = ByteLevelCounter::detect(&tokenizer);
|
||||
Ok(Self(Encoder::HuggingFace {
|
||||
tokenizer: Box::new(tokenizer),
|
||||
Self(Encoder::HuggingFace {
|
||||
tokenizer,
|
||||
byte_level,
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn from_cl100k_ranks(ranks: &str) -> Result<Self, Error> {
|
||||
|
|
@ -41,12 +50,36 @@ impl FastTokenizer {
|
|||
Self::from_ranks(SplitPattern::O200k, ranks)
|
||||
}
|
||||
|
||||
/// `cl100k_base` from ranks another loader already parsed.
|
||||
pub fn from_cl100k_pairs<'a>(
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::from_pairs(SplitPattern::Cl100k, pairs)
|
||||
}
|
||||
|
||||
/// `o200k_base` (and `o200k_harmony`, whose ordinary tokens are the same) from ranks
|
||||
/// another loader already parsed.
|
||||
pub fn from_o200k_pairs<'a>(
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::from_pairs(SplitPattern::O200k, pairs)
|
||||
}
|
||||
|
||||
fn from_ranks(split: SplitPattern, ranks: &str) -> Result<Self, Error> {
|
||||
TiktokenCounter::from_ranks(split, ranks)
|
||||
.map(Encoder::Tiktoken)
|
||||
.map(Self)
|
||||
}
|
||||
|
||||
fn from_pairs<'a>(
|
||||
split: SplitPattern,
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
TiktokenCounter::from_pairs(split, pairs)
|
||||
.map(Encoder::Tiktoken)
|
||||
.map(Self)
|
||||
}
|
||||
|
||||
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
|
||||
match &self.0 {
|
||||
Encoder::Tiktoken(counter) => Ok(counter.count(text)),
|
||||
|
|
|
|||
|
|
@ -39,8 +39,19 @@ pub(super) struct TiktokenCounter {
|
|||
|
||||
impl TiktokenCounter {
|
||||
pub(super) fn from_ranks(split: SplitPattern, rank_file: &str) -> Result<Self, Error> {
|
||||
Self::new(split, MergeRanks::parse(rank_file)?)
|
||||
}
|
||||
|
||||
pub(super) fn from_pairs<'a>(
|
||||
split: SplitPattern,
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::new(split, MergeRanks::from_pairs(pairs)?)
|
||||
}
|
||||
|
||||
fn new(split: SplitPattern, ranks: MergeRanks) -> Result<Self, Error> {
|
||||
Ok(Self {
|
||||
ranks: MergeRanks::parse(rank_file)?,
|
||||
ranks,
|
||||
piece_len: split.piece_len(),
|
||||
unicode_classes: UnicodeClasses::get().ok_or(Error::UnicodeClasses)?,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -22,11 +22,25 @@ pub(super) struct MergeRanks(FxHashMap<Box<[u8]>, Rank>);
|
|||
|
||||
impl MergeRanks {
|
||||
pub(super) fn parse(text: &str) -> Result<Self, Error> {
|
||||
let ranks = text
|
||||
.lines()
|
||||
.filter(|line| !line.is_empty())
|
||||
.map(parse_line)
|
||||
.collect::<Result<FxHashMap<_, _>, _>>()?;
|
||||
Self::from_entries(text.lines().filter(|line| !line.is_empty()).map(parse_line))
|
||||
}
|
||||
|
||||
/// The same table from ranks another loader already parsed.
|
||||
pub(super) fn from_pairs<'a>(
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], Rank)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::from_entries(pairs.into_iter().map(|(bytes, rank)| {
|
||||
if rank == NO_RANK {
|
||||
return Err(Error::Ranks(format!("rank {rank} is reserved")));
|
||||
}
|
||||
Ok((Box::from(bytes), rank))
|
||||
}))
|
||||
}
|
||||
|
||||
fn from_entries(
|
||||
entries: impl Iterator<Item = Result<(Box<[u8]>, Rank), Error>>,
|
||||
) -> Result<Self, Error> {
|
||||
let ranks = entries.collect::<Result<FxHashMap<_, _>, _>>()?;
|
||||
if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) {
|
||||
return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token")));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,5 +6,6 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokenizers.workspace = true
|
||||
|
|
|
|||
|
|
@ -6,4 +6,6 @@ pub enum Error {
|
|||
Load(#[source] tokenizers::Error),
|
||||
#[error("tokenization failed: {0}")]
|
||||
Encode(#[source] tokenizers::Error),
|
||||
#[error("token decoding failed: {0}")]
|
||||
Decode(#[source] tokenizers::Error),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,22 +2,243 @@
|
|||
|
||||
mod error;
|
||||
|
||||
pub use error::Error;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct HuggingFaceTokenizer(Box<tokenizers::Tokenizer>);
|
||||
pub use error::Error;
|
||||
use tokenizers::PostProcessor;
|
||||
pub use tokenizers::{
|
||||
AddedToken, EncodeInput, Encoding, InputSequence, PaddingDirection, PaddingParams,
|
||||
PaddingStrategy, TruncationDirection, TruncationParams,
|
||||
};
|
||||
|
||||
pub fn encoding_from_json(json: &str) -> Result<Encoding, Error> {
|
||||
serde_json::from_str(json).map_err(|error| Error::Load(error.into()))
|
||||
}
|
||||
|
||||
pub fn encoding_to_json(encoding: &Encoding) -> Result<String, Error> {
|
||||
serde_json::to_string(encoding).map_err(|error| Error::Load(error.into()))
|
||||
}
|
||||
|
||||
pub struct HuggingFaceTokenizer {
|
||||
tokenizer: Arc<tokenizers::Tokenizer>,
|
||||
special_token_ids: HashSet<u32>,
|
||||
}
|
||||
|
||||
impl HuggingFaceTokenizer {
|
||||
pub fn from_json(json: &str) -> Result<Self, Error> {
|
||||
json.parse::<tokenizers::Tokenizer>()
|
||||
.map(Box::new)
|
||||
.map(Self)
|
||||
.map(Self::new)
|
||||
.map_err(Error::Load)
|
||||
}
|
||||
|
||||
fn new(tokenizer: tokenizers::Tokenizer) -> Self {
|
||||
let special_token_ids: HashSet<u32> = tokenizer
|
||||
.get_added_tokens_decoder()
|
||||
.into_iter()
|
||||
.filter_map(|(id, token)| token.special.then_some(id))
|
||||
.collect();
|
||||
Self {
|
||||
tokenizer: Arc::new(tokenizer),
|
||||
special_token_ids,
|
||||
}
|
||||
}
|
||||
|
||||
/// The parsed model, for a count-only counter to share instead of parsing it again.
|
||||
pub fn shared(&self) -> Arc<tokenizers::Tokenizer> {
|
||||
Arc::clone(&self.tokenizer)
|
||||
}
|
||||
|
||||
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
|
||||
self.0
|
||||
self.tokenizer
|
||||
.encode_fast(text, true)
|
||||
.map(|encoding| encoding.len())
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
|
||||
self.tokenizer
|
||||
.encode_fast(text, true)
|
||||
.map(|encoding| encoding.get_ids().to_vec())
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn encode_result<'a>(
|
||||
&self,
|
||||
input: EncodeInput<'a>,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> Result<Encoding, Error> {
|
||||
if fast {
|
||||
return self
|
||||
.tokenizer
|
||||
.encode_fast(input, add_special_tokens)
|
||||
.map_err(Error::Encode);
|
||||
}
|
||||
self.tokenizer
|
||||
.encode_char_offsets(input, add_special_tokens)
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn encode_batch_result<'a>(
|
||||
&self,
|
||||
inputs: Vec<EncodeInput<'a>>,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> Result<Vec<Encoding>, Error> {
|
||||
if fast {
|
||||
return self
|
||||
.tokenizer
|
||||
.encode_batch_fast(inputs, add_special_tokens)
|
||||
.map_err(Error::Encode);
|
||||
}
|
||||
self.tokenizer
|
||||
.encode_batch_char_offsets(inputs, add_special_tokens)
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn to_json(&self, pretty: bool) -> Result<String, Error> {
|
||||
self.tokenizer.to_string(pretty).map_err(Error::Load)
|
||||
}
|
||||
|
||||
pub fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
self.tokenizer.token_to_id(token)
|
||||
}
|
||||
|
||||
pub fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
self.tokenizer.id_to_token(id)
|
||||
}
|
||||
|
||||
pub fn vocab(&self, with_added_tokens: bool) -> HashMap<String, u32> {
|
||||
self.tokenizer.get_vocab(with_added_tokens)
|
||||
}
|
||||
|
||||
pub fn vocab_size(&self, with_added_tokens: bool) -> usize {
|
||||
self.tokenizer.get_vocab_size(with_added_tokens)
|
||||
}
|
||||
|
||||
/// The added tokens by id, in id order.
|
||||
pub fn added_tokens_decoder(&self) -> Vec<(u32, AddedToken)> {
|
||||
let mut added: Vec<(u32, AddedToken)> = self
|
||||
.tokenizer
|
||||
.get_added_tokens_decoder()
|
||||
.into_iter()
|
||||
.collect();
|
||||
added.sort_unstable_by_key(|(id, _)| *id);
|
||||
added
|
||||
}
|
||||
|
||||
pub fn padding(&self) -> Option<&PaddingParams> {
|
||||
self.tokenizer.get_padding()
|
||||
}
|
||||
|
||||
pub fn truncation(&self) -> Option<&TruncationParams> {
|
||||
self.tokenizer.get_truncation()
|
||||
}
|
||||
|
||||
/// How many special tokens the post-processor adds to a single sequence or a pair.
|
||||
pub fn num_special_tokens_to_add(&self, is_pair: bool) -> usize {
|
||||
self.tokenizer
|
||||
.get_post_processor()
|
||||
.map_or(0, |processor| processor.added_tokens(is_pair))
|
||||
}
|
||||
|
||||
pub fn encode_special_tokens(&self) -> bool {
|
||||
self.tokenizer.get_encode_special_tokens()
|
||||
}
|
||||
|
||||
pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
|
||||
if !skip_special_tokens {
|
||||
return self.tokenizer.decode(ids, false).map_err(Error::Decode);
|
||||
}
|
||||
let filtered_ids: Vec<u32> = ids
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|id| !self.special_token_ids.contains(id))
|
||||
.collect();
|
||||
self.tokenizer
|
||||
.decode(&filtered_ids, true)
|
||||
.map_err(Error::Decode)
|
||||
}
|
||||
|
||||
pub fn name(&self) -> &str {
|
||||
"huggingface"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn codecs_round_trip_and_skip_special_tokens() {
|
||||
let json = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
|
||||
));
|
||||
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
|
||||
let ids = tokenizer.encode("<SOS>hello<EOT>").unwrap();
|
||||
|
||||
assert!(tokenizer.decode(&ids, false).unwrap().contains("<SOS>"));
|
||||
assert_eq!(tokenizer.decode(&ids, true).unwrap(), "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_filters_special_added_tokens() {
|
||||
let json = r#"{
|
||||
"version": "1.0",
|
||||
"truncation": null,
|
||||
"padding": null,
|
||||
"added_tokens": [
|
||||
{
|
||||
"id": 1,
|
||||
"content": "<s>",
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false,
|
||||
"special": true
|
||||
}
|
||||
],
|
||||
"normalizer": null,
|
||||
"pre_tokenizer": {"type": "Whitespace"},
|
||||
"post_processor": null,
|
||||
"decoder": null,
|
||||
"model": {
|
||||
"type": "WordLevel",
|
||||
"vocab": {"<unk>": 0, "<s>": 1, "hello": 2},
|
||||
"unk_token": "<unk>"
|
||||
}
|
||||
}"#;
|
||||
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
|
||||
|
||||
assert!(!tokenizer.decode(&[1, 2], true).unwrap().contains("<s>"));
|
||||
assert!(tokenizer.decode(&[1, 2], false).unwrap().contains("<s>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vocabulary_lookups_mirror_the_tokenizers_api() {
|
||||
let json = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
|
||||
));
|
||||
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
|
||||
let ids = tokenizer.encode("hello").unwrap();
|
||||
|
||||
let token = tokenizer.id_to_token(ids[0]).unwrap();
|
||||
assert_eq!(tokenizer.token_to_id(&token), Some(ids[0]));
|
||||
assert_eq!(tokenizer.id_to_token(u32::MAX), None);
|
||||
assert_eq!(tokenizer.vocab(true).len(), tokenizer.vocab_size(true));
|
||||
assert!(tokenizer.vocab_size(true) >= tokenizer.vocab_size(false));
|
||||
let added = tokenizer.added_tokens_decoder();
|
||||
assert!(added.windows(2).all(|pair| pair[0].0 < pair[1].0));
|
||||
assert!(added.iter().any(|(_, token)| token.special));
|
||||
assert!(tokenizer.padding().is_none());
|
||||
assert!(tokenizer.truncation().is_none());
|
||||
assert!(!tokenizer.encode_special_tokens());
|
||||
assert_eq!(
|
||||
tokenizer.num_special_tokens_to_add(false),
|
||||
tokenizer.encode("").unwrap().len()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,5 +6,8 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
base64.workspace = true
|
||||
once_cell = "1.21.3"
|
||||
rustc-hash = "2.1.3"
|
||||
thiserror.workspace = true
|
||||
tiktoken-rs.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,27 +1,125 @@
|
|||
#![forbid(unsafe_code)]
|
||||
|
||||
mod error;
|
||||
mod ranks;
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
pub use error::UnsupportedTokenizer;
|
||||
pub use ranks::{LoadError, Vocabulary};
|
||||
|
||||
pub struct TiktokenTokenizer(&'static tiktoken_rs::CoreBPE);
|
||||
pub struct TiktokenTokenizer {
|
||||
encoder: &'static tiktoken_rs::CoreBPE,
|
||||
/// Present for encodings built from a rank file; the embedded tiktoken-rs singletons
|
||||
/// behind [`from_name`](Self::from_name) keep their ranks private.
|
||||
vocabulary: Option<&'static Vocabulary>,
|
||||
name: &'static str,
|
||||
}
|
||||
|
||||
impl TiktokenTokenizer {
|
||||
/// Builds `name` from its packaged rank file (read through `load`), once per process.
|
||||
/// The tokenizer reports the requested name, so `gpt2` stays `gpt2` like tiktoken does.
|
||||
pub fn from_cached_ranks(
|
||||
name: &str,
|
||||
load: impl FnOnce(&str) -> std::io::Result<String>,
|
||||
) -> Result<Self, LoadError> {
|
||||
let (loaded, name) = ranks::load(name, load)?;
|
||||
Ok(Self {
|
||||
encoder: &loaded.bpe,
|
||||
vocabulary: Some(&loaded.vocabulary),
|
||||
name,
|
||||
})
|
||||
}
|
||||
|
||||
/// The encodings tiktoken-rs embeds, for hosts without the packaged rank files.
|
||||
pub fn from_name(name: &str) -> Result<Self, UnsupportedTokenizer> {
|
||||
let tokenizer = match name {
|
||||
"cl100k_base" => tiktoken_rs::cl100k_base_singleton(),
|
||||
"o200k_base" => tiktoken_rs::o200k_base_singleton(),
|
||||
"o200k_harmony" => tiktoken_rs::o200k_harmony_singleton(),
|
||||
"p50k_base" => tiktoken_rs::p50k_base_singleton(),
|
||||
"p50k_edit" => tiktoken_rs::p50k_edit_singleton(),
|
||||
"r50k_base" | "gpt2" => tiktoken_rs::r50k_base_singleton(),
|
||||
let (encoder, name) = match name {
|
||||
"cl100k_base" => (tiktoken_rs::cl100k_base_singleton(), "cl100k_base"),
|
||||
"o200k_base" => (tiktoken_rs::o200k_base_singleton(), "o200k_base"),
|
||||
"o200k_harmony" => (tiktoken_rs::o200k_harmony_singleton(), "o200k_harmony"),
|
||||
"p50k_base" => (tiktoken_rs::p50k_base_singleton(), "p50k_base"),
|
||||
"p50k_edit" => (tiktoken_rs::p50k_edit_singleton(), "p50k_edit"),
|
||||
"r50k_base" => (tiktoken_rs::r50k_base_singleton(), "r50k_base"),
|
||||
"gpt2" => (tiktoken_rs::r50k_base_singleton(), "gpt2"),
|
||||
_ => return Err(UnsupportedTokenizer(name.to_owned())),
|
||||
};
|
||||
Ok(Self(tokenizer))
|
||||
Ok(Self {
|
||||
encoder,
|
||||
vocabulary: None,
|
||||
name,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn vocabulary(&self) -> Option<&Vocabulary> {
|
||||
self.vocabulary
|
||||
}
|
||||
|
||||
pub fn count_tokens(&self, text: &str) -> usize {
|
||||
self.0.count_ordinary(text)
|
||||
self.encoder.count_ordinary(text)
|
||||
}
|
||||
|
||||
pub fn encode(&self, text: &str) -> Vec<u32> {
|
||||
self.encoder.encode_ordinary(text)
|
||||
}
|
||||
|
||||
pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result<Vec<u32>, String> {
|
||||
let allowed = allowed.iter().map(String::as_str).collect();
|
||||
self.encoder
|
||||
.encode(text, &allowed)
|
||||
.map(|(ids, _)| ids)
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
pub fn special_tokens(&self) -> HashSet<String> {
|
||||
self.encoder
|
||||
.special_tokens()
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// tiktoken's `encode_with_unstable`: the stable prefix of `text`'s tokens and every
|
||||
/// token sequence the unstable tail could still become, sorted for a stable order.
|
||||
pub fn encode_with_unstable(
|
||||
&self,
|
||||
text: &str,
|
||||
allowed: &[String],
|
||||
) -> (Vec<u32>, Vec<Vec<u32>>) {
|
||||
let allowed = allowed.iter().map(String::as_str).collect();
|
||||
let (stable, completions) = self.encoder._encode_unstable_native(text, &allowed);
|
||||
let mut completions: Vec<Vec<u32>> = completions.into_iter().collect();
|
||||
completions.sort_unstable();
|
||||
(stable, completions)
|
||||
}
|
||||
|
||||
pub fn decode_bytes(&self, ids: &[u32]) -> Result<Vec<u8>, String> {
|
||||
self.encoder
|
||||
.decode_bytes(ids)
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
pub fn decode(&self, ids: &[u32]) -> Result<String, String> {
|
||||
self.encoder
|
||||
.decode_bytes(ids)
|
||||
.map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
pub fn name(&self) -> &str {
|
||||
self.name
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encoding_for_model(model: &str) -> Option<&'static str> {
|
||||
match tiktoken_rs::tokenizer::get_tokenizer(model)? {
|
||||
tiktoken_rs::tokenizer::Tokenizer::Cl100kBase => Some("cl100k_base"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::O200kBase => Some("o200k_base"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::O200kHarmony => Some("o200k_harmony"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::P50kBase => Some("p50k_base"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::P50kEdit => Some("p50k_edit"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::R50kBase | tiktoken_rs::tokenizer::Tokenizer::Gpt2 => {
|
||||
Some("r50k_base")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -66,5 +164,75 @@ mod tests {
|
|||
panic!("unknown encoding must be rejected");
|
||||
};
|
||||
assert_eq!(name, "unknown-encoding");
|
||||
assert_eq!(TiktokenTokenizer::from_name("gpt2").unwrap().name(), "gpt2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codecs_round_trip_named_encodings() {
|
||||
let encodings = [
|
||||
"cl100k_base",
|
||||
"o200k_base",
|
||||
"o200k_harmony",
|
||||
"p50k_base",
|
||||
"p50k_edit",
|
||||
"r50k_base",
|
||||
"gpt2",
|
||||
];
|
||||
let texts = ["hello world", "café 漢字 مرحبا 🙂", "line one\nline two"];
|
||||
for name in encodings {
|
||||
let tokenizer = TiktokenTokenizer::from_name(name).unwrap();
|
||||
for text in texts {
|
||||
assert_eq!(
|
||||
tokenizer.decode(&tokenizer.encode(text)).unwrap(),
|
||||
text,
|
||||
"{name}: {text:?}",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoding_token_prefixes_replaces_incomplete_utf8() {
|
||||
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
|
||||
let reference = tiktoken_rs::cl100k_base_singleton();
|
||||
let ids = tokenizer.encode("🙂漢字");
|
||||
for end in 1..=ids.len() {
|
||||
let bytes = reference.decode_bytes(&ids[..end]).unwrap();
|
||||
assert_eq!(
|
||||
tokenizer.decode(&ids[..end]).unwrap(),
|
||||
String::from_utf8_lossy(&bytes),
|
||||
);
|
||||
}
|
||||
assert!(tokenizer.decode(&[u32::MAX]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unstable_encoding_prefixes_stay_consistent_with_full_encoding() {
|
||||
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
|
||||
let text = "hello fanta";
|
||||
let (stable, completions) = tokenizer.encode_with_unstable(text, &[]);
|
||||
assert!(
|
||||
text.as_bytes()
|
||||
.starts_with(&tokenizer.decode_bytes(&stable).unwrap())
|
||||
);
|
||||
assert!(!completions.is_empty());
|
||||
for completion in &completions {
|
||||
let mut ids = stable.clone();
|
||||
ids.extend(completion);
|
||||
assert!(
|
||||
tokenizer
|
||||
.decode_bytes(&ids)
|
||||
.unwrap()
|
||||
.starts_with(text.as_bytes())
|
||||
);
|
||||
}
|
||||
assert!(completions.windows(2).all(|pair| pair[0] < pair[1]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoding_for_model_maps_known_models() {
|
||||
assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base"));
|
||||
assert_eq!(encoding_for_model("text-davinci-003"), Some("p50k_base"));
|
||||
assert_eq!(encoding_for_model("unknown-model"), None);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
340
litellm-rust/crates/token-counter-tiktoken/src/ranks.rs
Normal file
340
litellm-rust/crates/token-counter-tiktoken/src/ranks.rs
Normal file
|
|
@ -0,0 +1,340 @@
|
|||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use once_cell::sync::OnceCell;
|
||||
use rustc_hash::FxHashMap;
|
||||
use thiserror::Error;
|
||||
use tiktoken_rs::{CoreBPE, O200K_BASE_PAT_STR, Rank};
|
||||
|
||||
use crate::UnsupportedTokenizer;
|
||||
|
||||
const CL100K: &str = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4";
|
||||
const O200K: &str = "fb374d419588a4632f3f557e76b4b70aebbca790";
|
||||
const P50K: &str = "ec7223a39ce59f226a68acc30dc1af2788490e15";
|
||||
const LEGACY_PATTERN: &str =
|
||||
r"'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s";
|
||||
const CL100K_PATTERN: &str = r"'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s";
|
||||
|
||||
static CL100K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static O200K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static HARMONY_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static P50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static EDIT_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static R50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
|
||||
/// One encoding built from a rank file: the BPE engine plus the vocabulary it was built
|
||||
/// from, kept because `CoreBPE` does not expose its ranks and tiktoken's Python API does
|
||||
/// (`token_byte_values`, `encode_single_token`, `max_token_value`, `_special_tokens`).
|
||||
pub(super) struct Loaded {
|
||||
pub(super) bpe: CoreBPE,
|
||||
pub(super) vocabulary: Vocabulary,
|
||||
}
|
||||
|
||||
/// The byte-level vocabulary of a tiktoken encoding.
|
||||
pub struct Vocabulary {
|
||||
ranks: FxHashMap<Vec<u8>, Rank>,
|
||||
special_tokens: FxHashMap<String, Rank>,
|
||||
max_token_value: Rank,
|
||||
}
|
||||
|
||||
impl Vocabulary {
|
||||
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
|
||||
pub fn token_byte_values(&self) -> Vec<Vec<u8>> {
|
||||
let mut values: Vec<Vec<u8>> = self.ranks.keys().cloned().collect();
|
||||
values.sort_unstable();
|
||||
values
|
||||
}
|
||||
|
||||
/// The rank of one whole token: a mergeable piece first, then a special token's text.
|
||||
pub fn encode_single_token(&self, piece: &[u8]) -> Option<Rank> {
|
||||
if let Some(rank) = self.ranks.get(piece) {
|
||||
return Some(*rank);
|
||||
}
|
||||
std::str::from_utf8(piece)
|
||||
.ok()
|
||||
.and_then(|text| self.special_tokens.get(text).copied())
|
||||
}
|
||||
|
||||
pub fn max_token_value(&self) -> Rank {
|
||||
self.max_token_value
|
||||
}
|
||||
|
||||
/// Every mergeable token with its rank, for building other tables from one parse.
|
||||
pub fn ranks(&self) -> impl Iterator<Item = (&[u8], Rank)> + '_ {
|
||||
self.ranks
|
||||
.iter()
|
||||
.map(|(bytes, rank)| (bytes.as_slice(), *rank))
|
||||
}
|
||||
|
||||
/// The special tokens with their ranks, tiktoken's `_special_tokens`.
|
||||
pub fn special_tokens(&self) -> impl Iterator<Item = (&str, Rank)> + '_ {
|
||||
self.special_tokens
|
||||
.iter()
|
||||
.map(|(token, rank)| (token.as_str(), *rank))
|
||||
}
|
||||
|
||||
pub fn is_special_token(&self, rank: Rank) -> bool {
|
||||
self.special_tokens.values().any(|special| *special == rank)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum LoadError {
|
||||
#[error(transparent)]
|
||||
Unsupported(#[from] UnsupportedTokenizer),
|
||||
#[error("failed to load tiktoken ranks: {0}")]
|
||||
Ranks(String),
|
||||
}
|
||||
|
||||
/// Loads `name` once per process. The returned name is the one requested (`gpt2` stays
|
||||
/// `gpt2`, as `tiktoken.get_encoding("gpt2").name` does), while `gpt2` and `r50k_base` share
|
||||
/// one cached encoder.
|
||||
pub(super) fn load(
|
||||
name: &str,
|
||||
load_file: impl FnOnce(&str) -> std::io::Result<String>,
|
||||
) -> Result<(&'static Loaded, &'static str), LoadError> {
|
||||
let (requested, canonical, file, cache) = match name {
|
||||
"cl100k_base" => ("cl100k_base", "cl100k_base", CL100K, &CL100K_ENCODER),
|
||||
"o200k_base" => ("o200k_base", "o200k_base", O200K, &O200K_ENCODER),
|
||||
"o200k_harmony" => ("o200k_harmony", "o200k_harmony", O200K, &HARMONY_ENCODER),
|
||||
"p50k_base" => ("p50k_base", "p50k_base", P50K, &P50K_ENCODER),
|
||||
"p50k_edit" => ("p50k_edit", "p50k_edit", P50K, &EDIT_ENCODER),
|
||||
"r50k_base" => ("r50k_base", "r50k_base", P50K, &R50K_ENCODER),
|
||||
"gpt2" => ("gpt2", "r50k_base", P50K, &R50K_ENCODER),
|
||||
_ => return Err(UnsupportedTokenizer(name.to_owned()).into()),
|
||||
};
|
||||
let loaded = cache.get_or_try_init(|| {
|
||||
let ranks = load_file(file).map_err(|error| LoadError::Ranks(error.to_string()))?;
|
||||
build(canonical, &ranks)
|
||||
})?;
|
||||
Ok((loaded, requested))
|
||||
}
|
||||
|
||||
fn build(name: &str, ranks: &str) -> Result<Loaded, LoadError> {
|
||||
let parsed = ranks
|
||||
.lines()
|
||||
.map(parse_rank)
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let encoder: FxHashMap<_, _> = parsed
|
||||
.into_iter()
|
||||
.filter(|(_, rank)| name != "r50k_base" || *rank < 50256)
|
||||
.collect();
|
||||
if encoder
|
||||
.values()
|
||||
.collect::<std::collections::HashSet<_>>()
|
||||
.len()
|
||||
!= encoder.len()
|
||||
|| (0..=u8::MAX).any(|byte| !encoder.contains_key(&[byte][..]))
|
||||
{
|
||||
return Err(LoadError::Ranks("invalid vocabulary ranks".into()));
|
||||
}
|
||||
let (pattern, specials): (&str, &[(&str, Rank)]) = match name {
|
||||
"cl100k_base" => (
|
||||
CL100K_PATTERN,
|
||||
&[
|
||||
("<|endoftext|>", 100257),
|
||||
("<|fim_prefix|>", 100258),
|
||||
("<|fim_middle|>", 100259),
|
||||
("<|fim_suffix|>", 100260),
|
||||
("<|endofprompt|>", 100276),
|
||||
],
|
||||
),
|
||||
"o200k_base" => (
|
||||
O200K_BASE_PAT_STR,
|
||||
&[("<|endoftext|>", 199999), ("<|endofprompt|>", 200018)],
|
||||
),
|
||||
"o200k_harmony" => (
|
||||
O200K_BASE_PAT_STR,
|
||||
&[
|
||||
("<|startoftext|>", 199998),
|
||||
("<|endoftext|>", 199999),
|
||||
("<|reserved_200000|>", 200000),
|
||||
("<|reserved_200001|>", 200001),
|
||||
("<|return|>", 200002),
|
||||
("<|constrain|>", 200003),
|
||||
("<|reserved_200004|>", 200004),
|
||||
("<|channel|>", 200005),
|
||||
("<|start|>", 200006),
|
||||
("<|end|>", 200007),
|
||||
("<|message|>", 200008),
|
||||
("<|reserved_200009|>", 200009),
|
||||
("<|reserved_200010|>", 200010),
|
||||
("<|reserved_200011|>", 200011),
|
||||
("<|call|>", 200012),
|
||||
],
|
||||
),
|
||||
"p50k_edit" => (
|
||||
LEGACY_PATTERN,
|
||||
&[
|
||||
("<|endoftext|>", 50256),
|
||||
("<|fim_prefix|>", 50281),
|
||||
("<|fim_middle|>", 50282),
|
||||
("<|fim_suffix|>", 50283),
|
||||
],
|
||||
),
|
||||
_ => (LEGACY_PATTERN, &[("<|endoftext|>", 50256)]),
|
||||
};
|
||||
let reserved = (200013..=201087)
|
||||
.filter(|_| name == "o200k_harmony")
|
||||
.map(|rank| (format!("<|reserved_{rank}|>"), rank));
|
||||
let special_tokens: FxHashMap<String, Rank> = specials
|
||||
.iter()
|
||||
.map(|(token, rank)| ((*token).to_owned(), *rank))
|
||||
.chain(reserved)
|
||||
.collect();
|
||||
let max_token_value = encoder
|
||||
.values()
|
||||
.chain(special_tokens.values())
|
||||
.copied()
|
||||
.max()
|
||||
.ok_or_else(|| LoadError::Ranks("empty vocabulary".into()))?;
|
||||
let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern)
|
||||
.map_err(|error| LoadError::Ranks(error.to_string()))?;
|
||||
Ok(Loaded {
|
||||
bpe,
|
||||
vocabulary: Vocabulary {
|
||||
ranks: encoder,
|
||||
special_tokens,
|
||||
max_token_value,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_rank(line: &str) -> Result<(Vec<u8>, Rank), LoadError> {
|
||||
let (token, rank) = line
|
||||
.split_once(' ')
|
||||
.ok_or_else(|| LoadError::Ranks("missing rank".into()))?;
|
||||
let bytes = STANDARD
|
||||
.decode(token)
|
||||
.map_err(|error| LoadError::Ranks(error.to_string()))?;
|
||||
let rank = rank
|
||||
.parse()
|
||||
.map_err(|error: std::num::ParseIntError| LoadError::Ranks(error.to_string()))?;
|
||||
Ok((bytes, rank))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::TiktokenTokenizer;
|
||||
|
||||
fn read_packaged_ranks(file: &str) -> std::io::Result<String> {
|
||||
std::fs::read_to_string(
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../../litellm/litellm_core_utils/tokenizers")
|
||||
.join(file),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packaged_encodings_match_embedded_encodings_and_reuse_successful_loads() {
|
||||
for name in [
|
||||
"cl100k_base",
|
||||
"o200k_base",
|
||||
"o200k_harmony",
|
||||
"p50k_base",
|
||||
"p50k_edit",
|
||||
"r50k_base",
|
||||
"gpt2",
|
||||
] {
|
||||
if name != "gpt2" {
|
||||
assert!(
|
||||
TiktokenTokenizer::from_cached_ranks(name, |_| {
|
||||
Err(std::io::Error::other("unreadable vocabulary"))
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let loads = std::sync::atomic::AtomicUsize::new(0);
|
||||
let barrier = std::sync::Barrier::new(4);
|
||||
let encoders = std::thread::scope(|scope| {
|
||||
let tasks: Vec<_> = (0..4)
|
||||
.map(|_| {
|
||||
scope.spawn(|| {
|
||||
barrier.wait();
|
||||
TiktokenTokenizer::from_cached_ranks(name, |file| {
|
||||
loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
read_packaged_ranks(file)
|
||||
})
|
||||
.unwrap()
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
tasks
|
||||
.into_iter()
|
||||
.map(|task| task.join().unwrap())
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
assert_eq!(loads.into_inner(), usize::from(name != "gpt2"));
|
||||
let actual = &encoders[0];
|
||||
let expected = TiktokenTokenizer::from_name(name).unwrap();
|
||||
assert_eq!(actual.special_tokens(), expected.special_tokens());
|
||||
let specials: Vec<_> = expected.special_tokens().into_iter().collect();
|
||||
let special_text = specials.join(" ");
|
||||
assert_eq!(
|
||||
actual.encode_special(&special_text, &specials).unwrap(),
|
||||
expected.encode_special(&special_text, &specials).unwrap()
|
||||
);
|
||||
for text in [
|
||||
"",
|
||||
"café 漢字 ع 🙂",
|
||||
"a\r\nb\t ",
|
||||
" hello 123456789",
|
||||
&special_text,
|
||||
] {
|
||||
let ids = expected.encode(text);
|
||||
assert_eq!(actual.encode(text), ids, "{name}: {text:?}");
|
||||
assert_eq!(actual.count_tokens(text), ids.len(), "{name}: {text:?}");
|
||||
assert_eq!(
|
||||
actual.decode_bytes(&ids).unwrap(),
|
||||
expected.decode_bytes(&ids).unwrap()
|
||||
);
|
||||
}
|
||||
let cached = TiktokenTokenizer::from_cached_ranks(name, |_| {
|
||||
panic!("reloaded cached vocabulary")
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(cached.encode("cached"), expected.encode("cached"));
|
||||
assert_eq!(cached.name(), name);
|
||||
assert!(expected.vocabulary().is_none());
|
||||
assert_vocabulary_lookups(name, actual);
|
||||
}
|
||||
}
|
||||
|
||||
/// The token-level lookups tiktoken's Python `Encoding` exposes, checked against the
|
||||
/// encoder itself and against the known vocabulary sizes.
|
||||
fn assert_vocabulary_lookups(name: &str, tokenizer: &TiktokenTokenizer) {
|
||||
let max_token_value = match name {
|
||||
"cl100k_base" => 100_276,
|
||||
"o200k_base" => 200_018,
|
||||
"o200k_harmony" => 201_087,
|
||||
"p50k_base" => 50_280,
|
||||
"p50k_edit" => 50_283,
|
||||
"r50k_base" | "gpt2" => 50_256,
|
||||
_ => unreachable!("{name}"),
|
||||
};
|
||||
let vocabulary = tokenizer.vocabulary().unwrap();
|
||||
assert_eq!(vocabulary.max_token_value(), max_token_value, "{name}");
|
||||
let values = vocabulary.token_byte_values();
|
||||
assert!(values.windows(2).all(|pair| pair[0] < pair[1]), "{name}");
|
||||
for piece in values.iter().step_by(997) {
|
||||
let rank = vocabulary.encode_single_token(piece).unwrap();
|
||||
assert_eq!(tokenizer.decode_bytes(&[rank]).unwrap(), *piece, "{name}");
|
||||
assert!(!vocabulary.is_special_token(rank), "{name}");
|
||||
}
|
||||
for (token, rank) in vocabulary.special_tokens() {
|
||||
assert_eq!(vocabulary.encode_single_token(token.as_bytes()), Some(rank));
|
||||
assert!(vocabulary.is_special_token(rank), "{name}: {token}");
|
||||
}
|
||||
assert_eq!(vocabulary.encode_single_token(b"<|not-a-token|>"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_ranks_return_errors_instead_of_panicking() {
|
||||
for ranks in ["", "IQ==", "IQ== x", "!!! 1", "IQ== 1"] {
|
||||
assert!(build("cl100k_base", ranks).is_err());
|
||||
}
|
||||
let repeated_rank = (0..=u8::MAX)
|
||||
.map(|byte| format!("{} 0\n", STANDARD.encode([byte])))
|
||||
.collect::<String>();
|
||||
assert!(build("cl100k_base", &repeated_rank).is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,10 @@
|
|||
# Token counting
|
||||
|
||||
`Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface
|
||||
`Tokenizer` is the text-counting interface. `TextCodec` adds encoding, decoding, and a name. `TokenCounter` applies LiteLLM request, message, and tool accounting using any `Tokenizer`
|
||||
|
||||
Counts follow the codec: tiktoken treats special-token spellings as ordinary text, while Hugging Face applies its added tokens, post-processing, padding, and truncation. `fast=True` preserves those semantics and requests acceleration where available. Unsupported configurations use the normal codec, including tiktoken encodings without a scanner and builds without the `fast` feature. Invalid input and process-guard errors still propagate. Runtime request counting currently uses the normal codec; the custom accelerator is retained for explicit use and testing
|
||||
|
||||
`FastCounter: TextCodec` exposes an optional accelerator over a loaded codec. `None` means callers should use that codec. The Python bridge caches this selection per immutable tokenizer, shares it with request counters, and initializes it with the GIL released. Hugging Face can also choose the full encoder per input when added tokens require it
|
||||
|
||||
The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation
|
||||
|
||||
|
|
@ -8,7 +12,9 @@ The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through t
|
|||
|
||||
The `tiktoken` feature provides `tiktoken::TiktokenTokenizer` through `tiktoken-rs`. Select an encoding with `TokenCounter::from_tiktoken`. The supported names are `cl100k_base`, `o200k_base`, `o200k_harmony`, `p50k_base`, `p50k_edit`, `r50k_base`, and `gpt2`
|
||||
|
||||
All three backends are enabled by default. The Python extension builds with `fast` only, which keeps the wheel at the size it had before the split. With `default-features = false`, callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
|
||||
All three backends are enabled by default in this crate and the Python extension. With `default-features = false`, Rust callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
|
||||
|
||||
Python `tiktoken` and `tokenizers` remain runtime dependencies and the default implementations. The catalog independently selects the tokenizer and request-counting routes. Enabling Rust changes factory dispatch; existing tokenizer objects keep their backend. Native Hugging Face wrappers provide an immutable encoding and decoding API, while training and mutable configuration remain available through the Python backend
|
||||
|
||||
Budget checks, cost calculation, and the `max_tokens` adjustment policy belong to `litellm-core-utils`. The counter does not own prices, budgets, or request limits
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ pub enum Error {
|
|||
JsonUtf8(#[source] FromUtf8Error),
|
||||
#[error("tokenization failed: {0}")]
|
||||
Encode(#[source] Box<dyn std::error::Error + Send + Sync>),
|
||||
#[error("token decoding failed: {0}")]
|
||||
Decode(String),
|
||||
#[error("token counting task failed: {0}")]
|
||||
Task(String),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,31 @@
|
|||
use litellm_token_counter_fast::Error as BackendError;
|
||||
pub use litellm_token_counter_fast::FastTokenizer;
|
||||
|
||||
use crate::{Error, TokenCounter, Tokenizer};
|
||||
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
|
||||
|
||||
pub trait FastCounter: TextCodec {
|
||||
fn fast_counter(&self) -> Option<FastTokenizer>;
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
impl FastCounter for crate::huggingface::HuggingFaceTokenizer {
|
||||
fn fast_counter(&self) -> Option<FastTokenizer> {
|
||||
Some(FastTokenizer::from_shared(self.shared()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
impl FastCounter for crate::tiktoken::TiktokenTokenizer {
|
||||
fn fast_counter(&self) -> Option<FastTokenizer> {
|
||||
let vocabulary = self.vocabulary()?;
|
||||
match self.name() {
|
||||
"cl100k_base" => FastTokenizer::from_cl100k_pairs(vocabulary.ranks()),
|
||||
"o200k_base" | "o200k_harmony" => FastTokenizer::from_o200k_pairs(vocabulary.ranks()),
|
||||
_ => return None,
|
||||
}
|
||||
.ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenCounter {
|
||||
pub fn from_json_fast(tokenizer_json: &str) -> Result<Self, Error> {
|
||||
|
|
@ -39,3 +63,65 @@ impl From<BackendError> for Error {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "huggingface", feature = "tiktoken"))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::huggingface::HuggingFaceTokenizer;
|
||||
use crate::tiktoken::TiktokenTokenizer;
|
||||
|
||||
const TEXTS: [&str; 4] = [
|
||||
"",
|
||||
"hello world <|endoftext|>",
|
||||
"café 漢字 ع 🙂 line\r\n indented 123456789",
|
||||
"<SOS>system<EOT> a\u{301} fi",
|
||||
];
|
||||
|
||||
fn packaged(file: &str) -> String {
|
||||
std::fs::read_to_string(format!(
|
||||
"{}/../../../litellm/litellm_core_utils/tokenizers/{file}",
|
||||
env!("CARGO_MANIFEST_DIR")
|
||||
))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fast_counters_derived_from_codecs_count_like_the_codecs() {
|
||||
let huggingface =
|
||||
HuggingFaceTokenizer::from_json(&packaged("anthropic_tokenizer.json")).unwrap();
|
||||
let fast = huggingface.fast_counter().unwrap();
|
||||
for text in TEXTS {
|
||||
assert_eq!(
|
||||
fast.count_tokens(text).unwrap(),
|
||||
Tokenizer::count_tokens(&huggingface, text).unwrap(),
|
||||
"{text:?}"
|
||||
);
|
||||
}
|
||||
|
||||
for name in ["cl100k_base", "o200k_base", "o200k_harmony"] {
|
||||
let tiktoken =
|
||||
TiktokenTokenizer::from_cached_ranks(name, |file| Ok(packaged(file))).unwrap();
|
||||
let fast = tiktoken.fast_counter().unwrap();
|
||||
for text in TEXTS {
|
||||
assert_eq!(
|
||||
fast.count_tokens(text).unwrap(),
|
||||
tiktoken.count_tokens(text),
|
||||
"{name}: {text:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encodings_without_a_fast_scanner_keep_the_codec() {
|
||||
let tiktoken =
|
||||
TiktokenTokenizer::from_cached_ranks("p50k_base", |file| Ok(packaged(file))).unwrap();
|
||||
assert!(tiktoken.fast_counter().is_none());
|
||||
assert!(
|
||||
TiktokenTokenizer::from_name("cl100k_base")
|
||||
.unwrap()
|
||||
.fast_counter()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
use litellm_token_counter_huggingface::Error as BackendError;
|
||||
pub use litellm_token_counter_huggingface::HuggingFaceTokenizer;
|
||||
pub use litellm_token_counter_huggingface::{
|
||||
AddedToken, EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection,
|
||||
PaddingParams, PaddingStrategy, TruncationDirection, TruncationParams, encoding_from_json,
|
||||
encoding_to_json,
|
||||
};
|
||||
|
||||
use crate::{Error, TokenCounter, Tokenizer};
|
||||
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
|
||||
|
||||
impl TokenCounter {
|
||||
pub fn from_json(tokenizer_json: &str) -> Result<Self, Error> {
|
||||
|
|
@ -17,11 +21,26 @@ impl Tokenizer for HuggingFaceTokenizer {
|
|||
}
|
||||
}
|
||||
|
||||
impl TextCodec for HuggingFaceTokenizer {
|
||||
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
|
||||
HuggingFaceTokenizer::encode(self, text).map_err(Error::from)
|
||||
}
|
||||
|
||||
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
|
||||
HuggingFaceTokenizer::decode(self, ids, skip_special_tokens).map_err(Error::from)
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
HuggingFaceTokenizer::name(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<BackendError> for Error {
|
||||
fn from(error: BackendError) -> Self {
|
||||
match error {
|
||||
BackendError::Load(source) => Self::Load(source),
|
||||
BackendError::Encode(source) => Self::Encode(source),
|
||||
BackendError::Decode(source) => Self::Decode(source.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,5 +20,5 @@ pub mod tiktoken;
|
|||
|
||||
pub use counter::{InputTokenCount, TokenCounter};
|
||||
pub use error::Error;
|
||||
pub use tokenizer::Tokenizer;
|
||||
pub use tokenizer::{TextCodec, Tokenizer};
|
||||
pub use types::CountableRequest;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
pub use litellm_token_counter_tiktoken::TiktokenTokenizer;
|
||||
use litellm_token_counter_tiktoken::UnsupportedTokenizer;
|
||||
use litellm_token_counter_tiktoken::{LoadError, UnsupportedTokenizer};
|
||||
pub use litellm_token_counter_tiktoken::{TiktokenTokenizer, Vocabulary, encoding_for_model};
|
||||
|
||||
use crate::{Error, TokenCounter, Tokenizer};
|
||||
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
|
||||
|
||||
impl TokenCounter {
|
||||
pub fn from_tiktoken(encoding: &str) -> Result<Self, Error> {
|
||||
|
|
@ -17,8 +17,31 @@ impl Tokenizer for TiktokenTokenizer {
|
|||
}
|
||||
}
|
||||
|
||||
impl TextCodec for TiktokenTokenizer {
|
||||
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
|
||||
Ok(TiktokenTokenizer::encode(self, text))
|
||||
}
|
||||
|
||||
fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result<String, Error> {
|
||||
TiktokenTokenizer::decode(self, ids).map_err(|error| Error::Decode(error.to_string()))
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
TiktokenTokenizer::name(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<UnsupportedTokenizer> for Error {
|
||||
fn from(error: UnsupportedTokenizer) -> Self {
|
||||
Self::UnsupportedTokenizer(error.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<LoadError> for Error {
|
||||
fn from(error: LoadError) -> Self {
|
||||
match error {
|
||||
LoadError::Unsupported(error) => error.into(),
|
||||
LoadError::Ranks(message) => Self::Ranks(message),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,12 @@ pub trait Tokenizer: Send + Sync {
|
|||
fn count_tokens(&self, text: &str) -> Result<usize, Error>;
|
||||
}
|
||||
|
||||
pub trait TextCodec: Tokenizer {
|
||||
fn encode(&self, text: &str) -> Result<Vec<u32>, Error>;
|
||||
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error>;
|
||||
fn name(&self) -> &str;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -58,7 +58,8 @@ from ._lazy_imports_registry import (
|
|||
|
||||
if TYPE_CHECKING:
|
||||
import httpx
|
||||
from tiktoken import Encoding
|
||||
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
def get_litellm_globals() -> dict[str, object]:
|
||||
|
|
@ -89,26 +90,11 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo
|
|||
# These are special lazy loaders for things that are used internally
|
||||
# They're separate from the main lazy import system because they have specific use cases
|
||||
|
||||
# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup
|
||||
_default_encoding: "Encoding | None" = None
|
||||
|
||||
def _get_default_encoding() -> "Tokenizer":
|
||||
from litellm.rust_bridge.tokenizer import get_encoding
|
||||
|
||||
def _get_default_encoding() -> "Encoding":
|
||||
"""
|
||||
Lazily load and cache the default OpenAI encoding.
|
||||
|
||||
This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken)
|
||||
at `litellm` import time. The encoding is cached after the first import.
|
||||
|
||||
This is used internally by utils.py functions that need the encoding but shouldn't
|
||||
trigger its import during module load.
|
||||
"""
|
||||
global _default_encoding
|
||||
if _default_encoding is None:
|
||||
from litellm.litellm_core_utils.default_encoding import encoding
|
||||
|
||||
_default_encoding = encoding
|
||||
return _default_encoding
|
||||
return get_encoding("cl100k_base")
|
||||
|
||||
|
||||
# Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time
|
||||
|
|
|
|||
|
|
@ -6,8 +6,9 @@ Core files:
|
|||
- `streaming_handler.py`: The core streaming logic + streaming related helper utils
|
||||
- `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`.
|
||||
- `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types.
|
||||
- `default_encoding.py`: code for loading the default encoding (tiktoken)
|
||||
- `default_encoding.py`: code for loading the default Python tokenizer and bundled cache
|
||||
- `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name.
|
||||
- `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
|
||||
- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
|
||||
|
||||
Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import os
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -15,20 +14,6 @@ except (ImportError, AttributeError):
|
|||
|
||||
filename = pkg_resources.resource_filename(__name__, "litellm_core_utils/tokenizers")
|
||||
|
||||
CL100K_BASE_RANK_FILE: Final = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
|
||||
O200K_BASE_RANK_FILE: Final = "fb374d419588a4632f3f557e76b4b70aebbca790"
|
||||
|
||||
|
||||
def cl100k_base_rank_file() -> str:
|
||||
"""The vendored tiktoken `cl100k_base` rank file (`base64(token) rank` lines)."""
|
||||
return Path(filename, CL100K_BASE_RANK_FILE).read_text(encoding="ascii")
|
||||
|
||||
|
||||
def o200k_base_rank_file() -> str:
|
||||
"""The vendored tiktoken `o200k_base` rank file (`base64(token) rank` lines)."""
|
||||
return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii")
|
||||
|
||||
|
||||
# Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory
|
||||
# unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR.
|
||||
# This keeps tiktoken fully offline-capable by default (see #1071).
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import anyio
|
|||
import anyio.lowlevel
|
||||
import httpx
|
||||
import tiktoken
|
||||
from tokenizers import Tokenizer
|
||||
from typing_extensions import ParamSpec, TypeVar
|
||||
|
||||
import litellm
|
||||
|
|
@ -30,8 +29,10 @@ from litellm.constants import (
|
|||
TOKEN_COUNTER_MAX_EXACT_CHARS,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding
|
||||
from litellm.litellm_core_utils.url_utils import safe_get
|
||||
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
|
||||
from litellm.rust_bridge.tokenizer import get_encoding
|
||||
from litellm.types.llms.anthropic import (
|
||||
AnthropicContentParamSource,
|
||||
AnthropicContentParamSourceFileId,
|
||||
|
|
@ -622,9 +623,11 @@ def _get_exact_count_function(
|
|||
if model is not None or custom_tokenizer is not None:
|
||||
tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model)
|
||||
if tokenizer_json["type"] == "huggingface_tokenizer":
|
||||
tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"]
|
||||
tokenizer: Final[HuggingFace] = tokenizer_json["tokenizer"]
|
||||
|
||||
def count_tokens(text: str) -> int:
|
||||
if isinstance(tokenizer, HuggingFaceTokenizer):
|
||||
return tokenizer.count(text)
|
||||
return len(tokenizer.encode_batch_fast([text])[0])
|
||||
|
||||
return count_tokens
|
||||
|
|
@ -632,31 +635,43 @@ def _get_exact_count_function(
|
|||
encoding: Final = openai_tokenizer_encoding(model)
|
||||
|
||||
def encode_length(text: str) -> int:
|
||||
return len(encoding.encode(text, disallowed_special=()))
|
||||
return _encoding_count(encoding, text)
|
||||
|
||||
return _get_tiktoken_count_function(encode_length)
|
||||
else:
|
||||
raise ValueError("Unsupported tokenizer type")
|
||||
else:
|
||||
default_encoding: Final = _get_default_encoding()
|
||||
|
||||
def encode_length(text: str) -> int:
|
||||
return len(_get_default_encoding().encode(text, disallowed_special=()))
|
||||
return _encoding_count(default_encoding, text)
|
||||
|
||||
return _get_tiktoken_count_function(encode_length)
|
||||
|
||||
|
||||
def openai_tokenizer_encoding(model: str) -> tiktoken.Encoding:
|
||||
"""The tiktoken encoding `token_counter` uses for a model on the `openai_tokenizer` path."""
|
||||
def _encoding_count(encoding: Encoding, text: str) -> int:
|
||||
if isinstance(encoding, OpenAIEncoding):
|
||||
return encoding.count(text)
|
||||
return len(encoding.encode(text, disallowed_special=()))
|
||||
|
||||
|
||||
def openai_tokenizer_encoding(model: str) -> Encoding:
|
||||
"""The encoding `token_counter` uses for a model on the `openai_tokenizer` path."""
|
||||
return get_encoding(openai_tokenizer_encoding_name(model))
|
||||
|
||||
|
||||
def openai_tokenizer_encoding_name(model: str) -> str:
|
||||
"""The tiktoken encoding name for `model`, without loading the encoding."""
|
||||
from litellm.utils import print_verbose
|
||||
|
||||
model_to_use: Final = _fix_model_name(model)
|
||||
if "gpt-4o" in model_to_use:
|
||||
return tiktoken.get_encoding("o200k_base")
|
||||
return "o200k_base"
|
||||
try:
|
||||
return tiktoken.encoding_for_model(model_to_use)
|
||||
return tiktoken.encoding_name_for_model(model_to_use)
|
||||
except KeyError:
|
||||
print_verbose("Warning: model not found. Using cl100k_base encoding.")
|
||||
return tiktoken.get_encoding("cl100k_base")
|
||||
return "cl100k_base"
|
||||
|
||||
|
||||
def uses_legacy_message_accounting(model: str) -> bool:
|
||||
|
|
|
|||
402
litellm/litellm_core_utils/tokenizer.py
Normal file
402
litellm/litellm_core_utils/tokenizer.py
Normal file
|
|
@ -0,0 +1,402 @@
|
|||
"""Python faces of the Rust text codecs.
|
||||
|
||||
``OpenAIEncoding`` mirrors ``tiktoken.Encoding`` and ``HuggingFaceTokenizer`` mirrors
|
||||
``tokenizers.Tokenizer``, so a caller holding ``litellm.encoding`` or the object returned by
|
||||
``litellm.create_tokenizer`` sees the same read-only surface whichever backend the Rust catalog
|
||||
selected. Both wrappers are immutable: ``tokenizers`` mutators (``enable_padding``,
|
||||
``enable_truncation``, ``add_tokens``) stay on the Python tokenizer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Collection, Mapping, Sequence, Set
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable
|
||||
|
||||
import tiktoken
|
||||
from tokenizers import AddedToken
|
||||
from tokenizers import Tokenizer as PythonHuggingFaceTokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
from litellm.rust_bridge._native import HuggingFaceEncoding
|
||||
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
|
||||
|
||||
SpecialTokens: TypeAlias = Literal["all"] | Collection[str]
|
||||
AllowedSpecial: TypeAlias = Literal["all"] | Set[str]
|
||||
HuggingFaceInput: TypeAlias = str | list[str] | tuple[str, ...]
|
||||
HuggingFaceBatchInput: TypeAlias = HuggingFaceInput | tuple[HuggingFaceInput, HuggingFaceInput] | list[HuggingFaceInput]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OpenAIEncoding:
|
||||
"""``tiktoken.Encoding`` over the Rust tiktoken codec."""
|
||||
|
||||
_native: NativeTokenizer
|
||||
_special_tokens: Mapping[str, int]
|
||||
|
||||
@staticmethod
|
||||
def wrap(native: NativeTokenizer) -> OpenAIEncoding:
|
||||
return OpenAIEncoding(native, MappingProxyType(native.special_tokens()))
|
||||
|
||||
@staticmethod
|
||||
def from_tiktoken(encoding: str) -> OpenAIEncoding:
|
||||
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
|
||||
|
||||
return OpenAIEncoding.wrap(NativeTokenizer.from_tiktoken(encoding))
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<Encoding {self.name!r}>"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._native.name
|
||||
|
||||
@property
|
||||
def max_token_value(self) -> int:
|
||||
return self._native.max_token_value()
|
||||
|
||||
@property
|
||||
def n_vocab(self) -> int:
|
||||
"""For backwards compatibility. Prefer to use `enc.max_token_value + 1`."""
|
||||
return self.max_token_value + 1
|
||||
|
||||
@property
|
||||
def eot_token(self) -> int:
|
||||
return self._special_tokens["<|endoftext|>"]
|
||||
|
||||
@property
|
||||
def special_tokens_set(self) -> set[str]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return set(self._special_tokens)
|
||||
|
||||
def is_special_token(self, token: int) -> bool:
|
||||
return self._native.is_special_token(token)
|
||||
|
||||
# ---- encoding -------------------------------------------------------------------------
|
||||
|
||||
def encode_ordinary(self, text: str) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._native.encode(text)
|
||||
|
||||
def encode(
|
||||
self,
|
||||
text: str,
|
||||
*,
|
||||
allowed_special: AllowedSpecial = frozenset(),
|
||||
disallowed_special: SpecialTokens = "all",
|
||||
) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
allowed: Final = self._allowed(text, allowed_special, disallowed_special)
|
||||
if not allowed:
|
||||
return self.encode_ordinary(text)
|
||||
return self._native.encode_special(text, tuple(allowed))
|
||||
|
||||
def encode_to_numpy(
|
||||
self,
|
||||
text: str,
|
||||
*,
|
||||
allowed_special: AllowedSpecial = frozenset(),
|
||||
disallowed_special: SpecialTokens = "all",
|
||||
) -> npt.NDArray[np.uint32]:
|
||||
import numpy
|
||||
|
||||
return numpy.asarray(
|
||||
self.encode(text, allowed_special=allowed_special, disallowed_special=disallowed_special),
|
||||
dtype=numpy.uint32,
|
||||
)
|
||||
|
||||
def encode_ordinary_batch(
|
||||
self, text: Sequence[str], *, num_threads: int = 8
|
||||
) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
with ThreadPoolExecutor(num_threads) as executor:
|
||||
return list( # mutable-ok: [LIT002] SDK returns a list
|
||||
executor.map(self.encode_ordinary, text)
|
||||
)
|
||||
|
||||
def encode_batch(
|
||||
self,
|
||||
text: Sequence[str],
|
||||
*,
|
||||
num_threads: int = 8,
|
||||
allowed_special: AllowedSpecial = frozenset(),
|
||||
disallowed_special: SpecialTokens = "all",
|
||||
) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
encode: Final = partial(self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special)
|
||||
with ThreadPoolExecutor(num_threads) as executor:
|
||||
return list( # mutable-ok: [LIT002] SDK returns a list
|
||||
executor.map(encode, text)
|
||||
)
|
||||
|
||||
def encode_with_unstable(
|
||||
self,
|
||||
text: str,
|
||||
*,
|
||||
allowed_special: AllowedSpecial = frozenset(),
|
||||
disallowed_special: SpecialTokens = "all",
|
||||
) -> tuple[list[int], list[list[int]]]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
"""The stable tokens of `text` and every completion its unstable tail could become.
|
||||
|
||||
Completions come back sorted; tiktoken returns them in hash order."""
|
||||
allowed: Final = self._allowed(text, allowed_special, disallowed_special)
|
||||
return self._native.encode_with_unstable(text, tuple(allowed))
|
||||
|
||||
def encode_single_token(self, text_or_bytes: str | bytes) -> int:
|
||||
"""The token of one whole piece, special tokens included. Raises `KeyError` otherwise."""
|
||||
piece: Final = text_or_bytes.encode("utf-8") if isinstance(text_or_bytes, str) else text_or_bytes
|
||||
return self._native.encode_single_token(piece)
|
||||
|
||||
def count(self, text: str, fast: bool = False) -> int:
|
||||
"""Count ordinary text; `fast` accelerates supported encodings and otherwise counts normally."""
|
||||
return self._native.count(text, fast)
|
||||
|
||||
# ---- decoding -------------------------------------------------------------------------
|
||||
|
||||
def decode_bytes(self, tokens: Sequence[int]) -> bytes:
|
||||
return self._native.decode_bytes(tokens)
|
||||
|
||||
def decode(self, tokens: Sequence[int], errors: str = "replace") -> str:
|
||||
return self.decode_bytes(tokens).decode("utf-8", errors=errors)
|
||||
|
||||
def decode_single_token_bytes(self, token: int) -> bytes:
|
||||
return self.decode_bytes((token,))
|
||||
|
||||
def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return [ # mutable-ok: [LIT002] SDK returns a list
|
||||
self.decode_single_token_bytes(token) for token in tokens
|
||||
]
|
||||
|
||||
def decode_with_offsets(
|
||||
self, tokens: Sequence[int]
|
||||
) -> tuple[str, list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
"""The decoded text and, per token, the index of the first character holding its bytes.
|
||||
|
||||
Like tiktoken, raises `UnicodeDecodeError` when the tokens do not decode to valid UTF-8."""
|
||||
token_bytes: Final = self.decode_tokens_bytes(tokens)
|
||||
text_len = 0
|
||||
offsets: Final[list[int]] = [] # mutable-ok: [LIT001] local accumulator
|
||||
for token in token_bytes:
|
||||
offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0)))
|
||||
text_len += sum(1 for c in token if not 0x80 <= c < 0xC0)
|
||||
return b"".join(token_bytes).decode("utf-8", errors="strict"), offsets
|
||||
|
||||
def decode_batch(
|
||||
self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8
|
||||
) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
with ThreadPoolExecutor(num_threads) as executor:
|
||||
return list( # mutable-ok: [LIT002] SDK returns a list
|
||||
executor.map(partial(self.decode, errors=errors), batch)
|
||||
)
|
||||
|
||||
def decode_bytes_batch(
|
||||
self, batch: Sequence[Sequence[int]], *, num_threads: int = 8
|
||||
) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
with ThreadPoolExecutor(num_threads) as executor:
|
||||
return list( # mutable-ok: [LIT002] SDK returns a list
|
||||
executor.map(self.decode_bytes, batch)
|
||||
)
|
||||
|
||||
def token_byte_values(self) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._native.token_byte_values()
|
||||
|
||||
def __reduce__(self) -> tuple[Callable[[str], OpenAIEncoding], tuple[str]]:
|
||||
return (OpenAIEncoding.from_tiktoken, (self.name,))
|
||||
|
||||
# ---- private --------------------------------------------------------------------------
|
||||
|
||||
def _allowed(self, text: str, allowed_special: AllowedSpecial, disallowed_special: SpecialTokens) -> frozenset[str]:
|
||||
"""tiktoken's special-token policy: which specials `text` may encode, after rejecting
|
||||
any it must not contain."""
|
||||
allowed: Final = frozenset(self._special_tokens) if allowed_special == "all" else frozenset(allowed_special)
|
||||
disallowed: Final = (
|
||||
frozenset(self._special_tokens) - allowed if disallowed_special == "all" else frozenset(disallowed_special)
|
||||
)
|
||||
for token in disallowed:
|
||||
if token in text:
|
||||
raise ValueError(
|
||||
f"Encountered text corresponding to disallowed special token {token!r}.\n"
|
||||
"If you want this text to be encoded as a special token, "
|
||||
f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n"
|
||||
"If you want this text to be encoded as normal text, disable the check for this token "
|
||||
f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n"
|
||||
"To disable this check for all special tokens, pass `disallowed_special=()`.\n"
|
||||
)
|
||||
return allowed
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HuggingFaceTokenizer:
|
||||
"""The read-only ``tokenizers.Tokenizer`` surface over the Rust Hugging Face codec."""
|
||||
|
||||
_native: NativeTokenizer
|
||||
|
||||
@staticmethod
|
||||
def from_str(json: str) -> HuggingFaceTokenizer:
|
||||
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
|
||||
|
||||
return HuggingFaceTokenizer(NativeTokenizer.from_json(json))
|
||||
|
||||
from_json = from_str
|
||||
|
||||
@staticmethod
|
||||
def from_buffer(buffer: bytes) -> HuggingFaceTokenizer:
|
||||
return HuggingFaceTokenizer.from_str(buffer.decode("utf-8"))
|
||||
|
||||
@staticmethod
|
||||
def from_file(path: str) -> HuggingFaceTokenizer:
|
||||
return HuggingFaceTokenizer.from_str(Path(path).read_text(encoding="utf-8"))
|
||||
|
||||
@staticmethod
|
||||
def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFaceTokenizer:
|
||||
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
|
||||
|
||||
return HuggingFaceTokenizer(NativeTokenizer.from_pretrained(identifier, revision=revision, token=token))
|
||||
|
||||
def to_str(self, pretty: bool = False) -> str:
|
||||
return self._native.to_json(pretty)
|
||||
|
||||
def save(self, path: str, pretty: bool = True) -> None:
|
||||
Path(path).write_text(self.to_str(pretty), encoding="utf-8")
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._native.name
|
||||
|
||||
# ---- vocabulary -----------------------------------------------------------------------
|
||||
|
||||
def token_to_id(self, token: str) -> int | None:
|
||||
return self._native.token_to_id(token)
|
||||
|
||||
def id_to_token(self, id: int) -> str | None:
|
||||
return self._native.id_to_token(id)
|
||||
|
||||
def get_vocab(
|
||||
self, with_added_tokens: bool = True
|
||||
) -> dict[str, int]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._native.get_vocab(with_added_tokens)
|
||||
|
||||
def get_vocab_size(self, with_added_tokens: bool = True) -> int:
|
||||
return self._native.get_vocab_size(with_added_tokens)
|
||||
|
||||
def get_added_tokens_decoder(self) -> dict[int, AddedToken]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return { # mutable-ok: [LIT002] SDK returns a dict
|
||||
token_id: AddedToken(
|
||||
content, single_word=single_word, lstrip=lstrip, rstrip=rstrip, normalized=normalized, special=special
|
||||
)
|
||||
for token_id, (
|
||||
content,
|
||||
single_word,
|
||||
lstrip,
|
||||
rstrip,
|
||||
normalized,
|
||||
special,
|
||||
) in self._native.added_tokens_decoder()
|
||||
}
|
||||
|
||||
def num_special_tokens_to_add(self, is_pair: bool) -> int:
|
||||
return self._native.num_special_tokens_to_add(is_pair)
|
||||
|
||||
@property
|
||||
def padding(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._native.padding()
|
||||
|
||||
@property
|
||||
def truncation(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._native.truncation()
|
||||
|
||||
@property
|
||||
def encode_special_tokens(self) -> bool:
|
||||
return self._native.encode_special_tokens()
|
||||
|
||||
# ---- encoding and decoding ------------------------------------------------------------
|
||||
|
||||
def encode(
|
||||
self,
|
||||
sequence: HuggingFaceInput,
|
||||
pair: HuggingFaceInput | None = None,
|
||||
is_pretokenized: bool = False,
|
||||
add_special_tokens: bool = True,
|
||||
) -> HuggingFaceEncoding:
|
||||
return self._native.encode_huggingface(sequence, pair, is_pretokenized, add_special_tokens)
|
||||
|
||||
def encode_batch(
|
||||
self,
|
||||
input: Sequence[HuggingFaceBatchInput],
|
||||
is_pretokenized: bool = False,
|
||||
add_special_tokens: bool = True,
|
||||
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=False)
|
||||
|
||||
def encode_batch_fast(
|
||||
self,
|
||||
input: Sequence[HuggingFaceBatchInput],
|
||||
is_pretokenized: bool = False,
|
||||
add_special_tokens: bool = True,
|
||||
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=True)
|
||||
|
||||
def _encode_batch(
|
||||
self, input: Sequence[HuggingFaceBatchInput], is_pretokenized: bool, add_special_tokens: bool, fast: bool
|
||||
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
sequences: Final = tuple(_batch_input(item, is_pretokenized) for item in input)
|
||||
return self._native.encode_batch_huggingface(sequences, is_pretokenized, add_special_tokens, fast)
|
||||
|
||||
def count(self, text: str, fast: bool = False) -> int:
|
||||
"""Count with this tokenizer's configuration; `fast` uses acceleration where supported."""
|
||||
return self._native.count(text, fast)
|
||||
|
||||
def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str:
|
||||
return self._native.decode(ids, skip_special_tokens=skip_special_tokens)
|
||||
|
||||
def decode_batch(
|
||||
self, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True
|
||||
) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return [ # mutable-ok: [LIT002] SDK returns a list
|
||||
self.decode(ids, skip_special_tokens=skip_special_tokens) for ids in sequences
|
||||
]
|
||||
|
||||
def __reduce__(self) -> tuple[Callable[[str], HuggingFaceTokenizer], tuple[str]]:
|
||||
return (HuggingFaceTokenizer.from_str, (self.to_str(),))
|
||||
|
||||
|
||||
def _batch_input(
|
||||
item: HuggingFaceBatchInput, is_pretokenized: bool
|
||||
) -> tuple[HuggingFaceInput, HuggingFaceInput | None]:
|
||||
if isinstance(item, str):
|
||||
return (item, None)
|
||||
if is_pretokenized and all(isinstance(word, str) for word in item):
|
||||
return (tuple(word for word in item if isinstance(word, str)), None)
|
||||
if len(item) != 2:
|
||||
raise TypeError("batch input must be a sequence or a pair of sequences")
|
||||
return (item[0], item[1])
|
||||
|
||||
|
||||
Encoding: TypeAlias = tiktoken.Encoding | OpenAIEncoding
|
||||
HuggingFace: TypeAlias = PythonHuggingFaceTokenizer | HuggingFaceTokenizer
|
||||
Tokenizer: TypeAlias = Encoding | HuggingFace
|
||||
|
||||
|
||||
class _AddedToken(Protocol):
|
||||
@property
|
||||
def special(self) -> bool: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _AddedTokenDecoder(Protocol):
|
||||
def get_added_tokens_decoder(self) -> Mapping[int, _AddedToken]: ...
|
||||
|
||||
|
||||
def strip_special_tokens(tokenizer: object, tokens: Sequence[int]) -> Sequence[int]:
|
||||
"""Drop the special added tokens before a Python `tokenizers` decode; the Rust codec's
|
||||
`decode(skip_special_tokens=True)` already does this itself."""
|
||||
if isinstance(tokenizer, HuggingFaceTokenizer) or not isinstance(tokenizer, _AddedTokenDecoder):
|
||||
return tokens
|
||||
try:
|
||||
added: Final = tokenizer.get_added_tokens_decoder()
|
||||
except Exception: # noqa: BLE001 # optional metadata failures historically fall back to decoding
|
||||
return tokens
|
||||
special_ids: Final = frozenset(token_id for token_id, token in added.items() if token.special)
|
||||
return tuple(token for token in tokens if token not in special_ids)
|
||||
|
|
@ -23,9 +23,8 @@ from ..common_utils import (
|
|||
from .streaming_iterator import A2AModelResponseIterator
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
_REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = (
|
||||
|
|
@ -292,7 +291,7 @@ class A2AConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -14,9 +14,8 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.utils import ImageObject, ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -171,7 +170,7 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -16,9 +16,8 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import Choices, ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -68,7 +67,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from litellm.types.utils import ModelResponse
|
|||
from ...openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AmazonNovaChatConfig(OpenAILikeChatConfig):
|
||||
|
|
@ -86,7 +86,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
|
|||
from litellm.types.utils import LiteLLMBatch, LlmProviders, ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LoggingClass = LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -290,7 +289,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -100,9 +100,8 @@ from ..common_utils import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LoggingClass = LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -2688,7 +2687,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AnthropicTextError(BaseLLMException):
|
||||
|
|
@ -185,7 +185,7 @@ class AnthropicTextConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -29,9 +29,8 @@ from ...base_llm.chat.transformation import BaseConfig
|
|||
from ..common_utils import AzureOpenAIError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LoggingClass = LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -304,7 +303,7 @@ class AzureOpenAIConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
|
|
@ -297,7 +296,7 @@ class AzureAIAgentsConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AzureModelRouterConfig(AzureAIStudioConfig):
|
||||
|
|
@ -59,7 +59,7 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ from litellm.types.utils import ModelResponse, ProviderField
|
|||
from litellm.utils import _add_path_to_api_base, supports_tool_choice
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AzureFoundryErrorStrings(str, enum.Enum):
|
||||
|
|
@ -305,7 +305,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -12,9 +12,10 @@ from litellm.types.utils import ImageResponse
|
|||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5)."""
|
||||
|
|
@ -245,7 +246,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -12,9 +12,8 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -121,7 +120,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@ from collections.abc import AsyncIterator, Iterator
|
|||
from typing import TYPE_CHECKING, Union
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from pydantic import BaseModel
|
||||
|
||||
from litellm import LiteLLMLoggingObj, ModelResponse
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
|
|
@ -39,7 +39,7 @@ class CompletionTransformationBridge(ABC):
|
|||
messages: list["AllMessageValues"],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> "ModelResponse":
|
||||
|
|
|
|||
|
|
@ -21,9 +21,8 @@ from litellm.types.llms.openai import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
from ..base_utils import (
|
||||
|
|
@ -344,7 +343,7 @@ class BaseConfig(ABC):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> "ModelResponse":
|
||||
|
|
|
|||
|
|
@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
|
|||
from litellm.types.utils import ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -68,7 +67,7 @@ class BaseTextCompletionConfig(BaseConfig, ABC):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
|
|||
from litellm.types.utils import EmbeddingResponse, ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -80,7 +79,7 @@ class BaseEmbeddingConfig(BaseConfig, ABC):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -21,9 +21,8 @@ from litellm.types.utils import LlmProviders, ModelResponse
|
|||
from ..chat.transformation import BaseConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.router import Router as _Router
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
|
|
@ -231,7 +230,7 @@ class BaseFilesConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -11,9 +11,8 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.utils import ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -93,7 +92,7 @@ class BaseImageGenerationConfig(ABC):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -17,9 +17,8 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -82,7 +81,7 @@ class BaseImageVariationConfig(BaseConfig, ABC):
|
|||
image: FileTypes,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
) -> ImageResponse:
|
||||
pass
|
||||
|
|
@ -98,7 +97,7 @@ class BaseImageVariationConfig(BaseConfig, ABC):
|
|||
image: FileTypes,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
) -> ImageResponse:
|
||||
pass
|
||||
|
|
@ -125,7 +124,7 @@ class BaseImageVariationConfig(BaseConfig, ABC):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -40,9 +40,8 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
|
|
@ -990,7 +989,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ from ..common_utils import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
# Computer use tool prefixes supported by Bedrock
|
||||
BEDROCK_COMPUTER_USE_TOOLS: Final = [
|
||||
|
|
@ -1920,7 +1920,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -37,9 +37,8 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -438,7 +437,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ from litellm.types.utils import (
|
|||
from .amazon_llama_transformation import AmazonLlamaConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AmazonDeepSeekR1Config(AmazonLlamaConfig):
|
||||
|
|
@ -39,7 +39,7 @@ class AmazonDeepSeekR1Config(AmazonLlamaConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -21,9 +21,8 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import Choices
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
|
|
@ -198,7 +197,7 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> "ModelResponse":
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ from ..converse_transformation import AmazonConverseConfig
|
|||
from .base_invoke_transformation import AmazonInvokeConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
_CachePointCarrier = TypeVar("_CachePointCarrier", SystemContentBlock, ContentBlock)
|
||||
_INJECTION_POINTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
|
@ -128,7 +128,7 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AmazonQwen2Config(AmazonQwen3Config):
|
||||
|
|
@ -44,7 +44,7 @@ class AmazonQwen2Config(AmazonQwen3Config):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
|
||||
|
|
@ -170,7 +170,7 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, Usage
|
|||
from litellm.utils import get_base64_str
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -190,7 +189,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -359,7 +358,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -34,9 +34,8 @@ from litellm.types.utils import ModelResponse, Usage
|
|||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -288,7 +287,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -29,9 +29,8 @@ from ..common_utils import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -258,7 +257,7 @@ class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""
|
||||
Brave Search API module.
|
||||
"""
|
||||
|
||||
from litellm.llms.brave.search.transformation import BraveSearchConfig
|
||||
|
||||
__all__ = ["BraveSearchConfig"]
|
||||
"""
|
||||
Brave Search API module.
|
||||
"""
|
||||
|
||||
from litellm.llms.brave.search.transformation import BraveSearchConfig
|
||||
|
||||
__all__ = ["BraveSearchConfig"]
|
||||
|
|
|
|||
|
|
@ -23,9 +23,8 @@ from litellm.utils import CustomStreamWrapper, ModelResponse, Usage
|
|||
from ..common_utils import API_BASE, BytezError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -187,7 +186,7 @@ class BytezChatConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse
|
|||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -87,7 +86,7 @@ class ClarifaiConfig(OpenAIGPTConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -15,9 +15,8 @@ from ..common_utils import ModelResponseIterator as CohereModelResponseIterator
|
|||
from ..common_utils import validate_environment as cohere_validate_environment
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -227,7 +226,7 @@ class CohereChatConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -20,9 +20,8 @@ from ..common_utils import CohereError, CohereV2ModelResponseIterator
|
|||
from ..common_utils import validate_environment as cohere_validate_environment
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -191,7 +190,7 @@ class CohereV2ChatConfig(OpenAIGPTConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from litellm.types.utils import EmbeddingResponse
|
|||
from .v1_transformation import CohereEmbeddingConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
def validate_environment(api_key, headers: dict):
|
||||
|
|
@ -60,7 +60,7 @@ async def async_embedding(
|
|||
api_base: str,
|
||||
api_key: str | None,
|
||||
headers: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
client: AsyncHTTPHandler | None = None,
|
||||
):
|
||||
## LOGGING
|
||||
|
|
@ -122,7 +122,7 @@ def embedding(
|
|||
logging_obj: LiteLLMLoggingObj,
|
||||
optional_params: dict,
|
||||
headers: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
data: dict | CohereEmbeddingRequest | None = None,
|
||||
complete_api_base: str | None = None,
|
||||
api_key: str | None = None,
|
||||
|
|
|
|||
|
|
@ -13,9 +13,8 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.utils import ImageObject, ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -132,7 +131,7 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -17,9 +17,8 @@ from litellm.types.utils import ModelResponse
|
|||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -66,7 +65,7 @@ class CompactifAIChatConfig(OpenAIGPTConfig):
|
|||
messages: Sequence[AllMessageValues],
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -26,9 +26,8 @@ from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProv
|
|||
from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -268,7 +267,7 @@ class BaseLLMAIOHTTPHandler:
|
|||
messages: list,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
client: ClientSession | None = None,
|
||||
):
|
||||
|
|
|
|||
|
|
@ -192,12 +192,12 @@ def _rust_responses_websocket_enabled(
|
|||
from .http_handler import get_shared_realtime_ssl_context
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from aiohttp import ClientSession
|
||||
from websockets.asyncio.client import ClientConnection
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
|
|
@ -493,7 +493,7 @@ class BaseLLMHTTPHandler:
|
|||
messages: list,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
client: AsyncHTTPHandler | None = None,
|
||||
json_mode: bool = False,
|
||||
|
|
@ -559,7 +559,7 @@ class BaseLLMHTTPHandler:
|
|||
api_base: str | None,
|
||||
custom_llm_provider: str,
|
||||
model_response: ModelResponse,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
optional_params: dict,
|
||||
timeout: float | httpx.Timeout,
|
||||
|
|
|
|||
|
|
@ -38,9 +38,8 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.utils import ImageObject, ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -165,7 +164,7 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -149,9 +149,8 @@ def _split_parallel_tool_calls(messages: list[AllMessageValues]) -> list[AllMess
|
|||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -189,7 +188,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
return "databricks"
|
||||
|
||||
@classmethod
|
||||
def get_config(cls):
|
||||
def get_config(cls, *, model: str | None = None):
|
||||
return super().get_config()
|
||||
|
||||
def get_required_params(self) -> list[ProviderField]:
|
||||
|
|
@ -651,7 +650,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -277,12 +277,7 @@ def completion(
|
|||
|
||||
## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
|
||||
prompt_tokens: Final = len(encoding.encode(prompt))
|
||||
completion_tokens: Final = len(
|
||||
encoding.encode(
|
||||
model_response["choices"][0]["message"]["content"],
|
||||
disallowed_special=(),
|
||||
)
|
||||
)
|
||||
completion_tokens: Final = len(encoding.encode(model_response["choices"][0]["message"]["content"]))
|
||||
|
||||
model_response.created = int(time.time())
|
||||
model_response.model = model
|
||||
|
|
|
|||
|
|
@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
|
|||
from ..common_utils import EdenAIException, reported_cost, resolve_api_base, resolve_api_key
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding
|
||||
|
||||
_OPTIONAL_MAPPING: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
|
||||
|
|
@ -97,7 +96,7 @@ class EdenAIChatConfig(OpenAIGPTConfig):
|
|||
messages: list[AllMessageValues], # mutable-ok: inherited contract
|
||||
optional_params: dict[str, object], # mutable-ok: inherited contract
|
||||
litellm_params: dict[str, object], # mutable-ok: inherited contract
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Encoding | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -19,9 +19,8 @@ from litellm.utils import convert_to_model_response_object
|
|||
from ..common_utils import EdenAIException, endpoint_url, json_headers, pick, reported_cost
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding
|
||||
|
||||
_SUPPORTED_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = (
|
||||
"background",
|
||||
|
|
@ -94,7 +93,7 @@ class EdenAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
request_data: dict[str, object], # mutable-ok: inherited contract
|
||||
optional_params: dict[str, object], # mutable-ok: inherited contract
|
||||
litellm_params: dict[str, object], # mutable-ok: inherited contract
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Encoding | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse
|
|||
from .transformation import FalAIBaseConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -187,7 +186,7 @@ class FalAIBriaConfig(FalAIBaseConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -8,9 +8,8 @@ from litellm.types.utils import ImageResponse
|
|||
from .transformation import FalAIBaseConfig, fal_images_to_image_objects
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -194,7 +193,7 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse
|
|||
from .transformation import FalAIBaseConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -150,7 +149,7 @@ class FalAIIdeogramV3Config(FalAIBaseConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse
|
|||
from .transformation import FalAIBaseConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -182,7 +181,7 @@ class FalAIImagen4Config(FalAIBaseConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse
|
|||
from .transformation import FalAIBaseConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -172,7 +171,7 @@ class FalAIRecraftV3Config(FalAIBaseConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse
|
|||
from .transformation import FalAIBaseConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -208,7 +207,7 @@ class FalAIStableDiffusionConfig(FalAIBaseConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -16,9 +16,8 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.utils import ImageObject, ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -117,7 +116,7 @@ class FalAIBaseConfig(BaseImageGenerationConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ from ..common_utils import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
def _map_reasoning_effort(value: object) -> object:
|
||||
|
|
@ -708,7 +708,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -24,9 +24,8 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.utils import ImageObject, ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -173,7 +172,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -26,9 +26,8 @@ from ..authenticator import get_access_token
|
|||
from ..file_handler import upload_file_sync
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -416,7 +415,7 @@ class GigaChatConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: tiktoken.Encoding | None,
|
||||
encoding: Tokenizer | None,
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, ServerToolUs
|
|||
from ...openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
GROQ_COMPOUND_MODELS: Final = frozenset({"compound", "compound-mini"})
|
||||
|
||||
|
|
@ -286,7 +286,7 @@ class GroqChatConfig(OpenAILikeChatConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import json
|
||||
import os
|
||||
from collections.abc import Sequence
|
||||
from typing import Final, Literal, Protocol, get_args
|
||||
|
||||
import httpx
|
||||
|
|
@ -32,7 +31,7 @@ hf_tasks_embeddings: Final = (
|
|||
class _SupportsTokenEncode(Protocol):
|
||||
"""Token encoder handle. Only ``encode`` is ever called on it here."""
|
||||
|
||||
def encode(self, text: str, *, disallowed_special: tuple[str, ...]) -> Sequence[int]: ...
|
||||
def encode(self, text: str) -> list[int]: ...
|
||||
|
||||
|
||||
def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> str | None:
|
||||
|
|
@ -214,7 +213,7 @@ class HuggingFaceEmbedding(BaseLLM):
|
|||
model_response.model = model
|
||||
input_tokens = 0
|
||||
for text in input:
|
||||
input_tokens += len(encoding.encode(text, disallowed_special=()))
|
||||
input_tokens += len(encoding.encode_ordinary(text))
|
||||
|
||||
setattr(
|
||||
model_response,
|
||||
|
|
|
|||
|
|
@ -25,9 +25,8 @@ from litellm.utils import token_counter
|
|||
from ..common_utils import HuggingFaceError, hf_task_list, hf_tasks, output_parser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LoggingClass = LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -479,7 +478,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
|
|
@ -225,7 +224,7 @@ class LangFlowConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -23,9 +23,8 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
|
|
@ -415,7 +414,7 @@ class LangGraphConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from litellm.types.utils import ModelResponse
|
|||
from ...openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class LemonadeChatConfig(OpenAILikeChatConfig):
|
||||
|
|
@ -231,7 +231,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream
|
|||
from litellm.utils import convert_to_model_response_object, supports_reasoning
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: str) -> str:
|
||||
|
|
@ -580,7 +580,7 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -14,9 +14,8 @@ from litellm.utils import ModelResponse, Usage
|
|||
from ..common_utils import NLPCloudError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LoggingClass = LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -175,7 +174,7 @@ class NLPCloudConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue