diff --git a/.github/actions/cache-cargo-build/action.yml b/.github/actions/cache-cargo-build/action.yml index c3b8ce22c68..222fad637fb 100644 --- a/.github/actions/cache-cargo-build/action.yml +++ b/.github/actions/cache-cargo-build/action.yml @@ -15,6 +15,12 @@ description: >- cache the same directory for different workloads, and a shared key would let whichever ran first deny the others a save. +inputs: + profile: + description: "Cargo profile the build uses (dev or release)" + required: false + default: "dev" + runs: using: composite steps: @@ -25,6 +31,6 @@ runs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-maturin-dev- + ${{ runner.os }}-maturin-${{ inputs.profile }}- diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py index 7913b25918b..691c10203bf 100644 --- a/.github/e2e-stack/secrets_to_env.py +++ b/.github/e2e-stack/secrets_to_env.py @@ -9,6 +9,7 @@ from pydantic import TypeAdapter, ValidationError secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str]) ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") MIN_MASKED_LENGTH: Final = 8 +ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS" def main() -> int: @@ -30,10 +31,15 @@ def main() -> int: f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n" ) return 1 - for value in secrets.values(): - if len(value) >= MIN_MASKED_LENGTH: - _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n") - sys.stdout.flush() + if os.environ.get(ACTIONS_RUNNER_FLAG) == "true": + _ = sys.stdout.write( + "".join( + f"::add-mask::{value.replace('%', '%25')}\n" + for value in secrets.values() + if len(value) >= MIN_MASKED_LENGTH + ) + ) + sys.stdout.flush() lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value) try: with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle: diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index dbc4ae8f5c2..2386b184e54 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -9,6 +9,7 @@ UNSUPPORTED: Final = re.compile( r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$" r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" + r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$" ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index ea6d2401084..f2b82f86b47 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -134,7 +134,16 @@ def main( uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members) native_path: Final = wheel.parent / "native" / Path(native_member.filename).name native_path.parent.mkdir(parents=True, exist_ok=True) - native_path.write_bytes(archive.read(native_member)) + native_bytes: Final = archive.read(native_member) + native_path.write_bytes(native_bytes) + duplicated_vocabularies: Final = tuple( + member.filename + for member in wheel_members + if member.filename.startswith("litellm/litellm_core_utils/tokenizers/") + and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name) + and member.file_size > 0 + and archive.read(member) in native_bytes + ) wheel_metadata_tags_match: Final = ( len(wheel_metadata_tags) == len(expanded_filename_tags) @@ -205,7 +214,7 @@ def main( native_module: Final = load_native_module(native_path) native_module_loads: Final = native_module is not None panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") - native_size_limit: Final = 40_000_000 + native_size_limit: Final = 35_000_000 native_size_within_limit: Final = native_member.file_size <= native_size_limit validations: Final = ( (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), @@ -223,6 +232,7 @@ def main( ("Native module loads", native_module_loads), ("Production module omits the panic test hook", panic_test_hook_absent), (f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit), + ("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies), ("Wheel contents are valid", not unexpected_members), ) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index fd7513a3937..ec7e211faa1 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -13,6 +13,7 @@ on: - ".github/workflows/codspeed.yml" - ".github/actions/setup-uv-with-retries/**" - ".github/actions/cache-cargo-build/**" + - ".github/scripts/uv_sync_with_retries.sh" pull_request: branches: - main @@ -25,6 +26,7 @@ on: - ".github/workflows/codspeed.yml" - ".github/actions/setup-uv-with-retries/**" - ".github/actions/cache-cargo-build/**" + - ".github/scripts/uv_sync_with_retries.sh" # Allow CodSpeed to trigger backtest performance analysis # in order to generate initial data workflow_dispatch: @@ -59,19 +61,27 @@ jobs: - name: Cache the Rust build uses: ./.github/actions/cache-cargo-build + with: + profile: release # Build the wheel and resolve every dependency outside the CodSpeed # runner: the same maturin build took 42 minutes inside `codspeed run` # versus under 3 minutes as a plain step (LIT-6183) - - name: Build environment + - name: Build the release wheel + run: uv build --wheel --out-dir dist + + - name: Install the wheel into the benchmark environment + run: | + UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12 + uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl + + - name: Collect benchmarks + env: + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + LITELLM_REQUIRE_INSTALLED_WHEEL: "1" run: > - env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 - uv run --frozen --no-default-groups - --with pytest==8.3.5 - --with pytest-codspeed==4.3.0 - --with "mcp>=2.2.0,<3.0" - --with "a2a-sdk>=1.1.0,<2.0" - pytest + "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest + --import-mode=importlib -p pytest_codspeed.plugin tests/benchmarks/ --codspeed @@ -82,13 +92,9 @@ jobs: with: mode: simulation run: > - env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 - uv run --frozen --no-default-groups - --with pytest==8.3.5 - --with pytest-codspeed==4.3.0 - --with "mcp>=2.2.0,<3.0" - --with "a2a-sdk>=1.1.0,<2.0" - pytest + env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1 + "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest + --import-mode=importlib -p pytest_codspeed.plugin tests/benchmarks/ --codspeed diff --git a/backend/Dockerfile b/backend/Dockerfile index 622fedcd70d..57e0a43a98d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra saml \ --python python3.13 +RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ + RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ prisma generate --schema=./schema.prisma diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 50aa40ba220..5842cf6f1ac 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -1,8 +1,11 @@ """Guard the cost map on pull requests. -Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file, -and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named -litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models. +Every pull request whose diff against its merge base touches one of the three cost map files gets the file +checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the +map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies +and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot +(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three +files and may only add or update models. """ from __future__ import annotations @@ -108,20 +111,37 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str ) +def touches_cost_map(changed_files: Sequence[str]) -> bool: + return any(path in GUARDED_PATHS for path in changed_files) + + +def contract_for(bot: bool, changed_files: Sequence[str]) -> str: + if bot: + return "bot contract enforced" + return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched" + + def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]: + if not bot and not touches_cost_map(changed_files): + return () head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH) if isinstance(head_map, str): return (head_map,) return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ())) -def _git(*args: str) -> str: +def _git(*args: str) -> str | None: result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True) - return result.stdout if result.returncode == 0 else "" + return result.stdout if result.returncode == 0 else None def snapshot(revision: str) -> Snapshot: - return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS)) + return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS)) + + +def changed_files(base: str, head: str) -> tuple[str, ...] | None: + diff: Final = _git("diff", "--name-only", "--no-renames", base, head) + return None if diff is None else tuple(diff.splitlines()) def main(argv: Sequence[str]) -> int: @@ -131,9 +151,12 @@ def main(argv: Sequence[str]) -> int: parser.add_argument("--head-ref", required=True, help="head branch name of the pull request") args: Final = parser.parse_args(argv) bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX) - changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines()) - failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot) - contract: Final = "bot contract enforced" if bot else "human PR, file checks only" + changed: Final = changed_files(args.base, args.head) + if changed is None: + print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown") + return 1 + failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot) + contract: Final = contract_for(bot, changed) if failures: print(f"cost map guard failed ({contract}):") print("\n".join(f"- {failure}" for failure in failures)) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 03e0dabbc17..37384cbfa53 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3253,6 +3253,7 @@ dependencies = [ name = "litellm-token-counter-huggingface" version = "0.1.0" dependencies = [ + "serde_json", "thiserror 2.0.19", "tokenizers", ] @@ -3261,6 +3262,9 @@ dependencies = [ name = "litellm-token-counter-tiktoken" version = "0.1.0" dependencies = [ + "base64 0.22.1", + "once_cell", + "rustc-hash", "thiserror 2.0.19", "tiktoken-rs", ] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 0e8941cb8e6..813d0713128 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -89,7 +89,7 @@ veil = "0.3.0" [profile.release] opt-level = 3 -lto = "thin" +lto = "fat" codegen-units = 1 panic = "unwind" debug = false diff --git a/litellm-rust/crates/host-python/src/execution.rs b/litellm-rust/crates/host-python/src/execution.rs index 083c184e37e..b435bf241d2 100644 --- a/litellm-rust/crates/host-python/src/execution.rs +++ b/litellm-rust/crates/host-python/src/execution.rs @@ -29,7 +29,7 @@ pyo3::create_exception!( static FORK_GATE: ForkGate = ForkGate::new(); -/// Whether this process has started the Tokio runtime. +/// Whether this process has entered process-bound native execution. pub fn runtime_started() -> bool { FORK_GATE.started(std::process::id()) } @@ -40,9 +40,8 @@ pub fn reserve_process_for_forking() -> Result<(), RuntimeAlreadyStarted> { FORK_GATE.reserve(std::process::id()) } -/// The only door to the Tokio runtime: every route reaches it through this module, which is -/// what lets the gate speak for the whole extension. `clippy.toml` disallows going around it. -fn enter_runtime() -> PyResult<()> { +/// Claims process-bound native state before runtime startup or tokenizer execution. +pub fn enter_native() -> PyResult<()> { FORK_GATE .enter(std::process::id()) .map_err(|refused| match refused { @@ -60,7 +59,7 @@ fn enter_runtime() -> PyResult<()> { #[expect(clippy::disallowed_methods, reason = "this is the gated door")] fn runtime() -> PyResult<&'static Runtime> { - enter_runtime()?; + enter_native()?; Ok(pyo3_async_runtimes::tokio::get_runtime()) } @@ -70,7 +69,7 @@ where F: Future> + Send + 'static, T: for<'py> IntoPyObject<'py> + Send + 'static, { - enter_runtime()?; + enter_native()?; pyo3_async_runtimes::tokio::future_into_py(py, future) } diff --git a/litellm-rust/crates/host-python/src/lib.rs b/litellm-rust/crates/host-python/src/lib.rs index 7d164ab7535..4a33975a918 100644 --- a/litellm-rust/crates/host-python/src/lib.rs +++ b/litellm-rust/crates/host-python/src/lib.rs @@ -20,7 +20,7 @@ pub use argument::lookup; pub use callable::wrap_failure; pub use driver::run_call; pub use execution::{ - ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value, + ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, enter_native, poll_async_value, reserve_process_for_forking, run_async, run_async_value, run_sync, run_sync_value, runtime_started, }; diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 4e6c510d104..7846beef28a 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -10,7 +10,7 @@ name = "_native" crate-type = ["cdylib"] [features] -default = ["abi3", "fast"] +default = ["abi3", "fast", "huggingface", "tiktoken"] abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] panic-test = [] diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index ed9bc90f650..b1fc5244d6f 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -12,7 +12,7 @@ mod routes; reason = "secret-manager foundations await rollout activation" )] mod secrets; -mod token_counter; +mod tokenizer; #[pymodule(gil_used = true)] mod _native { @@ -37,7 +37,12 @@ mod _native { #[pymodule_export] use crate::routes::responses::ResponsesWebSocketConnection; #[pymodule_export] - use crate::token_counter::TokenCounter; + use crate::routes::token_counter::TokenCounter; + #[cfg(feature = "huggingface")] + #[pymodule_export] + use crate::tokenizer::HuggingFaceEncoding; + #[pymodule_export] + use crate::tokenizer::Tokenizer; #[pymodule_export] use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking}; use pyo3::{prelude::*, types::PyModule}; @@ -83,10 +88,13 @@ mod tests { "achat_completions", "ResponsesWebSocketConnection", "TokenCounter", + "Tokenizer", "gil_stats", "process_state_started", "reserve_process_for_forking", ]; + #[cfg(feature = "huggingface")] + expected.push("HuggingFaceEncoding"); expected.sort_unstable(); let mut public_names: Vec = native_module(py) diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index 2d6b849a6b1..8a78a26423d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod chat_completions; pub(crate) mod messages; pub(crate) mod ocr; pub(crate) mod responses; +pub(crate) mod token_counter; #[cfg(test)] mod tests { diff --git a/litellm-rust/crates/python-bridge/src/routes/token_counter.rs b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs new file mode 100644 index 00000000000..168c4883b0b --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs @@ -0,0 +1,87 @@ +use std::sync::Arc; +use std::{num::NonZero, thread::available_parallelism}; + +use litellm_host_python::{enter_native, run_async}; +use litellm_token_counter::{ + CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, +}; +use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, + types::PyAny, +}; +use tokio::sync::Semaphore; + +use crate::errors::RustBridgeDeclined; +use crate::tokenizer::Tokenizer; + +/// Counts the input tokens of a raw request body off the Python event loop with +/// the GIL released. Python owns which requests get here and what to do with +/// the count. At most one encode per core runs at a time; the rest wait in the +/// async task, where a cancelled Python awaiter drops them before any blocking +/// work is scheduled. +#[pyclass(frozen)] +pub(crate) struct TokenCounter { + inner: Arc, + encode_slots: Arc, +} + +#[pymethods] +impl TokenCounter { + #[staticmethod] + #[pyo3(signature = (tokenizer, fast = false))] + fn from_tokenizer(py: Python<'_>, tokenizer: &Tokenizer, fast: bool) -> PyResult { + enter_native()?; + let inner = CoreTokenCounter::new(tokenizer.counter(py, fast)); + Ok(Self { + inner: Arc::new(inner), + encode_slots: Arc::new(Semaphore::new(encode_parallelism())), + }) + } + + fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult> { + let counter = Arc::clone(&self.inner); + let encode_slots = Arc::clone(&self.encode_slots); + let body = body.to_vec(); + run_async( + py, + async move { + let _slot = encode_slots + .acquire_owned() + .await + .map_err(|error| Error::Task(error.to_string()))?; + tokio::task::spawn_blocking(move || count_body(&counter, &body)) + .await + .map_err(|error| Error::Task(error.to_string()))? + }, + token_count_error_to_pyerr, + ) + } +} + +fn encode_parallelism() -> usize { + available_parallelism().map_or(1, NonZero::get) +} + +fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result { + let request = CountableRequest::parse(body)?; + counter.count_request(&request) +} + +pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr { + let message = error.to_string(); + match error { + Error::Load(_) + | Error::Ranks(_) + | Error::UnicodeClasses + | Error::UnsupportedTokenizer(_) => PyValueError::new_err(message), + Error::RequestParse(_) + | Error::MissingInput + | Error::FloatText + | Error::ContentBlock + | Error::ArrayItems + | Error::JsonSerialization(_) + | Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message), + Error::Encode(_) | Error::Decode(_) | Error::Task(_) => PyRuntimeError::new_err(message), + } +} diff --git a/litellm-rust/crates/python-bridge/src/token_counter.rs b/litellm-rust/crates/python-bridge/src/token_counter.rs deleted file mode 100644 index 244401e6696..00000000000 --- a/litellm-rust/crates/python-bridge/src/token_counter.rs +++ /dev/null @@ -1,158 +0,0 @@ -use std::sync::Arc; - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -use std::{num::NonZero, thread::available_parallelism}; - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -use litellm_host_python::release_gil; -use litellm_host_python::run_async; -use litellm_token_counter::{ - CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, -}; -use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, - prelude::*, - types::PyAny, -}; -use tokio::sync::Semaphore; - -use crate::errors::RustBridgeDeclined; - -/// Counts the input tokens of a raw request body off the Python event loop with -/// the GIL released. Python owns which requests get here and what to do with -/// the count. At most one encode per core runs at a time; the rest wait in the -/// async task, where a cancelled Python awaiter drops them before any blocking -/// work is scheduled. -#[pyclass(frozen)] -pub(crate) struct TokenCounter { - inner: Arc, - encode_slots: Arc, -} - -#[pymethods] -impl TokenCounter { - #[new] - fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_json_fast(tokenizer_json)) - } - #[cfg(all(not(feature = "fast"), feature = "huggingface"))] - { - Self::load(py, || CoreTokenCounter::from_json(tokenizer_json)) - } - #[cfg(not(any(feature = "fast", feature = "huggingface")))] - { - let _ = (py, tokenizer_json); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast or huggingface feature", - )) - } - } - - #[staticmethod] - fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file)) - } - #[cfg(not(feature = "fast"))] - { - let _ = (py, rank_file); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast feature", - )) - } - } - - #[staticmethod] - fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file)) - } - #[cfg(not(feature = "fast"))] - { - let _ = (py, rank_file); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast feature", - )) - } - } - - #[staticmethod] - fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { - #[cfg(feature = "tiktoken")] - { - Self::load(py, || CoreTokenCounter::from_tiktoken(encoding)) - } - #[cfg(not(feature = "tiktoken"))] - { - let _ = (py, encoding); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the tiktoken feature", - )) - } - } - - fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult> { - let counter = Arc::clone(&self.inner); - let encode_slots = Arc::clone(&self.encode_slots); - let body = body.to_vec(); - run_async( - py, - async move { - let _slot = encode_slots - .acquire_owned() - .await - .map_err(|error| Error::Task(error.to_string()))?; - tokio::task::spawn_blocking(move || count_body(&counter, &body)) - .await - .map_err(|error| Error::Task(error.to_string()))? - }, - token_count_error_to_pyerr, - ) - } -} - -impl TokenCounter { - #[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] - fn load( - py: Python<'_>, - load: impl FnOnce() -> Result + Send, - ) -> PyResult { - let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?; - Ok(Self { - inner: Arc::new(inner), - encode_slots: Arc::new(Semaphore::new(encode_parallelism())), - }) - } -} - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -fn encode_parallelism() -> usize { - available_parallelism().map_or(1, NonZero::get) -} - -fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result { - let request = CountableRequest::parse(body)?; - counter.count_request(&request) -} - -fn token_count_error_to_pyerr(error: Error) -> PyErr { - let message = error.to_string(); - match error { - Error::Load(_) - | Error::Ranks(_) - | Error::UnicodeClasses - | Error::UnsupportedTokenizer(_) => PyValueError::new_err(message), - Error::RequestParse(_) - | Error::MissingInput - | Error::FloatText - | Error::ContentBlock - | Error::ArrayItems - | Error::JsonSerialization(_) - | Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message), - Error::Encode(_) | Error::Task(_) => PyRuntimeError::new_err(message), - } -} diff --git a/litellm-rust/crates/python-bridge/src/tokenizer.rs b/litellm-rust/crates/python-bridge/src/tokenizer.rs new file mode 100644 index 00000000000..df219d55eb6 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/tokenizer.rs @@ -0,0 +1,713 @@ +//! The Python face of the text codecs: one `Tokenizer` class over the tiktoken and Hugging +//! Face backends, carrying the read-only surface of `tiktoken.Encoding` and +//! `tokenizers.Tokenizer` that `litellm/litellm_core_utils/tokenizer.py` wraps. +use std::borrow::Cow; +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +use std::collections::HashMap; +use std::sync::Arc; +#[cfg(feature = "fast")] +use std::sync::OnceLock; + +use litellm_host_python::{enter_native, release_gil}; +#[cfg(feature = "fast")] +use litellm_token_counter::fast::{FastCounter, FastTokenizer}; +use litellm_token_counter::{Error, TextCodec}; +use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString}; + +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +use pyo3::exceptions::PyValueError; +#[cfg(feature = "huggingface")] +use pyo3::{exceptions::PyIOError, types::PyDict}; +#[cfg(feature = "tiktoken")] +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError}, + types::PyBytes, +}; + +#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))] +use crate::errors::RustBridgeDeclined; +use crate::routes::token_counter::token_count_error_to_pyerr; + +#[cfg(feature = "huggingface")] +use litellm_token_counter::huggingface::{ + EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, PaddingStrategy, + TruncationDirection, encoding_from_json, encoding_to_json, +}; +#[cfg(feature = "tiktoken")] +use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary}; + +#[cfg(feature = "tiktoken")] +pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { + enter_native()?; + let resource: std::path::PathBuf = + PyModule::import(py, "litellm.litellm_core_utils.tokenizers")? + .getattr("__file__")? + .extract()?; + release_gil(py, || { + TiktokenTokenizer::from_cached_ranks(encoding, |file| { + std::fs::read_to_string(resource.with_file_name(file)) + }) + }) + .map_err(|error| token_count_error_to_pyerr(error.into())) +} + +pub(crate) enum Codec { + #[cfg(feature = "tiktoken")] + Tiktoken(TiktokenTokenizer), + #[cfg(feature = "huggingface")] + HuggingFace(HuggingFaceTokenizer), +} + +impl Codec { + pub(crate) fn codec(&self) -> &dyn TextCodec { + match *self { + #[cfg(feature = "tiktoken")] + Self::Tiktoken(ref tokenizer) => tokenizer, + #[cfg(feature = "huggingface")] + Self::HuggingFace(ref tokenizer) => tokenizer, + } + } + + #[cfg(feature = "fast")] + fn fast_counter(&self) -> Option { + match *self { + #[cfg(feature = "tiktoken")] + Self::Tiktoken(ref tokenizer) => tokenizer.fast_counter(), + #[cfg(feature = "huggingface")] + Self::HuggingFace(ref tokenizer) => tokenizer.fast_counter(), + } + } +} + +/// The loaded model is shared: `TokenCounter::from_tokenizer` counts with the same parse, +/// and the opt-in count-only counter is derived from it once, on first use. +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Tokenizer { + inner: Arc, + #[cfg(feature = "fast")] + fast: OnceLock>>, +} + +#[pymethods] +impl Tokenizer { + #[staticmethod] + fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { + #[cfg(feature = "tiktoken")] + { + let tokenizer = load_tiktoken(py, encoding)?; + Ok(Self::new(Codec::Tiktoken(tokenizer))) + } + #[cfg(not(feature = "tiktoken"))] + { + let _ = (py, encoding); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the tiktoken feature", + )) + } + } + + #[staticmethod] + fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult { + #[cfg(feature = "huggingface")] + { + enter_native()?; + let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json)) + .map_err(|error| token_count_error_to_pyerr(error.into()))?; + Ok(Self::new(Codec::HuggingFace(tokenizer))) + } + #[cfg(not(feature = "huggingface"))] + { + let _ = (py, tokenizer_json); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the huggingface feature", + )) + } + } + + #[staticmethod] + #[pyo3(signature = (identifier, revision = "main", token = None))] + fn from_pretrained( + py: Python<'_>, + identifier: &str, + revision: &str, + token: Option<&str>, + ) -> PyResult { + #[cfg(feature = "huggingface")] + { + enter_native()?; + let kwargs = PyDict::new(py); + kwargs.set_item("repo_id", identifier)?; + kwargs.set_item("filename", "tokenizer.json")?; + kwargs.set_item("revision", revision)?; + kwargs.set_item("token", token)?; + let path: String = PyModule::import(py, "huggingface_hub")? + .getattr("hf_hub_download")? + .call((), Some(&kwargs))? + .extract()?; + let json = + release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?; + Self::from_json(py, &json) + } + #[cfg(not(feature = "huggingface"))] + { + let _ = (py, identifier, revision, token); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the huggingface feature", + )) + } + } + + fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult> { + enter_native()?; + let text = self.text(text)?; + release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr) + } + + #[pyo3(signature = (ids, skip_special_tokens = true))] + fn decode(&self, py: Python<'_>, ids: Vec, skip_special_tokens: bool) -> PyResult { + enter_native()?; + release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens)) + .map_err(token_count_error_to_pyerr) + } + + #[pyo3(signature = (text, fast = false))] + fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>, fast: bool) -> PyResult { + enter_native()?; + let text = self.text(text)?; + let counter = self.counter(py, fast); + release_gil(py, || { + litellm_token_counter::Tokenizer::count_tokens(&counter, &text) + }) + .map_err(token_count_error_to_pyerr) + } + + #[getter] + fn name(&self) -> &str { + self.inner.codec().name() + } + + // ---- tiktoken: the `tiktoken.Encoding` surface ------------------------------------------ + + #[cfg(feature = "tiktoken")] + fn encode_special( + &self, + py: Python<'_>, + text: &Bound<'_, PyString>, + allowed: Vec, + ) -> PyResult> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let text = self.text(text)?; + release_gil(py, || tokenizer.encode_special(&text, &allowed)) + .map_err(PyRuntimeError::new_err) + } + + /// tiktoken's `encode_with_unstable`: `(stable_tokens, completions)`. + #[cfg(feature = "tiktoken")] + fn encode_with_unstable( + &self, + py: Python<'_>, + text: &Bound<'_, PyString>, + allowed: Vec, + ) -> PyResult<(Vec, Vec>)> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let text = self.text(text)?; + Ok(release_gil(py, || { + tokenizer.encode_with_unstable(&text, &allowed) + })) + } + + /// The special tokens by text: tiktoken's `_special_tokens`. + #[cfg(feature = "tiktoken")] + fn special_tokens(&self) -> PyResult> { + Ok(self + .vocabulary()? + .special_tokens() + .map(|(token, rank)| (token.to_owned(), rank)) + .collect()) + } + + #[cfg(feature = "tiktoken")] + fn max_token_value(&self) -> PyResult { + Ok(self.vocabulary()?.max_token_value()) + } + + #[cfg(feature = "tiktoken")] + fn is_special_token(&self, token: u32) -> PyResult { + Ok(self.vocabulary()?.is_special_token(token)) + } + + /// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`. + #[cfg(feature = "tiktoken")] + fn token_byte_values<'py>(&self, py: Python<'py>) -> PyResult>> { + let vocabulary = self.vocabulary()?; + let values = release_gil(py, || vocabulary.token_byte_values()); + Ok(values.iter().map(|value| PyBytes::new(py, value)).collect()) + } + + /// The token of one whole piece; `KeyError` when it is not in the vocabulary. + #[cfg(feature = "tiktoken")] + fn encode_single_token(&self, py: Python<'_>, piece: Vec) -> PyResult { + self.vocabulary()? + .encode_single_token(&piece) + .ok_or_else(|| PyKeyError::new_err(PyBytes::new(py, &piece).unbind())) + } + + #[cfg(feature = "tiktoken")] + fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec) -> PyResult> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let bytes = + release_gil(py, || tokenizer.decode_bytes(&ids)).map_err(PyKeyError::new_err)?; + Ok(PyBytes::new(py, &bytes)) + } + + // ---- Hugging Face: the `tokenizers.Tokenizer` surface ----------------------------------- + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))] + fn encode_huggingface( + &self, + py: Python<'_>, + sequence: Sequence, + pair: Option, + is_pretokenized: bool, + add_special_tokens: bool, + fast: bool, + ) -> PyResult { + enter_native()?; + let tokenizer = self.huggingface()?; + let sequence = sequence.input(is_pretokenized)?; + let input = match pair { + Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?), + None => EncodeInput::Single(sequence), + }; + release_gil(py, || { + tokenizer.encode_result(input, add_special_tokens, fast) + }) + .map(|inner| HuggingFaceEncoding { inner }) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))] + fn encode_batch_huggingface( + &self, + py: Python<'_>, + inputs: Vec<(Sequence, Option)>, + is_pretokenized: bool, + add_special_tokens: bool, + fast: bool, + ) -> PyResult> { + enter_native()?; + let tokenizer = self.huggingface()?; + let inputs = inputs + .into_iter() + .map(|(sequence, pair)| { + let sequence = sequence.input(is_pretokenized)?; + match pair { + Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)), + None => Ok(EncodeInput::Single(sequence)), + } + }) + .collect::>>()?; + release_gil(py, || { + tokenizer.encode_batch_result(inputs, add_special_tokens, fast) + }) + .map(|encodings| { + encodings + .into_iter() + .map(|inner| HuggingFaceEncoding { inner }) + .collect() + }) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (pretty = false))] + fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult { + enter_native()?; + let tokenizer = self.huggingface()?; + release_gil(py, || tokenizer.to_json(pretty)) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + fn token_to_id(&self, token: &str) -> PyResult> { + Ok(self.huggingface()?.token_to_id(token)) + } + + #[cfg(feature = "huggingface")] + fn id_to_token(&self, id: u32) -> PyResult> { + Ok(self.huggingface()?.id_to_token(id)) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (with_added_tokens = true))] + fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult> { + let tokenizer = self.huggingface()?; + Ok(release_gil(py, || tokenizer.vocab(with_added_tokens))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (with_added_tokens = true))] + fn get_vocab_size(&self, with_added_tokens: bool) -> PyResult { + Ok(self.huggingface()?.vocab_size(with_added_tokens)) + } + + /// The added tokens by id as `(id, (content, single_word, lstrip, rstrip, normalized, + /// special))`, for Python to rebuild as `tokenizers.AddedToken`. + #[cfg(feature = "huggingface")] + fn added_tokens_decoder(&self) -> PyResult> { + Ok(self + .huggingface()? + .added_tokens_decoder() + .into_iter() + .map(|(id, token)| { + ( + id, + ( + token.content, + token.single_word, + token.lstrip, + token.rstrip, + token.normalized, + token.special, + ), + ) + }) + .collect()) + } + + /// The padding parameters as `tokenizers.Tokenizer.padding` reports them. + #[cfg(feature = "huggingface")] + fn padding<'py>(&self, py: Python<'py>) -> PyResult>> { + let Some(params) = self.huggingface()?.padding() else { + return Ok(None); + }; + let padding = PyDict::new(py); + padding.set_item( + "length", + match params.strategy { + PaddingStrategy::BatchLongest => None, + PaddingStrategy::Fixed(length) => Some(length), + }, + )?; + padding.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?; + padding.set_item("pad_id", params.pad_id)?; + padding.set_item("pad_type_id", params.pad_type_id)?; + padding.set_item("pad_token", ¶ms.pad_token)?; + padding.set_item("direction", params.direction.as_ref())?; + Ok(Some(padding)) + } + + /// The truncation parameters as `tokenizers.Tokenizer.truncation` reports them. + #[cfg(feature = "huggingface")] + fn truncation<'py>(&self, py: Python<'py>) -> PyResult>> { + let Some(params) = self.huggingface()?.truncation() else { + return Ok(None); + }; + let truncation = PyDict::new(py); + truncation.set_item("max_length", params.max_length)?; + truncation.set_item("stride", params.stride)?; + truncation.set_item("strategy", params.strategy.as_ref())?; + truncation.set_item("direction", params.direction.as_ref())?; + Ok(Some(truncation)) + } + + #[cfg(feature = "huggingface")] + fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult { + Ok(self.huggingface()?.num_special_tokens_to_add(is_pair)) + } + + #[cfg(feature = "huggingface")] + fn encode_special_tokens(&self) -> PyResult { + Ok(self.huggingface()?.encode_special_tokens()) + } +} + +#[cfg(feature = "huggingface")] +type AddedTokenFields = (String, bool, bool, bool, bool, bool); + +impl Tokenizer { + fn new(inner: Codec) -> Self { + Self { + inner: Arc::new(inner), + #[cfg(feature = "fast")] + fast: OnceLock::new(), + } + } + + pub(crate) fn counter(&self, py: Python<'_>, fast: bool) -> SharedCounter { + #[cfg(feature = "fast")] + if fast { + let counter = self.fast.get().unwrap_or_else(|| { + release_gil(py, || { + self.fast + .get_or_init(|| self.inner.fast_counter().map(Arc::new)) + }) + }); + if let Some(counter) = counter { + return SharedCounter::Fast(Arc::clone(counter)); + } + } + #[cfg(not(feature = "fast"))] + let _ = (py, fast); + SharedCounter::Codec(Arc::clone(&self.inner)) + } + + /// A Python `str` as UTF-8. tiktoken replaces lone surrogates the way its Python `encode` + /// does; `tokenizers` rejects them, so that backend keeps the encode error. + fn text<'a>(&self, text: &'a Bound<'_, PyString>) -> PyResult> { + match text.to_cow() { + Ok(text) => Ok(text), + Err(error) => match *self.inner { + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(_) if error.is_instance_of::(text.py()) => { + text.call_method1("encode", ("utf-16", "surrogatepass"))? + .call_method1("decode", ("utf-16", "replace"))? + .extract::() + .map(Cow::Owned) + } + _ => Err(error), + }, + } + } + + #[cfg(feature = "tiktoken")] + fn tiktoken(&self) -> PyResult<&TiktokenTokenizer> { + match *self.inner { + Codec::Tiktoken(ref tokenizer) => Ok(tokenizer), + #[cfg(feature = "huggingface")] + Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")), + } + } + + #[cfg(feature = "tiktoken")] + fn vocabulary(&self) -> PyResult<&Vocabulary> { + self.tiktoken()?.vocabulary().ok_or_else(|| { + PyRuntimeError::new_err("this encoding was built without its vocabulary") + }) + } + + #[cfg(feature = "huggingface")] + fn huggingface(&self) -> PyResult<&HuggingFaceTokenizer> { + match *self.inner { + Codec::HuggingFace(ref tokenizer) => Ok(tokenizer), + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")), + } + } +} + +pub(crate) enum SharedCounter { + Codec(Arc), + #[cfg(feature = "fast")] + Fast(Arc), +} + +impl litellm_token_counter::Tokenizer for SharedCounter { + fn count_tokens(&self, text: &str) -> Result { + match self { + Self::Codec(codec) => codec.codec().count_tokens(text), + #[cfg(feature = "fast")] + Self::Fast(counter) => counter.count_tokens(text).map_err(Error::from), + } + } +} + +#[cfg(feature = "huggingface")] +#[derive(FromPyObject)] +pub(crate) enum Sequence { + Text(String), + Words(Vec), +} + +#[cfg(feature = "huggingface")] +impl Sequence { + fn input(self, is_pretokenized: bool) -> PyResult> { + match (self, is_pretokenized) { + (Self::Text(text), false) => Ok(text.into()), + (Self::Words(words), true) => Ok(words.into()), + _ => Err(pyo3::exceptions::PyTypeError::new_err( + "input must match is_pretokenized", + )), + } + } +} + +#[cfg(feature = "huggingface")] +fn direction(value: &str, left: T, right: T, what: &str) -> PyResult { + match value { + "left" => Ok(left), + "right" => Ok(right), + other => Err(PyValueError::new_err(format!( + "invalid {what} direction {other:?}: expected 'left' or 'right'" + ))), + } +} + +/// `tokenizers.Encoding`, mutable like the original: `pad`, `truncate` and `set_sequence_id` +/// change it in place. +#[cfg(feature = "huggingface")] +#[pyclass(module = "litellm.rust_bridge._native")] +pub(crate) struct HuggingFaceEncoding { + inner: Encoding, +} + +#[cfg(feature = "huggingface")] +#[pymethods] +impl HuggingFaceEncoding { + #[new] + #[pyo3(signature = (json = None))] + fn new(json: Option<&str>) -> PyResult { + let inner = match json { + Some(json) => encoding_from_json(json) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + None => Encoding::default(), + }; + Ok(Self { inner }) + } + + #[staticmethod] + #[pyo3(signature = (encodings, growing_offsets = true))] + fn merge(encodings: Vec>, growing_offsets: bool) -> Self { + Self { + inner: Encoding::merge( + encodings.iter().map(|encoding| encoding.inner.clone()), + growing_offsets, + ), + } + } + + fn __reduce__<'py>( + &self, + py: Python<'py>, + ) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> { + let json = encoding_to_json(&self.inner) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok((py.get_type::(), (json,))) + } + + fn __repr__(&self) -> String { + format!( + "Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \ + attention_mask, special_tokens_mask, overflowing])", + self.inner.len() + ) + } + + fn __len__(&self) -> usize { + self.inner.len() + } + #[getter] + fn ids(&self) -> Vec { + self.inner.get_ids().to_vec() + } + #[getter] + fn tokens(&self) -> Vec { + self.inner.get_tokens().to_vec() + } + #[getter] + fn offsets(&self) -> Vec<(usize, usize)> { + self.inner.get_offsets().to_vec() + } + #[getter] + fn type_ids(&self) -> Vec { + self.inner.get_type_ids().to_vec() + } + #[getter] + fn attention_mask(&self) -> Vec { + self.inner.get_attention_mask().to_vec() + } + #[getter] + fn special_tokens_mask(&self) -> Vec { + self.inner.get_special_tokens_mask().to_vec() + } + #[getter] + fn word_ids(&self) -> Vec> { + self.inner.get_word_ids().to_vec() + } + #[getter] + fn sequence_ids(&self) -> Vec> { + self.inner.get_sequence_ids() + } + #[getter] + fn overflowing(&self) -> Vec { + self.inner + .get_overflowing() + .iter() + .cloned() + .map(|inner| Self { inner }) + .collect() + } + #[getter] + fn n_sequences(&self) -> usize { + self.inner.n_sequences() + } + + #[pyo3(signature = (word_index, sequence_index = 0))] + fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> { + self.inner.word_to_tokens(word_index, sequence_index) + } + #[pyo3(signature = (word_index, sequence_index = 0))] + fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> { + self.inner.word_to_chars(word_index, sequence_index) + } + fn token_to_sequence(&self, token_index: usize) -> Option { + self.inner.token_to_sequence(token_index) + } + fn token_to_chars(&self, token_index: usize) -> Option<(usize, usize)> { + self.inner + .token_to_chars(token_index) + .map(|(_, offsets)| offsets) + } + fn token_to_word(&self, token_index: usize) -> Option { + self.inner.token_to_word(token_index).map(|(_, word)| word) + } + #[pyo3(signature = (char_pos, sequence_index = 0))] + fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option { + self.inner.char_to_token(char_pos, sequence_index) + } + #[pyo3(signature = (char_pos, sequence_index = 0))] + fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option { + self.inner.char_to_word(char_pos, sequence_index) + } + + fn set_sequence_id(&mut self, sequence_id: usize) { + self.inner.set_sequence_id(sequence_id); + } + + #[pyo3(signature = (length, direction = "right", pad_id = 0, pad_type_id = 0, pad_token = "[PAD]"))] + fn pad( + &mut self, + length: usize, + direction: &str, + pad_id: u32, + pad_type_id: u32, + pad_token: &str, + ) -> PyResult<()> { + let direction = self::direction( + direction, + PaddingDirection::Left, + PaddingDirection::Right, + "padding", + )?; + self.inner + .pad(length, pad_id, pad_type_id, pad_token, direction); + Ok(()) + } + + #[pyo3(signature = (max_length, stride = 0, direction = "right"))] + fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> { + let direction = self::direction( + direction, + TruncationDirection::Left, + TruncationDirection::Right, + "truncation", + )?; + self.inner.truncate(max_length, stride, direction); + Ok(()) + } +} diff --git a/litellm-rust/crates/token-counter-fast/src/lib.rs b/litellm-rust/crates/token-counter-fast/src/lib.rs index ce91af642ea..157ee847658 100644 --- a/litellm-rust/crates/token-counter-fast/src/lib.rs +++ b/litellm-rust/crates/token-counter-fast/src/lib.rs @@ -8,6 +8,8 @@ mod scanner; mod tiktoken; mod unicode_classes; +use std::sync::Arc; + use byte_level::ByteLevelCounter; use scanner::{SplitPattern, TiktokenCounter}; @@ -15,22 +17,29 @@ pub use error::Error; enum Encoder { HuggingFace { - tokenizer: Box, + tokenizer: Arc, byte_level: Option, }, Tiktoken(TiktokenCounter), } +/// A count-only tokenizer. Its model tables are immutable, so one built from an already +/// loaded model (`from_shared`, `from_*_pairs`) adds only the count-specific tables. pub struct FastTokenizer(Encoder); impl FastTokenizer { pub fn from_json(json: &str) -> Result { let tokenizer = json.parse::().map_err(Error::Load)?; + Ok(Self::from_shared(Arc::new(tokenizer))) + } + + /// Counts with a Hugging Face model another codec already holds; nothing is re-parsed. + pub fn from_shared(tokenizer: Arc) -> Self { let byte_level = ByteLevelCounter::detect(&tokenizer); - Ok(Self(Encoder::HuggingFace { - tokenizer: Box::new(tokenizer), + Self(Encoder::HuggingFace { + tokenizer, byte_level, - })) + }) } pub fn from_cl100k_ranks(ranks: &str) -> Result { @@ -41,12 +50,36 @@ impl FastTokenizer { Self::from_ranks(SplitPattern::O200k, ranks) } + /// `cl100k_base` from ranks another loader already parsed. + pub fn from_cl100k_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_pairs(SplitPattern::Cl100k, pairs) + } + + /// `o200k_base` (and `o200k_harmony`, whose ordinary tokens are the same) from ranks + /// another loader already parsed. + pub fn from_o200k_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_pairs(SplitPattern::O200k, pairs) + } + fn from_ranks(split: SplitPattern, ranks: &str) -> Result { TiktokenCounter::from_ranks(split, ranks) .map(Encoder::Tiktoken) .map(Self) } + fn from_pairs<'a>( + split: SplitPattern, + pairs: impl IntoIterator, + ) -> Result { + TiktokenCounter::from_pairs(split, pairs) + .map(Encoder::Tiktoken) + .map(Self) + } + pub fn count_tokens(&self, text: &str) -> Result { match &self.0 { Encoder::Tiktoken(counter) => Ok(counter.count(text)), diff --git a/litellm-rust/crates/token-counter-fast/src/scanner.rs b/litellm-rust/crates/token-counter-fast/src/scanner.rs index c2c3057aeeb..882ea91db81 100644 --- a/litellm-rust/crates/token-counter-fast/src/scanner.rs +++ b/litellm-rust/crates/token-counter-fast/src/scanner.rs @@ -39,8 +39,19 @@ pub(super) struct TiktokenCounter { impl TiktokenCounter { pub(super) fn from_ranks(split: SplitPattern, rank_file: &str) -> Result { + Self::new(split, MergeRanks::parse(rank_file)?) + } + + pub(super) fn from_pairs<'a>( + split: SplitPattern, + pairs: impl IntoIterator, + ) -> Result { + Self::new(split, MergeRanks::from_pairs(pairs)?) + } + + fn new(split: SplitPattern, ranks: MergeRanks) -> Result { Ok(Self { - ranks: MergeRanks::parse(rank_file)?, + ranks, piece_len: split.piece_len(), unicode_classes: UnicodeClasses::get().ok_or(Error::UnicodeClasses)?, }) diff --git a/litellm-rust/crates/token-counter-fast/src/tiktoken.rs b/litellm-rust/crates/token-counter-fast/src/tiktoken.rs index 16172b7a688..68f09b14a25 100644 --- a/litellm-rust/crates/token-counter-fast/src/tiktoken.rs +++ b/litellm-rust/crates/token-counter-fast/src/tiktoken.rs @@ -22,11 +22,25 @@ pub(super) struct MergeRanks(FxHashMap, Rank>); impl MergeRanks { pub(super) fn parse(text: &str) -> Result { - let ranks = text - .lines() - .filter(|line| !line.is_empty()) - .map(parse_line) - .collect::, _>>()?; + Self::from_entries(text.lines().filter(|line| !line.is_empty()).map(parse_line)) + } + + /// The same table from ranks another loader already parsed. + pub(super) fn from_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_entries(pairs.into_iter().map(|(bytes, rank)| { + if rank == NO_RANK { + return Err(Error::Ranks(format!("rank {rank} is reserved"))); + } + Ok((Box::from(bytes), rank)) + })) + } + + fn from_entries( + entries: impl Iterator, Rank), Error>>, + ) -> Result { + let ranks = entries.collect::, _>>()?; if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) { return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token"))); } diff --git a/litellm-rust/crates/token-counter-huggingface/Cargo.toml b/litellm-rust/crates/token-counter-huggingface/Cargo.toml index 6d8cb85e524..a5c2b1bb160 100644 --- a/litellm-rust/crates/token-counter-huggingface/Cargo.toml +++ b/litellm-rust/crates/token-counter-huggingface/Cargo.toml @@ -6,5 +6,6 @@ license.workspace = true repository.workspace = true [dependencies] +serde_json.workspace = true thiserror.workspace = true tokenizers.workspace = true diff --git a/litellm-rust/crates/token-counter-huggingface/src/error.rs b/litellm-rust/crates/token-counter-huggingface/src/error.rs index adc4551886f..e7f6260321b 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/error.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/error.rs @@ -6,4 +6,6 @@ pub enum Error { Load(#[source] tokenizers::Error), #[error("tokenization failed: {0}")] Encode(#[source] tokenizers::Error), + #[error("token decoding failed: {0}")] + Decode(#[source] tokenizers::Error), } diff --git a/litellm-rust/crates/token-counter-huggingface/src/lib.rs b/litellm-rust/crates/token-counter-huggingface/src/lib.rs index 8e05c2cca46..170a36aea05 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/lib.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/lib.rs @@ -2,22 +2,243 @@ mod error; -pub use error::Error; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; -pub struct HuggingFaceTokenizer(Box); +pub use error::Error; +use tokenizers::PostProcessor; +pub use tokenizers::{ + AddedToken, EncodeInput, Encoding, InputSequence, PaddingDirection, PaddingParams, + PaddingStrategy, TruncationDirection, TruncationParams, +}; + +pub fn encoding_from_json(json: &str) -> Result { + serde_json::from_str(json).map_err(|error| Error::Load(error.into())) +} + +pub fn encoding_to_json(encoding: &Encoding) -> Result { + serde_json::to_string(encoding).map_err(|error| Error::Load(error.into())) +} + +pub struct HuggingFaceTokenizer { + tokenizer: Arc, + special_token_ids: HashSet, +} impl HuggingFaceTokenizer { pub fn from_json(json: &str) -> Result { json.parse::() - .map(Box::new) - .map(Self) + .map(Self::new) .map_err(Error::Load) } + fn new(tokenizer: tokenizers::Tokenizer) -> Self { + let special_token_ids: HashSet = tokenizer + .get_added_tokens_decoder() + .into_iter() + .filter_map(|(id, token)| token.special.then_some(id)) + .collect(); + Self { + tokenizer: Arc::new(tokenizer), + special_token_ids, + } + } + + /// The parsed model, for a count-only counter to share instead of parsing it again. + pub fn shared(&self) -> Arc { + Arc::clone(&self.tokenizer) + } + pub fn count_tokens(&self, text: &str) -> Result { - self.0 + self.tokenizer .encode_fast(text, true) .map(|encoding| encoding.len()) .map_err(Error::Encode) } + + pub fn encode(&self, text: &str) -> Result, Error> { + self.tokenizer + .encode_fast(text, true) + .map(|encoding| encoding.get_ids().to_vec()) + .map_err(Error::Encode) + } + + pub fn encode_result<'a>( + &self, + input: EncodeInput<'a>, + add_special_tokens: bool, + fast: bool, + ) -> Result { + if fast { + return self + .tokenizer + .encode_fast(input, add_special_tokens) + .map_err(Error::Encode); + } + self.tokenizer + .encode_char_offsets(input, add_special_tokens) + .map_err(Error::Encode) + } + + pub fn encode_batch_result<'a>( + &self, + inputs: Vec>, + add_special_tokens: bool, + fast: bool, + ) -> Result, Error> { + if fast { + return self + .tokenizer + .encode_batch_fast(inputs, add_special_tokens) + .map_err(Error::Encode); + } + self.tokenizer + .encode_batch_char_offsets(inputs, add_special_tokens) + .map_err(Error::Encode) + } + + pub fn to_json(&self, pretty: bool) -> Result { + self.tokenizer.to_string(pretty).map_err(Error::Load) + } + + pub fn token_to_id(&self, token: &str) -> Option { + self.tokenizer.token_to_id(token) + } + + pub fn id_to_token(&self, id: u32) -> Option { + self.tokenizer.id_to_token(id) + } + + pub fn vocab(&self, with_added_tokens: bool) -> HashMap { + self.tokenizer.get_vocab(with_added_tokens) + } + + pub fn vocab_size(&self, with_added_tokens: bool) -> usize { + self.tokenizer.get_vocab_size(with_added_tokens) + } + + /// The added tokens by id, in id order. + pub fn added_tokens_decoder(&self) -> Vec<(u32, AddedToken)> { + let mut added: Vec<(u32, AddedToken)> = self + .tokenizer + .get_added_tokens_decoder() + .into_iter() + .collect(); + added.sort_unstable_by_key(|(id, _)| *id); + added + } + + pub fn padding(&self) -> Option<&PaddingParams> { + self.tokenizer.get_padding() + } + + pub fn truncation(&self) -> Option<&TruncationParams> { + self.tokenizer.get_truncation() + } + + /// How many special tokens the post-processor adds to a single sequence or a pair. + pub fn num_special_tokens_to_add(&self, is_pair: bool) -> usize { + self.tokenizer + .get_post_processor() + .map_or(0, |processor| processor.added_tokens(is_pair)) + } + + pub fn encode_special_tokens(&self) -> bool { + self.tokenizer.get_encode_special_tokens() + } + + pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result { + if !skip_special_tokens { + return self.tokenizer.decode(ids, false).map_err(Error::Decode); + } + let filtered_ids: Vec = ids + .iter() + .copied() + .filter(|id| !self.special_token_ids.contains(id)) + .collect(); + self.tokenizer + .decode(&filtered_ids, true) + .map_err(Error::Decode) + } + + pub fn name(&self) -> &str { + "huggingface" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codecs_round_trip_and_skip_special_tokens() { + let json = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json" + )); + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + let ids = tokenizer.encode("hello").unwrap(); + + assert!(tokenizer.decode(&ids, false).unwrap().contains("")); + assert_eq!(tokenizer.decode(&ids, true).unwrap(), "hello"); + } + + #[test] + fn decode_filters_special_added_tokens() { + let json = r#"{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 1, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": {"type": "Whitespace"}, + "post_processor": null, + "decoder": null, + "model": { + "type": "WordLevel", + "vocab": {"": 0, "": 1, "hello": 2}, + "unk_token": "" + } + }"#; + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + + assert!(!tokenizer.decode(&[1, 2], true).unwrap().contains("")); + assert!(tokenizer.decode(&[1, 2], false).unwrap().contains("")); + } + + #[test] + fn vocabulary_lookups_mirror_the_tokenizers_api() { + let json = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json" + )); + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + let ids = tokenizer.encode("hello").unwrap(); + + let token = tokenizer.id_to_token(ids[0]).unwrap(); + assert_eq!(tokenizer.token_to_id(&token), Some(ids[0])); + assert_eq!(tokenizer.id_to_token(u32::MAX), None); + assert_eq!(tokenizer.vocab(true).len(), tokenizer.vocab_size(true)); + assert!(tokenizer.vocab_size(true) >= tokenizer.vocab_size(false)); + let added = tokenizer.added_tokens_decoder(); + assert!(added.windows(2).all(|pair| pair[0].0 < pair[1].0)); + assert!(added.iter().any(|(_, token)| token.special)); + assert!(tokenizer.padding().is_none()); + assert!(tokenizer.truncation().is_none()); + assert!(!tokenizer.encode_special_tokens()); + assert_eq!( + tokenizer.num_special_tokens_to_add(false), + tokenizer.encode("").unwrap().len() + ); + } } diff --git a/litellm-rust/crates/token-counter-tiktoken/Cargo.toml b/litellm-rust/crates/token-counter-tiktoken/Cargo.toml index 494a9233e69..2fb3103e0c8 100644 --- a/litellm-rust/crates/token-counter-tiktoken/Cargo.toml +++ b/litellm-rust/crates/token-counter-tiktoken/Cargo.toml @@ -6,5 +6,8 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true +once_cell = "1.21.3" +rustc-hash = "2.1.3" thiserror.workspace = true tiktoken-rs.workspace = true diff --git a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs index ecdb3946eee..f049e90a1cb 100644 --- a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs +++ b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs @@ -1,27 +1,125 @@ #![forbid(unsafe_code)] mod error; +mod ranks; + +use std::collections::HashSet; pub use error::UnsupportedTokenizer; +pub use ranks::{LoadError, Vocabulary}; -pub struct TiktokenTokenizer(&'static tiktoken_rs::CoreBPE); +pub struct TiktokenTokenizer { + encoder: &'static tiktoken_rs::CoreBPE, + /// Present for encodings built from a rank file; the embedded tiktoken-rs singletons + /// behind [`from_name`](Self::from_name) keep their ranks private. + vocabulary: Option<&'static Vocabulary>, + name: &'static str, +} impl TiktokenTokenizer { + /// Builds `name` from its packaged rank file (read through `load`), once per process. + /// The tokenizer reports the requested name, so `gpt2` stays `gpt2` like tiktoken does. + pub fn from_cached_ranks( + name: &str, + load: impl FnOnce(&str) -> std::io::Result, + ) -> Result { + let (loaded, name) = ranks::load(name, load)?; + Ok(Self { + encoder: &loaded.bpe, + vocabulary: Some(&loaded.vocabulary), + name, + }) + } + + /// The encodings tiktoken-rs embeds, for hosts without the packaged rank files. pub fn from_name(name: &str) -> Result { - let tokenizer = match name { - "cl100k_base" => tiktoken_rs::cl100k_base_singleton(), - "o200k_base" => tiktoken_rs::o200k_base_singleton(), - "o200k_harmony" => tiktoken_rs::o200k_harmony_singleton(), - "p50k_base" => tiktoken_rs::p50k_base_singleton(), - "p50k_edit" => tiktoken_rs::p50k_edit_singleton(), - "r50k_base" | "gpt2" => tiktoken_rs::r50k_base_singleton(), + let (encoder, name) = match name { + "cl100k_base" => (tiktoken_rs::cl100k_base_singleton(), "cl100k_base"), + "o200k_base" => (tiktoken_rs::o200k_base_singleton(), "o200k_base"), + "o200k_harmony" => (tiktoken_rs::o200k_harmony_singleton(), "o200k_harmony"), + "p50k_base" => (tiktoken_rs::p50k_base_singleton(), "p50k_base"), + "p50k_edit" => (tiktoken_rs::p50k_edit_singleton(), "p50k_edit"), + "r50k_base" => (tiktoken_rs::r50k_base_singleton(), "r50k_base"), + "gpt2" => (tiktoken_rs::r50k_base_singleton(), "gpt2"), _ => return Err(UnsupportedTokenizer(name.to_owned())), }; - Ok(Self(tokenizer)) + Ok(Self { + encoder, + vocabulary: None, + name, + }) + } + + pub fn vocabulary(&self) -> Option<&Vocabulary> { + self.vocabulary } pub fn count_tokens(&self, text: &str) -> usize { - self.0.count_ordinary(text) + self.encoder.count_ordinary(text) + } + + pub fn encode(&self, text: &str) -> Vec { + self.encoder.encode_ordinary(text) + } + + pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result, String> { + let allowed = allowed.iter().map(String::as_str).collect(); + self.encoder + .encode(text, &allowed) + .map(|(ids, _)| ids) + .map_err(|error| error.to_string()) + } + + pub fn special_tokens(&self) -> HashSet { + self.encoder + .special_tokens() + .into_iter() + .map(str::to_owned) + .collect() + } + + /// tiktoken's `encode_with_unstable`: the stable prefix of `text`'s tokens and every + /// token sequence the unstable tail could still become, sorted for a stable order. + pub fn encode_with_unstable( + &self, + text: &str, + allowed: &[String], + ) -> (Vec, Vec>) { + let allowed = allowed.iter().map(String::as_str).collect(); + let (stable, completions) = self.encoder._encode_unstable_native(text, &allowed); + let mut completions: Vec> = completions.into_iter().collect(); + completions.sort_unstable(); + (stable, completions) + } + + pub fn decode_bytes(&self, ids: &[u32]) -> Result, String> { + self.encoder + .decode_bytes(ids) + .map_err(|error| error.to_string()) + } + + pub fn decode(&self, ids: &[u32]) -> Result { + self.encoder + .decode_bytes(ids) + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .map_err(|error| error.to_string()) + } + + pub fn name(&self) -> &str { + self.name + } +} + +pub fn encoding_for_model(model: &str) -> Option<&'static str> { + match tiktoken_rs::tokenizer::get_tokenizer(model)? { + tiktoken_rs::tokenizer::Tokenizer::Cl100kBase => Some("cl100k_base"), + tiktoken_rs::tokenizer::Tokenizer::O200kBase => Some("o200k_base"), + tiktoken_rs::tokenizer::Tokenizer::O200kHarmony => Some("o200k_harmony"), + tiktoken_rs::tokenizer::Tokenizer::P50kBase => Some("p50k_base"), + tiktoken_rs::tokenizer::Tokenizer::P50kEdit => Some("p50k_edit"), + tiktoken_rs::tokenizer::Tokenizer::R50kBase | tiktoken_rs::tokenizer::Tokenizer::Gpt2 => { + Some("r50k_base") + } } } @@ -66,5 +164,75 @@ mod tests { panic!("unknown encoding must be rejected"); }; assert_eq!(name, "unknown-encoding"); + assert_eq!(TiktokenTokenizer::from_name("gpt2").unwrap().name(), "gpt2"); + } + + #[test] + fn codecs_round_trip_named_encodings() { + let encodings = [ + "cl100k_base", + "o200k_base", + "o200k_harmony", + "p50k_base", + "p50k_edit", + "r50k_base", + "gpt2", + ]; + let texts = ["hello world", "café 漢字 مرحبا 🙂", "line one\nline two"]; + for name in encodings { + let tokenizer = TiktokenTokenizer::from_name(name).unwrap(); + for text in texts { + assert_eq!( + tokenizer.decode(&tokenizer.encode(text)).unwrap(), + text, + "{name}: {text:?}", + ); + } + } + } + + #[test] + fn decoding_token_prefixes_replaces_incomplete_utf8() { + let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap(); + let reference = tiktoken_rs::cl100k_base_singleton(); + let ids = tokenizer.encode("🙂漢字"); + for end in 1..=ids.len() { + let bytes = reference.decode_bytes(&ids[..end]).unwrap(); + assert_eq!( + tokenizer.decode(&ids[..end]).unwrap(), + String::from_utf8_lossy(&bytes), + ); + } + assert!(tokenizer.decode(&[u32::MAX]).is_err()); + } + + #[test] + fn unstable_encoding_prefixes_stay_consistent_with_full_encoding() { + let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap(); + let text = "hello fanta"; + let (stable, completions) = tokenizer.encode_with_unstable(text, &[]); + assert!( + text.as_bytes() + .starts_with(&tokenizer.decode_bytes(&stable).unwrap()) + ); + assert!(!completions.is_empty()); + for completion in &completions { + let mut ids = stable.clone(); + ids.extend(completion); + assert!( + tokenizer + .decode_bytes(&ids) + .unwrap() + .starts_with(text.as_bytes()) + ); + } + assert!(completions.windows(2).all(|pair| pair[0] < pair[1])); + } + + #[test] + fn encoding_for_model_maps_known_models() { + assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base")); + assert_eq!(encoding_for_model("text-davinci-003"), Some("p50k_base")); + assert_eq!(encoding_for_model("unknown-model"), None); } } diff --git a/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs b/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs new file mode 100644 index 00000000000..1f5e5262de5 --- /dev/null +++ b/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs @@ -0,0 +1,340 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; +use once_cell::sync::OnceCell; +use rustc_hash::FxHashMap; +use thiserror::Error; +use tiktoken_rs::{CoreBPE, O200K_BASE_PAT_STR, Rank}; + +use crate::UnsupportedTokenizer; + +const CL100K: &str = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"; +const O200K: &str = "fb374d419588a4632f3f557e76b4b70aebbca790"; +const P50K: &str = "ec7223a39ce59f226a68acc30dc1af2788490e15"; +const LEGACY_PATTERN: &str = + r"'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s"; +const CL100K_PATTERN: &str = r"'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s"; + +static CL100K_ENCODER: OnceCell = OnceCell::new(); +static O200K_ENCODER: OnceCell = OnceCell::new(); +static HARMONY_ENCODER: OnceCell = OnceCell::new(); +static P50K_ENCODER: OnceCell = OnceCell::new(); +static EDIT_ENCODER: OnceCell = OnceCell::new(); +static R50K_ENCODER: OnceCell = OnceCell::new(); + +/// One encoding built from a rank file: the BPE engine plus the vocabulary it was built +/// from, kept because `CoreBPE` does not expose its ranks and tiktoken's Python API does +/// (`token_byte_values`, `encode_single_token`, `max_token_value`, `_special_tokens`). +pub(super) struct Loaded { + pub(super) bpe: CoreBPE, + pub(super) vocabulary: Vocabulary, +} + +/// The byte-level vocabulary of a tiktoken encoding. +pub struct Vocabulary { + ranks: FxHashMap, Rank>, + special_tokens: FxHashMap, + max_token_value: Rank, +} + +impl Vocabulary { + /// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`. + pub fn token_byte_values(&self) -> Vec> { + let mut values: Vec> = self.ranks.keys().cloned().collect(); + values.sort_unstable(); + values + } + + /// The rank of one whole token: a mergeable piece first, then a special token's text. + pub fn encode_single_token(&self, piece: &[u8]) -> Option { + if let Some(rank) = self.ranks.get(piece) { + return Some(*rank); + } + std::str::from_utf8(piece) + .ok() + .and_then(|text| self.special_tokens.get(text).copied()) + } + + pub fn max_token_value(&self) -> Rank { + self.max_token_value + } + + /// Every mergeable token with its rank, for building other tables from one parse. + pub fn ranks(&self) -> impl Iterator + '_ { + self.ranks + .iter() + .map(|(bytes, rank)| (bytes.as_slice(), *rank)) + } + + /// The special tokens with their ranks, tiktoken's `_special_tokens`. + pub fn special_tokens(&self) -> impl Iterator + '_ { + self.special_tokens + .iter() + .map(|(token, rank)| (token.as_str(), *rank)) + } + + pub fn is_special_token(&self, rank: Rank) -> bool { + self.special_tokens.values().any(|special| *special == rank) + } +} + +#[derive(Debug, Error)] +pub enum LoadError { + #[error(transparent)] + Unsupported(#[from] UnsupportedTokenizer), + #[error("failed to load tiktoken ranks: {0}")] + Ranks(String), +} + +/// Loads `name` once per process. The returned name is the one requested (`gpt2` stays +/// `gpt2`, as `tiktoken.get_encoding("gpt2").name` does), while `gpt2` and `r50k_base` share +/// one cached encoder. +pub(super) fn load( + name: &str, + load_file: impl FnOnce(&str) -> std::io::Result, +) -> Result<(&'static Loaded, &'static str), LoadError> { + let (requested, canonical, file, cache) = match name { + "cl100k_base" => ("cl100k_base", "cl100k_base", CL100K, &CL100K_ENCODER), + "o200k_base" => ("o200k_base", "o200k_base", O200K, &O200K_ENCODER), + "o200k_harmony" => ("o200k_harmony", "o200k_harmony", O200K, &HARMONY_ENCODER), + "p50k_base" => ("p50k_base", "p50k_base", P50K, &P50K_ENCODER), + "p50k_edit" => ("p50k_edit", "p50k_edit", P50K, &EDIT_ENCODER), + "r50k_base" => ("r50k_base", "r50k_base", P50K, &R50K_ENCODER), + "gpt2" => ("gpt2", "r50k_base", P50K, &R50K_ENCODER), + _ => return Err(UnsupportedTokenizer(name.to_owned()).into()), + }; + let loaded = cache.get_or_try_init(|| { + let ranks = load_file(file).map_err(|error| LoadError::Ranks(error.to_string()))?; + build(canonical, &ranks) + })?; + Ok((loaded, requested)) +} + +fn build(name: &str, ranks: &str) -> Result { + let parsed = ranks + .lines() + .map(parse_rank) + .collect::, _>>()?; + let encoder: FxHashMap<_, _> = parsed + .into_iter() + .filter(|(_, rank)| name != "r50k_base" || *rank < 50256) + .collect(); + if encoder + .values() + .collect::>() + .len() + != encoder.len() + || (0..=u8::MAX).any(|byte| !encoder.contains_key(&[byte][..])) + { + return Err(LoadError::Ranks("invalid vocabulary ranks".into())); + } + let (pattern, specials): (&str, &[(&str, Rank)]) = match name { + "cl100k_base" => ( + CL100K_PATTERN, + &[ + ("<|endoftext|>", 100257), + ("<|fim_prefix|>", 100258), + ("<|fim_middle|>", 100259), + ("<|fim_suffix|>", 100260), + ("<|endofprompt|>", 100276), + ], + ), + "o200k_base" => ( + O200K_BASE_PAT_STR, + &[("<|endoftext|>", 199999), ("<|endofprompt|>", 200018)], + ), + "o200k_harmony" => ( + O200K_BASE_PAT_STR, + &[ + ("<|startoftext|>", 199998), + ("<|endoftext|>", 199999), + ("<|reserved_200000|>", 200000), + ("<|reserved_200001|>", 200001), + ("<|return|>", 200002), + ("<|constrain|>", 200003), + ("<|reserved_200004|>", 200004), + ("<|channel|>", 200005), + ("<|start|>", 200006), + ("<|end|>", 200007), + ("<|message|>", 200008), + ("<|reserved_200009|>", 200009), + ("<|reserved_200010|>", 200010), + ("<|reserved_200011|>", 200011), + ("<|call|>", 200012), + ], + ), + "p50k_edit" => ( + LEGACY_PATTERN, + &[ + ("<|endoftext|>", 50256), + ("<|fim_prefix|>", 50281), + ("<|fim_middle|>", 50282), + ("<|fim_suffix|>", 50283), + ], + ), + _ => (LEGACY_PATTERN, &[("<|endoftext|>", 50256)]), + }; + let reserved = (200013..=201087) + .filter(|_| name == "o200k_harmony") + .map(|rank| (format!("<|reserved_{rank}|>"), rank)); + let special_tokens: FxHashMap = specials + .iter() + .map(|(token, rank)| ((*token).to_owned(), *rank)) + .chain(reserved) + .collect(); + let max_token_value = encoder + .values() + .chain(special_tokens.values()) + .copied() + .max() + .ok_or_else(|| LoadError::Ranks("empty vocabulary".into()))?; + let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern) + .map_err(|error| LoadError::Ranks(error.to_string()))?; + Ok(Loaded { + bpe, + vocabulary: Vocabulary { + ranks: encoder, + special_tokens, + max_token_value, + }, + }) +} + +fn parse_rank(line: &str) -> Result<(Vec, Rank), LoadError> { + let (token, rank) = line + .split_once(' ') + .ok_or_else(|| LoadError::Ranks("missing rank".into()))?; + let bytes = STANDARD + .decode(token) + .map_err(|error| LoadError::Ranks(error.to_string()))?; + let rank = rank + .parse() + .map_err(|error: std::num::ParseIntError| LoadError::Ranks(error.to_string()))?; + Ok((bytes, rank)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TiktokenTokenizer; + + fn read_packaged_ranks(file: &str) -> std::io::Result { + std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../litellm/litellm_core_utils/tokenizers") + .join(file), + ) + } + + #[test] + fn packaged_encodings_match_embedded_encodings_and_reuse_successful_loads() { + for name in [ + "cl100k_base", + "o200k_base", + "o200k_harmony", + "p50k_base", + "p50k_edit", + "r50k_base", + "gpt2", + ] { + if name != "gpt2" { + assert!( + TiktokenTokenizer::from_cached_ranks(name, |_| { + Err(std::io::Error::other("unreadable vocabulary")) + }) + .is_err() + ); + } + let loads = std::sync::atomic::AtomicUsize::new(0); + let barrier = std::sync::Barrier::new(4); + let encoders = std::thread::scope(|scope| { + let tasks: Vec<_> = (0..4) + .map(|_| { + scope.spawn(|| { + barrier.wait(); + TiktokenTokenizer::from_cached_ranks(name, |file| { + loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + read_packaged_ranks(file) + }) + .unwrap() + }) + }) + .collect(); + tasks + .into_iter() + .map(|task| task.join().unwrap()) + .collect::>() + }); + assert_eq!(loads.into_inner(), usize::from(name != "gpt2")); + let actual = &encoders[0]; + let expected = TiktokenTokenizer::from_name(name).unwrap(); + assert_eq!(actual.special_tokens(), expected.special_tokens()); + let specials: Vec<_> = expected.special_tokens().into_iter().collect(); + let special_text = specials.join(" "); + assert_eq!( + actual.encode_special(&special_text, &specials).unwrap(), + expected.encode_special(&special_text, &specials).unwrap() + ); + for text in [ + "", + "café 漢字 ع 🙂", + "a\r\nb\t ", + " hello 123456789", + &special_text, + ] { + let ids = expected.encode(text); + assert_eq!(actual.encode(text), ids, "{name}: {text:?}"); + assert_eq!(actual.count_tokens(text), ids.len(), "{name}: {text:?}"); + assert_eq!( + actual.decode_bytes(&ids).unwrap(), + expected.decode_bytes(&ids).unwrap() + ); + } + let cached = TiktokenTokenizer::from_cached_ranks(name, |_| { + panic!("reloaded cached vocabulary") + }) + .unwrap(); + assert_eq!(cached.encode("cached"), expected.encode("cached")); + assert_eq!(cached.name(), name); + assert!(expected.vocabulary().is_none()); + assert_vocabulary_lookups(name, actual); + } + } + + /// The token-level lookups tiktoken's Python `Encoding` exposes, checked against the + /// encoder itself and against the known vocabulary sizes. + fn assert_vocabulary_lookups(name: &str, tokenizer: &TiktokenTokenizer) { + let max_token_value = match name { + "cl100k_base" => 100_276, + "o200k_base" => 200_018, + "o200k_harmony" => 201_087, + "p50k_base" => 50_280, + "p50k_edit" => 50_283, + "r50k_base" | "gpt2" => 50_256, + _ => unreachable!("{name}"), + }; + let vocabulary = tokenizer.vocabulary().unwrap(); + assert_eq!(vocabulary.max_token_value(), max_token_value, "{name}"); + let values = vocabulary.token_byte_values(); + assert!(values.windows(2).all(|pair| pair[0] < pair[1]), "{name}"); + for piece in values.iter().step_by(997) { + let rank = vocabulary.encode_single_token(piece).unwrap(); + assert_eq!(tokenizer.decode_bytes(&[rank]).unwrap(), *piece, "{name}"); + assert!(!vocabulary.is_special_token(rank), "{name}"); + } + for (token, rank) in vocabulary.special_tokens() { + assert_eq!(vocabulary.encode_single_token(token.as_bytes()), Some(rank)); + assert!(vocabulary.is_special_token(rank), "{name}: {token}"); + } + assert_eq!(vocabulary.encode_single_token(b"<|not-a-token|>"), None); + } + + #[test] + fn malformed_ranks_return_errors_instead_of_panicking() { + for ranks in ["", "IQ==", "IQ== x", "!!! 1", "IQ== 1"] { + assert!(build("cl100k_base", ranks).is_err()); + } + let repeated_rank = (0..=u8::MAX) + .map(|byte| format!("{} 0\n", STANDARD.encode([byte]))) + .collect::(); + assert!(build("cl100k_base", &repeated_rank).is_err()); + } +} diff --git a/litellm-rust/crates/token-counter/README.md b/litellm-rust/crates/token-counter/README.md index a6b2b50aac0..3c6381fcea7 100644 --- a/litellm-rust/crates/token-counter/README.md +++ b/litellm-rust/crates/token-counter/README.md @@ -1,6 +1,10 @@ # Token counting -`Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface +`Tokenizer` is the text-counting interface. `TextCodec` adds encoding, decoding, and a name. `TokenCounter` applies LiteLLM request, message, and tool accounting using any `Tokenizer` + +Counts follow the codec: tiktoken treats special-token spellings as ordinary text, while Hugging Face applies its added tokens, post-processing, padding, and truncation. `fast=True` preserves those semantics and requests acceleration where available. Unsupported configurations use the normal codec, including tiktoken encodings without a scanner and builds without the `fast` feature. Invalid input and process-guard errors still propagate. Runtime request counting currently uses the normal codec; the custom accelerator is retained for explicit use and testing + +`FastCounter: TextCodec` exposes an optional accelerator over a loaded codec. `None` means callers should use that codec. The Python bridge caches this selection per immutable tokenizer, shares it with request counters, and initializes it with the GIL released. Hugging Face can also choose the full encoder per input when added tokens require it The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation @@ -8,7 +12,9 @@ The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through t The `tiktoken` feature provides `tiktoken::TiktokenTokenizer` through `tiktoken-rs`. Select an encoding with `TokenCounter::from_tiktoken`. The supported names are `cl100k_base`, `o200k_base`, `o200k_harmony`, `p50k_base`, `p50k_edit`, `r50k_base`, and `gpt2` -All three backends are enabled by default. The Python extension builds with `fast` only, which keeps the wheel at the size it had before the split. With `default-features = false`, callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend +All three backends are enabled by default in this crate and the Python extension. With `default-features = false`, Rust callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend + +Python `tiktoken` and `tokenizers` remain runtime dependencies and the default implementations. The catalog independently selects the tokenizer and request-counting routes. Enabling Rust changes factory dispatch; existing tokenizer objects keep their backend. Native Hugging Face wrappers provide an immutable encoding and decoding API, while training and mutable configuration remain available through the Python backend Budget checks, cost calculation, and the `max_tokens` adjustment policy belong to `litellm-core-utils`. The counter does not own prices, budgets, or request limits diff --git a/litellm-rust/crates/token-counter/src/error.rs b/litellm-rust/crates/token-counter/src/error.rs index b05ce007e46..6a94b0e39b8 100644 --- a/litellm-rust/crates/token-counter/src/error.rs +++ b/litellm-rust/crates/token-counter/src/error.rs @@ -32,6 +32,8 @@ pub enum Error { JsonUtf8(#[source] FromUtf8Error), #[error("tokenization failed: {0}")] Encode(#[source] Box), + #[error("token decoding failed: {0}")] + Decode(String), #[error("token counting task failed: {0}")] Task(String), } diff --git a/litellm-rust/crates/token-counter/src/fast.rs b/litellm-rust/crates/token-counter/src/fast.rs index de3f86abd68..f601c3cbcca 100644 --- a/litellm-rust/crates/token-counter/src/fast.rs +++ b/litellm-rust/crates/token-counter/src/fast.rs @@ -1,7 +1,31 @@ use litellm_token_counter_fast::Error as BackendError; pub use litellm_token_counter_fast::FastTokenizer; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; + +pub trait FastCounter: TextCodec { + fn fast_counter(&self) -> Option; +} + +#[cfg(feature = "huggingface")] +impl FastCounter for crate::huggingface::HuggingFaceTokenizer { + fn fast_counter(&self) -> Option { + Some(FastTokenizer::from_shared(self.shared())) + } +} + +#[cfg(feature = "tiktoken")] +impl FastCounter for crate::tiktoken::TiktokenTokenizer { + fn fast_counter(&self) -> Option { + let vocabulary = self.vocabulary()?; + match self.name() { + "cl100k_base" => FastTokenizer::from_cl100k_pairs(vocabulary.ranks()), + "o200k_base" | "o200k_harmony" => FastTokenizer::from_o200k_pairs(vocabulary.ranks()), + _ => return None, + } + .ok() + } +} impl TokenCounter { pub fn from_json_fast(tokenizer_json: &str) -> Result { @@ -39,3 +63,65 @@ impl From for Error { } } } + +#[cfg(all(test, feature = "huggingface", feature = "tiktoken"))] +mod tests { + use super::*; + use crate::huggingface::HuggingFaceTokenizer; + use crate::tiktoken::TiktokenTokenizer; + + const TEXTS: [&str; 4] = [ + "", + "hello world <|endoftext|>", + "café 漢字 ع 🙂 line\r\n indented 123456789", + "system a\u{301} fi", + ]; + + fn packaged(file: &str) -> String { + std::fs::read_to_string(format!( + "{}/../../../litellm/litellm_core_utils/tokenizers/{file}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() + } + + #[test] + fn fast_counters_derived_from_codecs_count_like_the_codecs() { + let huggingface = + HuggingFaceTokenizer::from_json(&packaged("anthropic_tokenizer.json")).unwrap(); + let fast = huggingface.fast_counter().unwrap(); + for text in TEXTS { + assert_eq!( + fast.count_tokens(text).unwrap(), + Tokenizer::count_tokens(&huggingface, text).unwrap(), + "{text:?}" + ); + } + + for name in ["cl100k_base", "o200k_base", "o200k_harmony"] { + let tiktoken = + TiktokenTokenizer::from_cached_ranks(name, |file| Ok(packaged(file))).unwrap(); + let fast = tiktoken.fast_counter().unwrap(); + for text in TEXTS { + assert_eq!( + fast.count_tokens(text).unwrap(), + tiktoken.count_tokens(text), + "{name}: {text:?}" + ); + } + } + } + + #[test] + fn encodings_without_a_fast_scanner_keep_the_codec() { + let tiktoken = + TiktokenTokenizer::from_cached_ranks("p50k_base", |file| Ok(packaged(file))).unwrap(); + assert!(tiktoken.fast_counter().is_none()); + assert!( + TiktokenTokenizer::from_name("cl100k_base") + .unwrap() + .fast_counter() + .is_none() + ); + } +} diff --git a/litellm-rust/crates/token-counter/src/huggingface.rs b/litellm-rust/crates/token-counter/src/huggingface.rs index fb7683b373e..43613fac8ea 100644 --- a/litellm-rust/crates/token-counter/src/huggingface.rs +++ b/litellm-rust/crates/token-counter/src/huggingface.rs @@ -1,7 +1,11 @@ use litellm_token_counter_huggingface::Error as BackendError; -pub use litellm_token_counter_huggingface::HuggingFaceTokenizer; +pub use litellm_token_counter_huggingface::{ + AddedToken, EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, + PaddingParams, PaddingStrategy, TruncationDirection, TruncationParams, encoding_from_json, + encoding_to_json, +}; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; impl TokenCounter { pub fn from_json(tokenizer_json: &str) -> Result { @@ -17,11 +21,26 @@ impl Tokenizer for HuggingFaceTokenizer { } } +impl TextCodec for HuggingFaceTokenizer { + fn encode(&self, text: &str) -> Result, Error> { + HuggingFaceTokenizer::encode(self, text).map_err(Error::from) + } + + fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result { + HuggingFaceTokenizer::decode(self, ids, skip_special_tokens).map_err(Error::from) + } + + fn name(&self) -> &str { + HuggingFaceTokenizer::name(self) + } +} + impl From for Error { fn from(error: BackendError) -> Self { match error { BackendError::Load(source) => Self::Load(source), BackendError::Encode(source) => Self::Encode(source), + BackendError::Decode(source) => Self::Decode(source.to_string()), } } } diff --git a/litellm-rust/crates/token-counter/src/lib.rs b/litellm-rust/crates/token-counter/src/lib.rs index 446c91049de..84bf35ca2ba 100644 --- a/litellm-rust/crates/token-counter/src/lib.rs +++ b/litellm-rust/crates/token-counter/src/lib.rs @@ -20,5 +20,5 @@ pub mod tiktoken; pub use counter::{InputTokenCount, TokenCounter}; pub use error::Error; -pub use tokenizer::Tokenizer; +pub use tokenizer::{TextCodec, Tokenizer}; pub use types::CountableRequest; diff --git a/litellm-rust/crates/token-counter/src/tiktoken.rs b/litellm-rust/crates/token-counter/src/tiktoken.rs index 07c1c9f5b73..5883a629ea5 100644 --- a/litellm-rust/crates/token-counter/src/tiktoken.rs +++ b/litellm-rust/crates/token-counter/src/tiktoken.rs @@ -1,7 +1,7 @@ -pub use litellm_token_counter_tiktoken::TiktokenTokenizer; -use litellm_token_counter_tiktoken::UnsupportedTokenizer; +use litellm_token_counter_tiktoken::{LoadError, UnsupportedTokenizer}; +pub use litellm_token_counter_tiktoken::{TiktokenTokenizer, Vocabulary, encoding_for_model}; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; impl TokenCounter { pub fn from_tiktoken(encoding: &str) -> Result { @@ -17,8 +17,31 @@ impl Tokenizer for TiktokenTokenizer { } } +impl TextCodec for TiktokenTokenizer { + fn encode(&self, text: &str) -> Result, Error> { + Ok(TiktokenTokenizer::encode(self, text)) + } + + fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result { + TiktokenTokenizer::decode(self, ids).map_err(|error| Error::Decode(error.to_string())) + } + + fn name(&self) -> &str { + TiktokenTokenizer::name(self) + } +} + impl From for Error { fn from(error: UnsupportedTokenizer) -> Self { Self::UnsupportedTokenizer(error.0) } } + +impl From for Error { + fn from(error: LoadError) -> Self { + match error { + LoadError::Unsupported(error) => error.into(), + LoadError::Ranks(message) => Self::Ranks(message), + } + } +} diff --git a/litellm-rust/crates/token-counter/src/tokenizer.rs b/litellm-rust/crates/token-counter/src/tokenizer.rs index 88c29c672a7..146ac5b4d0d 100644 --- a/litellm-rust/crates/token-counter/src/tokenizer.rs +++ b/litellm-rust/crates/token-counter/src/tokenizer.rs @@ -4,6 +4,12 @@ pub trait Tokenizer: Send + Sync { fn count_tokens(&self, text: &str) -> Result; } +pub trait TextCodec: Tokenizer { + fn encode(&self, text: &str) -> Result, Error>; + fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result; + fn name(&self) -> &str; +} + #[cfg(test)] mod tests { use super::*; diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index fe3c7c264ee..29fb46fa125 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -58,7 +58,8 @@ from ._lazy_imports_registry import ( if TYPE_CHECKING: import httpx - from tiktoken import Encoding + + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def get_litellm_globals() -> dict[str, object]: @@ -89,26 +90,11 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo # These are special lazy loaders for things that are used internally # They're separate from the main lazy import system because they have specific use cases -# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup -_default_encoding: "Encoding | None" = None +def _get_default_encoding() -> "Tokenizer": + from litellm.rust_bridge.tokenizer import get_encoding -def _get_default_encoding() -> "Encoding": - """ - Lazily load and cache the default OpenAI encoding. - - This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken) - at `litellm` import time. The encoding is cached after the first import. - - This is used internally by utils.py functions that need the encoding but shouldn't - trigger its import during module load. - """ - global _default_encoding - if _default_encoding is None: - from litellm.litellm_core_utils.default_encoding import encoding - - _default_encoding = encoding - return _default_encoding + return get_encoding("cl100k_base") # Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 917bfbd5ae9..3a28d65e47c 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -7,6 +7,7 @@ "bash_20250124": null, "code-execution-2025-08-25": "code-execution-2025-08-25", "compact-2026-01-12": "compact-2026-01-12", + "compact-2026-09-04": "compact-2026-09-04", "computer-use-2025-01-24": "computer-use-2025-01-24", "computer-use-2025-11-24": "computer-use-2025-11-24", "context-1m-2025-08-07": "context-1m-2025-08-07", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 37743a9ce33..7b02a5ede00 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -115,6 +115,7 @@ from litellm.types.utils import ( LlmProviders, LlmProvidersSet, ModelInfo, + ModelInfoBase, PromptTokensDetailsWrapper, ServiceTier, StandardBuiltInToolsParams, @@ -322,6 +323,48 @@ class OCRPricing(TypedDict, total=False): annotation_cost_per_page: ReadOnly[float | None] +_WALL_CLOCK_PRICED_MODES: Final = frozenset({"chat", "completion", "embedding", "responses"}) + + +def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool: + return ( + (model_info.get("input_cost_per_token") or 0.0) > 0 + or (model_info.get("output_cost_per_token") or 0.0) > 0 + or model_info.get("tiered_pricing") is not None + ) + + +def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool: + mode: Final = model_info.get("mode") + return mode is None or mode in _WALL_CLOCK_PRICED_MODES + + +def _per_second_pricing_cost( + model: str, + custom_llm_provider: str | None, + response_time_ms: float | None, +) -> tuple[float, float] | None: + try: + model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model + return None + if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): + return None + input_cost_per_second: Final = model_info.get("input_cost_per_second") + output_cost_per_second: Final = model_info.get("output_cost_per_second") + if input_cost_per_second is None and output_cost_per_second is None: + return None + seconds: Final = (response_time_ms or 0.0) / 1000 + verbose_logger.debug( + "For model=%s - input_cost_per_second: %s; output_cost_per_second: %s; response time: %s", + model, + input_cost_per_second, + output_cost_per_second, + response_time_ms, + ) + return (input_cost_per_second or 0.0) * seconds, (output_cost_per_second or 0.0) * seconds + + def cost_per_token( model: str = "", prompt_tokens: int = 0, @@ -448,9 +491,6 @@ def cost_per_token( if response_cost is not None: return response_cost[0], response_cost[1] - # given - prompt_tokens_cost_usd_dollar: float = 0 - completion_tokens_cost_usd_dollar: float = 0 model_cost_ref: Final = litellm.model_cost # Only callers that explicitly pass `custom_llm_provider` get the # dedup/prefix-join treatment. When provider is omitted, preserve legacy @@ -611,6 +651,14 @@ def cost_per_token( number_of_queries=number_of_queries or 1, optional_params=(getattr(response, "_hidden_params", None) if response else None), ) + elif ( + per_second_cost := _per_second_pricing_cost( + model=model, + custom_llm_provider=custom_llm_provider, + response_time_ms=response_time_ms, + ) + ) is not None: + return per_second_cost elif custom_llm_provider == "vertex_ai": cost_router: Final = google_cost_router( model=model_without_prefix, @@ -685,12 +733,7 @@ def cost_per_token( ) else: model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) - - if ( - (model_info.get("input_cost_per_token") or 0.0) > 0 - or (model_info.get("output_cost_per_token") or 0.0) > 0 - or model_info.get("tiered_pricing") is not None - ): + if _has_token_or_tiered_pricing(model_info): return generic_cost_per_token( model=model, usage=usage_block, @@ -698,36 +741,8 @@ def cost_per_token( service_tier=service_tier, data_residency=data_residency, ) - - input_cost_per_second: Final = model_info.get("input_cost_per_second") - if input_cost_per_second is not None and response_time_ms is not None: - verbose_logger.debug( - "For model=%s - input_cost_per_second: %s; response time: %s", - model, - input_cost_per_second, - response_time_ms, - ) - ## COST PER SECOND ## - prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000 - - output_cost_per_second: Final = model_info.get("output_cost_per_second") - if output_cost_per_second is not None and response_time_ms is not None: - verbose_logger.debug( - "For model=%s - output_cost_per_second: %s; response time: %s", - model, - output_cost_per_second, - response_time_ms, - ) - ## COST PER SECOND ## - completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000 - - verbose_logger.debug( - "Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s", - model, - prompt_tokens_cost_usd_dollar, - completion_tokens_cost_usd_dollar, - ) - return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar + verbose_logger.debug("No per-token, tiered, or per-second pricing for model=%s; cost is 0", model) + return 0.0, 0.0 def get_replicate_completion_pricing(completion_response: dict, total_time=0.0): @@ -1222,6 +1237,21 @@ def _split_responses_ws_logging_object_by_service_tier( ) +def _response_time_ms_for_cost( + completion_response: object, + litellm_logging_obj: LitellmLoggingObject | None, + total_time: float | None, +) -> float: + stamped: Final = getattr(completion_response, "_response_ms", None) + if isinstance(stamped, (int, float)): + return float(stamped) + if total_time: + return total_time + if litellm_logging_obj is not None: + return litellm_logging_obj.get_response_ms() + return 0.0 + + def completion_cost( completion_response: object | None = None, model: str | None = None, @@ -1443,8 +1473,6 @@ def completion_cost( prompt_tokens_details = _usage.get("prompt_tokens_details") or {} cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0) - total_time = getattr(completion_response, "_response_ms", 0) - hidden_params = getattr(completion_response, "_hidden_params", None) if hidden_params is not None: custom_llm_provider = hidden_params.get("custom_llm_provider", custom_llm_provider or None) @@ -1676,6 +1704,11 @@ def completion_cost( ) return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj) + response_time_ms = _response_time_ms_for_cost( + completion_response=completion_response, + litellm_logging_obj=litellm_logging_obj, + total_time=total_time, + ) # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" @@ -1686,7 +1719,7 @@ def completion_cost( # see https://replicate.com/pricing elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost: # for unmapped replicate model, default to replicate's time tracking logic - return get_replicate_completion_pricing(completion_response, total_time) + return get_replicate_completion_pricing(completion_response, response_time_ms) if model is None: raise ValueError( @@ -1718,7 +1751,7 @@ def completion_cost( prompt_tokens=prompt_tokens or 0, completion_tokens=completion_tokens or 0, custom_llm_provider=custom_llm_provider, - response_time_ms=total_time, + response_time_ms=response_time_ms, region_name=None if explicit_pricing else region_name, custom_cost_per_second=custom_cost_per_second, custom_cost_per_token=custom_cost_per_token, diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index d3ad7234d93..7ecbeea255c 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -427,7 +427,7 @@ class LLMCallSpanData: # plain ``.get`` — no repeated ``isinstance`` guards. raw_response: Final = payload.get("response") response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {}) - choices_out: Final = _dicts(response.get("choices")) or _responses_choices(response) or _ocr_choices(response) + choices_out: Final = _output_choices(response) # ``finish_reasons`` is metadata, not content, so derive it from # ``choices_out`` before gating. The raw message/choice bodies are only # retained when content capture is enabled (see ``capture_span_content``); @@ -752,20 +752,99 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: return (choice,) -def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: - markdowns: Final = tuple( - text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None +def _output_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + """The response output as chat-shaped choices; images and binary bodies become size summaries, never bytes.""" + return ( + _completion_choices(response) + or _responses_choices(response) + or _ocr_choices(response) + or _transcription_choices(response) + or _moderation_choices(response) + or _image_choices(response) + or _binary_choices(response) ) - if not markdowns: + + +def _text_choice(content: str, finish_reason: str | None = None) -> _Choice: + message: Final[_AssistantMessage] = {"role": "assistant", "content": content, "refusal": None, "tool_calls": None} + return {"message": message, "finish_reason": finish_reason} + + +def _joined_choice(parts: tuple[str, ...]) -> tuple[_Choice, ...]: + return (_text_choice("\n\n".join(parts)),) if parts else () + + +def _completion_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return tuple( + _text_choice(text, as_str(choice.get("finish_reason"))) + if "message" not in choice and isinstance(text := choice.get("text"), str) + else choice + for choice in _dicts(response.get("choices")) + ) + + +def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple(text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None) + ) + + +def _transcription_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + text: Final = response.get("text") + return (_text_choice(text),) if isinstance(text, str) and text else () + + +def _moderation_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple( + _moderation_verdict(flagged, result.get("categories")) + for result in _dicts(response.get("results")) + if isinstance(flagged := result.get("flagged"), bool) + ) + ) + + +def _moderation_verdict(flagged: bool, categories: object) -> str: + if not flagged: + return "not flagged" + hits: Final = ( + tuple(name for name, hit in cast(Mapping[str, object], categories).items() if hit is True) + if isinstance(categories, dict) + else () + ) + return f"flagged: {', '.join(hits)}" if hits else "flagged" + + +def _image_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple(summary for item in _dicts(response.get("data")) if (summary := _image_summary(item)) is not None) + ) + + +def _image_summary(item: Mapping[str, object]) -> str | None: + location: Final = _image_location(item) + if location is None: + return None + revised: Final = as_str(item.get("revised_prompt")) + return f"{revised}\n{location}" if revised else location + + +def _image_location(item: Mapping[str, object]) -> str | None: + url: Final = as_str(item.get("url")) + if url is not None: + return url + encoded: Final = item.get("b64_json") + if not isinstance(encoded, str): + return None + return f"b64_json image ({len(encoded) * 3 // 4 - encoded[-2:].count('=')} bytes)" + + +def _binary_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + size: Final = as_int(response.get("num_bytes")) + if size is None: return () - message: Final[_AssistantMessage] = { - "role": "assistant", - "content": "\n\n".join(markdowns), - "refusal": None, - "tool_calls": None, - } - choice: Final[_Choice] = {"message": message, "finish_reason": None} - return (choice,) + content_type: Final = as_str(response.get("content_type")) + return (_text_choice(f"{content_type} ({size} bytes)" if content_type else f"{size} bytes"),) def _responses_parts_text(parts: tuple[Mapping[str, object], ...], part_type: str, field: str) -> str | None: diff --git a/litellm/litellm_core_utils/README.md b/litellm/litellm_core_utils/README.md index b61c8982762..a5f5e8326b9 100644 --- a/litellm/litellm_core_utils/README.md +++ b/litellm/litellm_core_utils/README.md @@ -6,8 +6,9 @@ Core files: - `streaming_handler.py`: The core streaming logic + streaming related helper utils - `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`. - `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types. -- `default_encoding.py`: code for loading the default encoding (tiktoken) +- `default_encoding.py`: code for loading the default Python tokenizer and bundled cache - `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name. - `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s" - `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion]) +Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer. diff --git a/litellm/litellm_core_utils/bug_report.py b/litellm/litellm_core_utils/bug_report.py new file mode 100644 index 00000000000..4a8f4bc65ad --- /dev/null +++ b/litellm/litellm_core_utils/bug_report.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import os +import platform +import traceback +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal +from urllib.parse import urlencode + +import litellm +from litellm._version import version as litellm_version +from litellm.types.utils import LlmProviders + +ISSUE_URL_BASE: Final = "https://github.com/BerriAI/litellm/issues/new" +MAX_URL_LENGTH: Final = 6000 +MAX_FRAMES: Final = 12 +DISABLE_ENV_VAR: Final = "LITELLM_DISABLE_BUG_REPORT_LINK" +NOTICE_PREFIX: Final = "This looks like a bug in LiteLLM rather than in your request." +KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders) + +Surface = Literal["sdk", "proxy"] + + +@dataclass(frozen=True, slots=True) +class BugReport: + surface: Surface + exception_type: str + litellm_frames: tuple[str, ...] + litellm_version: str + python_version: str + call_type: str | None + custom_llm_provider: str | None + stream: bool | None + config_lines: tuple[str, ...] + + +def bug_report_enabled() -> bool: + return os.getenv(DISABLE_ENV_VAR, "").lower() != "true" + + +def should_report_bug(exc: object) -> bool: + return bug_report_enabled() and isinstance(exc, BaseException) and getattr(exc, "status_code", None) is None + + +def _format_frame(frame: traceback.FrameSummary, package_dir: Path, package_parent: Path) -> str | None: + frame_path: Final = Path(frame.filename).resolve() + try: + frame_path.relative_to(package_dir) + relative_path: Final = frame_path.relative_to(package_parent) + except ValueError: + return None + return f"{relative_path.as_posix()}:{frame.lineno} in {frame.name}" + + +def _get_litellm_frames(exc: BaseException) -> tuple[str, ...]: + if exc.__traceback__ is None: + return () + package_dir: Final = Path(litellm.__file__).resolve().parent + package_parent: Final = package_dir.parent + return tuple( + frame_text + for frame in traceback.extract_tb(exc.__traceback__) + if (frame_text := _format_frame(frame, package_dir, package_parent)) is not None + )[-MAX_FRAMES:] + + +def allowlisted(value: object, allowed: frozenset[str]) -> str | None: + return value if isinstance(value, str) and value in allowed else None + + +def build_bug_report( + exc: BaseException, + *, + surface: Surface, + call_type: str | None = None, + custom_llm_provider: object = None, + stream: object = None, + config_lines: tuple[str, ...] = (), +) -> BugReport: + return BugReport( + surface=surface, + exception_type=type(exc).__name__, + litellm_frames=_get_litellm_frames(exc), + litellm_version=litellm_version, + python_version=platform.python_version(), + call_type=call_type, + custom_llm_provider=allowlisted(custom_llm_provider, KNOWN_PROVIDERS), + stream=stream if isinstance(stream, bool) else None, + config_lines=config_lines, + ) + + +def _domain(report: BugReport) -> str: + if report.surface == "sdk": + return "Python SDK: the litellm package itself" + if report.custom_llm_provider is not None: + return "LLM translation: a specific provider's request or response" + return "Proxy core: startup, config, health checks, endpoints" + + +def _title(report: BugReport, frames: tuple[str, ...]) -> str: + location: Final = frames[-1].split(":", 1)[0] if frames else "litellm" + return f"[Bug]: {report.exception_type} in {location}" + + +def _description(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str: + frame_block: Final = "LiteLLM frames:\n```\n" + "\n".join(frames) + "\n```\n\n" if frames else "" + stream_line: Final = "" if report.stream is None else f"Stream: {str(report.stream).lower()}\n" + config_block: Final = ( + "\nConfig (true/false flags and LiteLLM-defined values only):\n```\n" + "\n".join(config_lines) + "\n```\n" + if config_lines + else "" + ) + return ( + "Auto-generated by LiteLLM's bug report link. It carries no request data or error text. " + "Please describe what you were doing, and paste the error message from your log below " + "if it contains nothing sensitive.\n\n" + "```\n\n```\n\n" + f"Exception: `{report.exception_type}`\n\n" + f"{frame_block}" + f"Surface: {report.surface}\n" + f"Endpoint / call: {report.call_type or 'unknown'}\n" + f"Provider: {report.custom_llm_provider or 'unknown'}\n" + f"LiteLLM: {report.litellm_version}\n" + f"Python: {report.python_version}\n" + f"{stream_line}" + f"{config_block}" + ) + + +def _issue_url(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str: + deployment: Final[tuple[tuple[str, str], ...]] = ( + (("deployment", "pip / Python SDK"),) + if report.surface == "sdk" + else (("deployment", "Docker"),) + if os.path.exists("/.dockerenv") + else () + ) + fields: Final = ( + ("template", "bug_report.yml"), + ("labels", "bug"), + ("title", _title(report, frames)), + ("version", report.litellm_version), + ("domain", _domain(report)), + ("description", _description(report, frames, config_lines)), + ) + deployment + return f"{ISSUE_URL_BASE}?{urlencode(fields)}" + + +def bug_report_issue_url(report: BugReport) -> str: + frames: Final = report.litellm_frames + config_lines: Final = report.config_lines + candidates: Final = ( + *((frames, config_lines[:count]) for count in range(len(config_lines), -1, -1)), + *((frames[index:], ()) for index in range(1, len(frames) + 1)), + ) + return next( + ( + url + for candidate_frames, candidate_config in candidates + if len(url := _issue_url(report, candidate_frames, candidate_config)) <= MAX_URL_LENGTH + ), + _issue_url(report, (), ()), + ) + + +def strip_bug_report_notice(message: str) -> str: + index: Final = message.find(NOTICE_PREFIX) + if index == -1: + return message + head: Final = message[:index] + return head.removesuffix("\n") + + +def bug_report_notice(report: BugReport) -> str: + return ( + f"{NOTICE_PREFIX} File it with one click " + f"(prefilled, no request data or error text, review before submitting): {bug_report_issue_url(report)}" + ) diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index 71b30614d8d..c3b6a008411 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -1,5 +1,4 @@ import os -from pathlib import Path from typing import Final import litellm @@ -15,20 +14,6 @@ except (ImportError, AttributeError): filename = pkg_resources.resource_filename(__name__, "litellm_core_utils/tokenizers") -CL100K_BASE_RANK_FILE: Final = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4" -O200K_BASE_RANK_FILE: Final = "fb374d419588a4632f3f557e76b4b70aebbca790" - - -def cl100k_base_rank_file() -> str: - """The vendored tiktoken `cl100k_base` rank file (`base64(token) rank` lines).""" - return Path(filename, CL100K_BASE_RANK_FILE).read_text(encoding="ascii") - - -def o200k_base_rank_file() -> str: - """The vendored tiktoken `o200k_base` rank file (`base64(token) rank` lines).""" - return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii") - - # Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory # unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR. # This keeps tiktoken fully offline-capable by default (see #1071). diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 425714d730f..da2f11f2593 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -10,6 +10,11 @@ import httpx import litellm from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger +from litellm.litellm_core_utils.bug_report import ( + bug_report_notice, + build_bug_report, + should_report_bug, +) from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.types.utils import LlmProviders @@ -2673,7 +2678,21 @@ def exception_type( ) else: raise APIConnectionError( - message=f"{original_exception}\n{_redact_string(traceback.format_exc())}", + message=( + f"{original_exception}\n{_redact_string(traceback.format_exc())}" + + ( + "\n" + + bug_report_notice( + build_bug_report( + original_exception, + surface="sdk", + custom_llm_provider=custom_llm_provider, + ) + ) + if should_report_bug(original_exception) + else "" + ) + ), llm_provider=custom_llm_provider, model=model, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index e4744079622..34d0ded618c 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -1,10 +1,24 @@ import re -from collections.abc import Iterator, Mapping +from collections.abc import Generator, Iterator, Mapping +from contextlib import contextmanager +from contextvars import ContextVar from typing import Any, Final from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams _CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata") +_inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar( + "inherited_message_logging_disabled", default=False +) + + +@contextmanager +def inherit_message_logging_privacy(disabled: bool) -> Generator[None]: + token: Final = _inherited_message_logging_disabled.set(_inherited_message_logging_disabled.get() or disabled) + try: + yield + finally: + _inherited_message_logging_disabled.reset(token) def iter_client_callback_metadata_dicts( @@ -143,7 +157,7 @@ def get_trusted_callback_params(kwargs: Mapping[str, Any] | None) -> tuple[tuple def initialize_standard_callback_dynamic_params( - kwargs: dict | None = None, + kwargs: dict[str, object] | None = None, ) -> StandardCallbackDynamicParams: """ Initialize the standard callback dynamic params from the kwargs @@ -179,4 +193,10 @@ def initialize_standard_callback_dynamic_params( if param in _trusted_overlay_callback_params: standard_callback_dynamic_params[param] = trusted_value + if _inherited_message_logging_disabled.get(): + private_params: Final[StandardCallbackDynamicParams] = { + **standard_callback_dynamic_params, + "turn_off_message_logging": True, + } + return private_params return standard_callback_dynamic_params diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4249cd4573d..cc124348d9b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -496,6 +496,14 @@ def mask_api_base_credentials(api_base: str) -> str: return api_base[:key_end] + "*" * 5 + api_base[-4:] +def _timestamp_seconds(moment: object) -> float | None: + if isinstance(moment, datetime.datetime): + return moment.timestamp() + if isinstance(moment, (int, float)): + return float(moment) + return None + + class Logging(LiteLLMLoggingBaseClass): global \ supabaseClient, \ @@ -1634,10 +1642,12 @@ class Logging(LiteLLMLoggingBaseClass): return response.mcp_tool_call_response def get_response_ms(self) -> float: - return ( - self.model_call_details.get("end_time", datetime.datetime.now()) - - self.model_call_details.get("start_time", datetime.datetime.now()) - ).total_seconds() * 1000 + now: Final = datetime.datetime.now() + start_seconds: Final = _timestamp_seconds(self.model_call_details.get("start_time", now)) + end_seconds: Final = _timestamp_seconds(self.model_call_details.get("end_time", now)) + if start_seconds is None or end_seconds is None: + return 0.0 + return (end_seconds - start_seconds) * 1000 def set_cost_breakdown( self, @@ -6277,6 +6287,8 @@ def _extract_response_obj_and_hidden_params( hidden_params = getattr(init_response_obj, "_hidden_params", None) elif isinstance(init_response_obj, dict): response_obj = init_response_obj + elif isinstance(init_response_obj, HttpxBinaryResponseContent): + response_obj = dict(init_response_obj.logging_summary()) else: response_obj = {} diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 4a3c8de78c5..503814cc143 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -251,6 +251,6 @@ def update_response_metadata( return metadata: Final = ResponseMetadata(result) - metadata.set_hidden_params(logging_obj, model, kwargs) metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead) + metadata.set_hidden_params(logging_obj, model, kwargs) metadata.apply() diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index b409b181a79..15bccf0301e 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -245,6 +245,8 @@ def _redact_model_response_dict_choices(choices, redacted_str: str): if "audio" in choice["delta"]: choice["delta"]["audio"] = None _redact_tool_calls_dict(choice["delta"]) + elif choice.get("text") is not None: + choice["text"] = redacted_str else: _redact_choice_content(choice) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index bf37b1be2e4..5d7956059e4 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -10,7 +10,6 @@ import anyio import anyio.lowlevel import httpx import tiktoken -from tokenizers import Tokenizer from typing_extensions import ParamSpec, TypeVar import litellm @@ -30,8 +29,10 @@ from litellm.constants import ( TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import _get_httpx_client +from litellm.rust_bridge.tokenizer import get_encoding from litellm.types.llms.anthropic import ( AnthropicContentParamSource, AnthropicContentParamSourceFileId, @@ -622,9 +623,11 @@ def _get_exact_count_function( if model is not None or custom_tokenizer is not None: tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model) if tokenizer_json["type"] == "huggingface_tokenizer": - tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"] + tokenizer: Final[HuggingFace] = tokenizer_json["tokenizer"] def count_tokens(text: str) -> int: + if isinstance(tokenizer, HuggingFaceTokenizer): + return tokenizer.count(text) return len(tokenizer.encode_batch_fast([text])[0]) return count_tokens @@ -632,31 +635,43 @@ def _get_exact_count_function( encoding: Final = openai_tokenizer_encoding(model) def encode_length(text: str) -> int: - return len(encoding.encode(text, disallowed_special=())) + return _encoding_count(encoding, text) return _get_tiktoken_count_function(encode_length) else: raise ValueError("Unsupported tokenizer type") else: + default_encoding: Final = _get_default_encoding() def encode_length(text: str) -> int: - return len(_get_default_encoding().encode(text, disallowed_special=())) + return _encoding_count(default_encoding, text) return _get_tiktoken_count_function(encode_length) -def openai_tokenizer_encoding(model: str) -> tiktoken.Encoding: - """The tiktoken encoding `token_counter` uses for a model on the `openai_tokenizer` path.""" +def _encoding_count(encoding: Encoding, text: str) -> int: + if isinstance(encoding, OpenAIEncoding): + return encoding.count(text) + return len(encoding.encode(text, disallowed_special=())) + + +def openai_tokenizer_encoding(model: str) -> Encoding: + """The encoding `token_counter` uses for a model on the `openai_tokenizer` path.""" + return get_encoding(openai_tokenizer_encoding_name(model)) + + +def openai_tokenizer_encoding_name(model: str) -> str: + """The tiktoken encoding name for `model`, without loading the encoding.""" from litellm.utils import print_verbose model_to_use: Final = _fix_model_name(model) if "gpt-4o" in model_to_use: - return tiktoken.get_encoding("o200k_base") + return "o200k_base" try: - return tiktoken.encoding_for_model(model_to_use) + return tiktoken.encoding_name_for_model(model_to_use) except KeyError: print_verbose("Warning: model not found. Using cl100k_base encoding.") - return tiktoken.get_encoding("cl100k_base") + return "cl100k_base" def uses_legacy_message_accounting(model: str) -> bool: diff --git a/litellm/litellm_core_utils/tokenizer.py b/litellm/litellm_core_utils/tokenizer.py new file mode 100644 index 00000000000..aea187fa08e --- /dev/null +++ b/litellm/litellm_core_utils/tokenizer.py @@ -0,0 +1,402 @@ +"""Python faces of the Rust text codecs. + +``OpenAIEncoding`` mirrors ``tiktoken.Encoding`` and ``HuggingFaceTokenizer`` mirrors +``tokenizers.Tokenizer``, so a caller holding ``litellm.encoding`` or the object returned by +``litellm.create_tokenizer`` sees the same read-only surface whichever backend the Rust catalog +selected. Both wrappers are immutable: ``tokenizers`` mutators (``enable_padding``, +``enable_truncation``, ``add_tokens``) stay on the Python tokenizer. +""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping, Sequence, Set +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable + +import tiktoken +from tokenizers import AddedToken +from tokenizers import Tokenizer as PythonHuggingFaceTokenizer + +if TYPE_CHECKING: + import numpy as np + import numpy.typing as npt + + from litellm.rust_bridge._native import HuggingFaceEncoding + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + +SpecialTokens: TypeAlias = Literal["all"] | Collection[str] +AllowedSpecial: TypeAlias = Literal["all"] | Set[str] +HuggingFaceInput: TypeAlias = str | list[str] | tuple[str, ...] +HuggingFaceBatchInput: TypeAlias = HuggingFaceInput | tuple[HuggingFaceInput, HuggingFaceInput] | list[HuggingFaceInput] + + +@dataclass(frozen=True, slots=True) +class OpenAIEncoding: + """``tiktoken.Encoding`` over the Rust tiktoken codec.""" + + _native: NativeTokenizer + _special_tokens: Mapping[str, int] + + @staticmethod + def wrap(native: NativeTokenizer) -> OpenAIEncoding: + return OpenAIEncoding(native, MappingProxyType(native.special_tokens())) + + @staticmethod + def from_tiktoken(encoding: str) -> OpenAIEncoding: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return OpenAIEncoding.wrap(NativeTokenizer.from_tiktoken(encoding)) + + def __repr__(self) -> str: + return f"" + + @property + def name(self) -> str: + return self._native.name + + @property + def max_token_value(self) -> int: + return self._native.max_token_value() + + @property + def n_vocab(self) -> int: + """For backwards compatibility. Prefer to use `enc.max_token_value + 1`.""" + return self.max_token_value + 1 + + @property + def eot_token(self) -> int: + return self._special_tokens["<|endoftext|>"] + + @property + def special_tokens_set(self) -> set[str]: # mutable-ok: [LIT001, LIT002] SDK return type + return set(self._special_tokens) + + def is_special_token(self, token: int) -> bool: + return self._native.is_special_token(token) + + # ---- encoding ------------------------------------------------------------------------- + + def encode_ordinary(self, text: str) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.encode(text) + + def encode( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type + allowed: Final = self._allowed(text, allowed_special, disallowed_special) + if not allowed: + return self.encode_ordinary(text) + return self._native.encode_special(text, tuple(allowed)) + + def encode_to_numpy( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> npt.NDArray[np.uint32]: + import numpy + + return numpy.asarray( + self.encode(text, allowed_special=allowed_special, disallowed_special=disallowed_special), + dtype=numpy.uint32, + ) + + def encode_ordinary_batch( + self, text: Sequence[str], *, num_threads: int = 8 + ) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(self.encode_ordinary, text) + ) + + def encode_batch( + self, + text: Sequence[str], + *, + num_threads: int = 8, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + encode: Final = partial(self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special) + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(encode, text) + ) + + def encode_with_unstable( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> tuple[list[int], list[list[int]]]: # mutable-ok: [LIT001, LIT002] SDK return type + """The stable tokens of `text` and every completion its unstable tail could become. + + Completions come back sorted; tiktoken returns them in hash order.""" + allowed: Final = self._allowed(text, allowed_special, disallowed_special) + return self._native.encode_with_unstable(text, tuple(allowed)) + + def encode_single_token(self, text_or_bytes: str | bytes) -> int: + """The token of one whole piece, special tokens included. Raises `KeyError` otherwise.""" + piece: Final = text_or_bytes.encode("utf-8") if isinstance(text_or_bytes, str) else text_or_bytes + return self._native.encode_single_token(piece) + + def count(self, text: str, fast: bool = False) -> int: + """Count ordinary text; `fast` accelerates supported encodings and otherwise counts normally.""" + return self._native.count(text, fast) + + # ---- decoding ------------------------------------------------------------------------- + + def decode_bytes(self, tokens: Sequence[int]) -> bytes: + return self._native.decode_bytes(tokens) + + def decode(self, tokens: Sequence[int], errors: str = "replace") -> str: + return self.decode_bytes(tokens).decode("utf-8", errors=errors) + + def decode_single_token_bytes(self, token: int) -> bytes: + return self.decode_bytes((token,)) + + def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + return [ # mutable-ok: [LIT002] SDK returns a list + self.decode_single_token_bytes(token) for token in tokens + ] + + def decode_with_offsets( + self, tokens: Sequence[int] + ) -> tuple[str, list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + """The decoded text and, per token, the index of the first character holding its bytes. + + Like tiktoken, raises `UnicodeDecodeError` when the tokens do not decode to valid UTF-8.""" + token_bytes: Final = self.decode_tokens_bytes(tokens) + text_len = 0 + offsets: Final[list[int]] = [] # mutable-ok: [LIT001] local accumulator + for token in token_bytes: + offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0))) + text_len += sum(1 for c in token if not 0x80 <= c < 0xC0) + return b"".join(token_bytes).decode("utf-8", errors="strict"), offsets + + def decode_batch( + self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8 + ) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(partial(self.decode, errors=errors), batch) + ) + + def decode_bytes_batch( + self, batch: Sequence[Sequence[int]], *, num_threads: int = 8 + ) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(self.decode_bytes, batch) + ) + + def token_byte_values(self) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.token_byte_values() + + def __reduce__(self) -> tuple[Callable[[str], OpenAIEncoding], tuple[str]]: + return (OpenAIEncoding.from_tiktoken, (self.name,)) + + # ---- private -------------------------------------------------------------------------- + + def _allowed(self, text: str, allowed_special: AllowedSpecial, disallowed_special: SpecialTokens) -> frozenset[str]: + """tiktoken's special-token policy: which specials `text` may encode, after rejecting + any it must not contain.""" + allowed: Final = frozenset(self._special_tokens) if allowed_special == "all" else frozenset(allowed_special) + disallowed: Final = ( + frozenset(self._special_tokens) - allowed if disallowed_special == "all" else frozenset(disallowed_special) + ) + for token in disallowed: + if token in text: + raise ValueError( + f"Encountered text corresponding to disallowed special token {token!r}.\n" + "If you want this text to be encoded as a special token, " + f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n" + "If you want this text to be encoded as normal text, disable the check for this token " + f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n" + "To disable this check for all special tokens, pass `disallowed_special=()`.\n" + ) + return allowed + + +@dataclass(frozen=True, slots=True) +class HuggingFaceTokenizer: + """The read-only ``tokenizers.Tokenizer`` surface over the Rust Hugging Face codec.""" + + _native: NativeTokenizer + + @staticmethod + def from_str(json: str) -> HuggingFaceTokenizer: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return HuggingFaceTokenizer(NativeTokenizer.from_json(json)) + + from_json = from_str + + @staticmethod + def from_buffer(buffer: bytes) -> HuggingFaceTokenizer: + return HuggingFaceTokenizer.from_str(buffer.decode("utf-8")) + + @staticmethod + def from_file(path: str) -> HuggingFaceTokenizer: + return HuggingFaceTokenizer.from_str(Path(path).read_text(encoding="utf-8")) + + @staticmethod + def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFaceTokenizer: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return HuggingFaceTokenizer(NativeTokenizer.from_pretrained(identifier, revision=revision, token=token)) + + def to_str(self, pretty: bool = False) -> str: + return self._native.to_json(pretty) + + def save(self, path: str, pretty: bool = True) -> None: + Path(path).write_text(self.to_str(pretty), encoding="utf-8") + + @property + def name(self) -> str: + return self._native.name + + # ---- vocabulary ----------------------------------------------------------------------- + + def token_to_id(self, token: str) -> int | None: + return self._native.token_to_id(token) + + def id_to_token(self, id: int) -> str | None: + return self._native.id_to_token(id) + + def get_vocab( + self, with_added_tokens: bool = True + ) -> dict[str, int]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.get_vocab(with_added_tokens) + + def get_vocab_size(self, with_added_tokens: bool = True) -> int: + return self._native.get_vocab_size(with_added_tokens) + + def get_added_tokens_decoder(self) -> dict[int, AddedToken]: # mutable-ok: [LIT001, LIT002] SDK return type + return { # mutable-ok: [LIT002] SDK returns a dict + token_id: AddedToken( + content, single_word=single_word, lstrip=lstrip, rstrip=rstrip, normalized=normalized, special=special + ) + for token_id, ( + content, + single_word, + lstrip, + rstrip, + normalized, + special, + ) in self._native.added_tokens_decoder() + } + + def num_special_tokens_to_add(self, is_pair: bool) -> int: + return self._native.num_special_tokens_to_add(is_pair) + + @property + def padding(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.padding() + + @property + def truncation(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.truncation() + + @property + def encode_special_tokens(self) -> bool: + return self._native.encode_special_tokens() + + # ---- encoding and decoding ------------------------------------------------------------ + + def encode( + self, + sequence: HuggingFaceInput, + pair: HuggingFaceInput | None = None, + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> HuggingFaceEncoding: + return self._native.encode_huggingface(sequence, pair, is_pretokenized, add_special_tokens) + + def encode_batch( + self, + input: Sequence[HuggingFaceBatchInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=False) + + def encode_batch_fast( + self, + input: Sequence[HuggingFaceBatchInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=True) + + def _encode_batch( + self, input: Sequence[HuggingFaceBatchInput], is_pretokenized: bool, add_special_tokens: bool, fast: bool + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + sequences: Final = tuple(_batch_input(item, is_pretokenized) for item in input) + return self._native.encode_batch_huggingface(sequences, is_pretokenized, add_special_tokens, fast) + + def count(self, text: str, fast: bool = False) -> int: + """Count with this tokenizer's configuration; `fast` uses acceleration where supported.""" + return self._native.count(text, fast) + + def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str: + return self._native.decode(ids, skip_special_tokens=skip_special_tokens) + + def decode_batch( + self, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True + ) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type + return [ # mutable-ok: [LIT002] SDK returns a list + self.decode(ids, skip_special_tokens=skip_special_tokens) for ids in sequences + ] + + def __reduce__(self) -> tuple[Callable[[str], HuggingFaceTokenizer], tuple[str]]: + return (HuggingFaceTokenizer.from_str, (self.to_str(),)) + + +def _batch_input( + item: HuggingFaceBatchInput, is_pretokenized: bool +) -> tuple[HuggingFaceInput, HuggingFaceInput | None]: + if isinstance(item, str): + return (item, None) + if is_pretokenized and all(isinstance(word, str) for word in item): + return (tuple(word for word in item if isinstance(word, str)), None) + if len(item) != 2: + raise TypeError("batch input must be a sequence or a pair of sequences") + return (item[0], item[1]) + + +Encoding: TypeAlias = tiktoken.Encoding | OpenAIEncoding +HuggingFace: TypeAlias = PythonHuggingFaceTokenizer | HuggingFaceTokenizer +Tokenizer: TypeAlias = Encoding | HuggingFace + + +class _AddedToken(Protocol): + @property + def special(self) -> bool: ... + + +@runtime_checkable +class _AddedTokenDecoder(Protocol): + def get_added_tokens_decoder(self) -> Mapping[int, _AddedToken]: ... + + +def strip_special_tokens(tokenizer: object, tokens: Sequence[int]) -> Sequence[int]: + """Drop the special added tokens before a Python `tokenizers` decode; the Rust codec's + `decode(skip_special_tokens=True)` already does this itself.""" + if isinstance(tokenizer, HuggingFaceTokenizer) or not isinstance(tokenizer, _AddedTokenDecoder): + return tokens + try: + added: Final = tokenizer.get_added_tokens_decoder() + except Exception: # noqa: BLE001 # optional metadata failures historically fall back to decoding + return tokens + special_ids: Final = frozenset(token_id for token_id, token in added.items() if token.special) + return tuple(token for token in tokens if token not in special_ids) diff --git a/litellm/llms/a2a/chat/transformation.py b/litellm/llms/a2a/chat/transformation.py index 77f26b65de0..c5a71daaba5 100644 --- a/litellm/llms/a2a/chat/transformation.py +++ b/litellm/llms/a2a/chat/transformation.py @@ -23,9 +23,8 @@ from ..common_utils import ( from .streaming_iterator import A2AModelResponseIterator if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer _REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = ( @@ -292,7 +291,7 @@ class A2AConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/aiml/image_generation/transformation.py b/litellm/llms/aiml/image_generation/transformation.py index 4f4cd074165..b585d35a2ac 100644 --- a/litellm/llms/aiml/image_generation/transformation.py +++ b/litellm/llms/aiml/image_generation/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -171,7 +170,7 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/aiohttp_openai/chat/transformation.py b/litellm/llms/aiohttp_openai/chat/transformation.py index 530896bf9b0..a06c670e3f1 100644 --- a/litellm/llms/aiohttp_openai/chat/transformation.py +++ b/litellm/llms/aiohttp_openai/chat/transformation.py @@ -16,9 +16,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -68,7 +67,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/amazon_nova/chat/transformation.py b/litellm/llms/amazon_nova/chat/transformation.py index 7551fb28c21..a93fbf1e933 100644 --- a/litellm/llms/amazon_nova/chat/transformation.py +++ b/litellm/llms/amazon_nova/chat/transformation.py @@ -17,7 +17,7 @@ from litellm.types.utils import ModelResponse from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonNovaChatConfig(OpenAILikeChatConfig): @@ -86,7 +86,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/batches/transformation.py b/litellm/llms/anthropic/batches/transformation.py index 4f4d39f09b0..7dee7513538 100644 --- a/litellm/llms/anthropic/batches/transformation.py +++ b/litellm/llms/anthropic/batches/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest from litellm.types.utils import LiteLLMBatch, LlmProviders, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -290,7 +289,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 545e920156e..b7c2ce3c568 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -96,13 +96,13 @@ from ..common_utils import ( AnthropicModelInfo, eager_input_streaming_flag, process_anthropic_headers, + requires_native_compaction_beta, strip_advisor_blocks_from_messages, ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -1771,7 +1771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return tools - def _ensure_beta_header(self, headers: dict, beta_value: str) -> None: + def _ensure_beta_header(self, headers: dict[str, str], beta_value: str) -> None: """ Ensure a beta header value is present in the anthropic-beta header. Merges with existing values instead of overriding them. @@ -1780,13 +1780,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): headers: Dictionary of headers to update beta_value: The beta header value to add """ - existing_beta: Final = headers.get("anthropic-beta") - if existing_beta is None: - headers["anthropic-beta"] = beta_value - return - existing_values: Final = [beta.strip() for beta in existing_beta.split(",")] - if beta_value not in existing_values: - headers["anthropic-beta"] = f"{existing_beta}, {beta_value}" + existing_values: Final = tuple( + beta.strip() + for key, value in headers.items() + if key.lower() == "anthropic-beta" + for beta in value.split(",") + if beta.strip() + ) + for key in tuple(headers): + if key.lower() == "anthropic-beta": + headers.pop(key) + headers["anthropic-beta"] = ", ".join(dict.fromkeys((*existing_values, beta_value))) def _ensure_context_management_beta_header(self, headers: dict, context_management: object) -> None: """ @@ -1824,7 +1828,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, ) - def update_headers_with_optional_anthropic_beta(self, headers: dict, optional_params: dict) -> dict: + def update_headers_with_optional_anthropic_beta( + self, headers: dict, optional_params: dict, messages: Sequence[object] = () + ) -> dict: """Update headers with optional anthropic beta.""" # Skip adding beta headers for Vertex requests @@ -1833,6 +1839,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if is_vertex_request: return headers + if requires_native_compaction_beta(self._resolved_provider, optional_params, messages): + self._ensure_beta_header(headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) + _tools: Final = optional_params.get("tools", []) for tool in _tools: if tool.get("type", None) and tool.get("type").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value): @@ -1929,8 +1938,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): custom_llm_provider=self._resolved_provider, ) - headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params) - # === Tool-name sanitization (single chokepoint) === # Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We # sanitize *here* -- not in map_openai_params -- because: @@ -1977,6 +1984,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): message=f"{e}\nReceived Messages={messages}", ) # don't use verbose_logger.exception, if exception is raised + self.update_headers_with_optional_anthropic_beta( + headers=headers, optional_params=optional_params, messages=anthropic_messages + ) + ## Auto-strip advisor blocks from history if advisor tool is absent. ## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool. _all_tools: Final = optional_params.get("tools") or [] @@ -2688,7 +2699,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index c6015e7884e..0c3c8996789 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -79,6 +79,27 @@ _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") +def requires_native_compaction_beta( + custom_llm_provider: str, + optional_params: Mapping[str, object], + messages: Sequence[object], +) -> bool: + return custom_llm_provider == "anthropic" and ( + optional_params.get("compaction") is not None + or any( + isinstance(block, Mapping) + and block.get("type") == "compaction" + and isinstance(block.get("signature"), str) + and bool(block.get("signature")) + for message in messages + if isinstance(message, Mapping) + for content in (message.get("content"),) + if isinstance(content, (list, tuple)) + for block in content + ) + ) + + def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool: from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.utils import supports_prompt_caching diff --git a/litellm/llms/anthropic/compaction.py b/litellm/llms/anthropic/compaction.py new file mode 100644 index 00000000000..cd09f936248 --- /dev/null +++ b/litellm/llms/anthropic/compaction.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter + +from litellm.llms.compaction import CompactionProtocol +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicCompaction + +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_HEADERS: Final = TypeAdapter(dict[str, str]) +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_CONFLICTS: Final = ("context_management", "response_format", "stop", "stop_sequences", "tool_choice") + + +def supports_native_compaction(params: Mapping[str, object]) -> bool: + from litellm.utils import get_model_info + + if params.get("custom_llm_provider") not in (None, "anthropic", "openai"): + return False + model: Final = str(params.get("model", "")).removeprefix("openai/").removeprefix("anthropic/") + try: + return get_model_info(model=model, custom_llm_provider="anthropic").get("supports_anthropic_compaction") is True + except Exception: + return False + + +def compatible_defaults(payload: Mapping[str, object]) -> bool: + return all(payload.get(key) is None for key in _CONFLICTS) + + +def request_kwargs() -> Mapping[str, object]: + operation: Final[AnthropicCompaction] = {"type": "summarize"} + return MappingProxyType( + { + "compaction": operation, + "extra_headers": _HEADERS.validate_python( + MappingProxyType({"anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}) + ), + } + ) + + +def _native_blocks(protocol: CompactionProtocol, response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + if protocol == "messages": + return ( + _OBJECTS.validate_python(response.get("content", ())) if response.get("stop_reason") == "compaction" else () + ) + choices: Final = _OBJECTS.validate_python(response.get("choices", ())) + choice: Final = choices[0] if len(choices) == 1 else _EMPTY + message: Final = _MAPPING.validate_python(choice.get("message", _EMPTY)) + fields: Final = _MAPPING.validate_python(message.get("provider_specific_fields") or _EMPTY) + return _OBJECTS.validate_python(fields.get("compaction_blocks", ())) + + +def extract_summary(protocol: CompactionProtocol, response: Mapping[str, object]) -> str | None: + blocks: Final = _native_blocks(protocol, response) + block: Final = blocks[0] if len(blocks) == 1 else _EMPTY + content: Final = block.get("content") + return ( + content + if block.get("type") == "compaction" + and isinstance(block.get("signature"), str) + and block.get("signature") + and isinstance(content, str) + and content.strip() + else None + ) diff --git a/litellm/llms/anthropic/completion/transformation.py b/litellm/llms/anthropic/completion/transformation.py index b15b0159bd9..46ab27ab0c7 100644 --- a/litellm/llms/anthropic/completion/transformation.py +++ b/litellm/llms/anthropic/completion/transformation.py @@ -33,7 +33,7 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AnthropicTextError(BaseLLMException): @@ -185,7 +185,7 @@ class AnthropicTextConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 1a85cf80bff..85431a5a637 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -2,8 +2,11 @@ import copy import hashlib import json from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast +from pydantic import JsonValue, TypeAdapter + import litellm from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, @@ -16,6 +19,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64 TOOL_NAME_HASH_LENGTH: Final = 8 TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55 PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"}) +_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue]) def _optional_attr(source: object, name: str) -> object: @@ -36,6 +40,20 @@ def _thought_signature(provider_specific_fields: object) -> str | None: return signature if isinstance(signature, str) else None +def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]: + fields: Final = _as_string_mapping(provider_specific_fields) + raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None + return ( + tuple( + block + for raw_block in raw_blocks + if (block := _as_string_mapping(raw_block)) is not None and block.get("type") == "compaction" + ) + if isinstance(raw_blocks, (list, tuple)) + else () + ) + + _ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset( {"name", "type", "input_schema", "description", "cache_control", "strict"} ) @@ -1330,7 +1348,11 @@ class LiteLLMAnthropicMessagesAdapter: tool_name_mapping: dict[str, str] | None = None, ) -> list[dict[str, Any]]: new_content: Final[list[dict[str, Any]]] = [] - for choice in choices: + for choice, compaction_blocks in ( + (choice, _compaction_blocks(_optional_attr(choice.message, "provider_specific_fields"))) + for choice in choices + ): + new_content.extend(_COMPACTION_BLOCK.validate_python(block) for block in compaction_blocks) # Handle thinking blocks first if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks: for thinking_block in choice.message.thinking_blocks: @@ -1365,7 +1387,7 @@ class LiteLLMAnthropicMessagesAdapter: ) # Handle text content - if choice.message.content is not None: + if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks): new_content.append( AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump() ) @@ -1545,21 +1567,35 @@ class LiteLLMAnthropicMessagesAdapter: openai_finish_reason=openai_finish_reason ) anthropic_finish_reason: Final = ( - "refusal" + "compaction" + if len(anthropic_content) == 1 and anthropic_content[0].get("type") == "compaction" + else "refusal" if refusal_text is not None and translated_finish_reason != "max_tokens" else translated_finish_reason ) # extract usage usage: Final[Usage] = getattr(response, "usage") - anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage) - - if polyfill_result is not None and polyfill_result.iterations_usage is not None: - message_iteration: Final[UsageIteration] = { - "type": "message", - "input_tokens": anthropic_usage["input_tokens"], - "output_tokens": usage.completion_tokens or 0, - } - anthropic_usage["iterations"] = list(polyfill_result.iterations_usage) + [message_iteration] + message_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage) + polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None + anthropic_usage: Final[AnthropicUsage] = ( + TypeAdapter(AnthropicUsage).validate_python( + MappingProxyType( + { + **message_usage, + "iterations": ( + *polyfill_iterations, + UsageIteration( + type="message", + input_tokens=message_usage.get("input_tokens", 0), + output_tokens=usage.completion_tokens or 0, + ), + ), + } + ) + ) + if polyfill_iterations is not None + else message_usage + ) translated_obj: Final = AnthropicMessagesResponse( id=response.id, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index d87cb0a64f5..ac4240690c1 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -580,7 +580,9 @@ def anthropic_messages_handler( ) if anthropic_messages_provider_config is None: # Route to Responses API for OpenAI / Azure, chat/completions for everything else. - if _should_route_to_responses_api(custom_llm_provider, original_model, model): + if kwargs.get("compaction") is None and _should_route_to_responses_api( + custom_llm_provider, original_model, model + ): return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( max_tokens=max_tokens, messages=messages, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index eed30c2698c..a83e23d83d5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -24,6 +24,7 @@ from ...common_utils import ( AnthropicError, AnthropicModelInfo, optionally_handle_anthropic_oauth, + requires_native_compaction_beta, strip_advisor_blocks_from_messages, strip_encrypted_reasoning_blocks_from_anthropic_messages, ) @@ -74,6 +75,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "tool_choice", "thinking", "context_management", + *(("compaction",) if self._resolved_provider == "anthropic" else ()), "output_format", "inference_geo", "speed", @@ -637,6 +639,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ) beta_values.update(existing_beta) + if requires_native_compaction_beta(custom_llm_provider, optional_params, messages): + beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) + # Check for context management context_management_param: Final = optional_params.get("context_management") if context_management_param is not None: diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 424422612db..355714c0daf 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -29,9 +29,8 @@ from ...base_llm.chat.transformation import BaseConfig from ..common_utils import AzureOpenAIError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -304,7 +303,7 @@ class AzureOpenAIConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/agents/transformation.py b/litellm/llms/azure_ai/agents/transformation.py index 60ce81a23c7..baba3149963 100644 --- a/litellm/llms/azure_ai/agents/transformation.py +++ b/litellm/llms/azure_ai/agents/transformation.py @@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -297,7 +296,7 @@ class AzureAIAgentsConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/azure_model_router/transformation.py b/litellm/llms/azure_ai/azure_model_router/transformation.py index 9e35e396e15..0e4c8ca0d15 100644 --- a/litellm/llms/azure_ai/azure_model_router/transformation.py +++ b/litellm/llms/azure_ai/azure_model_router/transformation.py @@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AzureModelRouterConfig(AzureAIStudioConfig): @@ -59,7 +59,7 @@ class AzureModelRouterConfig(AzureAIStudioConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 00e1c1e25ba..779a86629e2 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -30,7 +30,7 @@ from litellm.types.utils import ModelResponse, ProviderField from litellm.utils import _add_path_to_api_base, supports_tool_choice if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AzureFoundryErrorStrings(str, enum.Enum): @@ -305,7 +305,7 @@ class AzureAIStudioConfig(OpenAIConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 67b1a8bcab3..18b4b6f456a 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -12,9 +12,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): """Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5).""" @@ -245,7 +246,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 2296909cfe1..e4bf148abf3 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -12,9 +12,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -121,7 +120,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/bridges/completion_transformation.py b/litellm/llms/base_llm/bridges/completion_transformation.py index 87b55152d09..a5c03705088 100644 --- a/litellm/llms/base_llm/bridges/completion_transformation.py +++ b/litellm/llms/base_llm/bridges/completion_transformation.py @@ -7,10 +7,10 @@ from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Union if TYPE_CHECKING: - import tiktoken from pydantic import BaseModel from litellm import LiteLLMLoggingObj, ModelResponse + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.types.llms.openai import AllMessageValues @@ -39,7 +39,7 @@ class CompletionTransformationBridge(ABC): messages: list["AllMessageValues"], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 7bfc87a30d6..7decf1b4186 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -21,9 +21,8 @@ from litellm.types.llms.openai import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.types.utils import ModelResponse from ..base_utils import ( @@ -344,7 +343,7 @@ class BaseConfig(ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/base_llm/completion/transformation.py b/litellm/llms/base_llm/completion/transformation.py index fb472dfa63b..b8ebbfed12f 100644 --- a/litellm/llms/base_llm/completion/transformation.py +++ b/litellm/llms/base_llm/completion/transformation.py @@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -68,7 +67,7 @@ class BaseTextCompletionConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/embedding/transformation.py b/litellm/llms/base_llm/embedding/transformation.py index da87dcc7f98..46ac3ccf433 100644 --- a/litellm/llms/base_llm/embedding/transformation.py +++ b/litellm/llms/base_llm/embedding/transformation.py @@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -80,7 +79,7 @@ class BaseEmbeddingConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/files/transformation.py b/litellm/llms/base_llm/files/transformation.py index 254995c028f..5ecf033fb2c 100644 --- a/litellm/llms/base_llm/files/transformation.py +++ b/litellm/llms/base_llm/files/transformation.py @@ -21,9 +21,8 @@ from litellm.types.utils import LlmProviders, ModelResponse from ..chat.transformation import BaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.router import Router as _Router from litellm.types.llms.openai import HttpxBinaryResponseContent @@ -231,7 +230,7 @@ class BaseFilesConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/image_generation/transformation.py b/litellm/llms/base_llm/image_generation/transformation.py index 4616441133e..7ac440c6f48 100644 --- a/litellm/llms/base_llm/image_generation/transformation.py +++ b/litellm/llms/base_llm/image_generation/transformation.py @@ -11,9 +11,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -93,7 +92,7 @@ class BaseImageGenerationConfig(ABC): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/base_llm/image_variations/transformation.py b/litellm/llms/base_llm/image_variations/transformation.py index d3e02139e0e..15a4e0f243c 100644 --- a/litellm/llms/base_llm/image_variations/transformation.py +++ b/litellm/llms/base_llm/image_variations/transformation.py @@ -17,9 +17,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -82,7 +81,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: pass @@ -98,7 +97,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: pass @@ -125,7 +124,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index e1a9a807abc..29133bcfaf9 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -40,9 +40,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -990,7 +989,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 4f9b1f56a5b..21830eb0d8e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -99,7 +99,7 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer # Computer use tool prefixes supported by Bedrock BEDROCK_COMPUTER_USE_TOOLS: Final = [ @@ -1920,7 +1920,7 @@ class AmazonConverseConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/litellm/llms/bedrock/chat/invoke_agent/transformation.py index d489e47c3b5..6877af74494 100644 --- a/litellm/llms/bedrock/chat/invoke_agent/transformation.py +++ b/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -37,9 +37,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -438,7 +437,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py index 5a3f4f17b8b..5699f94d084 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -25,7 +25,7 @@ from litellm.types.utils import ( from .amazon_llama_transformation import AmazonLlamaConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonDeepSeekR1Config(AmazonLlamaConfig): @@ -39,7 +39,7 @@ class AmazonDeepSeekR1Config(AmazonLlamaConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py index 5d39b68d9d5..f8b730b6cd0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py @@ -21,9 +21,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.types.utils import ModelResponse LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -198,7 +197,7 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py index bc97551d57a..8d1ff1d2bd9 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py @@ -28,7 +28,7 @@ from ..converse_transformation import AmazonConverseConfig from .base_invoke_transformation import AmazonInvokeConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer _CachePointCarrier = TypeVar("_CachePointCarrier", SystemContentBlock, ContentBlock) _INJECTION_POINTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) @@ -128,7 +128,7 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py index c78375c37bb..67364ccfda0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py @@ -21,7 +21,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonQwen2Config(AmazonQwen3Config): @@ -44,7 +44,7 @@ class AmazonQwen2Config(AmazonQwen3Config): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py index e251fb15725..2e19dbf77af 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py @@ -19,7 +19,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): @@ -170,7 +170,7 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py index 39cded4ed64..fe287111fdd 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, Usage from litellm.utils import get_base64_str if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -190,7 +189,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 1326dc22ca0..a8b94fb5703 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -359,7 +358,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 90a2692f68a..dcc5e249d8a 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -34,9 +34,8 @@ from litellm.types.utils import ModelResponse, Usage from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -288,7 +287,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/black_forest_labs/image_generation/transformation.py b/litellm/llms/black_forest_labs/image_generation/transformation.py index 119ffff1c34..e5c2bdf59e9 100644 --- a/litellm/llms/black_forest_labs/image_generation/transformation.py +++ b/litellm/llms/black_forest_labs/image_generation/transformation.py @@ -29,9 +29,8 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -258,7 +257,7 @@ class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/brave/search/__init__.py b/litellm/llms/brave/search/__init__.py index cc1168d7ef8..de70c62e040 100644 --- a/litellm/llms/brave/search/__init__.py +++ b/litellm/llms/brave/search/__init__.py @@ -1,7 +1,7 @@ -""" -Brave Search API module. -""" - -from litellm.llms.brave.search.transformation import BraveSearchConfig - -__all__ = ["BraveSearchConfig"] +""" +Brave Search API module. +""" + +from litellm.llms.brave.search.transformation import BraveSearchConfig + +__all__ = ["BraveSearchConfig"] diff --git a/litellm/llms/bytez/chat/transformation.py b/litellm/llms/bytez/chat/transformation.py index 7977db0f056..e622761dd7f 100644 --- a/litellm/llms/bytez/chat/transformation.py +++ b/litellm/llms/bytez/chat/transformation.py @@ -23,9 +23,8 @@ from litellm.utils import CustomStreamWrapper, ModelResponse, Usage from ..common_utils import API_BASE, BytezError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -187,7 +186,7 @@ class BytezChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/clarifai/chat/transformation.py b/litellm/llms/clarifai/chat/transformation.py index 76d35467497..a0946254de0 100644 --- a/litellm/llms/clarifai/chat/transformation.py +++ b/litellm/llms/clarifai/chat/transformation.py @@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -87,7 +86,7 @@ class ClarifaiConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/chat/transformation.py b/litellm/llms/cohere/chat/transformation.py index fa46bd7f6cf..a26cdc81695 100644 --- a/litellm/llms/cohere/chat/transformation.py +++ b/litellm/llms/cohere/chat/transformation.py @@ -15,9 +15,8 @@ from ..common_utils import ModelResponseIterator as CohereModelResponseIterator from ..common_utils import validate_environment as cohere_validate_environment if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -227,7 +226,7 @@ class CohereChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/chat/v2_transformation.py b/litellm/llms/cohere/chat/v2_transformation.py index 4252e7d02e9..37c53640d18 100644 --- a/litellm/llms/cohere/chat/v2_transformation.py +++ b/litellm/llms/cohere/chat/v2_transformation.py @@ -20,9 +20,8 @@ from ..common_utils import CohereError, CohereV2ModelResponseIterator from ..common_utils import validate_environment as cohere_validate_environment if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -191,7 +190,7 @@ class CohereV2ChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/embed/handler.py b/litellm/llms/cohere/embed/handler.py index 3cebf6b9a90..6d9543d7c30 100644 --- a/litellm/llms/cohere/embed/handler.py +++ b/litellm/llms/cohere/embed/handler.py @@ -20,7 +20,7 @@ from litellm.types.utils import EmbeddingResponse from .v1_transformation import CohereEmbeddingConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def validate_environment(api_key, headers: dict): @@ -60,7 +60,7 @@ async def async_embedding( api_base: str, api_key: str | None, headers: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", client: AsyncHTTPHandler | None = None, ): ## LOGGING @@ -122,7 +122,7 @@ def embedding( logging_obj: LiteLLMLoggingObj, optional_params: dict, headers: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", data: dict | CohereEmbeddingRequest | None = None, complete_api_base: str | None = None, api_key: str | None = None, diff --git a/litellm/llms/cometapi/image_generation/transformation.py b/litellm/llms/cometapi/image_generation/transformation.py index 03c820de198..4432c151a64 100644 --- a/litellm/llms/cometapi/image_generation/transformation.py +++ b/litellm/llms/cometapi/image_generation/transformation.py @@ -13,9 +13,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -132,7 +131,7 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/compactifai/chat/transformation.py b/litellm/llms/compactifai/chat/transformation.py index 3c5a889ce63..a02db2338d8 100644 --- a/litellm/llms/compactifai/chat/transformation.py +++ b/litellm/llms/compactifai/chat/transformation.py @@ -17,9 +17,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -66,7 +65,7 @@ class CompactifAIChatConfig(OpenAIGPTConfig): messages: Sequence[AllMessageValues], optional_params: Mapping[str, object], litellm_params: Mapping[str, object], - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/compaction.py b/litellm/llms/compaction.py new file mode 100644 index 00000000000..16afa2e2c25 --- /dev/null +++ b/litellm/llms/compaction.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias + +from pydantic import TypeAdapter + +from litellm.types.llms.openai import AllMessageValues + +if TYPE_CHECKING: + from litellm.router import Router + +CompactionProtocol: TypeAlias = Literal["chat", "messages"] +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_MESSAGES: Final = TypeAdapter(list[AllMessageValues]) + + +class NativeCompactionProvider(Protocol): + def supports_native_compaction(self, params: Mapping[str, object]) -> bool: ... + + def compatible_defaults(self, payload: Mapping[str, object]) -> bool: ... + + def request_kwargs(self) -> Mapping[str, object]: ... + + def extract_summary(self, protocol: CompactionProtocol, response: Mapping[str, object]) -> str | None: ... + + +def get_native_compaction_provider(params: Mapping[str, object]) -> NativeCompactionProvider | None: + from litellm.llms.anthropic import compaction + + return compaction if compaction.supports_native_compaction(params) else None + + +async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mapping[str, object]) -> Mapping[str, object]: + if protocol == "messages": + return _MAPPING.validate_python( + await router.aanthropic_messages(custom_llm_provider=None, client=None, **payload) + ) + response: Final = await router.acompletion( + model=str(payload["model"]), + messages=_MESSAGES.validate_python(payload["messages"]), + stream=False, + **MappingProxyType( + {key: value for key, value in payload.items() if key not in ("model", "messages", "stream")} + ), + ) + return _MAPPING.validate_python(response.model_dump()) diff --git a/litellm/llms/custom_httpx/aiohttp_handler.py b/litellm/llms/custom_httpx/aiohttp_handler.py index 0809ef5274f..034c9514092 100644 --- a/litellm/llms/custom_httpx/aiohttp_handler.py +++ b/litellm/llms/custom_httpx/aiohttp_handler.py @@ -26,9 +26,8 @@ from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProv from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -268,7 +267,7 @@ class BaseLLMAIOHTTPHandler: messages: list, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, client: ClientSession | None = None, ): diff --git a/litellm/llms/custom_httpx/asgi_handler.py b/litellm/llms/custom_httpx/asgi_handler.py new file mode 100644 index 00000000000..ab704ab12a6 --- /dev/null +++ b/litellm/llms/custom_httpx/asgi_handler.py @@ -0,0 +1,46 @@ +from collections.abc import Generator +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Final + +import httpx +from starlette.types import ASGIApp, Receive, Scope, Send + +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared cache retains its legacy parameter mapping +) +from litellm.types.llms.custom_http import httpxSpecialProvider + + +@dataclass(frozen=True, slots=True) +class _ASGITarget: + app: ASGIApp + root_path: str + client: tuple[str, int] | None + + +_target: Final[ContextVar[_ASGITarget]] = ContextVar("httpx_asgi_target") + + +async def _dispatch(scope: Scope, receive: Receive, send: Send) -> None: + target: Final = _target.get() + await target.app({**scope, "root_path": target.root_path, "client": target.client}, receive, send) + + +_TRANSPORT: Final = httpx.ASGITransport(app=_dispatch, raise_app_exceptions=False) + + +@contextmanager +def get_async_asgi_client( + app: ASGIApp, root_path: str = "", client: tuple[str, int] | None = None +) -> Generator[httpx.AsyncClient]: + handler: Final = get_async_httpx_client( + llm_provider=httpxSpecialProvider.ASGI, + params={"transport": _TRANSPORT, "timeout": httpx.Timeout(None), "follow_redirects": False}, + ) + token: Final = _target.set(_ASGITarget(app, root_path, client)) + try: + yield handler.client + finally: + _target.reset(token) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index b7b2477e85c..fcc05e54bc5 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -614,11 +614,15 @@ class AsyncHTTPHandler: client_alias: str | None = None, # name for client in logs ssl_verify: VerifyTypes | None = None, shared_session: Optional["ClientSession"] = None, + transport: httpx.AsyncBaseTransport | None = None, + follow_redirects: bool = True, ): self.timeout = timeout self.event_hooks = event_hooks self.ssl_verify = ssl_verify self.shared_session = shared_session + self.transport = transport + self.follow_redirects = follow_redirects self._owns_client = True self._client = self.create_client( timeout=timeout, @@ -651,6 +655,16 @@ class AsyncHTTPHandler: ssl_verify: VerifyTypes | None = None, shared_session: Optional["ClientSession"] = None, ) -> httpx.AsyncClient: + if self.transport is not None: + return httpx.AsyncClient( + transport=self.transport, + event_hooks=event_hooks, + timeout=timeout if timeout is not None else _DEFAULT_TIMEOUT, + headers=get_default_headers(), + cookies=blocked_cookie_jar(), + follow_redirects=self.follow_redirects, + trust_env=False, + ) # Get unified SSL configuration ssl_config: Final = get_ssl_configuration(ssl_verify) @@ -680,7 +694,7 @@ class AsyncHTTPHandler: cert=cert, headers=default_headers, cookies=blocked_cookie_jar(), - follow_redirects=True, + follow_redirects=self.follow_redirects, http2=http2_enabled(), ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 18d4c27fec1..333ce523e34 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -192,12 +192,12 @@ def _rust_responses_websocket_enabled( from .http_handler import get_shared_realtime_ssl_context if TYPE_CHECKING: - import tiktoken from aiohttp import ClientSession from websockets.asyncio.client import ClientConnection from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) @@ -493,7 +493,7 @@ class BaseLLMHTTPHandler: messages: list, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, client: AsyncHTTPHandler | None = None, json_mode: bool = False, @@ -559,7 +559,7 @@ class BaseLLMHTTPHandler: api_base: str | None, custom_llm_provider: str, model_response: ModelResponse, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", logging_obj: LiteLLMLoggingObj, optional_params: dict, timeout: float | httpx.Timeout, diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index c0e278a96ef..ffa60a3d9bc 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -38,9 +38,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -165,7 +164,7 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 30144d29f51..538904b34e6 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -149,9 +149,8 @@ def _split_parallel_tool_calls(messages: list[AllMessageValues]) -> list[AllMess if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -189,7 +188,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): return "databricks" @classmethod - def get_config(cls): + def get_config(cls, *, model: str | None = None): return super().get_config() def get_required_params(self) -> list[ProviderField]: @@ -651,7 +650,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/deprecated_providers/aleph_alpha.py b/litellm/llms/deprecated_providers/aleph_alpha.py index 2ad9ce4edc8..8f5c35f32f2 100644 --- a/litellm/llms/deprecated_providers/aleph_alpha.py +++ b/litellm/llms/deprecated_providers/aleph_alpha.py @@ -277,12 +277,7 @@ def completion( ## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here. prompt_tokens: Final = len(encoding.encode(prompt)) - completion_tokens: Final = len( - encoding.encode( - model_response["choices"][0]["message"]["content"], - disallowed_special=(), - ) - ) + completion_tokens: Final = len(encoding.encode(model_response["choices"][0]["message"]["content"])) model_response.created = int(time.time()) model_response.model = model diff --git a/litellm/llms/edenai/chat/transformation.py b/litellm/llms/edenai/chat/transformation.py index 4fd5a9d550b..67d308d9e38 100644 --- a/litellm/llms/edenai/chat/transformation.py +++ b/litellm/llms/edenai/chat/transformation.py @@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, Usage from ..common_utils import EdenAIException, reported_cost, resolve_api_base, resolve_api_key if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding _OPTIONAL_MAPPING: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) @@ -97,7 +96,7 @@ class EdenAIChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], # mutable-ok: inherited contract optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract - encoding: "tiktoken.Encoding | None", + encoding: "Encoding | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/edenai/image_generation/transformation.py b/litellm/llms/edenai/image_generation/transformation.py index 2965c4041de..7f729cd7fbb 100644 --- a/litellm/llms/edenai/image_generation/transformation.py +++ b/litellm/llms/edenai/image_generation/transformation.py @@ -19,9 +19,8 @@ from litellm.utils import convert_to_model_response_object from ..common_utils import EdenAIException, endpoint_url, json_headers, pick, reported_cost if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding _SUPPORTED_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = ( "background", @@ -94,7 +93,7 @@ class EdenAIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict[str, object], # mutable-ok: inherited contract optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract - encoding: "tiktoken.Encoding | None", + encoding: "Encoding | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/bria_transformation.py b/litellm/llms/fal_ai/image_generation/bria_transformation.py index c528550811a..53da48e62ca 100644 --- a/litellm/llms/fal_ai/image_generation/bria_transformation.py +++ b/litellm/llms/fal_ai/image_generation/bria_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -187,7 +186,7 @@ class FalAIBriaConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py index 6b8558b8124..7c63e1077f1 100644 --- a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py +++ b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageResponse from .transformation import FalAIBaseConfig, fal_images_to_image_objects if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -194,7 +193,7 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py index 04b4f426878..ad1852a622b 100644 --- a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -150,7 +149,7 @@ class FalAIIdeogramV3Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py index 8a6665b2585..3624b76a4a3 100644 --- a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py +++ b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -182,7 +181,7 @@ class FalAIImagen4Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py index 4880dfec7e3..934ce420d53 100644 --- a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -172,7 +171,7 @@ class FalAIRecraftV3Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py index bc3a4d07282..79d8800773b 100644 --- a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py +++ b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -208,7 +207,7 @@ class FalAIStableDiffusionConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/transformation.py b/litellm/llms/fal_ai/image_generation/transformation.py index fd8e280da1c..8f081c7228d 100644 --- a/litellm/llms/fal_ai/image_generation/transformation.py +++ b/litellm/llms/fal_ai/image_generation/transformation.py @@ -16,9 +16,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -117,7 +116,7 @@ class FalAIBaseConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 28ebb39a303..196022b3558 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -46,7 +46,7 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def _map_reasoning_effort(value: object) -> object: @@ -708,7 +708,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index d009fe4cd72..bb8d7455031 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -24,9 +24,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -173,7 +172,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index d9250ea8836..c047dc0c881 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -26,9 +26,8 @@ from ..authenticator import get_access_token from ..file_handler import upload_file_sync if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -416,7 +415,7 @@ class GigaChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: tiktoken.Encoding | None, + encoding: Tokenizer | None, api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 41a2df17c6f..1da7ad0a7b5 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -27,7 +27,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, ServerToolUs from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer GROQ_COMPOUND_MODELS: Final = frozenset({"compound", "compound-mini"}) @@ -286,7 +286,7 @@ class GroqChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index 57d1357ee46..60917c68221 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -1,6 +1,5 @@ import json import os -from collections.abc import Sequence from typing import Final, Literal, Protocol, get_args import httpx @@ -32,7 +31,7 @@ hf_tasks_embeddings: Final = ( class _SupportsTokenEncode(Protocol): """Token encoder handle. Only ``encode`` is ever called on it here.""" - def encode(self, text: str, *, disallowed_special: tuple[str, ...]) -> Sequence[int]: ... + def encode(self, text: str) -> list[int]: ... def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> str | None: @@ -214,7 +213,7 @@ class HuggingFaceEmbedding(BaseLLM): model_response.model = model input_tokens = 0 for text in input: - input_tokens += len(encoding.encode(text, disallowed_special=())) + input_tokens += len(encoding.encode_ordinary(text)) setattr( model_response, diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index 33b0e21e326..3fdd4abda73 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -25,9 +25,8 @@ from litellm.utils import token_counter from ..common_utils import HuggingFaceError, hf_task_list, hf_tasks, output_parser if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -479,7 +478,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/langflow/chat/transformation.py b/litellm/llms/langflow/chat/transformation.py index 17ae7017cf6..a53887b36af 100644 --- a/litellm/llms/langflow/chat/transformation.py +++ b/litellm/llms/langflow/chat/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -225,7 +224,7 @@ class LangFlowConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index 84d79e6bd31..c9388ee472f 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -23,9 +23,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -415,7 +414,7 @@ class LangGraphConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index c01ad2a0edd..341e8dd2e12 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -19,7 +19,7 @@ from litellm.types.utils import ModelResponse from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class LemonadeChatConfig(OpenAILikeChatConfig): @@ -231,7 +231,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index f77e828b59a..33b567e9710 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -32,7 +32,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: str) -> str: @@ -580,7 +580,7 @@ class MistralConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/nlp_cloud/chat/transformation.py b/litellm/llms/nlp_cloud/chat/transformation.py index 17c547618d3..2ff48894619 100644 --- a/litellm/llms/nlp_cloud/chat/transformation.py +++ b/litellm/llms/nlp_cloud/chat/transformation.py @@ -14,9 +14,8 @@ from litellm.utils import ModelResponse, Usage from ..common_utils import NLPCloudError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -175,7 +174,7 @@ class NLPCloudConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 8e4e41b4ac1..ecff823a18d 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -65,9 +65,8 @@ from litellm.types.utils import ( from litellm.utils import supports_reasoning if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -603,7 +602,7 @@ class OCIChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index bcde8a041a6..cb3080e6534 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -31,9 +31,8 @@ from litellm.types.utils import ModelResponse, ModelResponseStream from ..common_utils import OllamaError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -321,7 +320,7 @@ class OllamaChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 3eb2c833094..0fc1cd926b8 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -35,9 +35,8 @@ from litellm.types.utils import ( from ..common_utils import OllamaError, OllamaModelInfo, _convert_image if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -252,7 +251,7 @@ class OllamaConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/oobabooga/chat/transformation.py b/litellm/llms/oobabooga/chat/transformation.py index 43d627102b6..05383a35389 100644 --- a/litellm/llms/oobabooga/chat/transformation.py +++ b/litellm/llms/oobabooga/chat/transformation.py @@ -11,9 +11,8 @@ from litellm.types.utils import ModelResponse, Usage from ..common_utils import OobaboogaError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -39,7 +38,7 @@ class OobaboogaConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 9dbcf0cc089..b63684db782 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -60,9 +60,8 @@ from litellm.utils import convert_to_model_response_object from ..common_utils import OpenAIError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.types.llms.openai import ChatCompletionToolParam @@ -671,7 +670,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai/image_generation/dall_e_2_transformation.py b/litellm/llms/openai/image_generation/dall_e_2_transformation.py index 74936cf1895..ffc6f1d5fe9 100644 --- a/litellm/llms/openai/image_generation/dall_e_2_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_2_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class DallE2ImageGenerationConfig(BaseImageGenerationConfig): """ @@ -52,7 +53,7 @@ class DallE2ImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_generation/dall_e_3_transformation.py b/litellm/llms/openai/image_generation/dall_e_3_transformation.py index 5c561d011a9..90b7eaedf2f 100644 --- a/litellm/llms/openai/image_generation/dall_e_3_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_3_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class DallE3ImageGenerationConfig(BaseImageGenerationConfig): """ @@ -52,7 +53,7 @@ class DallE3ImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py index 8dc4d8953ea..c3a826616ed 100644 --- a/litellm/llms/openai/image_generation/gpt_transformation.py +++ b/litellm/llms/openai/image_generation/gpt_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class GPTImageGenerationConfig(BaseImageGenerationConfig): """ @@ -61,7 +62,7 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_variations/transformation.py b/litellm/llms/openai/image_variations/transformation.py index afd2909b697..73b44d5ea6a 100644 --- a/litellm/llms/openai/image_variations/transformation.py +++ b/litellm/llms/openai/image_variations/transformation.py @@ -12,7 +12,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ..common_utils import OpenAIError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class OpenAIImageVariationConfig(BaseImageVariationConfig): @@ -53,7 +53,7 @@ class OpenAIImageVariationConfig(BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: return model_response @@ -68,7 +68,7 @@ class OpenAIImageVariationConfig(BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: return model_response diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 869ad387c5a..7ac0d988074 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -6,9 +6,10 @@ from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx if TYPE_CHECKING: - import tiktoken from aiohttp import ClientSession + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + import openai from openai import AsyncOpenAI, OpenAI from openai._base_client import make_request_options @@ -277,7 +278,7 @@ class OpenAIConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai_like/chat/transformation.py b/litellm/llms/openai_like/chat/transformation.py index 030710c8b2d..e5d6cbb7e5e 100644 --- a/litellm/llms/openai_like/chat/transformation.py +++ b/litellm/llms/openai_like/chat/transformation.py @@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -131,7 +130,7 @@ class OpenAILikeChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 77a902149d9..08d43c169f4 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -23,9 +23,8 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig from ..common_utils import OpenRouterException if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class CacheControlSupportedModels(str, Enum): @@ -182,7 +181,7 @@ class OpenrouterConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openrouter/image_generation/transformation.py b/litellm/llms/openrouter/image_generation/transformation.py index 6bbda324336..67d90d027ec 100644 --- a/litellm/llms/openrouter/image_generation/transformation.py +++ b/litellm/llms/openrouter/image_generation/transformation.py @@ -50,9 +50,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer else: LiteLLMLoggingObj = Any @@ -319,7 +318,7 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/perplexity/chat/transformation.py b/litellm/llms/perplexity/chat/transformation.py index 354f7692fd5..dca2f9857b8 100644 --- a/litellm/llms/perplexity/chat/transformation.py +++ b/litellm/llms/perplexity/chat/transformation.py @@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues, ChatCompletionAnnotation from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class PerplexityChatConfig(OpenAIGPTConfig): @@ -75,7 +75,7 @@ class PerplexityChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/petals/completion/transformation.py b/litellm/llms/petals/completion/transformation.py index 3e0de14a7b2..ee20c2b12d5 100644 --- a/litellm/llms/petals/completion/transformation.py +++ b/litellm/llms/petals/completion/transformation.py @@ -14,7 +14,7 @@ from litellm.types.utils import ModelResponse from ..common_utils import PetalsError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class PetalsConfig(BaseConfig): @@ -112,7 +112,7 @@ class PetalsConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/predibase/chat/transformation.py b/litellm/llms/predibase/chat/transformation.py index 2a63c489395..69924396a1e 100644 --- a/litellm/llms/predibase/chat/transformation.py +++ b/litellm/llms/predibase/chat/transformation.py @@ -18,9 +18,8 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage from ..common_utils import PredibaseError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -150,7 +149,7 @@ class PredibaseConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/recraft/image_generation/transformation.py b/litellm/llms/recraft/image_generation/transformation.py index 3a04e0a62b4..f65bf1e7292 100644 --- a/litellm/llms/recraft/image_generation/transformation.py +++ b/litellm/llms/recraft/image_generation/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.recraft import RecraftImageGenerationRequestParams from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -122,7 +121,7 @@ class RecraftImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/replicate/chat/transformation.py b/litellm/llms/replicate/chat/transformation.py index 769160c6ced..f7e09b7bec0 100644 --- a/litellm/llms/replicate/chat/transformation.py +++ b/litellm/llms/replicate/chat/transformation.py @@ -19,9 +19,8 @@ from litellm.utils import token_counter from ..common_utils import ReplicateError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -237,7 +236,7 @@ class ReplicateConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/runwayml/image_generation/transformation.py b/litellm/llms/runwayml/image_generation/transformation.py index 5913709c8a0..e5e988328d8 100644 --- a/litellm/llms/runwayml/image_generation/transformation.py +++ b/litellm/llms/runwayml/image_generation/transformation.py @@ -22,9 +22,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -308,7 +307,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: @@ -383,7 +382,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/sagemaker/completion/transformation.py b/litellm/llms/sagemaker/completion/transformation.py index 576018f0046..e1bc496a82d 100644 --- a/litellm/llms/sagemaker/completion/transformation.py +++ b/litellm/llms/sagemaker/completion/transformation.py @@ -24,9 +24,8 @@ from litellm.utils import token_counter from ..common_utils import SagemakerError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -198,7 +197,7 @@ class SagemakerConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index d64d7a57281..4c73ccacc16 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -15,9 +15,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -57,7 +56,7 @@ def validate_dict(data: dict, model) -> dict: return model(**data).model_dump(by_alias=True, exclude_unset=True) -def _messages_to_sap_template(messages: list[dict[str, str]]) -> list: +def _messages_to_sap_template(messages: list[AllMessageValues]) -> list: template: Final = [] for message in messages: if message["role"] == "user": @@ -311,7 +310,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): def transform_request( self, model: str, - messages: list[dict[str, str]], + messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, headers: dict, @@ -383,7 +382,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/stability/image_generation/transformation.py b/litellm/llms/stability/image_generation/transformation.py index cf3576a9404..656ffe395c8 100644 --- a/litellm/llms/stability/image_generation/transformation.py +++ b/litellm/llms/stability/image_generation/transformation.py @@ -26,9 +26,8 @@ from litellm.types.llms.stability import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -207,7 +206,7 @@ class StabilityImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/topaz/image_variations/transformation.py b/litellm/llms/topaz/image_variations/transformation.py index f4753c8ba17..94f60d29cb9 100644 --- a/litellm/llms/topaz/image_variations/transformation.py +++ b/litellm/llms/topaz/image_variations/transformation.py @@ -23,7 +23,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ..common_utils import TopazException, TopazModelInfo if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): @@ -139,7 +139,7 @@ class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: image_content: Final = await raw_response.read() @@ -158,7 +158,7 @@ class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: image_content: Final = raw_response.content diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 3c868b3a96f..b37bbd78f2b 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -29,7 +29,7 @@ from litellm.types.utils import ( from ..common_utils import TritonError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class TritonConfig(BaseConfig): @@ -95,7 +95,7 @@ class TritonConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: @@ -215,7 +215,7 @@ class TritonGenerateConfig(TritonConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: @@ -280,7 +280,7 @@ class TritonInferConfig(TritonConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index c5ca9f38144..b37bf473731 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -29,9 +29,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -285,7 +284,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index b2c52c53580..17ccf16837b 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -24,9 +24,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -284,7 +283,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py index 8faf7b0d484..ae6e08611ae 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py @@ -20,9 +20,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -214,7 +213,7 @@ class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 508f68b3eca..2fab6f438f6 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -14,7 +14,7 @@ from ....anthropic.chat.transformation import AnthropicConfig from .output_params_utils import sanitize_vertex_anthropic_output_params if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class VertexAIError(Exception): @@ -197,7 +197,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py index 89a5b8a570e..f2d2c0896d2 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py @@ -21,7 +21,7 @@ from litellm.types.utils import ( from ...common_utils import VertexAIError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class VertexAILlama3Config(OpenAIGPTConfig): @@ -112,7 +112,7 @@ class VertexAILlama3Config(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py index 67b01c2dc43..2ae8b4cd188 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py @@ -24,9 +24,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_model_iterator import MockResponseIterator @@ -276,7 +275,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): litellm_params: dict, client: HTTPHandler | httpx.Client | None = None, timeout: float | httpx.Timeout | None = None, - encoding: "tiktoken.Encoding | None" = None, + encoding: "Tokenizer | None" = None, ): """Synchronous completion request""" from litellm.utils import convert_to_model_response_object @@ -366,7 +365,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): litellm_params: dict, client: AsyncHTTPHandler | httpx.AsyncClient | None = None, timeout: float | httpx.Timeout | None = None, - encoding: "tiktoken.Encoding | None" = None, + encoding: "Tokenizer | None" = None, ): """Asynchronous completion request""" from litellm.utils import convert_to_model_response_object diff --git a/litellm/llms/watsonx/completion/transformation.py b/litellm/llms/watsonx/completion/transformation.py index 2be007336b4..2fec8485cf9 100644 --- a/litellm/llms/watsonx/completion/transformation.py +++ b/litellm/llms/watsonx/completion/transformation.py @@ -21,9 +21,8 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -280,7 +279,7 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/main.py b/litellm/main.py index 2570b93455f..7d231a1bf7a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -37,7 +37,6 @@ if TYPE_CHECKING: import dotenv import httpx import openai -import tiktoken from pydantic import BaseModel from typing_extensions import overload @@ -100,6 +99,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) +from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, foundry_chat_rejects_function_tools_while_reasoning, @@ -7489,7 +7489,9 @@ def text_completion( if isinstance(prompt, list): import concurrent.futures - tokenizer: Final = tiktoken.encoding_for_model("text-davinci-003") + from litellm.rust_bridge.tokenizer import get_encoding + + tokenizer: Final = get_encoding("p50k_base") ## if it's a 2d list - each element in the list is a text_completion() request if len(prompt) > 0 and isinstance(prompt[0], list): responses: Final = [None for x in prompt] # init responses @@ -9259,7 +9261,7 @@ async def acount_tokens( except Exception as e: verbose_logger.debug("Provider token counting failed for model=%s, falling back to local: %s", model, e) - # Fallback to local tiktoken-based token counting + # Fallback to local token counting fallback_messages = messages or [] if system and fallback_messages: fallback_messages = [{"role": "system", "content": system}] + fallback_messages @@ -9278,16 +9280,16 @@ async def acount_tokens( # Cache for encoding to avoid repeated __getattr__ calls -_encoding_cache: tiktoken.Encoding | None = None +_encoding_cache: Tokenizer | None = None -def _load_module_encoding() -> tiktoken.Encoding: +def _load_module_encoding() -> Tokenizer: import sys return sys.modules[__name__].encoding -def _get_encoding() -> tiktoken.Encoding: +def _get_encoding() -> Tokenizer: """Get encoding, loading it lazily if needed.""" global _encoding_cache if _encoding_cache is None: @@ -9296,18 +9298,15 @@ def _get_encoding() -> tiktoken.Encoding: return _encoding_cache -def _load_default_encoding() -> tiktoken.Encoding: +def _load_default_encoding() -> Tokenizer: from litellm._lazy_imports import _get_default_encoding return _get_default_encoding() -def __getattr__(name: str) -> tiktoken.Encoding: +def __getattr__(name: str) -> Tokenizer: """Lazy import handler for main module""" if name == "encoding": - # Use _get_default_encoding which properly sets TIKTOKEN_CACHE_DIR - # before loading tiktoken, ensuring the local cache is used - # instead of downloading from the internet _encoding: Final = _load_default_encoding() # Cache it in the module's __dict__ for subsequent accesses import sys diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 08cb2ddb8f7..b5106670a2b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14516,6 +14516,7 @@ "source": "https://docs.anthropic.com/en/docs/about-claude/pricing" }, "claude-sonnet-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -14555,6 +14556,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-sonnet-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -14770,6 +14772,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14808,6 +14811,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6-20260205": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14845,6 +14849,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14884,6 +14889,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-7-20260416": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14922,6 +14928,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -14961,6 +14968,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-fable-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -15001,6 +15009,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15043,6 +15052,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-8": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -62148,6 +62158,7 @@ "supports_audio_output": true }, "claude-mythos-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -62187,6 +62198,7 @@ } }, "claude-mythos-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -62227,6 +62239,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-mythos-preview": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -77528,5 +77541,45 @@ "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true + }, + "openrouter/nex-agi/nex-n2.5-mini": { + "cache_read_input_token_cost": 2.5e-09, + "input_cost_per_token": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false + }, + "openrouter/nex-agi/nex-n2.5-pro": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": false } } diff --git a/litellm/models/mcp_server.py b/litellm/models/mcp_server.py index 7ccff9434a7..9125d708e79 100644 --- a/litellm/models/mcp_server.py +++ b/litellm/models/mcp_server.py @@ -50,6 +50,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase): """Represents a LiteLLM_MCPServerTable record""" server_id: str + is_config: bool = Field(default=False, description="Whether this server is defined in config and is read-only.") server_name: str | None = None alias: str | None = None description: str | None = None diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 4a2713cb19c..20c114a2f3e 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -7081,6 +7081,7 @@ class MCPServerManager: def _build_mcp_server_table(self, server: MCPServer) -> LiteLLM_MCPServerTable: return LiteLLM_MCPServerTable( server_id=server.server_id, + is_config=self.is_config_declared_server(server.server_id) and server.server_id not in self.registry, server_name=server.server_name, alias=server.alias, description=(server.mcp_info.get("description") if server.mcp_info else None), diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 03122f25870..4dbca14b917 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -24767,6 +24767,12 @@ "title": "Is Byok", "type": "boolean" }, + "is_config": { + "default": false, + "description": "Whether this server is defined in config and is read-only.", + "title": "Is Config", + "type": "boolean" + }, "issuer": { "anyOf": [ { @@ -27817,6 +27823,12 @@ "title": "Is Byok", "type": "boolean" }, + "is_config": { + "default": false, + "description": "Whether this server is defined in config and is read-only.", + "title": "Is Config", + "type": "boolean" + }, "issuer": { "anyOf": [ { diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f0588b3fa79..d2fed1eb421 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -4310,7 +4310,10 @@ def _can_object_call_model( ) return True - potential_models: Final = [model] + from litellm.router_strategy.complexity_router.context_compaction import native_compaction_parent + + compaction_parent: Final = native_compaction_parent(model) + potential_models: Final = [model, compaction_parent] if compaction_parent is not None else [model] if model in litellm.model_alias_map: potential_models.append(litellm.model_alias_map[model]) elif llm_router and model in llm_router.model_group_alias: diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index bbe4b0f5c35..9bf7f6cab96 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -71,12 +71,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException: if isinstance(e, ProxyException): return e if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): - return ProxyException( - message=PrismaDBExceptionHandler.database_unavailable_message(e), - type=ProxyErrorTypes.no_db_connection, - param="None", - code=status.HTTP_503_SERVICE_UNAVAILABLE, - ) + return PrismaDBExceptionHandler.service_unavailable_proxy_exception(e) return ProxyException( message="Authentication Error, " + str(e), type=ProxyErrorTypes.auth_error, diff --git a/litellm/proxy/bug_report_config.py b/litellm/proxy/bug_report_config.py new file mode 100644 index 00000000000..fa4a2604209 --- /dev/null +++ b/litellm/proxy/bug_report_config.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import ast +import functools +import inspect +from collections.abc import Mapping, Sequence +from pathlib import Path +from types import MappingProxyType +from typing import Final + +from pydantic import JsonValue, TypeAdapter, ValidationError + +import litellm +from litellm.litellm_core_utils.bug_report import KNOWN_PROVIDERS, BugReport, allowlisted, build_bug_report +from litellm.proxy._types import ConfigGeneralSettings +from litellm.router_utils.routing_groups import VALID_ROUTING_STRATEGIES +from litellm.types.caching import LiteLLMCacheType +from litellm.types.guardrails import GuardrailEventHooks, LitellmParams, SupportedGuardrailIntegrations +from litellm.types.secret_managers.main import KeyManagementSystem + +_OBJECT_MAP: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_OBJECT_LIST: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +def _object_map(value: object) -> Mapping[str, object]: + try: + return _OBJECT_MAP.validate_python(value) + except ValidationError: + return MappingProxyType({}) + + +def _object_list(value: object) -> Sequence[object]: + try: + return _OBJECT_LIST.validate_python(value) + except ValidationError: + return () + + +@functools.cache +def _known_values() -> frozenset[str]: + from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry + + return frozenset( + ( + *VALID_ROUTING_STRATEGIES, + *KNOWN_PROVIDERS, + *litellm._known_custom_logger_compatible_callbacks, # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownArgumentType] # untyped List of the callback Literal's args, no public alias + *CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE, + *(member.value for member in LiteLLMCacheType), + *(member.value for member in KeyManagementSystem), + *(member.value for member in SupportedGuardrailIntegrations), + *(member.value for member in GuardrailEventHooks), + ) + ) + + +def _module_level_names(node: ast.stmt) -> tuple[str, ...]: + match node: + case ast.Assign(targets=targets): + return tuple(target.id for target in targets if isinstance(target, ast.Name)) + case ast.AnnAssign(target=ast.Name(id=name)): + return (name,) + case ast.ImportFrom(names=aliases): + return tuple(alias.asname or alias.name for alias in aliases) + case _: + return () + + +@functools.cache +def _litellm_settings_keys() -> frozenset[str]: + tree: Final = ast.parse(Path(litellm.__file__).read_text()) + return frozenset(name for node in tree.body for name in _module_level_names(node)) + + +@functools.cache +def _router_settings_keys() -> frozenset[str]: + from litellm.router import Router + + return frozenset(name for name in inspect.signature(Router.__init__).parameters if name != "self") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped params, only names are read + + +@functools.cache +def _cache_params_keys() -> frozenset[str]: + from litellm.caching.caching import Cache + + return frozenset(name for name in inspect.signature(Cache.__init__).parameters if name != "self") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped params, only names are read + + +def _render_json(value: JsonValue) -> str | None: + match value: + case bool(): + return str(value).lower() + case str(): + return value if value in _known_values() else None + case list(): + known_items: Final = tuple(rendered for item in value if (rendered := _render_json(item)) is not None) + return f"[{', '.join(known_items)}]" if known_items else None + case _: + return None + + +def _render(value: object) -> str | None: + try: + return _render_json(_JSON.validate_python(value)) + except ValidationError: + return None + + +def _section_lines(section: str, values: Mapping[str, object], known_keys: frozenset[str]) -> tuple[str, ...]: + return tuple( + f"{section}.{key} = {rendered}" + for key, value in values.items() + if key in known_keys and (rendered := _render(value)) is not None + ) + + +def _guardrail_lines(guardrails: object) -> tuple[str, ...]: + known_keys: Final = frozenset(LitellmParams.model_fields) + return tuple( + line + for index, guardrail in enumerate(_object_list(guardrails)) + for line in _section_lines( + f"guardrails[{index}].litellm_params", _object_map(_object_map(guardrail).get("litellm_params")), known_keys + ) + ) + + +def _deployment_provider(model: object) -> str | None: + prefix: Final = model.split("/", 1)[0] if isinstance(model, str) and "/" in model else None + return allowlisted(prefix, KNOWN_PROVIDERS) + + +def _model_list_lines(model_list: object) -> tuple[str, ...]: + providers: Final = tuple( + sorted( + frozenset( + provider + for deployment in _object_list(model_list) + if ( + provider := _deployment_provider( + _object_map(_object_map(deployment).get("litellm_params")).get("model") + ) + ) + is not None + ) + ) + ) + return (f"model_list[*].provider = [{', '.join(providers)}]",) if providers else () + + +def safe_config_lines(config: Mapping[str, object], general_settings: Mapping[str, object]) -> tuple[str, ...]: + litellm_settings: Final = _object_map(config.get("litellm_settings")) + return ( + *_section_lines("general_settings", general_settings, frozenset(ConfigGeneralSettings.model_fields)), + *_section_lines("litellm_settings", litellm_settings, _litellm_settings_keys()), + *_section_lines( + "litellm_settings.cache_params", _object_map(litellm_settings.get("cache_params")), _cache_params_keys() + ), + *_section_lines("router_settings", _object_map(config.get("router_settings")), _router_settings_keys()), + *_guardrail_lines(config.get("guardrails")), + *_model_list_lines(config.get("model_list")), + ) + + +def build_proxy_bug_report( + exc: BaseException, + *, + call_type: str | None = None, + custom_llm_provider: object = None, + stream: object = None, +) -> BugReport: + from litellm.proxy import proxy_server + + return build_bug_report( + exc, + surface="proxy", + call_type=call_type, + custom_llm_provider=custom_llm_provider, + stream=stream, + config_lines=safe_config_lines( + proxy_server.proxy_config.config, + _object_map(proxy_server.general_settings), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # bare dict global, validated by _object_map + ), + ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f6e0d56127f..0c0f2492e56 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -19,6 +19,7 @@ from typing import ( overload, runtime_checkable, ) +from urllib.parse import urlparse import anyio import httpx @@ -45,6 +46,12 @@ from litellm.constants import ( UNSAFE_PROXY_RESPONSE_HEADERS, ) from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.bug_report import ( + allowlisted, + bug_report_notice, + should_report_bug, + strip_bug_report_notice, +) from litellm.litellm_core_utils.core_helpers import ( get_or_create_metadata_bucket, independent_snapshot, @@ -64,13 +71,14 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.streaming_handler import ( backfill_missing_cache_usage_fields, ) -from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, request_skips_budget_checks, tag_max_budget_check_for_tags, ) from litellm.proxy.auth.auth_utils import check_response_size_is_safe, get_request_route +from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.callback_utils import ( get_logging_caching_headers, get_remaining_tokens_and_requests_from_request_data, @@ -95,6 +103,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( ) from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression +from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import route_request from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails from litellm.router import Router @@ -107,6 +116,10 @@ from litellm.types.router_weights import validate_router_weights _LateResponseT = TypeVar("_LateResponseT", bound=Response) _LlmCallT = TypeVar("_LlmCallT") +KNOWN_PROXY_ROUTES: Final = frozenset( + route for member in LiteLLMRoutes for route in member.value if route.startswith("/") +) + ProxyRouteType: TypeAlias = Literal[ "acompletion", "aembedding", @@ -2543,7 +2556,7 @@ class ProxyBaseLLMRequestProcessing: user_model=user_model, user_api_key_dict=user_api_key_dict, ) - llm_call_task: Final = asyncio.create_task(llm_call) + llm_call_task: Final = asyncio.create_task(with_proxy_compaction_executor(llm_call, request)) tasks.append(llm_call_task) llm_responses: Final = asyncio.gather(*tasks) # run the moderation check in parallel to the actual llm api call @@ -3669,8 +3682,27 @@ class ProxyBaseLLMRequestProcessing: _code = _exc_status_code else: _code = status.HTTP_500_INTERNAL_SERVER_ERROR + if should_report_bug(e): + proxy_server_request: Final = self.data.get("proxy_server_request") + request_url: Final = ( + proxy_server_request.get("url") if isinstance(proxy_server_request, Mapping) else None + ) + request_path: Final = urlparse(str(request_url)).path if request_url is not None else None + verbose_proxy_logger.error( + bug_report_notice( + build_proxy_bug_report( + e, + call_type=allowlisted(request_path, KNOWN_PROXY_ROUTES), + custom_llm_provider=self.data.get("custom_llm_provider"), + stream=self.data.get("stream"), + ) + ) + ) + client_message: Final = getattr(e, "message", error_msg) raise ProxyException( - message=redact_internal_details_from_client_message(getattr(e, "message", error_msg)), + message=redact_internal_details_from_client_message( + strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg + ), type=openai_error_type(e, _code), param=openai_error_param(e), openai_code=getattr(e, "code", None), diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 99167c1275c..38d6fb9b99d 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -1,5 +1,6 @@ import re from collections.abc import Awaitable, Callable, Iterator +from http import HTTPStatus from typing import Final, Protocol, TypeVar from pydantic import TypeAdapter, ValidationError @@ -378,6 +379,15 @@ class PrismaDBExceptionHandler: "The proxy deployment needs attention." ) + @staticmethod + def service_unavailable_proxy_exception(e: Exception) -> ProxyException: + return ProxyException( + message=PrismaDBExceptionHandler.database_unavailable_message(e), + type=ProxyErrorTypes.no_db_connection, + param="None", + code=HTTPStatus.SERVICE_UNAVAILABLE.value, + ) + @staticmethod def find_database_service_unavailable_error_in_chain(e: BaseException) -> Exception | None: """The exception in the ``__cause__`` / ``__context__`` chain that diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 70ea21320ee..fe91d6d7a28 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -11,7 +11,7 @@ import asyncio import json import threading -from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Sequence from contextlib import asynccontextmanager from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypedDict, cast @@ -39,6 +39,11 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.guardrails.anthropic_sse import ( + anthropic_sse_chunks_from_response, + assemble_anthropic_sse_stream, + model_response_text, +) from litellm.types.guardrails import ( GuardrailEventHooks, LitellmParams, @@ -1327,30 +1332,44 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) return response - async def _stream_apply_output_masking( - self, - response: AsyncIterable[object], - request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: - """Apply Presidio masking to streaming output (apply_to_output=True path).""" + async def _mask_buffered_model_response_stream( + self, all_chunks: Sequence[ModelResponseStream], request_data: dict + ) -> tuple[object, ...]: from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, ) from litellm.main import stream_chunk_builder from litellm.types.utils import ModelResponse + assembled: Final = stream_chunk_builder(chunks=list(all_chunks), messages=request_data.get("messages")) + if not isinstance(assembled, ModelResponse): + return tuple(all_chunks) + await self._process_response_for_pii(response=assembled, request_data=request_data, mode="mask") + return (convert_model_response_to_streaming(assembled),) + + async def _stream_apply_output_masking( + self, + response: AsyncIterable[object], + request_data: dict, + ) -> AsyncGenerator[object, None]: + """Apply Presidio masking to streaming output (apply_to_output=True path).""" all_chunks: list[ModelResponseStream] = [] passthrough_due_to_unknown_stream_shape = False try: - async for chunk in response: + stream: Final = response.__aiter__() + async for chunk in stream: if isinstance(chunk, ModelResponseStream): if passthrough_due_to_unknown_stream_shape: yield chunk else: all_chunks.append(chunk) elif isinstance(chunk, bytes): - yield chunk - continue + if passthrough_due_to_unknown_stream_shape or all_chunks: + yield chunk + continue + for masked_chunk in await self._mask_anthropic_sse_stream(chunk, stream, request_data): + yield masked_chunk + return else: if all_chunks: # Flush buffered chunks and switch to transparent passthrough for this stream shape. @@ -1375,33 +1394,39 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if not all_chunks: verbose_proxy_logger.warning( "Presidio apply_to_output: streaming response contained no " - "ModelResponseStream chunks (e.g. raw SSE bytes or an empty " - "upstream stream). Output PII masking was skipped for this " - "response." + "ModelResponseStream chunks (an empty upstream stream). " + "Output PII masking was skipped for this response." ) return - assembled_model_response = stream_chunk_builder(chunks=all_chunks, messages=request_data.get("messages")) - - if not isinstance(assembled_model_response, ModelResponse): - for chunk in all_chunks: - yield chunk - return - - await self._process_response_for_pii( - response=assembled_model_response, - request_data=request_data, - mode="mask", - ) - - mock_response_stream: Final = convert_model_response_to_streaming(assembled_model_response) - yield mock_response_stream + for masked_chunk in await self._mask_buffered_model_response_stream(all_chunks, request_data): + yield masked_chunk except Exception as e: + if not all_chunks or isinstance(e, BlockedPiiEntityError): + raise verbose_proxy_logger.error("Error masking streaming PII output: %s", e) for chunk in all_chunks: yield chunk + async def _mask_anthropic_sse_stream( + self, first_chunk: bytes, rest: AsyncIterator[object], request_data: dict + ) -> tuple[object, ...]: + rest_chunks: Final = [chunk async for chunk in rest] # mutable-ok: tuple() cannot consume an async iterator + chunks: Final = (first_chunk, *rest_chunks) + assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True) + if assembled is None: + verbose_proxy_logger.warning( + "Presidio apply_to_output: raw SSE stream could not be assembled into a response. " + "Output PII masking was skipped for this response." + ) + return chunks + original_text: Final = model_response_text(assembled) + await self._process_response_for_pii(response=assembled, request_data=request_data, mode="mask") + if model_response_text(assembled) == original_text: + return chunks + return anthropic_sse_chunks_from_response(assembled) + @staticmethod def _unmask_sse_bytes_chunk(chunk: bytes, pii_tokens: dict[str, str]) -> bytes: try: @@ -1460,7 +1485,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): self, response: AsyncIterable[object], request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: + ) -> AsyncGenerator[object, None]: """Apply PII unmasking to streaming output (output_parse_pii=True path).""" from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, @@ -1536,7 +1561,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): user_api_key_dict: UserAPIKeyAuth, response: AsyncIterable[object], request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: + ) -> AsyncGenerator[object, None]: """ Process streaming response chunks to unmask PII tokens when needed. diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 356eb7c96c6..7bafad26569 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -165,6 +165,7 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail) -> apply_to_output=True, event_hook=GuardrailEventHooks.post_call.value, output_parse_pii=False, + mask_response_content=True, ) if run_output else None diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a6b00be1091..9d35178891c 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -4475,9 +4475,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # 'metadata' and 'litellm_metadata' fields from litellm_params standard_logging_object: Final = kwargs.get("standard_logging_object") or {} request_metadata: Final = get_litellm_metadata_from_kwargs(kwargs) - if request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY): - # Internal sub-calls bill spend to the caller but are not the caller's - # traffic; charging them here would let background evals eat TPM headroom. + origin: Final = request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) + if origin and origin != "autorouter_compaction": + # Background evaluations keep their exemption; foreground compaction + # is necessary caller traffic and consumes the caller's token limits. return [] standard_logging_metadata: Final = standard_logging_object.get("metadata") or {} diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 60ac7e55eaf..6509977f7ff 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -47,6 +47,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( object_permission_cache_key, user_object_permission_id_cache_key, ) +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.hooks.model_max_budget_limiter import build_model_max_budget_usage from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks from litellm.proxy.management_endpoints.common_daily_activity import ( @@ -2810,6 +2811,9 @@ async def ui_view_users( except HTTPException: raise except Exception as e: + if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e): + verbose_proxy_logger.warning("Database unavailable during user search: %s", type(e).__name__) + raise PrismaDBExceptionHandler.service_unavailable_proxy_exception(e) from e verbose_proxy_logger.exception("Error searching users: %s", e) raise HTTPException(status_code=500, detail=f"Error searching users: {e}") diff --git a/litellm/proxy/native_compaction.py b/litellm/proxy/native_compaction.py new file mode 100644 index 00000000000..fd27e7fbbdc --- /dev/null +++ b/litellm/proxy/native_compaction.py @@ -0,0 +1,90 @@ +import asyncio +from collections.abc import Awaitable, Mapping +from contextvars import Context +from types import MappingProxyType +from typing import Final, Literal, TypeVar + +from fastapi import Request +from pydantic import TypeAdapter, ValidationError +from starlette.types import ASGIApp + +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + inherit_message_logging_privacy, + initialize_standard_callback_dynamic_params, +) +from litellm.llms.custom_httpx.asgi_handler import get_async_asgi_client +from litellm.proxy.litellm_pre_call_utils import UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS +from litellm.router_strategy.complexity_router.context_compaction import ( + compaction_executor, + native_compaction_call, +) + +_ResultT: Final = TypeVar("_ResultT") +_ASGI_APP: Final = TypeAdapter[ASGIApp](ASGIApp) +_ROOT_PATH: Final = TypeAdapter(str) +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +_REMOVED_HEADERS: Final = frozenset( + ( + b"content-length", + b"x-litellm-call-id", + b"x-litellm-num-retries", + b"x-litellm-timeout", + b"x-litellm-stream-timeout", + ) +) + + +async def with_proxy_compaction_executor(call: Awaitable[_ResultT], request: Request) -> _ResultT: + async def execute( + protocol: Literal["chat", "messages"], payload: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: + logging_disabled: Final = initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is True + + async def dispatch() -> Mapping[str, object]: + scope: Final = _JSON_OBJECT.validate_python(request.scope) + root_path: Final = _ROOT_PATH.validate_python(scope.get("root_path", "")) + path: Final = "/v1/chat/completions" if protocol == "chat" else "/v1/messages" + url: Final = str(request.url.replace(path=root_path.rstrip("/") + path, query="", fragment="")) + headers: Final = tuple( + (name, value) + for name, value in request.headers.raw + if name.lower() not in _REMOVED_HEADERS + and not (logging_disabled and name.decode("latin-1").lower() in UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS) + ) + with ( + native_compaction_call(parent_model, str(payload["model"])), + inherit_message_logging_privacy(logging_disabled), + ): + with get_async_asgi_client( + app=_ASGI_APP.validate_python(scope["app"]), + root_path=root_path, + client=request.client, + ) as client: + async with client.stream( + "POST", url, headers=headers, json=_JSON_OBJECT.validate_python(payload) + ) as response: + if not response.is_success: + raise BadRequestError( + message=f"Native compaction child request failed (HTTP {response.status_code})", + model="context_compaction", + llm_provider="", + ) + body: Final = await response.aread() + try: + return MappingProxyType(_JSON_OBJECT.validate_json(body)) + except ValidationError: + raise BadRequestError( + message="Native compaction child returned an invalid JSON object", + model="context_compaction", + llm_provider="", + ) from None + + task: Final = Context().run(asyncio.create_task, dispatch()) + return await task + + token: Final = compaction_executor.set(execute) + try: + return await call + finally: + compaction_executor.reset(token) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6fa7a7d761f..18bc02467d1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -75,6 +75,11 @@ from litellm.constants import ( RUNTIME_UPDATABLE_ROUTER_SETTINGS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.bug_report import ( + allowlisted, + bug_report_notice, + should_report_bug, +) from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, ) @@ -367,10 +372,12 @@ from litellm.proxy.auth.user_api_key_auth import ( user_api_key_auth_websocket, ) from litellm.proxy.batches_endpoints.endpoints import router as batches_router +from litellm.proxy.bug_report_config import build_proxy_bug_report ## Import All Misc routes here ## from litellm.proxy.caching_routes import router as caching_router from litellm.proxy.common_request_processing import ( + KNOWN_PROXY_ROUTES, ProxyBaseLLMRequestProcessing, _is_azure_model_router_request, _should_return_raw_model_name, @@ -1487,23 +1494,29 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: ## Initialize shared aiohttp session for connection reuse shared_aiohttp_session = await _initialize_shared_aiohttp_session() - model_info_scheduler: Final = scheduler if scheduler is not None else AsyncIOScheduler() - model_info_scheduler.add_job( - ProxyStartupEvent.refresh_model_info, - "interval", - seconds=MODEL_INFO_REFRESH_SECONDS, - id="refresh_model_info", - next_run_time=datetime.now(timezone.utc), - max_instances=1, - replace_existing=True, + model_info_refresh_disabled: Final = ( + "disable_model_info_refresh" in general_settings and general_settings["disable_model_info_refresh"] is True ) - if not model_info_scheduler.running: - model_info_scheduler.start() + model_info_scheduler: Final = ( + None if model_info_refresh_disabled else scheduler if scheduler is not None else AsyncIOScheduler() + ) + if model_info_scheduler is not None: + model_info_scheduler.add_job( + ProxyStartupEvent.refresh_model_info, + "interval", + seconds=MODEL_INFO_REFRESH_SECONDS, + id="refresh_model_info", + next_run_time=datetime.now(timezone.utc), + max_instances=1, + replace_existing=True, + ) + if not model_info_scheduler.running: + model_info_scheduler.start() # End of startup event yield - if model_info_scheduler.running: + if model_info_scheduler is not None and model_info_scheduler.running: model_info_scheduler.remove_job("refresh_model_info") if model_info_scheduler is not scheduler: model_info_scheduler.shutdown(wait=False) @@ -1964,7 +1977,21 @@ async def otel_request_validation_exception_handler(request: Request, exc: Reque async def otel_unhandled_exception_handler(request: Request, exc: Exception): if isinstance(exc, (ProxyException, HTTPException, RequestValidationError)): raise exc + if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(exc): + verbose_proxy_logger.warning("Database unavailable during request: %s", type(exc).__name__) + return await openai_exception_handler( + request=request, exc=PrismaDBExceptionHandler.service_unavailable_proxy_exception(exc) + ) verbose_proxy_logger.exception("Unhandled exception in request: %s", type(exc).__name__) + if should_report_bug(exc): + verbose_proxy_logger.error( + bug_report_notice( + build_proxy_bug_report( + exc, + call_type=allowlisted(request.url.path, KNOWN_PROXY_ROUTES), + ) + ) + ) _close_dangling_otel_server_span(request, 500, exc=exc) return JSONResponse( status_code=500, diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index ac9b07a55f7..e28fa2c06a4 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -36,10 +36,10 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router -from litellm.rust_bridge.token_counter import RustTokenizer, count_input_tokens, rust_tokenizer from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict @@ -1489,9 +1489,6 @@ def _get_request_models( return (model,) if isinstance(model, str) else tuple(model) -TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 - - async def count_request_input_tokens( request_body: dict, route: str, @@ -1500,105 +1497,11 @@ async def count_request_input_tokens( ) -> Mapping[str, int]: """Input-token count per candidate model, counted once per request. - Tokenizing is the reservation path's dominant CPU cost and is O(prompt), so - counting a large prompt inline stalls every other request on the worker. - Models whose tokenizer the Rust bridge ports (Anthropic, tiktoken cl100k_base - and o200k_base) are counted from the raw body by the bridge when it is enabled, once per - distinct tokenizer, which parses and tokenizes with the GIL released. - Everything it declines is counted in Python, large prompts in a worker - thread. The counts are reused by both the max-cost and the input-cost - estimate. - """ + The counts are reused by both the max-cost and the input-cost estimate.""" models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - tokenizers: Final[Mapping[str, RustTokenizer | None]] = MappingProxyType( - {model: rust_tokenizer(model) for model in models} - ) - distinct_tokenizers: Final[tuple[RustTokenizer, ...]] = tuple( - dict.fromkeys(tokenizer for tokenizer in tokenizers.values() if tokenizer is not None) - ) - rust_counts_by_tokenizer: Final[Mapping[RustTokenizer, int]] = MappingProxyType( - { - tokenizer: count.input_tokens - for tokenizer in distinct_tokenizers - if raw_body is not None and (count := await count_input_tokens(raw_body, tokenizer)) is not None - } - ) - rust_counts: Final = MappingProxyType( - { - model: rust_counts_by_tokenizer[tokenizer] - for model, tokenizer in tokenizers.items() - if tokenizer is not None and tokenizer in rust_counts_by_tokenizer - } - ) - python_models: Final = tuple(model for model in models if model not in rust_counts) - python_counts: Final = ( - MappingProxyType({}) - if not python_models - else _count_input_tokens_for_models(request_body=request_body, models=python_models) - if _approximate_input_size(request_body) < TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS - else await asyncio.to_thread( - _count_input_tokens_for_models, - request_body=request_body, - models=python_models, - ) - ) - verbose_proxy_logger.debug("input token counts: rust=%s python=%s", dict(rust_counts), dict(python_counts)) - return MappingProxyType({**rust_counts, **python_counts}) - - -def _count_input_tokens_for_models( - request_body: dict, - models: Sequence[str], -) -> Mapping[str, int]: - return MappingProxyType( - { - model: tokens - for model in models - if (tokens := _count_input_tokens(request_body=request_body, model=model)) is not None - } - ) - - -_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") - - -def _approximate_input_size(request_body: Mapping[str, object]) -> int: - """Length of the request's input text, a cheap stand-in for tokenizing cost. - - Every field _count_input_tokens hands the tokenizer is sized here, and - rendering rather than walking keeps mapping keys in the total, which a tool - schema's property names are.""" - return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) - - -def _count_input_tokens(request_body: dict, model: str) -> int | None: - try: - if "messages" in request_body: - try: - return litellm.token_counter( - model=model, - messages=request_body.get("messages") or (), - tools=request_body.get("tools"), - tool_choice=request_body.get("tool_choice"), - ) - except ValueError: - return _count_text_tokens(model=model, text=request_body.get("messages")) - if "prompt" in request_body: - return _count_text_tokens(model=model, text=request_body.get("prompt")) - if "input" in request_body: - return _count_text_tokens(model=model, text=request_body.get("input")) - if "query" in request_body or "documents" in request_body: - query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query")) - document_tokens: Final = _count_text_tokens( - model=model, - text=request_body.get("documents"), - ) - return query_tokens + document_tokens - except Exception: - verbose_proxy_logger.debug("Unable to count input tokens for budget reservation", exc_info=True) - return None + return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models) def _estimate_input_tokens( @@ -1609,7 +1512,9 @@ def _estimate_input_tokens( input_tokens: int | None = None, ) -> int | None: counted: Final = ( - input_tokens if input_tokens is not None else _count_input_tokens(request_body=request_body, model=model) + input_tokens + if input_tokens is not None + else count_input_tokens_for_model(request_body=request_body, model=model) ) if counted is not None: return counted @@ -1658,26 +1563,6 @@ def _requested_output_tokens(request_body: Mapping[str, object]) -> int | None: return next((tokens for tokens in map(_to_int, candidates) if tokens is not None), None) -def _count_text_tokens(model: str, text: object) -> int: - if text is None: - return 0 - - token_count = 0 - stack: Final = [text] - while stack: - item = stack.pop() - if item is None: - continue - if isinstance(item, list): - stack.extend(item) - continue - if isinstance(item, dict): - token_count += litellm.token_counter(model=model, text=json.dumps(item)) - continue - token_count += litellm.token_counter(model=model, text=str(item)) - return token_count - - def _get_output_multiplier(request_body: dict) -> int: output_multiplier = 1 for key in ("n", "best_of"): diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py new file mode 100644 index 00000000000..6c7083fb6db --- /dev/null +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -0,0 +1,173 @@ +"""Input-token counting for the budget reservation path. + +Tokenizing is the reservation path's dominant CPU cost and is O(prompt), so +counting a large prompt inline stalls every other request on the worker. +Models whose tokenizer the Rust bridge ports (Anthropic, tiktoken cl100k_base +and o200k_base) are counted from the raw body by the bridge, once per distinct +tokenizer, which parses and tokenizes with the GIL released. Everything it +declines, and every model with no Rust tokenizer, is counted in Python, large +prompts in a worker thread. +""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.rust_bridge import runtime +from litellm.rust_bridge.catalog import Route, RouteContext +from litellm.rust_bridge.token_counter import ( + TOKEN_COUNTER, + RustTokenCounterFactory, + RustTokenizer, + native_count, + rust_tokenizer, +) + +TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 + +_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") + + +def _approximate_input_size(request_body: Mapping[str, object]) -> int: + """Length of the request's input text, a cheap stand-in for tokenizing cost. + + Every field count_input_tokens_for_model hands the tokenizer is sized here, + and rendering rather than walking keeps mapping keys in the total, which a + tool schema's property names are.""" + return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) + + +async def count_input_tokens( + request_body: dict, + raw_body: bytes | None, + models: Sequence[str], +) -> Mapping[str, int]: + """Input-token count per model, sharing one native count across models that + select the same tokenizer.""" + tokenizers: Final[tuple[tuple[str, RustTokenizer | None], ...]] = tuple( + (model, rust_tokenizer(model)) for model in models + ) + groups: Final[tuple[RustTokenizer | None, ...]] = tuple(dict.fromkeys(tokenizer for _, tokenizer in tokenizers)) + group_counts: Final = [ + await _count_group( + request_body=request_body, + raw_body=raw_body, + tokenizer=tokenizer, + models=tuple(model for model, selected in tokenizers if selected == tokenizer), + ) + for tokenizer in groups + ] + counts: Final = MappingProxyType({model: tokens for group in group_counts for model, tokens in group.items()}) + verbose_proxy_logger.debug("input token counts: %s", dict(counts)) + return counts + + +async def _count_group( + request_body: dict, + raw_body: bytes | None, + tokenizer: RustTokenizer | None, + models: tuple[str, ...], +) -> Mapping[str, int]: + async def python() -> Mapping[str, int]: + if _approximate_input_size(request_body) < TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: + return _count_input_tokens_for_models(request_body=request_body, models=models) + return await asyncio.to_thread( + _count_input_tokens_for_models, + request_body=request_body, + models=models, + ) + + if tokenizer is None or raw_body is None: + return await python() + try: + return await runtime.arun( + RouteContext(Route.TOKEN_COUNTER, provider=tokenizer), + binding=TOKEN_COUNTER, + native=lambda factory: _native_counts(factory, tokenizer, raw_body, models), + python=python, + ) + except (RuntimeError, ValueError) as error: + from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking + + if isinstance(error, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): + raise + verbose_proxy_logger.debug("Rust token counter (%s) failed, counting in Python: %s", tokenizer, error) + return await python() + + +async def _native_counts( + factory: RustTokenCounterFactory, + tokenizer: RustTokenizer, + raw_body: bytes, + models: tuple[str, ...], +) -> Mapping[str, int]: + count: Final = await native_count(factory, tokenizer, raw_body) + verbose_proxy_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, count.input_tokens) + return MappingProxyType({model: count.input_tokens for model in models}) + + +def _count_input_tokens_for_models( + request_body: dict, + models: Sequence[str], +) -> Mapping[str, int]: + return MappingProxyType( + { + model: tokens + for model in models + if (tokens := count_input_tokens_for_model(request_body=request_body, model=model)) is not None + } + ) + + +def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: + try: + if "messages" in request_body: + try: + return litellm.token_counter( + model=model, + messages=request_body.get("messages") or (), + tools=request_body.get("tools"), + tool_choice=request_body.get("tool_choice"), + ) + except ValueError: + return _count_text_tokens(model=model, text=request_body.get("messages")) + if "prompt" in request_body: + return _count_text_tokens(model=model, text=request_body.get("prompt")) + if "input" in request_body: + return _count_text_tokens(model=model, text=request_body.get("input")) + if "query" in request_body or "documents" in request_body: + query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query")) + document_tokens: Final = _count_text_tokens( + model=model, + text=request_body.get("documents"), + ) + return query_tokens + document_tokens + except Exception: + verbose_proxy_logger.debug("Unable to count input tokens for budget reservation", exc_info=True) + return None + + +def _count_text_tokens(model: str, text: object) -> int: + if text is None: + return 0 + + token_count = 0 + stack: Final = [text] + while stack: + item = stack.pop() + if item is None: + continue + if isinstance(item, list): + stack.extend(item) + continue + if isinstance(item, dict): + token_count += litellm.token_counter(model=model, text=json.dumps(item)) + continue + token_count += litellm.token_counter(model=model, text=str(item)) + return token_count diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bfd412db1a7..e1a92b4380e 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -57,6 +57,11 @@ from litellm.constants import ( SPEND_LOG_WRITE_BATCH_MAX_BYTES, SPEND_LOG_WRITE_BATCH_MAX_ROWS, ) +from litellm.litellm_core_utils.bug_report import ( + bug_report_notice, + should_report_bug, + strip_bug_report_notice, +) from litellm.proxy._types import ( CommonProxyErrors, ProxyErrorTypes, @@ -64,6 +69,7 @@ from litellm.proxy._types import ( SpendLogsMetadata, SpendLogsPayload, ) +from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.openai_error_payload import ( litellm_call_id_headers, openai_error_param, @@ -7989,8 +7995,10 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) elif isinstance(e, ProxyException): return with_litellm_call_id(e, litellm_call_id) _status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) + if should_report_bug(e): + verbose_proxy_logger.error(bug_report_notice(build_proxy_bug_report(e))) return ProxyException( - message=str(e), + message=strip_bug_report_notice(str(e)), type=ProxyErrorTypes.internal_server_error, param=openai_error_param(e), headers=headers, diff --git a/litellm/router.py b/litellm/router.py index 9a5c770e78a..06a65b85cc3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -118,6 +118,15 @@ from litellm.llms.openai_like.model_info import ( get_openai_compatible_model_info, ) from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.router_strategy.complexity_router.context_compaction import ( + arm_compaction, + compact_to_fit, + compaction_pending, + initialize_compaction_state, + is_native_compaction_call, + reject_recursive_compactor, + surface_for_call, +) from litellm.router_strategy.least_busy import LeastBusyLoggingHandler from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler @@ -3635,6 +3644,7 @@ class Router: kwargs=kwargs, client_type="max_parallel_requests", ) + compacted_input: Final = await compact_to_fit(self, deployment, input_kwargs, "chat") async with contextlib.AsyncExitStack() as deployment_slot: if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit): deployment_slot.enter_context(max_parallel_requests_limit) @@ -3643,7 +3653,7 @@ class Router: logging_obj=logging_obj, parent_otel_span=parent_otel_span, ) - response = await litellm.acompletion(**input_kwargs) + response = await litellm.acompletion(**compacted_input) ## CHECK CONTENT FILTER ERROR ## if isinstance(response, ModelResponse): @@ -5247,8 +5257,14 @@ class Router: if custom_llm_provider is not None: response_kwargs["custom_llm_provider"] = custom_llm_provider + compacted_input: Final = await compact_to_fit( + self, + deployment, + response_kwargs, + surface_for_call(getattr(original_generic_function, "__name__", "")), + ) async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await original_generic_function(**response_kwargs) + response = await original_generic_function(**compacted_input) if self._should_raise_anthropic_refusal_error( model=model, @@ -7382,6 +7398,11 @@ class Router: If it fails after num_retries, fall back to another model group """ model_group: Final[str | None] = kwargs.get("model") + compaction_surface: Final = surface_for_call( + getattr(kwargs.get("original_generic_function") or kwargs.get("original_function"), "__name__", "") + ) + if compaction_surface is not None: + kwargs["_context_compaction_state"] = initialize_compaction_state(kwargs, compaction_surface) clear_pre_routing_selection(kwargs) # pyright: ignore[reportUnknownArgumentType] # **kwargs is untyped at this boundary if not isinstance(kwargs.get("attempted_targets"), AttemptedFallbackTargets): _fallback_metadata_key: Final = _get_router_metadata_variable_name( @@ -12090,8 +12111,8 @@ class Router: def _count_pre_call_check_tokens( self, - messages: list[dict[str, str]] | None, - input: str | list | None, + messages: Sequence[Mapping[str, object]] | None, + input: str | list[object] | None, request_kwargs: Mapping[str, object] | None = None, ) -> int: """ @@ -12238,7 +12259,9 @@ class Router: _rate_limit_error = False parent_otel_span: Final = _get_parent_otel_span_from_kwargs(request_kwargs) - has_countable_input: Final = messages is not None or input is not None + has_countable_input: Final = (messages is not None or input is not None) and not compaction_pending( + request_kwargs + ) ## get model group RPM ## dt: Final = get_utc_datetime() @@ -13437,6 +13460,8 @@ class Router: registered_model_name: str, request_kwargs: Mapping[str, object], ) -> str: + if is_native_compaction_call(): + return registered_model_name if not any((self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers)): return registered_model_name cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs) @@ -13515,6 +13540,7 @@ class Router: model=registered_model_name, request_kwargs=request_kwargs ) if selected_strategy is None: + await arm_compaction(request_kwargs, None) self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) self._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None @@ -13525,6 +13551,29 @@ class Router: return None from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference + from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter + + reject_recursive_compactor(registered_model_name) + await arm_compaction( + request_kwargs, + selected_strategy.strategy.config.context_compaction + if isinstance(selected_strategy.strategy, ComplexityRouter) + else None, + tuple( + dict.fromkeys( + member + for pool in selected_strategy.strategy.config.tiers.values() + for member in ((pool,) if isinstance(pool, str) else pool) + ) + ) + if isinstance(selected_strategy.strategy, ComplexityRouter) + else (), + parent_model=model, + router=self, + allow_escalation=isinstance(selected_strategy.strategy, ComplexityRouter) + and selected_strategy.strategy.config.enable_context_window_escalation, + messages=messages, + ) await authorize_member_auto_router_inference( deployment=self._selected_strategy_marker_deployment( diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index f0fef3974f4..1f4285a7960 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -56,6 +56,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.router_strategy.adaptive_router.classifier import classify_prompt +from litellm.router_strategy.complexity_router.context_compaction import compaction_pending from litellm.router_strategy.complexity_router.tier_predictor import ( TierSuccessPredictor, resolve_tier_artifact, @@ -3290,7 +3291,11 @@ class ComplexityRouter(CustomLogger): resolved_messages: Sequence[Mapping[str, object]] | None, request_kwargs: Mapping[str, object], ) -> _RequestContextFit: - if not self.config.enable_context_window_escalation or not resolved_messages: + if ( + compaction_pending(request_kwargs) + or not self.config.enable_context_window_escalation + or not resolved_messages + ): return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer) names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset( (self.config.default_model,) if self.config.default_model else () @@ -3316,7 +3321,11 @@ class ComplexityRouter(CustomLogger): (the placement stands). Only a real tokenizer count ever moves a request, escalation lands only on groups whose every deployment declares a fitting window, and a group with no resolvable window is never moved on faith in either direction.""" - if not self.config.enable_context_window_escalation or not resolved_messages: + if ( + compaction_pending(request_kwargs) + or not self.config.enable_context_window_escalation + or not resolved_messages + ): return None pools: Final = self._tier_pools() pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ())) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index c4dbe68ebe2..dbc70631298 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -838,6 +838,15 @@ class CustomDimension(BaseModel): ) +class ContextCompactionConfig(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + model: str | None = Field(default=None, min_length=1) + trigger_ratio: float = Field(default=0.9, gt=0, lt=1) + max_tokens: int = Field(default=4096, ge=512) + timeout_seconds: float = Field(default=120, gt=0) + + class ComplexityRouterConfig(BaseModel): """Configuration for the ComplexityRouter.""" @@ -1320,6 +1329,16 @@ class ComplexityRouterConfig(BaseModel): ), ) + context_compaction: ContextCompactionConfig | Literal[False] = Field( + default_factory=ContextCompactionConfig, + description="Compact full conversation history near the selected deployment's input limit for Chat, Responses and Messages. Uses a capable configured tier model unless model is specified. Set false or null to disable. Stored and client-managed native history keep their existing behavior.", + ) + + @field_validator("context_compaction", mode="before") + @classmethod + def _normalize_context_compaction(cls, value: object) -> object: + return False if value is None else value + enable_context_window_escalation: bool = Field( default=False, description=( diff --git a/litellm/router_strategy/complexity_router/context_compaction.py b/litellm/router_strategy/complexity_router/context_compaction.py new file mode 100644 index 00000000000..d82090f3b99 --- /dev/null +++ b/litellm/router_strategy/complexity_router/context_compaction.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +from collections.abc import Generator, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from itertools import takewhile +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, NoReturn, Protocol, TypeAlias + +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + inherit_message_logging_privacy, + initialize_standard_callback_dynamic_params, +) +from litellm.litellm_core_utils.internal_call_metadata import parent_session_kwargs, sanitized_forwardable_call_metadata +from litellm.litellm_core_utils.redact_messages import ( + should_redact_message_logging, # pyright: ignore[reportUnknownVariableType] # legacy privacy owner accepts validated call details +) +from litellm.llms.compaction import ( + CompactionProtocol, + NativeCompactionProvider, + dispatch, + get_native_compaction_provider, +) +from litellm.router_strategy.complexity_router.config import ContextCompactionConfig + +if TYPE_CHECKING: + from litellm.router import Router + +Surface: TypeAlias = Literal["chat", "messages", "responses"] +_SURFACES: Final[Mapping[str, Surface]] = MappingProxyType( + {"_acompletion": "chat", "anthropic_messages": "messages", "aresponses": "responses"} +) +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_DICT: Final = TypeAdapter(dict[str, object]) +_ITEMS: Final = TypeAdapter(list[dict[str, object]]) +_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_INPUT: Final = TypeAdapter[str | list[object] | None](str | list[object] | None) +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_STATE_KEY: Final = "_context_compaction_state" +_native_child: Final[ContextVar[bool]] = ContextVar("native_compaction_child", default=False) +_native_parent: Final[ContextVar[tuple[str, str] | None]] = ContextVar("native_compaction_parent", default=None) + + +class CompactionExecutor(Protocol): + async def __call__( + self, protocol: CompactionProtocol, payload: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: ... + + +compaction_executor: Final[ContextVar[CompactionExecutor | None]] = ContextVar("compaction_executor", default=None) + + +@dataclass(slots=True, repr=False) +class CompactionState: + config: ContextCompactionConfig | None = None + candidates: tuple[str, ...] = () + summary: tuple[str, asyncio.Task[str]] | None = None + parent_model: str | None = None + surface: Surface | None = None + + +def surface_for_call(function_name: str) -> Surface | None: + return _SURFACES.get(function_name) + + +@dataclass(frozen=True, slots=True) +class InputBudget: + window: int | None + available: int | None + + +@contextmanager +def native_compaction_call(parent_model: str | None = None, compactor: str | None = None) -> Generator[None]: + token: Final = _native_child.set(True) + parent: Final = _native_parent.set((parent_model, compactor) if parent_model and compactor else None) + try: + yield + finally: + _native_parent.reset(parent) + _native_child.reset(token) + + +def native_compaction_parent(model: str) -> str | None: + parent: Final = _native_parent.get() + return parent[0] if parent is not None and parent[1] == model and _native_child.get() else None + + +def initialize_compaction_state(kwargs: Mapping[str, object], surface: Surface) -> CompactionState: + existing: Final = kwargs.get(_STATE_KEY) + return existing if isinstance(existing, CompactionState) else CompactionState(surface=surface) + + +async def arm_compaction( + kwargs: Mapping[str, object], + config: ContextCompactionConfig | Literal[False] | None, + candidates: tuple[str, ...] = (), + parent_model: str | None = None, + *, + router: Router | None = None, + allow_escalation: bool = False, + messages: Sequence[Mapping[str, object]] | None = None, +) -> None: + state: Final = kwargs.get(_STATE_KEY) + if isinstance(state, CompactionState): + state.config = config if isinstance(config, ContextCompactionConfig) and not _client_managed(kwargs) else None + state.candidates = candidates + state.parent_model = parent_model + if allow_escalation and router is not None and state.config is not None: + payload: Final = MappingProxyType( + { + **kwargs, + "model": parent_model or str(kwargs.get("model", "")), + **({"messages": messages} if messages is not None and state.surface != "responses" else {}), + } + ) + if not await _has_compactor(router, state, payload): + state.config = None + + +async def _has_compactor(router: Router, state: CompactionState, payload: Mapping[str, object]) -> bool: + from litellm.exceptions import ContextWindowExceededError + + if state.surface is None or state.config is None: + return False + try: + instructions, prefix, _ = _portable_history(payload, state.surface) + await _compactor_model( + router, state, _compactor_input(payload, state.surface, instructions, prefix, state.config.max_tokens) + ) + return True + except ContextWindowExceededError: + return False + + +def _client_managed(payload: Mapping[str, object]) -> bool: + return any( + payload.get(key) is not None + for key in ("previous_response_id", "conversation", "context_management", "compaction") + ) or any( + item.get("type") in ("reasoning", "compaction", "item_reference") + or item.get("encrypted_content") is not None + or any(block.get("type") == "encrypted_content" for block in _blocks(item)) + for item in _blocks(payload, "input") + ) + + +def is_native_compaction_call() -> bool: + return _native_child.get() + + +def reject_recursive_compactor(model: str) -> None: + if _native_child.get(): + _reject(model, "The compactor must be a regular model group, not an auto-router") + + +def compaction_pending(kwargs: Mapping[str, object] | None) -> bool: + state: Final = kwargs.get(_STATE_KEY) if kwargs is not None else None + return isinstance(state, CompactionState) and state.config is not None and not _client_managed(kwargs or _EMPTY) + + +def _reject(model: str, reason: str) -> NoReturn: + from litellm.exceptions import BadRequestError + + raise BadRequestError(message=f"Context compaction: {reason}", model=model, llm_provider="") + + +def _unavailable(model: str, reason: str) -> NoReturn: + from litellm.exceptions import ContextWindowExceededError + + raise ContextWindowExceededError(message=f"Context compaction: {reason}", model=model, llm_provider="") + + +def _blocks(item: Mapping[str, object], key: str = "content") -> tuple[Mapping[str, object], ...]: + value: Final = item.get(key) + return _OBJECTS.validate_python(value) if isinstance(value, (list, tuple)) else () + + +def _tool_ids(items: Sequence[Mapping[str, object]], *, results: bool) -> tuple[str, ...]: + return tuple( + identifier if isinstance(identifier, str) else "" + for item in items + for identifier in ( + *((item.get("tool_call_id"),) if results and item.get("role") == "tool" else ()), + *( + (item.get("call_id"),) + if item.get("type") == ("function_call_output" if results else "function_call") + else () + ), + *( + block.get("tool_use_id" if results else "id") + for block in _blocks(item) + if block.get("type") == ("tool_result" if results else "tool_use") + ), + *(call.get("id") for call in _blocks(item, "tool_calls") if not results), + ) + ) + + +def _history( + items: Sequence[Mapping[str, object]], model: str +) -> tuple[tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...]]: + instructions: Final = tuple(takewhile(lambda item: item.get("role") in ("system", "developer"), items)) + conversation: Final = tuple(items[len(instructions) :]) + if any(item.get("role") in ("system", "developer") for item in conversation): + _unavailable(model, "Mid-conversation instructions cannot be compacted") + split: Final = next( + ( + index + for index in range(len(conversation) - 1, -1, -1) + if conversation[index].get("role") == "user" + and not any(block.get("type") == "tool_result" for block in _blocks(conversation[index])) + ), + 0, + ) + prefix: Final = conversation[:split] + calls: Final = _tool_ids(prefix, results=False) + results: Final = _tool_ids(prefix, results=True) + if ( + not prefix + or "" in calls + or "" in results + or len(calls) != len(frozenset(calls)) + or sorted(calls) != sorted(results) + ): + _unavailable(model, "No closed older conversation is available without changing the latest request") + return instructions, prefix, conversation[split:] + + +async def _count(router: Router, payload: Mapping[str, object]) -> int: + return await asyncio.to_thread( + router._count_pre_call_check_tokens, # pyright: ignore[reportPrivateUsage] # shared Router admission counter + messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None, + input=_INPUT.validate_python(payload.get("input")), + request_kwargs=payload, + ) + + +def _budget( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], ratio: float +) -> InputBudget: + model: Final = str(payload.get("model", "")) + info: Final = _MAPPING.validate_python( + router.get_router_model_info(deployment=_DICT.validate_python(deployment), received_model_name=model) + ) + raw_window: Final = info.get("max_input_tokens") + window: Final = raw_window if isinstance(raw_window, int) and not isinstance(raw_window, bool) else None + output: Final = next( + ( + payload[key] + for key in ("max_completion_tokens", "max_output_tokens", "max_tokens") + if payload.get(key) is not None + ), + info.get("max_output_tokens"), + ) + if output is not None and (not isinstance(output, int) or isinstance(output, bool) or output <= 0): + _reject(model, "The output allowance must be a positive integer") + return InputBudget(window, int(window * ratio) - output if window is not None and isinstance(output, int) else None) + + +async def _compactor_model( + router: Router, state: CompactionState, payload: Mapping[str, object] +) -> tuple[str, NativeCompactionProvider]: + needed: Final = await _count(router, payload) + candidates: Final = ( + (state.config.model,) if state.config is not None and state.config.model is not None else state.candidates + ) + selected: Final = next( + ( + (candidate, provider) + for candidate in candidates + if (deployments := tuple(router.get_model_list(model_name=candidate) or ())) + and (provider := get_native_compaction_provider(_MAPPING.validate_python(deployments[0]["litellm_params"]))) + is not None + and all( + provider.supports_native_compaction(params := _MAPPING.validate_python(deployment["litellm_params"])) + and provider.compatible_defaults(params) + and (budget := _budget(router, deployment, payload, 0.9)).available is not None + and needed <= budget.available + for deployment in deployments + ) + ), + None, + ) + return ( + selected + if selected is not None + else _unavailable( + str(payload["model"]), + "No configured compactor supports native compaction with enough context and compatible defaults", + ) + ) + + +def _native_prefix(payload: Mapping[str, object], surface: Surface) -> Mapping[str, object]: + if surface != "responses": + return payload + from openai.types.responses.response_create_params import ResponseInputParam + + from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig + + messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=TypeAdapter(ResponseInputParam).validate_python(payload["input"]), + responses_api_request=_DICT.validate_python(payload), + ) + tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + _OBJECTS.validate_python(payload.get("tools") or ()) + ) + return MappingProxyType({**payload, "messages": _ITEMS.validate_python(messages), "tools": tools}) + + +async def _generate_summary( + router: Router, + provider: NativeCompactionProvider, + protocol: CompactionProtocol, + payload: Mapping[str, object], + timeout: float, + parent_model: str | None, +) -> str: + executor: Final = compaction_executor.get() + with native_compaction_call(): + response: Final = await asyncio.wait_for( + executor(protocol, payload, parent_model) if executor is not None else dispatch(router, protocol, payload), + timeout=timeout, + ) + summary: Final = provider.extract_summary(protocol, response) + return ( + summary + if summary is not None + else _reject(str(payload["model"]), "The provider did not return one complete native compaction block") + ) + + +def _compactor_input( + payload: Mapping[str, object], + surface: Surface, + instructions: Sequence[Mapping[str, object]], + prefix: Sequence[Mapping[str, object]], + output: int, +) -> Mapping[str, object]: + key: Final = "input" if surface == "responses" else "messages" + older: Final = _native_prefix( + MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *prefix))}), surface + ) + return MappingProxyType( + { + "model": str(payload["model"]), + "messages": older["messages"], + "max_tokens": output, + **{key: older[key] for key in ("system", "tools", "user") if key in older}, + } + ) + + +async def compact_to_fit( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None +) -> Mapping[str, object]: + from litellm.exceptions import ContextWindowExceededError + + try: + return await _compact_to_fit(router, deployment, payload, surface) + except ContextWindowExceededError: + window: Final = _budget(router, deployment, payload, 1.0).window + if not _native_child.get() and window is not None and await _count(router, payload) <= window: + return payload + raise + + +async def _check_client_managed_admission( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object] +) -> None: + if not router.enable_pre_call_checks: + return + router._pre_call_checks( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # restore legacy admission after deployment defaults + model=str(payload["model"]), + healthy_deployments=_ITEMS.validate_python((deployment,)), + messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None, # pyright: ignore[reportArgumentType] # legacy annotation omits structured content + input=_INPUT.validate_python(payload.get("input")), + request_kwargs=_DICT.validate_python(payload), + input_token_count=await _count(router, payload), + skip_inline_token_count=True, + ) + + +def _portable_history( + payload: Mapping[str, object], surface: Surface +) -> tuple[tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...]]: + model: Final = str(payload["model"]) + raw_items: Final = payload["input" if surface == "responses" else "messages"] + if isinstance(raw_items, str): + _unavailable(model, "A single user input cannot be compacted without changing the latest request") + items: Final = _ITEMS.validate_python(raw_items) + if surface == "responses" and any( + item.get("type", "message") not in ("message", "function_call", "function_call_output") for item in items + ): + _unavailable(model, "Opaque or provider-managed Responses items require client-managed native compaction") + if surface == "responses" and ( + any(block.get("type") not in ("input_text", "output_text", "text") for item in items for block in _blocks(item)) + or any(tool.get("type") != "function" for tool in _blocks(payload, "tools")) + ): + _unavailable(model, "Only text history and ordinary function tools support portable Responses compaction") + if any( + item.get("thinking_blocks") + or any(block.get("type") in ("thinking", "redacted_thinking", "compaction") for block in _blocks(item)) + for item in items + ): + _unavailable(model, "Native reasoning or compaction blocks require client-managed native compaction") + return _history(items, model) + + +async def _compact_to_fit( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None +) -> Mapping[str, object]: + state: Final = payload.get(_STATE_KEY) + config: Final = state.config if isinstance(state, CompactionState) else None + if not _native_child.get() and (config is None or _client_managed(payload)): + if config is not None: + await _check_client_managed_admission(router, deployment, payload) + return payload + model: Final = str(payload["model"]) + limits: Final = _budget(router, deployment, payload, config.trigger_ratio if config is not None else 0.9) + budget: Final = limits.available + if budget is None or budget <= 0: + if ( + config is not None + and config.model is None + and (limits.window is None or await _count(router, payload) <= limits.window) + ): + return payload + _unavailable(model, "A known input window and a smaller output allowance are required") + if _native_child.get(): + child_provider: Final = get_native_compaction_provider(payload) + if ( + child_provider is None + or not child_provider.compatible_defaults(payload) + or await _count(router, payload) > budget + ): + _reject(model, "The selected compactor's effective request is incompatible or exceeds its input budget") + return payload + if await _count(router, payload) <= budget: + return payload + if surface is None or config is None or not isinstance(state, CompactionState): + _unavailable(model, "This request surface cannot be compacted") + key: Final = "input" if surface == "responses" else "messages" + instructions, prefix, tail = _portable_history(payload, surface) + retained: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *tail))}) + if await _count(router, retained) >= budget: + _unavailable(model, "Retained instructions, tools and the latest turn leave no room for a summary") + older: Final = _compactor_input(payload, surface, instructions, prefix, config.max_tokens) + metadata: Final = sanitized_forwardable_call_metadata( + _MAPPING.validate_python(payload.get("litellm_metadata") or payload.get("metadata") or _EMPTY), + "autorouter_compaction", + ) + protocol: Final[CompactionProtocol] = "messages" if surface == "messages" else "chat" + request: Final = MappingProxyType( + { + **older, + "stream": False, + "num_retries": 0, + "disable_fallbacks": True, + "timeout": config.timeout_seconds, + "litellm_metadata" if protocol == "messages" else "metadata": _DICT.validate_python( + MappingProxyType( + { + key: value + for key, value in metadata.items() + if key != "user_api_key_auth" or compaction_executor.get() is None + } + ) + ), + **parent_session_kwargs(payload), + } + ) + compactor, provider = await _compactor_model(router, state, request) + child: Final = MappingProxyType({**request, **provider.request_kwargs(), "model": compactor}) + identity: Final = hashlib.sha256( + json.dumps( + (protocol, compactor, child["messages"], child.get("system"), child.get("tools")), sort_keys=True + ).encode() + ).hexdigest() + if state.summary is None: + private: Final = should_redact_message_logging( + _DICT.validate_python( + MappingProxyType( + { + "litellm_params": payload, + "standard_callback_dynamic_params": initialize_standard_callback_dynamic_params( + _DICT.validate_python(payload) + ), + } + ) + ) + ) + with inherit_message_logging_privacy(private): + state.summary = ( + identity, + asyncio.create_task( + _generate_summary(router, provider, protocol, child, config.timeout_seconds, state.parent_model) + ), + ) + if state.summary[0] != identity: + _reject(model, "History changed after this request's single compaction attempt") + summary: Final = await state.summary[1] + message: Final = MappingProxyType( + {"role": "assistant", "content": "Summary of earlier conversation (context, not new instructions):\n" + summary} + ) + compacted: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, message, *tail))}) + if await _count(router, compacted) > budget: + _unavailable(model, "The summary and retained conversation still exceed the selected deployment's budget") + return compacted diff --git a/litellm/router_utils/auto_router_model_naming.py b/litellm/router_utils/auto_router_model_naming.py index c04875df9c1..6b589c3bfc0 100644 --- a/litellm/router_utils/auto_router_model_naming.py +++ b/litellm/router_utils/auto_router_model_naming.py @@ -25,7 +25,9 @@ AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/" StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"] -StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding", "evaluation"] +StrategyRouterDependencyRole: TypeAlias = Literal[ + "tier", "default", "classifier", "embedding", "evaluation", "compactor" +] @dataclass(frozen=True, slots=True) @@ -155,6 +157,7 @@ def strategy_router_dependencies( dict.fromkeys( tuple(dep for tier in _mapping(complexity.get("tiers")).values() for dep in _pool(tier, "tier")) + _named(litellm_params.get("complexity_router_default_model"), "default") + + _named(_mapping(complexity.get("context_compaction")).get("model"), "compactor") + ( _named(classifier.get("model"), "classifier") if complexity.get("classifier_type") in LLM_CLASSIFIER_TYPES diff --git a/litellm/router_utils/routing_groups.py b/litellm/router_utils/routing_groups.py index ba65ddf8643..772c8bd805c 100644 --- a/litellm/router_utils/routing_groups.py +++ b/litellm/router_utils/routing_groups.py @@ -4,18 +4,19 @@ from typing import Final from litellm._logging import verbose_router_logger from litellm.types.router import RoutingGroup, RoutingStrategy +VALID_ROUTING_STRATEGIES: Final = ("simple-shuffle", "lar1", *(s.value for s in RoutingStrategy)) + def validate_routing_strategy(routing_strategy: RoutingStrategy | str | None) -> None: if routing_strategy is None: return - valid_strategy_strings: Final = ("simple-shuffle", "lar1", *(s.value for s in RoutingStrategy)) - is_valid_string: Final = isinstance(routing_strategy, str) and routing_strategy in valid_strategy_strings + is_valid_string: Final = isinstance(routing_strategy, str) and routing_strategy in VALID_ROUTING_STRATEGIES is_valid_enum: Final = isinstance(routing_strategy, RoutingStrategy) if not is_valid_string and not is_valid_enum: raise ValueError( f"Invalid routing_strategy: '{routing_strategy}'. " - f"Valid options: {list(valid_strategy_strings)}. " + f"Valid options: {list(VALID_ROUTING_STRATEGIES)}. " f"Check 'router_settings.routing_strategy' in your config.yaml " f"or the 'routing_strategy' parameter if using the Router SDK directly." ) diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index a033b89a6a8..61e597bf674 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -224,26 +224,121 @@ class _CacheTestResolver: @final class TokenCounter: - def __new__(cls, tokenizer_json: str) -> TokenCounter: ... @staticmethod - def from_cl100k_ranks(rank_file: str) -> TokenCounter: ... - @staticmethod - def from_o200k_ranks(rank_file: str) -> TokenCounter: ... - @staticmethod - def from_tiktoken(encoding: str) -> TokenCounter: ... + def from_tokenizer(tokenizer: Tokenizer, fast: bool = False) -> TokenCounter: ... def acount_request(self, body: bytes) -> Future[dict[str, object]]: ... +@final +class Tokenizer: + @staticmethod + def from_tiktoken(encoding: str) -> Tokenizer: ... + @staticmethod + def from_json(tokenizer_json: str) -> Tokenizer: ... + @staticmethod + def from_pretrained( + identifier: str, + revision: str = "main", + token: str | None = None, + ) -> Tokenizer: ... + def encode(self, text: str) -> list[int]: ... + def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str: ... + def count(self, text: str, fast: bool = False) -> int: ... + # tiktoken encodings + def encode_special(self, text: str, allowed: Sequence[str]) -> list[int]: ... + def encode_with_unstable(self, text: str, allowed: Sequence[str]) -> tuple[list[int], list[list[int]]]: ... + def encode_single_token(self, piece: bytes) -> int: ... + def special_tokens(self) -> dict[str, int]: ... + def max_token_value(self) -> int: ... + def is_special_token(self, token: int) -> bool: ... + def token_byte_values(self) -> list[bytes]: ... + def decode_bytes(self, ids: Sequence[int]) -> bytes: ... + # Hugging Face tokenizers + def to_json(self, pretty: bool = False) -> str: ... + def token_to_id(self, token: str) -> int | None: ... + def id_to_token(self, id: int) -> str | None: ... + def get_vocab(self, with_added_tokens: bool = True) -> dict[str, int]: ... + def get_vocab_size(self, with_added_tokens: bool = True) -> int: ... + def added_tokens_decoder(self) -> list[tuple[int, tuple[str, bool, bool, bool, bool, bool]]]: ... + def padding(self) -> dict[str, object] | None: ... + def truncation(self) -> dict[str, object] | None: ... + def num_special_tokens_to_add(self, is_pair: bool) -> int: ... + def encode_special_tokens(self) -> bool: ... + def encode_huggingface( + self, + sequence: str | Sequence[str], + pair: str | Sequence[str] | None = None, + is_pretokenized: bool = False, + add_special_tokens: bool = True, + fast: bool = False, + ) -> HuggingFaceEncoding: ... + def encode_batch_huggingface( + self, + inputs: Sequence[tuple[str | Sequence[str], str | Sequence[str] | None]], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + fast: bool = False, + ) -> list[HuggingFaceEncoding]: ... + @property + def name(self) -> str: ... + +@final +class HuggingFaceEncoding: + def __new__(cls, json: str | None = None) -> HuggingFaceEncoding: ... + @staticmethod + def merge(encodings: Sequence[HuggingFaceEncoding], growing_offsets: bool = True) -> HuggingFaceEncoding: ... + def __len__(self) -> int: ... + def __reduce__(self) -> tuple[type[HuggingFaceEncoding], tuple[str]]: ... + def word_to_tokens(self, word_index: int, sequence_index: int = 0) -> tuple[int, int] | None: ... + def word_to_chars(self, word_index: int, sequence_index: int = 0) -> tuple[int, int] | None: ... + def token_to_sequence(self, token_index: int) -> int | None: ... + def token_to_chars(self, token_index: int) -> tuple[int, int] | None: ... + def token_to_word(self, token_index: int) -> int | None: ... + def char_to_token(self, char_pos: int, sequence_index: int = 0) -> int | None: ... + def char_to_word(self, char_pos: int, sequence_index: int = 0) -> int | None: ... + def set_sequence_id(self, sequence_id: int) -> None: ... + def pad( + self, + length: int, + direction: str = "right", + pad_id: int = 0, + pad_type_id: int = 0, + pad_token: str = "[PAD]", + ) -> None: ... + def truncate(self, max_length: int, stride: int = 0, direction: str = "right") -> None: ... + @property + def ids(self) -> list[int]: ... + @property + def tokens(self) -> list[str]: ... + @property + def offsets(self) -> list[tuple[int, int]]: ... + @property + def type_ids(self) -> list[int]: ... + @property + def attention_mask(self) -> list[int]: ... + @property + def special_tokens_mask(self) -> list[int]: ... + @property + def word_ids(self) -> list[int | None]: ... + @property + def sequence_ids(self) -> list[int | None]: ... + @property + def overflowing(self) -> list[HuggingFaceEncoding]: ... + @property + def n_sequences(self) -> int: ... + def gil_stats() -> dict[str, int]: ... def process_state_started() -> bool: ... def reserve_process_for_forking() -> None: ... __all__ = [ "ForkedAfterNativeRuntimeStarted", + "HuggingFaceEncoding", "ProcessReservedForForking", "ResponsesWebSocketConnection", "RustBridgeDeclined", "RustUpstreamError", "TokenCounter", + "Tokenizer", "achat_completions", "amessages", "aocr", diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 74ceb1ba123..d7479631e04 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -22,6 +22,8 @@ class Route(str, Enum): RESPONSES = "responses" TRANSCRIPTION = "transcription" OCR = "ocr" + TOKEN_COUNTER = "token_counter" + TOKENIZER = "tokenizer" class Delivery(Enum): @@ -92,6 +94,8 @@ RULES: Final[Rules] = ( RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), RouteRule(Route.OCR, Rollout.RUST_OPT_OUT), RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN), + RouteRule(Route.TOKEN_COUNTER, Rollout.RUST_OPT_IN), + RouteRule(Route.TOKENIZER, Rollout.RUST_OPT_IN), RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.LOCAL})), CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.REDIS})), diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index cd468c80655..152ba632996 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -2,6 +2,7 @@ from __future__ import annotations import os from enum import Enum, auto +from functools import lru_cache from typing import Final from pydantic import TypeAdapter, ValidationError @@ -32,7 +33,9 @@ class _RustConfiguration: _CONFIGURATION: Final = _RustConfiguration() +@lru_cache(maxsize=16) def _parse_env_bool(value: str | None) -> bool | None: + """`LITELLM_RUST` as a bool; cached by raw value because `decision` runs per tokenizer call.""" if value is None: return None try: diff --git a/litellm/rust_bridge/token_counter.py b/litellm/rust_bridge/token_counter.py index d36234f56c1..250ad18d44c 100644 --- a/litellm/rust_bridge/token_counter.py +++ b/litellm/rust_bridge/token_counter.py @@ -5,18 +5,19 @@ from __future__ import annotations from collections.abc import Awaitable from dataclasses import dataclass from functools import lru_cache -from typing import Final, Literal, Protocol, cast # noqa: TID251 # native extension exposes untyped callables +from typing import TYPE_CHECKING, Final, Literal, Protocol, cast # noqa: TID251 # PyO3 binding validation from pydantic import TypeAdapter +from typing_extensions import assert_never import litellm -from litellm._logging import verbose_logger -from litellm.litellm_core_utils.default_encoding import cl100k_base_rank_file, o200k_base_rank_file -from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding, uses_legacy_message_accounting +from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding_name, uses_legacy_message_accounting +from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.configuration import rust_enabled -from litellm.rust_bridge.runtime import BridgeErrorContext, RustHandled, aattempt -from litellm.utils import claude_json_str, huggingface_tokenizer_kind +from litellm.utils import huggingface_tokenizer_kind + +if TYPE_CHECKING: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer RustTokenizer = Literal["anthropic", "cl100k_base", "o200k_base"] @@ -27,13 +28,7 @@ class RustTokenCounter(Protocol): class RustTokenCounterFactory(Protocol): - def __call__(self, tokenizer_json: str) -> RustTokenCounter: - raise NotImplementedError - - def from_cl100k_ranks(self, rank_file: str) -> RustTokenCounter: - raise NotImplementedError - - def from_o200k_ranks(self, rank_file: str) -> RustTokenCounter: + def from_tokenizer(self, tokenizer: NativeTokenizer, fast: bool = False) -> RustTokenCounter: raise NotImplementedError @@ -51,7 +46,7 @@ def _as_factory(value: object) -> RustTokenCounterFactory | None: cast( # cast-ok: native extension protocol is runtime-defined RustTokenCounterFactory, value ) - if callable(value) + if callable(getattr(value, "from_tokenizer", None)) else None ) @@ -73,7 +68,7 @@ def rust_tokenizer(model: str) -> RustTokenizer | None: return "anthropic" if kind is not None or uses_legacy_message_accounting(model): return None - match openai_tokenizer_encoding(model).name: + match openai_tokenizer_encoding_name(model): case "cl100k_base": return "cl100k_base" case "o200k_base": @@ -84,31 +79,26 @@ def rust_tokenizer(model: str) -> RustTokenizer | None: @lru_cache(maxsize=4) def _counter(factory: RustTokenCounterFactory, tokenizer: RustTokenizer) -> RustTokenCounter: + return factory.from_tokenizer(_native_tokenizer(tokenizer)) + + +def _native_tokenizer(tokenizer: RustTokenizer) -> NativeTokenizer: match tokenizer: case "anthropic": - return factory(claude_json_str) - case "cl100k_base": - return factory.from_cl100k_ranks(cl100k_base_rank_file()) - case "o200k_base": - return factory.from_o200k_ranks(o200k_base_rank_file()) + native = tokenizer_dispatch.native_anthropic() + case "cl100k_base" | "o200k_base": + native = tokenizer_dispatch.native_encoding(tokenizer) + case _: + assert_never(tokenizer) + if native is None: + raise RuntimeError(f"native {tokenizer} tokenizer is unavailable") + return native -async def count_input_tokens(body: bytes, tokenizer: RustTokenizer) -> InputTokenCount | None: - if not rust_enabled(): - return None - factory: Final = TOKEN_COUNTER.load() - if factory is None: - return None - try: - attempt: Final = await aattempt( - native_call=lambda: _counter(factory, tokenizer).acount_request(body), - adapt=_INPUT_TOKEN_COUNT.validate_python, - context=BridgeErrorContext(route="token_counter", provider=tokenizer, model=""), - ) - except (RuntimeError, ValueError) as error: - verbose_logger.debug("Rust token counter (%s) failed, counting in Python: %s", tokenizer, error) - return None - if not isinstance(attempt, RustHandled): - return None - verbose_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, attempt.value.input_tokens) - return attempt.value +async def native_count(factory: RustTokenCounterFactory, tokenizer: RustTokenizer, body: bytes) -> InputTokenCount: + """One native count, validated into the public shape. + + ``RustBridgeDeclined`` and upstream errors propagate so the caller's route + runner can map them onto its fallback policy; other failures (RuntimeError, + ValueError) propagate as-is.""" + return _INPUT_TOKEN_COUNT.validate_python(await _counter(factory, tokenizer).acount_request(body)) diff --git a/litellm/rust_bridge/tokenizer.py b/litellm/rust_bridge/tokenizer.py new file mode 100644 index 00000000000..5ed89813620 --- /dev/null +++ b/litellm/rust_bridge/tokenizer.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # native class is validated at the binding boundary + +import tiktoken +from tokenizers import Tokenizer as PythonHuggingFaceTokenizer + +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding +from litellm.rust_bridge import runtime +from litellm.rust_bridge.bindings import NativeBinding +from litellm.rust_bridge.catalog import Route, RouteContext + +if TYPE_CHECKING: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + +def _as_factory(value: object) -> type[NativeTokenizer] | None: + return ( + cast(type["NativeTokenizer"], value) # cast-ok: PyO3 class validated at the native boundary + if isinstance(value, type) + else None + ) + + +TOKENIZER: Final = NativeBinding("Tokenizer", validate=_as_factory) + +# The catalog contexts the tokenizer factories dispatch on. Callers that cache a tokenizer per +# backend key their cache on `decision(...)` of the same context, so key and dispatch agree. +TIKTOKEN_CONTEXT: Final = RouteContext(Route.TOKENIZER, provider="tiktoken") +HUGGINGFACE_CONTEXT: Final = RouteContext(Route.TOKENIZER, provider="huggingface") + + +@lru_cache(maxsize=8) +def _native_tiktoken(factory: type[NativeTokenizer], name: str) -> NativeTokenizer: + return factory.from_tiktoken(name) + + +@lru_cache(maxsize=1) +def _native_anthropic(factory: type[NativeTokenizer]) -> NativeTokenizer: + from litellm.utils import claude_json_str + + return factory.from_json(claude_json_str) + + +@lru_cache(maxsize=8) +def _native_encoding(factory: type[NativeTokenizer], name: str) -> OpenAIEncoding: + return OpenAIEncoding.wrap(_native_tiktoken(factory, name)) + + +def native_encoding(name: str) -> NativeTokenizer | None: + """The native tiktoken encoding behind `get_encoding(name)`, for a Rust route that counts + with the same loaded model; `None` without the extension.""" + factory: Final = TOKENIZER.load() + return None if factory is None else _native_tiktoken(factory, name) + + +def native_anthropic() -> NativeTokenizer | None: + """The native packaged Anthropic tokenizer behind `anthropic()`, parsed once per process.""" + factory: Final = TOKENIZER.load() + return None if factory is None else _native_anthropic(factory) + + +def _python_encoding(name: str) -> tiktoken.Encoding: + from litellm.litellm_core_utils.default_encoding import encoding + + return encoding if name == encoding.name else tiktoken.get_encoding(name) + + +def get_encoding(name: str) -> Encoding: + return runtime.run( + TIKTOKEN_CONTEXT, + binding=TOKENIZER, + native=lambda factory: _native_encoding(factory, name), + python=lambda: _python_encoding(name), + ) + + +def anthropic() -> HuggingFace: + """The packaged Anthropic tokenizer on the selected backend.""" + from litellm.utils import claude_json_str + + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer(_native_anthropic(factory)), + python=lambda: PythonHuggingFaceTokenizer.from_str(claude_json_str), + ) + + +def from_str(json: str) -> HuggingFace: + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer(factory.from_json(json)), + python=lambda: PythonHuggingFaceTokenizer.from_str(json), + ) + + +def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFace: + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer( + factory.from_pretrained(identifier, revision=revision, token=token) + ), + python=lambda: PythonHuggingFaceTokenizer.from_pretrained(identifier, revision=revision, token=token), + ) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index b38684f1856..042df6f37fa 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -216,11 +216,20 @@ class AnthropicMessagesToolUseParam(TypedDict, total=False): caller: ToolCaller | None +class CompactionBlock(TypedDict, total=False): + """Native compaction block, signed for on-demand compaction.""" + + type: Required[ReadOnly[Literal["compaction"]]] + content: ReadOnly[str | None] + signature: ReadOnly[str] + + AnthropicMessagesAssistantMessageValues = ( AnthropicMessagesTextParam | AnthropicMessagesToolUseParam | ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock + | CompactionBlock ) @@ -390,6 +399,11 @@ AllAnthropicPassThroughMessageValues: TypeAlias = ( ) +class AnthropicCompaction(TypedDict, total=False): + type: Required[ReadOnly[Literal["summarize"]]] + instructions: ReadOnly[str] + + class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): max_tokens: int | None metadata: AnthropicMetadata | dict | None @@ -405,6 +419,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): top_p: float | None mcp_servers: list[AnthropicMcpServerTool] | None context_management: dict[str, Any] | None + compaction: ReadOnly[AnthropicCompaction | None] container: dict[str, Any] | None # Container config with skills for code execution output_format: AnthropicOutputSchema | None # Structured outputs support speed: str | None # Fast mode support for Opus models @@ -566,13 +581,6 @@ class ContextManagementResponse(TypedDict, total=False): applied_edits: list[AppliedEdit] -class CompactionBlock(TypedDict, total=False): - """Synthesized ``compaction`` content block (compact_20260112).""" - - type: Required[Literal["compaction"]] - content: str | None - - class UsageIteration(TypedDict, total=False): """One sampling iteration's token usage (compact_20260112).""" @@ -746,6 +754,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): WEB_SEARCH_2025_03_05 = "web-search-2025-03-05" CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27" COMPACT_2026_01_12 = "compact-2026-01-12" + COMPACT_2026_09_04 = "compact-2026-09-04" STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13" ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20" FAST_MODE_2026_02_01 = "fast-mode-2026-02-01" diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 1d4c3cdc864..5cf988bd19e 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from typing import Any, Literal, TypeAlias from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -6,8 +7,10 @@ from litellm.types.llms.anthropic import ( AnthropicResponseContentBlockText, AnthropicResponseContentBlockToolUse, AnthropicStopDetails, + CompactionBlock, ContextManagementResponse, ServerToolUsage, + UsageIteration, ) @@ -56,6 +59,7 @@ AnthropicResponseContentBlock: TypeAlias = ( | AnthropicResponseToolUseBlock | AnthropicResponseThinkingBlock | AnthropicResponseRedactedThinkingBlock + | CompactionBlock ) @@ -66,6 +70,7 @@ class AnthropicUsage(TypedDict, total=False): input_tokens: int output_tokens: int + iterations: ReadOnly[Sequence[UsageIteration]] """ Cache Tokens Used @@ -91,7 +96,9 @@ class AnthropicMessagesResponse(TypedDict, total=False): id: str model: str | None # This represents the Model type from Anthropic role: Literal["assistant"] | None - stop_reason: Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"] | None + stop_reason: ReadOnly[ + Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] | None + ] stop_details: NotRequired[ReadOnly[AnthropicStopDetails | None]] stop_sequence: str | None type: Literal["message"] | None diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py index 793893451df..06982a16755 100644 --- a/litellm/types/llms/custom_http.py +++ b/litellm/types/llms/custom_http.py @@ -32,6 +32,7 @@ class httpxSpecialProvider(str, Enum): Sandbox = "sandbox" ModelCostMap = "model_cost_map" PasswordBreachCheck = "password_breach_check" + ASGI = "asgi" VerifyTypes = str | bool | ssl.SSLContext diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index bc44eb5b5b7..599b1a76249 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -110,6 +110,14 @@ FileTypes = ( EmbeddingInput = str | list[str] +class BinaryResponseSummary(TypedDict): + """What logging keeps of a binary response (speech audio, file content): size and media type, never the bytes.""" + + object: ReadOnly[Literal["binary"]] + content_type: ReadOnly[str | None] + num_bytes: ReadOnly[int] + + class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): _hidden_params: dict @@ -117,6 +125,19 @@ class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): super().__init__(response) self._hidden_params = {} # mutable-ok: mutable-dict contract shared with ModelResponse logging consumers + def logging_summary(self) -> BinaryResponseSummary: + return { + "object": "binary", + "content_type": self.response.headers.get("content-type"), + "num_bytes": self._num_bytes(), + } + + def _num_bytes(self) -> int: + try: + return len(self.response.content) + except httpx.ResponseNotRead: + return self.response.num_bytes_downloaded + def set_response_cost(self, response_cost: float | None) -> None: if response_cost is None: self._hidden_params.pop("response_cost", None) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2e8c869b89d..346181d9d9e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -104,6 +104,8 @@ def _nested_selector( if TYPE_CHECKING: + from litellm.litellm_core_utils.tokenizer import Tokenizer + from .vector_stores import VectorStoreSearchResponse else: VectorStoreSearchResponse = Any @@ -196,6 +198,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_output_config: bool | None supports_image_size: bool | None supports_anthropic_thinking_payload: ReadOnly[bool | None] + supports_anthropic_compaction: ReadOnly[bool | None] supported_audio_formats: ReadOnly[Sequence[Literal["mp3", "wav"]] | None] vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"] | None] bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None @@ -3023,6 +3026,7 @@ RoutingDecisionCause = Literal[ InternalCallOrigin = Literal[ "autorouter_classifier", + "autorouter_compaction", "shadow_eval_router", "shadow_eval_judge", "llm_as_a_judge_guardrail", @@ -3936,6 +3940,7 @@ all_litellm_params = ( agentic_loop_internal_litellm_params + [TRUSTED_CALLBACK_VARS_FIELD, ADDRESSED_RESPONSE_ID_FIELD, *bedrock_batch_litellm_params] + [ + "_context_compaction_state", "metadata", "litellm_metadata", "keepalive_seconds", @@ -4406,7 +4411,7 @@ class ProviderSpecificHeader(TypedDict): class SelectTokenizerResponse(TypedDict): type: Literal["openai_tokenizer", "huggingface_tokenizer"] - tokenizer: Any + tokenizer: ReadOnly["Tokenizer"] class LiteLLMFineTuningJob(FineTuningJob): diff --git a/litellm/utils.py b/litellm/utils.py index 812299560c7..df2a6adbe1f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -40,14 +40,11 @@ from types import MappingProxyType import dotenv import httpx import openai -import tiktoken from httpx import Proxy from httpx._utils import get_environment_proxies from openai.lib import _parsing, _pydantic from openai.types.chat.completion_create_params import ResponseFormat from pydantic import BaseModel -from tiktoken import Encoding -from tokenizers import Tokenizer import litellm import litellm.litellm_core_utils @@ -91,6 +88,10 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_fill_missing_generalizations, ) from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, strip_special_tokens +from litellm.rust_bridge import tokenizer as tokenizer_dispatch +from litellm.rust_bridge.catalog import decision +from litellm.rust_bridge.configuration import Decision _CachingHandlerResponse = None _LLMCachingHandler = None @@ -289,6 +290,8 @@ import importlib.metadata from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast, get_args +from typing_extensions import assert_never + from litellm import utils as litellm_utils # These are lazy loaded via __getattr__ @@ -2254,17 +2257,27 @@ def _select_tokenizer(model: str, custom_tokenizer: CustomHuggingfaceTokenizer | identifier=custom_tokenizer["identifier"], revision=custom_tokenizer["revision"], auth_token=custom_tokenizer["auth_token"], + backend=_huggingface_tokenizer_backend(), ) return _select_tokenizer_helper(model=model) +def _huggingface_tokenizer_backend() -> Decision: + """The backend `tokenizer_dispatch.from_str` / `from_pretrained` will select right now. + + Cached HuggingFace tokenizers are keyed on it, so flipping `LITELLM_RUST` or + `litellm.rust(...)` reaches a fresh object instead of the other backend's.""" + return decision(tokenizer_dispatch.HUGGINGFACE_CONTEXT) + + @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) -def _select_custom_tokenizer_helper(identifier: str, revision: str, auth_token: str | None) -> SelectTokenizerResponse: +def _select_custom_tokenizer_helper( + identifier: str, revision: str, auth_token: str | None, backend: Decision +) -> SelectTokenizerResponse: verbose_logger.debug("Loading custom HuggingFace tokenizer %s (revision %s)", identifier, revision) return create_pretrained_tokenizer(identifier=identifier, revision=revision, auth_token=auth_token) -@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: if litellm.disable_hf_tokenizer_download is True: return _return_openai_tokenizer(model) @@ -2274,6 +2287,10 @@ def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: if result is not None: return result except Exception as e: + from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking + + if isinstance(e, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): + raise verbose_logger.debug("Error selecting tokenizer: %s", e) # default - tiktoken @@ -2308,19 +2325,26 @@ def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | None: kind: Final = huggingface_tokenizer_kind(model) if kind is None: return None - return {"type": "huggingface_tokenizer", "tokenizer": _load_huggingface_tokenizer(kind)} + return { + "type": "huggingface_tokenizer", + "tokenizer": _load_huggingface_tokenizer(kind, _huggingface_tokenizer_backend()), + } -def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind) -> Tokenizer: +@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) +def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decision) -> HuggingFace: + """One tokenizer per kind and backend; `backend` is the cache key, the dispatch re-derives it.""" match kind: case "cohere": - return Tokenizer.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") + return tokenizer_dispatch.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") case "anthropic": - return Tokenizer.from_str(claude_json_str) + return tokenizer_dispatch.anthropic() case "llama2": - return Tokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") + return tokenizer_dispatch.from_pretrained("hf-internal-testing/llama-tokenizer") case "llama3": - return Tokenizer.from_pretrained("Xenova/llama-3-tokenizer") + return tokenizer_dispatch.from_pretrained("Xenova/llama-3-tokenizer") + case _: + assert_never(kind) def encode(model="", text="", custom_tokenizer: dict | None = None): @@ -2336,15 +2360,13 @@ def encode(model="", text="", custom_tokenizer: dict | None = None): enc: The encoded text. """ tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model=model) - if isinstance(tokenizer_json["tokenizer"], Encoding): - enc = tokenizer_json["tokenizer"].encode(text, disallowed_special=()) - else: - enc = tokenizer_json["tokenizer"].encode(text) - # Normalize: HuggingFace Tokenizer.encode() returns an Encoding object; - # extract .ids so the return type is always List[int]. - if hasattr(enc, "ids"): - return enc.ids - return enc + if tokenizer_json["type"] == "openai_tokenizer": + openai_tokenizer: Final = cast( # cast-ok: [LIT006] caller's explicit type tag selects this interface + Encoding, tokenizer_json["tokenizer"] + ) + return openai_tokenizer.encode(text, disallowed_special=()) + encoded: Final = tokenizer_json["tokenizer"].encode(text) + return encoded.ids if hasattr(encoded, "ids") else encoded def decode( @@ -2363,26 +2385,12 @@ def decode( """ tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model=model) if tokenizer_json["type"] == "huggingface_tokenizer": - if skip_special_tokens: - tokens = _strip_huggingface_special_token_ids(tokenizer_json["tokenizer"], tokens) - dec = tokenizer_json["tokenizer"].decode(tokens, skip_special_tokens=skip_special_tokens) - return dec - dec = tokenizer_json["tokenizer"].decode(tokens) - return dec - - -def _strip_huggingface_special_token_ids(tokenizer: Tokenizer, tokens: Sequence[int]) -> Sequence[int]: - try: - added_tokens_decoder: Final = tokenizer.get_added_tokens_decoder() - except Exception: - return tokens - - special_token_ids: Final = { - token_id for token_id, added_token in added_tokens_decoder.items() if getattr(added_token, "special", False) - } - if not special_token_ids: - return tokens - return [token for token in tokens if token not in special_token_ids] + ids: Final = strip_special_tokens(tokenizer_json["tokenizer"], tokens) if skip_special_tokens else tokens + hf_tokenizer: Final = cast( # cast-ok: [LIT006] caller's explicit type tag selects this interface + HuggingFace, tokenizer_json["tokenizer"] + ) + return hf_tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) + return tokenizer_json["tokenizer"].decode(tokens) def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: str | None = None): @@ -2398,7 +2406,7 @@ def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: st dict: A dictionary with the tokenizer and its type. """ - tokenizer: Final = Tokenizer.from_pretrained(identifier, revision=revision, token=auth_token) + tokenizer: Final = tokenizer_dispatch.from_pretrained(identifier, revision=revision, token=auth_token) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} @@ -2413,7 +2421,7 @@ def create_tokenizer(json: str): dict: A dictionary with the tokenizer and its type. """ - tokenizer: Final = Tokenizer.from_str(json) + tokenizer: Final = tokenizer_dispatch.from_str(json) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} @@ -6150,6 +6158,7 @@ def _get_model_info_helper( supports_tool_search=_model_info.get("supports_tool_search", None), supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None), + supports_anthropic_compaction=_model_info.get("supports_anthropic_compaction", None), supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None), supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None), supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None), diff --git a/migrations/Dockerfile b/migrations/Dockerfile index c6d1b0cc46e..f34940c0ce0 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -67,6 +67,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra extra_proxy \ --python python3.13 +RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ + COPY migrations/run.py /app/run.py # Pre-warm the Prisma binary cache so the Job pod doesn't reach the diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 08cb2ddb8f7..b5106670a2b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14516,6 +14516,7 @@ "source": "https://docs.anthropic.com/en/docs/about-claude/pricing" }, "claude-sonnet-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -14555,6 +14556,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-sonnet-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -14770,6 +14772,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14808,6 +14811,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6-20260205": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14845,6 +14849,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14884,6 +14889,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-7-20260416": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14922,6 +14928,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -14961,6 +14968,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-fable-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -15001,6 +15009,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15043,6 +15052,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-8": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -62148,6 +62158,7 @@ "supports_audio_output": true }, "claude-mythos-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -62187,6 +62198,7 @@ } }, "claude-mythos-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -62227,6 +62239,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-mythos-preview": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -77528,5 +77541,45 @@ "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true + }, + "openrouter/nex-agi/nex-n2.5-mini": { + "cache_read_input_token_cost": 2.5e-09, + "input_cost_per_token": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false + }, + "openrouter/nex-agi/nex-n2.5-pro": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": false } } diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index f3a4e614f59..16b762f7803 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -839,6 +839,9 @@ "supports_adaptive_thinking": { "type": "boolean" }, + "supports_anthropic_compaction": { + "type": "boolean" + }, "supports_anthropic_thinking_payload": { "type": "boolean" }, diff --git a/pyproject.toml b/pyproject.toml index a1b276e4e8c..f447343ff33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,11 +19,13 @@ dependencies = [ "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<3.0.0", "python-dotenv>=1.0.0,<2.0", + "pyyaml>=6.0.3,<7.0", + "packaging>=24.0", + "importlib-metadata>=8.0.0,<9.0", "tiktoken>=0.8.0,<1.0; python_version < '3.14'", "tiktoken>=0.12.0,<1.0; python_version >= '3.14'", - "importlib-metadata>=8.0.0,<9.0", - "packaging>=24.0", "tokenizers>=0.21.0,<1.0", + "huggingface-hub>=0.34.0,<2.0", "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", @@ -191,6 +193,7 @@ litellm-proxy = "litellm.proxy.client.cli:litellm_proxy_cli" [dependency-groups] dev = [ + "numpy>=1.26.0,<3.0", "diff-cover==9.7.2", "hypothesis==6.165.10", "reportlab==5.0.1", @@ -290,6 +293,12 @@ healthcheck = [ "httpx==0.28.1", "pyyaml==6.0.3", ] +benchmarks = [ + "pytest==9.0.3", + "pytest-codspeed==4.3.0", + "mcp>=2.2.0,<3", + "a2a-sdk==1.1.0", +] [build-system] requires = ["maturin==1.15.0"] diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index c9b31cfb7d7..309ecab9991 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -8,8 +8,11 @@ flipping results between runs. Running the executor inline keeps each benchmark's cost self-contained and deterministic. """ +import os +import sys from collections.abc import Callable, Iterator from concurrent.futures import Future +from pathlib import Path from typing import ParamSpec, TypeVar import pytest @@ -20,6 +23,21 @@ P = ParamSpec("P") R = TypeVar("R") +def pytest_configure(config: pytest.Config) -> None: + if os.environ.get("LITELLM_REQUIRE_INSTALLED_WHEEL") != "1": + return + + import litellm + import litellm.rust_bridge._native as native + + prefix = Path(sys.prefix).resolve() + for name, module_file in (("litellm", litellm.__file__), ("litellm.rust_bridge._native", native.__file__)): + path = Path(module_file).resolve() + if not path.is_relative_to(prefix): + raise pytest.UsageError(f"{name} resolved outside the benchmark environment: {path}") + print(f"{name}: {path}") # noqa: T201 # provenance evidence must be visible in CI logs + + def _submit_inline(fn: Callable[P, R], /, *args: P.args, **kwargs: P.kwargs) -> Future[R]: future: Future[R] = Future() try: diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index 78e6562a4a8..758d579f67d 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -106,10 +106,11 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat env_path: Final = tmp_path / ".env" result: Final = subprocess.run( - [sys.executable, str(SECRETS_TO_ENV), str(env_path)], + [sys.executable, "-I", str(SECRETS_TO_ENV), str(env_path)], input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}', capture_output=True, text=True, + env={**os.environ, "GITHUB_ACTIONS": "true"}, ) assert result.returncode == 0, result.stderr @@ -117,6 +118,24 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" +def test_outside_actions_no_value_is_printed(tmp_path: Path) -> None: + env_path: Final = tmp_path / ".env" + local_env: Final = {key: value for key, value in os.environ.items() if key != "GITHUB_ACTIONS"} + + result: Final = subprocess.run( + [sys.executable, "-I", str(SECRETS_TO_ENV), str(env_path)], + input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}', + capture_output=True, + text=True, + env=local_env, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout == "" + assert "sk-0123456789abcdef" not in result.stderr + assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" + + def redact_output(tmp_path: Path, values: tuple[str, ...], text: str) -> tuple[subprocess.CompletedProcess[str], Path]: env_path: Final = tmp_path / ".env" _ = env_path.write_text("".join(f"{name}='{value}'\n" for name, value in zip(("A", "B", "C"), values))) @@ -212,6 +231,11 @@ def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]: (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()), (("tests/e2e/guardrails/test_presidio_masking_e2e.py",), ()), (("tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py",), ()), + (("tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py",), ()), + ( + ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), + ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), + ), ( ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 2b59e8770b5..15c2d6763d6 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -105,7 +105,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 78cee641eba..214fe594f00 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -31,6 +31,7 @@ from e2e_config import ( FIXTURE_MODE_RAW, MANAGED_FILES_OPT_IN_ENV, MCP_OAUTH_LIVE_OPT_IN_ENV, + OTEL_V2_OPT_IN_ENV, PROMPT_CACHING_OPT_IN_ENV, PROVIDER_EDGE_HOST_OPT_IN_ENV, PROXY_BASE_URL, @@ -63,6 +64,7 @@ OPT_IN_MARKERS: Final = MappingProxyType( "mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV, "provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV, "callback_cost_capture": CALLBACK_COST_CAPTURE_OPT_IN_ENV, + "otel_v2": OTEL_V2_OPT_IN_ENV, } ) @@ -158,6 +160,10 @@ def pytest_configure(config: pytest.Config) -> None: "logging.callback_cost_capture.capture CustomLogger, so the gateway and pytest must share a filesystem; " "deselected unless E2E_CALLBACK_COST_CAPTURE_FILE is set", ) + config.addinivalue_line( + "markers", + "otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml index 86eb44f6cb1..4fe9f949fae 100644 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ b/tests/e2e/coverage_registry/guardrail.yaml @@ -2,6 +2,7 @@ # Rolls up into the "Logging & Guardrails" dashboard module together with logging.* - {id: guardrail.presidio.pre_call.masks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "PII masking pre-call; data-leak blast radius"} - {id: guardrail.presidio.post_call.masks, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Mask PII in model output"} +- {id: guardrail.presidio.post_call.masks_generated_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, anthropic_messages_stream], source: "guardrail_hooks/presidio.py", rationale: "Mask model-generated credit-card output with the UI default scope"} - {id: guardrail.presidio.logging_only.masks, module: guardrail, tier: P0, hook_point: logging_only, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Redact in logs without blocking"} - {id: guardrail.presidio.pre_call.logs_masked_entities, module: guardrail, tier: P0, hook_point: pre_call, assertions: [logs_masked_entities], exercised_on: [chat_completions], source: "guardrail_hooks/presidio.py", rationale: "A masking run must record itself on the spend log: the dashboard's guardrail panel renders the masked-entity counts and per-entity scores straight off metadata.guardrail_information, so a run that masks but records nothing leaves an operator unable to audit it"} - {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"} diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 49d4d92ff0b..e4e1ac2c7b6 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -94,3 +94,15 @@ - {id: llm.messages.together_ai.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together over /v1/messages streaming"} - {id: llm.messages.together_ai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over /v1/messages"} - {id: llm.messages.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip over /v1/messages"} +- {id: llm.chat_completions.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /chat/completions: cost header and spend row agree"} +- {id: llm.chat_completions.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /chat/completions"} +- {id: llm.messages.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /v1/messages"} +- {id: llm.messages.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI models served on the Anthropic Messages contract"} +- {id: llm.messages.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI over /v1/messages streams the Anthropic event grammar"} +- {id: llm.messages.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI over /v1/messages: cost header and spend row agree"} +- {id: llm.messages.openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI tool calls translated to Anthropic tool_use blocks"} +- {id: llm.messages.openai.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI tool result round trip over /v1/messages"} +- {id: llm.responses.openai.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI function_call_output round trip over /v1/responses"} +- {id: llm.responses.anthropic.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses streams Responses events"} +- {id: llm.responses.anthropic.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses: cost header and spend row agree"} +- {id: llm.responses.anthropic.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic function_call_output round trip over /v1/responses"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 518aee83fa6..5055d36c0dc 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -148,6 +148,7 @@ CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE" PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE" CALLBACK_COST_CAPTURE_OPT_IN_ENV: Final = "E2E_CALLBACK_COST_CAPTURE_FILE" +OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/gateway/record_replay_ci_config.yml b/tests/e2e/gateway/record_replay_ci_config.yml index 08972969cf0..5b3db0ff530 100644 --- a/tests/e2e/gateway/record_replay_ci_config.yml +++ b/tests/e2e/gateway/record_replay_ci_config.yml @@ -1,3 +1,4 @@ general_settings: master_key: os.environ/LITELLM_MASTER_KEY store_model_in_db: true + disable_model_info_refresh: true diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 3ceac737399..97ecac0290f 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -46,7 +46,7 @@ class BlockedWordBody(BaseModel): class GuardrailParamsBase(BaseModel): - mode: GuardrailMode + mode: GuardrailMode | list[GuardrailMode] default_on: bool @@ -381,6 +381,26 @@ class GuardrailsClient: ), ) + def messages_stream_raw( + self, + key: str, + model: str, + text: str, + *, + guardrails: list[str] | None = None, + max_tokens: int = 64, + ) -> StreamingResponse: + return self.proxy.messages_stream( + key, + AnthropicMessagesBody( + model=model, + messages=[ChatMessage(role="user", content=text)], + max_tokens=max_tokens, + stream=True, + guardrails=guardrails, + ), + ) + def responses( self, key: str, diff --git a/tests/e2e/guardrails/test_presidio_masking_e2e.py b/tests/e2e/guardrails/test_presidio_masking_e2e.py index 49d698938ce..d47d64be9e3 100644 --- a/tests/e2e/guardrails/test_presidio_masking_e2e.py +++ b/tests/e2e/guardrails/test_presidio_masking_e2e.py @@ -30,6 +30,7 @@ this suite deliberately requires the detected-entity details to remain visible. from __future__ import annotations import os +import re import time from collections.abc import Callable from typing import Final, Literal @@ -65,9 +66,13 @@ GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS = 5.0 # angle brackets; the logged payload keeps the placeholder verbatim. MASKED_EMAIL_TOKEN = "EMAIL_ADDRESS" MASKED_PHONE_TOKEN = "PHONE_NUMBER" +MASKED_CREDIT_CARD_TOKEN = "CREDIT_CARD" # Fictional NANP 555 number; a standard format Presidio's phone recognizer detects. FAKE_PHONE = "+1 415-555-0134" +FAKE_VISA_TEST_CARD = "4111 1111 1111 1111" + +_CARD_DIGIT_RUN: Final = re.compile(r"(?:\d[ -]?){13,19}") def _presidio_bases() -> tuple[str, str]: @@ -86,8 +91,8 @@ def _register_presidio( resources: ResourceManager, *, name: str, - mode: GuardrailMode = "pre_call", - filter_scope: Literal["input", "output", "both"] = "input", + mode: GuardrailMode | list[GuardrailMode] = "pre_call", + filter_scope: Literal["input", "output", "both"] | None = "input", entities: dict[PiiEntity, PiiAction] | None = None, ) -> None: analyzer, anonymizer = _presidio_bases() @@ -123,6 +128,74 @@ def _first_content(response: ChatResponse) -> str: return (message.content if message else None) or "" +class _StreamDelta(BaseModel): + content: str | None = None + + +class _StreamChoice(BaseModel): + delta: _StreamDelta + + +class _StreamChunk(BaseModel): + choices: tuple[_StreamChoice, ...] = () + + +class _AnthropicStreamDelta(BaseModel): + type: str | None = None + text: str | None = None + + +class _AnthropicStreamEvent(BaseModel): + type: str + delta: _AnthropicStreamDelta | None = None + + +def _credit_card_prompt(marker: str) -> str: + return ( + f"{marker} Reply with only the well known Visa sandbox test card number that starts with 4111, " + "the 16 digits grouped in fours separated by spaces, and nothing else." + ) + + +def _passes_luhn(digits: str) -> bool: + checksum = sum( + digit if position % 2 == 0 else (digit * 2 - 9 if digit * 2 > 9 else digit * 2) + for position, digit in enumerate(int(char) for char in reversed(digits)) + ) + return checksum % 10 == 0 + + +def _contains_card_number(text: str) -> bool: + """Presidio's CREDIT_CARD recognizer only reports Luhn-valid digit runs, so a + Luhn-invalid number the model hallucinates is not something masking can catch.""" + return any( + 13 <= len(digits) <= 19 and _passes_luhn(digits) + for digits in (re.sub(r"[ -]", "", match.group()) for match in _CARD_DIGIT_RUN.finditer(text)) + ) + + +def _stream_content(result: StreamingResponse) -> str: + return "".join( + choice.delta.content + for event in result.stream_events + if event != "[DONE]" + for choice in _StreamChunk.model_validate_json(event).choices[:1] + if choice.delta.content + ) + + +def _anthropic_stream_content(result: StreamingResponse) -> str: + return "".join( + event.delta.text + for payload in result.stream_events + for event in [_AnthropicStreamEvent.model_validate_json(payload)] + if event.type == "content_block_delta" + and event.delta is not None + and event.delta.type == "text_delta" + and event.delta.text + ) + + def _messages_text(response: AnthropicMessagesResponse) -> str: """The text of a /v1/messages answer, whichever shape the proxy produced (Anthropic-native content blocks or OpenAI-normalized choices).""" @@ -290,6 +363,124 @@ class TestPresidioPostCallMasking: time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) +def _assert_eventually_masks_generated_card(fetch: Callable[[], str | None]) -> None: + deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS + last: str = "" + while True: + content = fetch() + if content is not None: + last = content + if _contains_card_number(content): + pytest.fail( + "the post_call output masking let a card number through: " + f"{content[:300]!r}" + ) + if MASKED_CREDIT_CARD_TOKEN in content: + return + if time.monotonic() >= deadline: + pytest.fail( + "presidio post_call output masking never masked the generated card within " + f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}" + ) + time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) + + +class TestPresidioCreditCardOutputMasking: + """Proves the UI-default Presidio scope masks model-generated card output.""" + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["chat_completions"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-chat-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=512) + match result: + case Success(data=data): + return _first_content(data) + case _: + return None + + _assert_eventually_masks_generated_card(fetch) + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["chat_completions_stream"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-stream-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.chat_stream_raw( + scoped_key, + MODEL, + prompt, + guardrails=[name], + max_tokens=512, + ) + if not result.ok or result.stream_error: + return None + return _stream_content(result) + + _assert_eventually_masks_generated_card(fetch) + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["anthropic_messages_stream"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_anthropic_messages( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-messages-stream-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.messages_stream_raw( + scoped_key, + MODEL, + prompt, + guardrails=[name], + max_tokens=512, + ) + if not result.ok or result.stream_error: + return None + return _anthropic_stream_content(result) + + _assert_eventually_masks_generated_card(fetch) + + _LOGGED_ENTITIES: dict[PiiEntity, PiiAction] = {"EMAIL_ADDRESS": "MASK", "PHONE_NUMBER": "MASK"} _ENTITY_LIST_ADAPTER: Final = TypeAdapter(list[GuardrailEntityMatch]) diff --git a/tests/e2e/llm_translation/conversational_matrix.py b/tests/e2e/llm_translation/conversational_matrix.py new file mode 100644 index 00000000000..0d6f6ed3d4e --- /dev/null +++ b/tests/e2e/llm_translation/conversational_matrix.py @@ -0,0 +1,544 @@ +"""The endpoint x deployment x auth matrix behind test_conversational_matrix_e2e.py. + +One conversation, three wire formats. Each `Surface` speaks its own API through +the customer SDK (chat completions and Responses through the OpenAI SDK, Messages +through the Anthropic SDK) and folds what came back into the surface-neutral +`Reply` / `StreamedReply`, so a single behavior test asserts the same contract on +every cell. A new model, from an existing or a new provider, is one `Deployment` row +in DEPLOYMENTS; a new way of handing the proxy a provider credential is one +`AuthMethod`. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal, Protocol + +import anthropic +import openai +import pytest +from _pytest.mark.structures import ParameterSet +from anthropic.types import ( + MessageParam, + RawMessageStreamEvent, + TextBlock, + ToolChoiceToolParam, + ToolParam, + ToolResultBlockParam, + ToolUseBlock, + ToolUseBlockParam, +) +from e2e_config import provider_edge_base, unique_marker +from lifecycle import ResourceManager +from llm_translation.sdk_clients import NO_PROXY_CACHE, SdkClients, response_header +from models import CredentialCreateBody, LiteLLMParamsBody +from openai.types.chat import ( + ChatCompletionAssistantMessageParam, + ChatCompletionChunk, + ChatCompletionMessageFunctionToolCallParam, + ChatCompletionMessageParam, + ChatCompletionNamedToolChoiceParam, + ChatCompletionToolMessageParam, + ChatCompletionToolParam, +) +from openai.types.chat.chat_completion_message_function_tool_call import ChatCompletionMessageFunctionToolCall +from openai.types.responses import ( + FunctionToolParam, + ResponseFunctionToolCall, + ResponseFunctionToolCallParam, + ResponseInputParam, + ResponseStreamEvent, + ToolChoiceFunctionParam, +) +from openai.types.responses.response_input_param import FunctionCallOutput +from proxy_client import ProxyClient +from pydantic import BaseModel + +SurfaceName = Literal["chat_completions", "messages", "responses"] +AuthMethod = Literal["env_ref", "stored_credential"] +Capability = Literal["basic", "tool_use", "multi_turn"] +Streaming = Literal["stream", "nonstream"] +Assertion = Literal["works", "cost_logged"] +ToolMode = Literal["none", "forced", "offered"] + +SURFACES: Final[tuple[SurfaceName, ...]] = ("chat_completions", "messages", "responses") +AUTH_METHODS: Final[tuple[AuthMethod, ...]] = ("env_ref", "stored_credential") + +MAX_OUTPUT_TOKENS: Final = 512 +INSTRUCTIONS: Final = "You are a terse assistant. Answer in one short sentence." +GREETING_PROMPT: Final = "Say hello." +WEATHER_PROMPT: Final = "What is the weather in Paris right now? Use the get_weather tool." +WEATHER_REPORT: Final = "Paris: 22 degrees Celsius, clear skies" +WEATHER_TOOL_NAME: Final = "get_weather" +WEATHER_TOOL_DESCRIPTION: Final = "Current weather for a city" +WEATHER_TOOL_SCHEMA: Final[Mapping[str, object]] = MappingProxyType( + { + "type": "object", + "properties": {"location": {"type": "string", "description": "City name"}}, + "required": ["location"], + } +) + + +@dataclass(frozen=True, slots=True) +class Deployment: + """One deployment target: the litellm backend string plus how to wire it.""" + + route: Literal["openai", "anthropic"] + label: str + backend: str + api_key_env: str + edge_mount: str + edge_suffix: str + + def api_base(self) -> str | None: + base: Final = provider_edge_base(self.edge_mount) + return None if base is None else f"{base}{self.edge_suffix}" + + def api_key(self) -> str: + key: Final = os.environ.get(self.api_key_env, "") + assert key, f"{self.api_key_env} is not set in the test process environment" + return key + + +DEPLOYMENTS: Final[tuple[Deployment, ...]] = ( + Deployment( + route="openai", + label="gpt-4o-mini", + backend="openai/gpt-4o-mini", + api_key_env="OPENAI_API_KEY", + edge_mount="openai", + edge_suffix="/v1", + ), + Deployment( + route="openai", + label="gpt-5.4-mini", + backend="openai/gpt-5.4-mini", + api_key_env="OPENAI_API_KEY", + edge_mount="openai", + edge_suffix="/v1", + ), + Deployment( + route="anthropic", + label="claude-haiku-4-5", + backend="anthropic/claude-haiku-4-5", + api_key_env="ANTHROPIC_API_KEY", + edge_mount="anthropic", + edge_suffix="", + ), +) + + +@dataclass(frozen=True, slots=True) +class Cell: + surface: SurfaceName + deployment: Deployment + auth: AuthMethod + + @property + def id(self) -> str: + return f"{self.surface}-{self.deployment.label}-{self.auth}" + + def registry_id(self, capability: Capability, streaming: Streaming, assertion: Assertion) -> str: + return f"llm.{self.surface}.{self.deployment.route}.{capability}.{streaming}.{assertion}" + + +CELLS: Final[tuple[Cell, ...]] = tuple( + Cell(surface=surface, deployment=deployment, auth=auth) + for surface in SURFACES + for deployment in DEPLOYMENTS + for auth in AUTH_METHODS +) + + +def cells_covering(capability: Capability, streaming: Streaming, assertion: Assertion) -> tuple[ParameterSet, ...]: + """Every cell as a pytest param carrying the registry id its test proves.""" + return tuple( + pytest.param(cell, id=cell.id, marks=pytest.mark.covers(cell.registry_id(capability, streaming, assertion))) + for cell in CELLS + ) + + +DeploymentKey = tuple[str, AuthMethod] + + +@dataclass(frozen=True, slots=True) +class Deployments: + """Model aliases registered on the proxy, one per (deployment, auth).""" + + aliases: Mapping[DeploymentKey, str] + + def alias(self, cell: Cell) -> str: + return self.aliases[(cell.deployment.label, cell.auth)] + + +def _litellm_params(deployment: Deployment, auth: AuthMethod, credential_name: str) -> LiteLLMParamsBody: + match auth: + case "env_ref": + return LiteLLMParamsBody( + model=deployment.backend, api_key=f"os.environ/{deployment.api_key_env}", api_base=deployment.api_base() + ) + case "stored_credential": + return LiteLLMParamsBody( + model=deployment.backend, litellm_credential_name=credential_name, api_base=deployment.api_base() + ) + + +def _register(proxy: ProxyClient, resources: ResourceManager, deployment: Deployment, auth: AuthMethod) -> str: + marker: Final = unique_marker() + credential_name: Final = f"e2e-matrix-{deployment.label}-{marker}" + if auth == "stored_credential": + proxy.create_credential( + CredentialCreateBody(credential_name=credential_name, credential_values={"api_key": deployment.api_key()}) + ) + resources.defer(lambda: proxy.delete_credential(credential_name)) + alias: Final = f"e2e-matrix-{deployment.label}-{auth}-{marker}" + model_id: Final = proxy.create_model(alias, _litellm_params(deployment, auth, credential_name)) + resources.defer(lambda: proxy.delete_model(model_id)) + return alias + + +def register_deployments(proxy: ProxyClient) -> Iterator[Deployments]: + resources: Final = ResourceManager(client=proxy) + try: + yield Deployments( + aliases=MappingProxyType( + { + (deployment.label, auth): _register(proxy, resources, deployment, auth) + for deployment in DEPLOYMENTS + for auth in AUTH_METHODS + } + ) + ) + finally: + resources.teardown() + + +class WeatherArgs(BaseModel): + location: str + + +@dataclass(frozen=True, slots=True) +class ToolCall: + call_id: str + name: str + arguments: str + + def parsed(self) -> WeatherArgs: + return WeatherArgs.model_validate_json(self.arguments) + + +@dataclass(frozen=True, slots=True) +class Usage: + input_tokens: int + output_tokens: int + + +@dataclass(frozen=True, slots=True) +class Reply: + """What every surface owes the caller for one non-streamed turn.""" + + response_id: str + text: str + tool_calls: tuple[ToolCall, ...] + usage: Usage | None + call_id_header: str | None + cost_header: str | None + + +@dataclass(frozen=True, slots=True) +class StreamedReply: + """The reassembled stream: its text, whether the surface's own terminal event + arrived, and whether usage was reported anywhere in the stream.""" + + text: str + finished: bool + usage_reported: bool + event_count: int + + +class Surface(Protocol): + @property + def name(self) -> SurfaceName: ... + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: ... + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: ... + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: ... + + +def _chat_tool() -> ChatCompletionToolParam: + return { + "type": "function", + "function": { + "name": WEATHER_TOOL_NAME, + "description": WEATHER_TOOL_DESCRIPTION, + "parameters": dict(WEATHER_TOOL_SCHEMA), + }, + } + + +def _messages_tool() -> ToolParam: + return { + "name": WEATHER_TOOL_NAME, + "description": WEATHER_TOOL_DESCRIPTION, + "input_schema": dict(WEATHER_TOOL_SCHEMA), + } + + +def _responses_tool() -> FunctionToolParam: + return { + "type": "function", + "name": WEATHER_TOOL_NAME, + "description": WEATHER_TOOL_DESCRIPTION, + "parameters": dict(WEATHER_TOOL_SCHEMA), + "strict": False, + } + + +def _chat_tool_choice() -> ChatCompletionNamedToolChoiceParam: + return {"type": "function", "function": {"name": WEATHER_TOOL_NAME}} + + +def _messages_tool_choice() -> ToolChoiceToolParam: + return {"type": "tool", "name": WEATHER_TOOL_NAME, "disable_parallel_tool_use": True} + + +def _responses_tool_choice() -> ToolChoiceFunctionParam: + return {"type": "function", "name": WEATHER_TOOL_NAME} + + +def _usage(input_tokens: int | None, output_tokens: int | None) -> Usage | None: + if input_tokens is None or output_tokens is None: + return None + return Usage(input_tokens=input_tokens, output_tokens=output_tokens) + + +@dataclass(frozen=True, slots=True) +class ChatCompletionsSurface: + sdk: SdkClients + name: SurfaceName = "chat_completions" + + def _turn(self, key: str, model: str, messages: Sequence[ChatCompletionMessageParam], tool: ToolMode) -> Reply: + raw: Final = self.sdk.openai(key).chat.completions.with_raw_response.create( + model=model, + messages=list(messages), + max_completion_tokens=MAX_OUTPUT_TOKENS, + tools=openai.omit if tool == "none" else [_chat_tool()], + tool_choice=_chat_tool_choice() if tool == "forced" else openai.omit, + parallel_tool_calls=False if tool == "forced" else openai.omit, + extra_body=NO_PROXY_CACHE, + ) + completion: Final = raw.parse() + message: Final = completion.choices[0].message + calls: Final = tuple( + ToolCall(call_id=call.id, name=call.function.name, arguments=call.function.arguments) + for call in message.tool_calls or () + if isinstance(call, ChatCompletionMessageFunctionToolCall) + ) + return Reply( + response_id=completion.id, + text=message.content or "", + tool_calls=calls, + usage=None + if completion.usage is None + else _usage(completion.usage.prompt_tokens, completion.usage.completion_tokens), + call_id_header=response_header(raw.headers, "x-litellm-call-id"), + cost_header=response_header(raw.headers, "x-litellm-response-cost"), + ) + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: + return self._turn(key, model, _chat_history(prompt), "forced" if with_tool else "none") + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: + chunks: Final[tuple[ChatCompletionChunk, ...]] = tuple( + self.sdk.openai(key).chat.completions.create( + model=model, + messages=_chat_history(prompt), + max_completion_tokens=MAX_OUTPUT_TOKENS, + stream=True, + stream_options={"include_usage": True}, + extra_body=NO_PROXY_CACHE, + ) + ) + return StreamedReply( + text="".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices), + finished=any(chunk.choices[0].finish_reason is not None for chunk in chunks if chunk.choices), + usage_reported=any(chunk.usage is not None for chunk in chunks), + event_count=len(chunks), + ) + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: + tool_call: Final[ChatCompletionMessageFunctionToolCallParam] = { + "id": call.call_id, + "type": "function", + "function": {"name": call.name, "arguments": call.arguments}, + } + assistant: Final[ChatCompletionAssistantMessageParam] = {"role": "assistant", "tool_calls": [tool_call]} + tool_result: Final[ChatCompletionToolMessageParam] = { + "role": "tool", + "tool_call_id": call.call_id, + "content": result, + } + return self._turn(key, model, (*_chat_history(prompt), assistant, tool_result), "offered") + + +def _chat_history(prompt: str) -> tuple[ChatCompletionMessageParam, ...]: + return ({"role": "system", "content": INSTRUCTIONS}, {"role": "user", "content": prompt}) + + +@dataclass(frozen=True, slots=True) +class MessagesSurface: + sdk: SdkClients + name: SurfaceName = "messages" + + def _turn(self, key: str, model: str, messages: Sequence[MessageParam], tool: ToolMode) -> Reply: + raw: Final = self.sdk.anthropic(key).messages.with_raw_response.create( + model=model, + max_tokens=MAX_OUTPUT_TOKENS, + system=INSTRUCTIONS, + messages=list(messages), + tools=anthropic.omit if tool == "none" else [_messages_tool()], + tool_choice=_messages_tool_choice() if tool == "forced" else anthropic.omit, + extra_body=NO_PROXY_CACHE, + ) + message: Final = raw.parse() + return Reply( + response_id=message.id, + text="".join(block.text for block in message.content if isinstance(block, TextBlock)), + tool_calls=tuple( + ToolCall(call_id=block.id, name=block.name, arguments=json.dumps(block.input)) + for block in message.content + if isinstance(block, ToolUseBlock) + ), + usage=_usage(message.usage.input_tokens, message.usage.output_tokens), + call_id_header=response_header(raw.headers, "x-litellm-call-id"), + cost_header=response_header(raw.headers, "x-litellm-response-cost"), + ) + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: + return self._turn(key, model, ({"role": "user", "content": prompt},), "forced" if with_tool else "none") + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: + events: Final[tuple[RawMessageStreamEvent, ...]] = tuple( + self.sdk.anthropic(key).messages.create( + model=model, + max_tokens=MAX_OUTPUT_TOKENS, + system=INSTRUCTIONS, + messages=[{"role": "user", "content": prompt}], + stream=True, + extra_body=NO_PROXY_CACHE, + ) + ) + return StreamedReply( + text="".join( + event.delta.text + for event in events + if event.type == "content_block_delta" and event.delta.type == "text_delta" + ), + finished=any(event.type == "message_stop" for event in events), + usage_reported=any(event.type == "message_delta" and event.usage.output_tokens > 0 for event in events), + event_count=len(events), + ) + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: + tool_use: Final[ToolUseBlockParam] = { + "type": "tool_use", + "id": call.call_id, + "name": call.name, + "input": call.parsed().model_dump(), + } + tool_result: Final[ToolResultBlockParam] = { + "type": "tool_result", + "tool_use_id": call.call_id, + "content": result, + } + history: Final[tuple[MessageParam, ...]] = ( + {"role": "user", "content": prompt}, + {"role": "assistant", "content": [tool_use]}, + {"role": "user", "content": [tool_result]}, + ) + return self._turn(key, model, history, "offered") + + +@dataclass(frozen=True, slots=True) +class ResponsesSurface: + sdk: SdkClients + name: SurfaceName = "responses" + + def _turn(self, key: str, model: str, history: ResponseInputParam, tool: ToolMode) -> Reply: + raw: Final = self.sdk.openai(key).responses.with_raw_response.create( + model=model, + input=history, + instructions=INSTRUCTIONS, + max_output_tokens=MAX_OUTPUT_TOKENS, + tools=openai.omit if tool == "none" else [_responses_tool()], + tool_choice=_responses_tool_choice() if tool == "forced" else openai.omit, + parallel_tool_calls=False if tool == "forced" else openai.omit, + extra_body=NO_PROXY_CACHE, + ) + response: Final = raw.parse() + return Reply( + response_id=response.id, + text=response.output_text, + tool_calls=tuple( + ToolCall(call_id=item.call_id, name=item.name, arguments=item.arguments) + for item in response.output + if isinstance(item, ResponseFunctionToolCall) + ), + usage=None if response.usage is None else _usage(response.usage.input_tokens, response.usage.output_tokens), + call_id_header=response_header(raw.headers, "x-litellm-call-id"), + cost_header=response_header(raw.headers, "x-litellm-response-cost"), + ) + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: + return self._turn(key, model, [{"role": "user", "content": prompt}], "forced" if with_tool else "none") + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: + events: Final[tuple[ResponseStreamEvent, ...]] = tuple( + self.sdk.openai(key).responses.create( + model=model, + input=prompt, + instructions=INSTRUCTIONS, + max_output_tokens=MAX_OUTPUT_TOKENS, + stream=True, + extra_body=NO_PROXY_CACHE, + ) + ) + return StreamedReply( + text="".join(event.delta for event in events if event.type == "response.output_text.delta"), + finished=bool(events) and events[-1].type == "response.completed", + usage_reported=any( + event.type == "response.completed" and event.response.usage is not None for event in events + ), + event_count=len(events), + ) + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: + function_call: Final[ResponseFunctionToolCallParam] = { + "type": "function_call", + "call_id": call.call_id, + "name": call.name, + "arguments": call.arguments, + } + output: Final[FunctionCallOutput] = { + "type": "function_call_output", + "call_id": call.call_id, + "output": result, + } + return self._turn(key, model, [{"role": "user", "content": prompt}, function_call, output], "offered") + + +def build_surfaces(sdk: SdkClients) -> Mapping[SurfaceName, Surface]: + return MappingProxyType[SurfaceName, Surface]( + { + "chat_completions": ChatCompletionsSurface(sdk), + "messages": MessagesSurface(sdk), + "responses": ResponsesSurface(sdk), + } + ) diff --git a/tests/e2e/llm_translation/test_conversational_matrix_e2e.py b/tests/e2e/llm_translation/test_conversational_matrix_e2e.py new file mode 100644 index 00000000000..a5cfc9ce70b --- /dev/null +++ b/tests/e2e/llm_translation/test_conversational_matrix_e2e.py @@ -0,0 +1,163 @@ +"""The same conversation contract on every (endpoint, deployment, auth) cell. +A deployment is one provider model (openai/gpt-4o-mini, anthropic/claude-haiku-4-5, ...). + +/chat/completions, /v1/messages and /v1/responses each have their own +translation code in the proxy, so a bug fixed on one surface tends to survive +on the others. Every test here runs once per cell in `CELLS` +(conversational_matrix.py), so a change to a shared helper is proven against all +surfaces and providers at once, and a new model or provider is one row in `DEPLOYMENTS`. + +Edge-wired: OpenAI and Anthropic traffic goes through the provider edge in +record and replay, so the whole matrix replays with zero provider calls. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from typing import Final + +import pytest +from e2e_config import unique_marker +from lifecycle import ResourceManager +from llm_translation.conversational_matrix import ( + GREETING_PROMPT, + WEATHER_PROMPT, + WEATHER_REPORT, + WEATHER_TOOL_NAME, + Cell, + Deployments, + Surface, + SurfaceName, + ToolCall, + build_surfaces, + cells_covering, + register_deployments, +) +from llm_translation.sdk_clients import SdkClients +from models import SpendLogRow +from proxy_client import ProxyClient + +pytestmark = [pytest.mark.e2e, pytest.mark.replayable] + + +def _approx_equal(actual: float, expected: float) -> bool: + return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) + + +@pytest.fixture(scope="module") +def deployments(proxy: ProxyClient) -> Iterator[Deployments]: + yield from register_deployments(proxy) + + +@pytest.fixture(scope="module") +def surfaces(sdk: SdkClients) -> Mapping[SurfaceName, Surface]: + return build_surfaces(sdk) + + +def _weather_call(surface: Surface, key: str, model: str) -> ToolCall: + first: Final = surface.reply(key, model, WEATHER_PROMPT, with_tool=True) + assert len(first.tool_calls) == 1, ( + f"{surface.name} forced tool_choice={WEATHER_TOOL_NAME} with parallel calls off, " + f"got {len(first.tool_calls)} tool call(s): {first.tool_calls} text={first.text!r}" + ) + call: Final = first.tool_calls[0] + assert call.name == WEATHER_TOOL_NAME, f"{surface.name} called {call.name!r}, not the forced {WEATHER_TOOL_NAME!r}" + assert call.call_id, f"{surface.name} tool call has no id, so the caller cannot answer it: {call}" + assert "paris" in call.parsed().location.lower(), f"{surface.name} tool arguments lost the location: {call}" + return call + + +class TestConversationalMatrix: + @pytest.mark.parametrize("cell", cells_covering("basic", "nonstream", "works")) + def test_reply_carries_assistant_text_and_usage( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + surface: Final = surfaces[cell.surface] + reply: Final = surface.reply(resources.key(), deployments.alias(cell), GREETING_PROMPT) + + assert reply.response_id, f"{cell.id}: response has no id" + assert reply.text.strip(), f"{cell.id}: response carried no assistant text" + assert reply.usage is not None and reply.usage.input_tokens > 0 and reply.usage.output_tokens > 0, ( + f"{cell.id}: usage missing or zero, so the caller cannot account for this call: {reply.usage}" + ) + assert reply.call_id_header, f"{cell.id}: x-litellm-call-id header missing" + + @pytest.mark.parametrize("cell", cells_covering("basic", "stream", "works")) + def test_stream_delivers_text_usage_and_a_terminal_event( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + surface: Final = surfaces[cell.surface] + streamed: Final = surface.stream(resources.key(), deployments.alias(cell), GREETING_PROMPT) + + assert streamed.event_count > 1, f"{cell.id}: stream arrived as {streamed.event_count} event(s), not a stream" + assert streamed.text.strip(), f"{cell.id}: stream carried no text deltas" + assert streamed.finished, f"{cell.id}: stream never sent its terminal event" + assert streamed.usage_reported, f"{cell.id}: stream never reported usage" + + @pytest.mark.parametrize("cell", cells_covering("basic", "nonstream", "cost_logged")) + def test_cost_header_matches_the_spend_log( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + proxy: ProxyClient, + ) -> None: + key: Final = resources.key() + surface: Final = surfaces[cell.surface] + reply: Final = surface.reply(key, deployments.alias(cell), f"{GREETING_PROMPT} {unique_marker()}") + + assert reply.cost_header is not None, f"{cell.id}: x-litellm-response-cost header missing" + header_cost: Final = float(reply.cost_header) + assert header_cost > 0, f"{cell.id}: x-litellm-response-cost is not positive: {header_cost}" + + def _priced(rows: list[SpendLogRow]) -> bool: + return any(row.spend is not None and row.spend > 0 for row in rows) + + rows: Final = proxy.poll_logs_for_key(key, predicate=_priced) + priced: Final = tuple(row for row in rows if row.spend is not None and row.spend > 0) + assert len(priced) == 1, f"{cell.id}: expected exactly one priced spend row for a fresh key, got {rows}" + row: Final = priced[0] + assert (row.prompt_tokens or 0) > 0 and (row.completion_tokens or 0) > 0, ( + f"{cell.id}: spend row has no token counts, so the cost is not real usage: {row}" + ) + assert row.spend is not None and _approx_equal(row.spend, header_cost), ( + f"{cell.id}: logged spend {row.spend} disagrees with x-litellm-response-cost {header_cost}" + ) + assert row.model and cell.deployment.backend.endswith(row.model), ( + f"{cell.id}: spend row logged model {row.model!r}, not the deployment's {cell.deployment.backend!r}" + ) + + @pytest.mark.parametrize("cell", cells_covering("tool_use", "nonstream", "works")) + def test_tool_call_is_returned_named_and_addressable( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + _weather_call(surfaces[cell.surface], resources.key(), deployments.alias(cell)) + + @pytest.mark.parametrize("cell", cells_covering("multi_turn", "nonstream", "works")) + def test_tool_result_round_trip_reaches_the_model( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + key: Final = resources.key() + model: Final = deployments.alias(cell) + surface: Final = surfaces[cell.surface] + call: Final = _weather_call(surface, key, model) + + answer: Final = surface.reply_to_tool_result(key, model, WEATHER_PROMPT, call, WEATHER_REPORT) + assert "22" in answer.text, f"{cell.id}: the model never saw the tool result: {answer.text!r}" diff --git a/tests/e2e/logging/logging_client.py b/tests/e2e/logging/logging_client.py index 66dfa233ec4..c2f987ea33d 100644 --- a/tests/e2e/logging/logging_client.py +++ b/tests/e2e/logging/logging_client.py @@ -146,6 +146,29 @@ class LangfuseObservationList(BaseModel): data: list[LangfuseObservation] = [] +class LangfuseOtelMetadata(BaseModel): + """Langfuse stores every OTel span attribute under metadata.attributes.""" + + model_config = ConfigDict(extra="ignore") + + attributes: dict[str, str] = {} + + +def otel_attributes(obs: LangfuseObservation) -> dict[str, str]: + try: + return LangfuseOtelMetadata.model_validate(obs.metadata).attributes + except ValidationError: + return {} + + +def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool: + attributes = otel_attributes(obs) + return ( + attributes.get("langfuse.observation.type") == "generation" + and attributes.get("litellm.metadata.user_api_key_alias") == key_alias + ) + + class LangfuseListParams(BaseModel): model_config = ConfigDict(populate_by_name=True) @@ -630,6 +653,18 @@ class LoggingClient: time.sleep(POLL_INTERVAL) return last + def poll_langfuse_generation( + self, creds: LangfuseCreds, *, key_alias: str, from_start_time: str + ) -> LangfuseObservation | None: + """The OTel v2 generation the proxy exported for one key alias since from_start_time.""" + deadline = time.monotonic() + POLL_TIMEOUT + while time.monotonic() < deadline: + for obs in self.list_langfuse_observations(creds, from_start_time=from_start_time): + if is_otel_v2_generation(obs, key_alias=key_alias): + return obs + time.sleep(POLL_INTERVAL) + return None + def poll_langfuse_trace_observations( self, creds: LangfuseCreds, diff --git a/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py new file mode 100644 index 00000000000..71008d94557 --- /dev/null +++ b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py @@ -0,0 +1,210 @@ +"""Live e2e: the OTel v2 Langfuse generation carries output for every non-chat endpoint (LIT-8309). + +With LITELLM_OTEL_V2=true the proxy exports one generation per request to the +team's Langfuse destination. Chat, Responses, embeddings and OCR already fill +its output; this file pins the remaining five families. Each test registers a +real OpenAI deployment, drives the endpoint through the shared transport, then +reads the generation back from Langfuse and asserts its output reflects what +the caller received: the completion text, the transcript, the moderation +verdict, and for images and speech a bounded summary that never carries the +raw base64 or audio bytes. +""" + +from __future__ import annotations + +import base64 +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from logging_client import LangfuseCreds, LangfuseObservation, LoggingClient, load_langfuse_creds +from models import ( + CompletionBody, + CompletionResponse, + ImageGenerationBody, + ImageGenerationResponse, + LiteLLMParamsBody, + ModerationBody, + ModerationResponse, + SpeechBody, + TranscriptionForm, + TranscriptionResponse, +) +from pydantic import BaseModel, TypeAdapter, ValidationError + +pytestmark = [pytest.mark.e2e, pytest.mark.otel_v2] + +WEATHER_WAV: Final = ( + Path(__file__).resolve().parent.parent / "llm_translation" / "realtime" / "fixtures" / "weather_question_24k.wav" +) +BOUNDED_OUTPUT_CHARS: Final = 1024 + + +class _OutputMessage(BaseModel): + """One assistant message of the Langfuse generation output; only the text is read.""" + + content: str = "" + + +_OUTPUT_MESSAGES: Final = TypeAdapter(list[_OutputMessage]) + + +@pytest.fixture(scope="session") +def langfuse_creds() -> LangfuseCreds: + return load_langfuse_creds() + + +def _langfuse_key( + client: LoggingClient, creds: LangfuseCreds, resources: ResourceManager, params: LiteLLMParamsBody +) -> tuple[str, str, str]: + """A model registered for this run plus a key on a team whose Langfuse callback is `creds`.""" + model: Final = f"e2e-otel-out-{unique_marker()}" + model_id: Final = client.proxy.create_model(model, params) + resources.defer(lambda: client.proxy.delete_model(model_id)) + team_id: Final = client.create_team(f"otel-out-team-{unique_marker()}", models=[model]) + resources.defer(lambda: client.delete_team(team_id)) + client.add_team_langfuse_callback(team_id, creds) + alias: Final = f"otel-out-key-{unique_marker()}" + key: Final = client.key_with_alias(alias, models=[model], team_id=team_id) + resources.defer(lambda: client.delete_key(key)) + return model, key, alias + + +def _generation(client: LoggingClient, creds: LangfuseCreds, *, alias: str, started: datetime) -> LangfuseObservation: + since: Final = (started - timedelta(seconds=5)).isoformat() + observation: Final = client.poll_langfuse_generation(creds, key_alias=alias, from_start_time=since) + assert observation is not None, f"no OTel v2 generation reached Langfuse for key alias {alias!r}" + return observation + + +def _output_text(observation: LangfuseObservation) -> str: + assert observation.output not in (None, "", [], {}), f"generation output is empty: {observation!r}" + try: + messages: Final = _OUTPUT_MESSAGES.validate_python(observation.output) + except ValidationError: + pytest.fail(f"generation output is not a list of assistant messages: {observation!r}") + assert messages, f"generation output is empty: {observation!r}" + return "\n".join(message.content for message in messages) + + +def _openai(model: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody(model=model, api_key="os.environ/OPENAI_API_KEY") + + +class TestOtelV2LangfuseGenerationOutput: + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["completions"]) + def test_completions_output_is_the_completion_text( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-3.5-turbo-instruct")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/completions", + headers=client.proxy.transport.bearer(key), + json=CompletionBody(model=model, prompt=f"Repeat exactly: {unique_marker()}", n=2), + response_type=CompletionResponse, + ) + ) + texts: Final = tuple(choice.text.strip() for choice in response.choices) + assert len(texts) == 2 and all(texts), f"/v1/completions returned no text: {response!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert all(text in output for text in texts), ( + f"generation output lacks the completion texts {texts!r}: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["images_generations"]) + def test_images_output_is_a_bounded_summary_without_base64( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-image-1-mini")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/images/generations", + headers=client.proxy.transport.bearer(key), + json=ImageGenerationBody(model=model, prompt=f"a plain red square {unique_marker()}"), + response_type=ImageGenerationResponse, + timeout=180.0, + ) + ) + assert response.data, f"/v1/images/generations returned no data: {response!r}" + encoded: Final = response.data[0].b64_json or "" + assert encoded, f"expected a b64_json image from gpt-image-1-mini: {response.data[0].url!r}" + image_bytes: Final = len(base64.b64decode(encoded)) + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert len(output) <= BOUNDED_OUTPUT_CHARS, f"image generation output is not bounded ({len(output)} chars)" + assert encoded[:64] not in output, "image generation output leaks the raw base64 payload" + assert output == f"b64_json image ({image_bytes} bytes)", ( + f"image generation output does not report the {image_bytes} decoded bytes: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["audio_speech"]) + def test_speech_output_is_a_bounded_summary_without_audio_bytes( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-4o-mini-tts")) + started: Final = datetime.now(timezone.utc) + audio: Final = client.proxy.transport.stream_binary( + "/v1/audio/speech", + headers=client.proxy.transport.bearer(key), + json=SpeechBody(model=model, input=f"hello {unique_marker()}"), + ) + assert audio.ok and audio.total_bytes > 0, f"/v1/audio/speech returned no audio: {audio!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert len(output) <= BOUNDED_OUTPUT_CHARS, f"speech output is not bounded ({len(output)} chars)" + assert output.endswith(f" ({audio.total_bytes} bytes)"), ( + f"speech output does not report the {audio.total_bytes} audio bytes the caller received: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["audio_transcriptions"]) + def test_transcription_output_is_the_transcript( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-4o-mini-transcribe")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.upload( + "/v1/audio/transcriptions", + headers=client.proxy.transport.bearer(key), + form=TranscriptionForm(model=model), + filename=WEATHER_WAV.name, + content=WEATHER_WAV.read_bytes(), + file_content_type="audio/wav", + response_type=TranscriptionResponse, + ) + ) + transcript: Final = response.text.strip() + assert transcript, f"/v1/audio/transcriptions returned no text: {response!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert transcript in output, f"generation output lacks the transcript {transcript!r}: {output!r}" + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["moderations"]) + def test_moderations_output_is_the_verdict( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/omni-moderation-latest")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/moderations", + headers=client.proxy.transport.bearer(key), + json=ModerationBody(model=model, input=f"I will find you and hurt you badly {unique_marker()}"), + response_type=ModerationResponse, + ) + ) + assert response.results, f"/v1/moderations returned no results: {response!r}" + verdict: Final = "flagged: " if response.results[0].flagged else "not flagged" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert output.startswith(verdict), ( + f"generation output does not carry the moderation verdict {verdict!r}: {output!r}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 2608da2e4fc..d0b1e8824da 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -760,6 +760,77 @@ class OcrResponse(BaseModel): pages: list[OcrPage] = [] +# ---------- completions ---------- + + +class CompletionBody(BaseModel): + model: str + prompt: str + max_tokens: int = 8 + n: int = 1 + + +class CompletionChoice(BaseModel): + text: str = "" + + +class CompletionResponse(BaseModel): + choices: list[CompletionChoice] = [] + + +# ---------- images ---------- + + +class ImageGenerationBody(BaseModel): + model: str + prompt: str + n: int = 1 + size: str = "1024x1024" + quality: str = "low" + + +class ImageDatum(BaseModel): + url: str | None = None + b64_json: str | None = None + + +class ImageGenerationResponse(BaseModel): + data: list[ImageDatum] = [] + + +# ---------- audio ---------- + + +class SpeechBody(BaseModel): + model: str + input: str + voice: str = "alloy" + + +class TranscriptionForm(BaseModel): + model: str + + +class TranscriptionResponse(BaseModel): + text: str = "" + + +# ---------- moderations ---------- + + +class ModerationBody(BaseModel): + model: str + input: str + + +class ModerationResult(BaseModel): + flagged: bool + + +class ModerationResponse(BaseModel): + results: list[ModerationResult] = [] + + # ---------- spend logs ---------- diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 742f809b9e9..a84562e04d0 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -15,3 +15,4 @@ markers = mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set callback_cost_capture: reads the JSONL written by the gateway's registered logging.callback_cost_capture.capture CustomLogger, so the gateway and pytest must share a filesystem; deselected unless E2E_CALLBACK_COST_CAPTURE_FILE is set + otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set diff --git a/tests/proxy_unit_tests/test_custom_tokenizer_bug.py b/tests/proxy_unit_tests/test_custom_tokenizer_bug.py index c4b1f4f3afd..23b6b302202 100644 --- a/tests/proxy_unit_tests/test_custom_tokenizer_bug.py +++ b/tests/proxy_unit_tests/test_custom_tokenizer_bug.py @@ -22,10 +22,8 @@ from litellm.proxy.proxy_server import token_counter def _fake_hf_tokenizer(num_tokens: int) -> MagicMock: - encoding = MagicMock() - encoding.__len__.return_value = num_tokens tokenizer = MagicMock() - tokenizer.encode_batch_fast.return_value = [encoding] + tokenizer.encode_batch_fast.return_value = [[0] * num_tokens] return tokenizer @@ -58,7 +56,7 @@ async def test_custom_tokenizer_from_model_info_is_used(monkeypatch): ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - with patch.object(litellm.utils, "Tokenizer") as mock_tokenizer_cls: + with patch.object(litellm.utils, "tokenizer_dispatch") as mock_tokenizer_cls: mock_tokenizer_cls.from_pretrained.return_value = _fake_hf_tokenizer(7) response = await token_counter( @@ -92,7 +90,7 @@ async def test_model_without_custom_tokenizer_uses_default(monkeypatch): ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - with patch.object(litellm.utils, "Tokenizer") as mock_tokenizer_cls: + with patch.object(litellm.utils, "tokenizer_dispatch") as mock_tokenizer_cls: response = await token_counter( request=TokenCountRequest( model="gpt-4", diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 01f2a13d252..70ab4fe07de 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -4,6 +4,7 @@ and the typed StandardLoggingPayload adapter. These need no OTel SDK.""" import json import logging import re +from collections.abc import Mapping from pathlib import Path from typing import Final @@ -151,9 +152,7 @@ def test_llm_call_span_name(): def _all_constants(cls): return { - getattr(cls, name) - for name in vars(cls) - if not name.startswith("__") and isinstance(getattr(cls, name), str) + getattr(cls, name) for name in vars(cls) if not name.startswith("__") and isinstance(getattr(cls, name), str) } @@ -465,9 +464,7 @@ def test_mcp_tool_call_content_gated_off_by_default(): off = MCPToolCallSpanData.from_standard_logging_payload(_mcp_payload()) assert off.arguments_json is None and off.result_json is None - on = MCPToolCallSpanData.from_standard_logging_payload( - _mcp_payload(), capture_content=True - ) + on = MCPToolCallSpanData.from_standard_logging_payload(_mcp_payload(), capture_content=True) assert on.arguments_json is not None and '"Paris"' in on.arguments_json assert on.result_json is not None and "21" in on.result_json @@ -689,9 +686,7 @@ def test_content_capture_gated_off_by_default(): payload = _sample_payload( messages=[{"role": "user", "content": "secret prompt"}], ) - payload["response"]["choices"] = [ - {"finish_reason": "stop", "message": {"role": "assistant", "content": "secret"}} - ] + payload["response"]["choices"] = [{"finish_reason": "stop", "message": {"role": "assistant", "content": "secret"}}] data = LLMCallSpanData.from_standard_logging_payload(payload) assert data.messages_in == () assert data.choices_out == () @@ -911,6 +906,221 @@ def test_ocr_pages_without_markdown_stay_empty(): assert data.choices_out == () +def _assistant_choice(content: str, finish_reason: str | None = None) -> dict[str, object]: + return { + "message": {"role": "assistant", "content": content, "refusal": None, "tool_calls": None}, + "finish_reason": finish_reason, + } + + +def _route_payload(call_type: str, model: str, response: Mapping[str, object]) -> dict[str, object]: + return _sample_payload(call_type=call_type, model=model, messages=None, response=response) + + +def test_text_completion_choices_become_assistant_messages_in_choice_order() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "atext_completion", + "gpt-3.5-turbo-instruct", + { + "id": "cmpl-1", + "object": "text_completion", + "choices": [ + {"index": 0, "text": " first", "finish_reason": "length", "logprobs": None}, + {"index": 1, "text": " second", "finish_reason": "stop", "logprobs": None}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice(" first", "length"), _assistant_choice(" second", "stop")) + assert data.finish_reasons == ("length", "stop") + assert data.response_id == "cmpl-1" + + +def test_text_completion_choices_follow_the_content_capture_gate_but_finish_reasons_do_not() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "atext_completion", "gpt-3.5-turbo-instruct", {"choices": [{"text": "x", "finish_reason": "stop"}]} + ) + ) + + assert data.choices_out == () + assert data.finish_reasons == ("stop",) + + +def test_chat_choices_with_a_message_are_passed_through_untouched_even_beside_a_stray_text_key() -> None: + choice: Final = { + "index": 0, + "finish_reason": "stop", + "text": "no", + "message": {"role": "assistant", "content": "chat"}, + } + data: Final = LLMCallSpanData.from_standard_logging_payload( + _sample_payload(response={"choices": [choice]}), capture_content=True + ) + + assert data.choices_out == (choice,) + + +def test_transcription_text_becomes_one_assistant_choice() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("atranscription", "gpt-4o-mini-transcribe", {"text": "What is the weather like?", "task": "x"}), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("What is the weather like?"),) + assert data.finish_reasons == () + + +def test_empty_transcription_text_stays_empty() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("atranscription", "gpt-4o-mini-transcribe", {"text": ""}), capture_content=True + ) + + assert data.choices_out == () + + +def test_moderation_results_become_one_verdict_per_input_naming_the_hit_categories() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "amoderation", + "omni-moderation-latest", + { + "id": "modr-1", + "results": [ + { + "flagged": True, + "categories": {"harassment": False, "violence": True, "self-harm": True}, + "category_scores": {"harassment": 0.01, "violence": 0.98, "self-harm": 0.7}, + }, + {"flagged": False, "categories": {"violence": False}}, + {"flagged": True}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("flagged: violence, self-harm\n\nnot flagged\n\nflagged"),) + assert data.response_id == "modr-1" + + +def test_moderation_output_follows_the_content_capture_gate() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("amoderation", "omni-moderation-latest", {"results": [{"flagged": True}]}) + ) + + assert data.choices_out == () + + +def test_moderation_results_without_a_verdict_produce_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("amoderation", "omni-moderation-latest", {"results": [{"categories": {"violence": True}}]}), + capture_content=True, + ) + + assert data.choices_out == () + + +def test_image_data_becomes_a_size_summary_and_never_carries_the_base64_payload() -> None: + encoded: Final = "QUJDRA==" + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "aimage_generation", + "gpt-image-1-mini", + { + "created": 1, + "data": [ + {"b64_json": encoded, "revised_prompt": "a red bicycle"}, + {"url": "https://images.example/cat.png"}, + {"b64_json": "QUJDREVGR0g="}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == ( + _assistant_choice( + "a red bicycle\nb64_json image (4 bytes)\n\nhttps://images.example/cat.png\n\nb64_json image (8 bytes)" + ), + ) + assert encoded not in json.dumps(data.choices_out) + + +def test_image_data_without_a_url_or_payload_stays_empty_and_embeddings_are_not_images() -> None: + images: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aimage_generation", "gpt-image-1-mini", {"data": [{"revised_prompt": "x"}]}), + capture_content=True, + ) + embeddings: Final = LLMCallSpanData.from_standard_logging_payload( + _embedding_payload([[0.1, 0.2]]), capture_content=True + ) + + assert images.choices_out == () + assert embeddings.choices_out == () + + +def test_speech_summary_becomes_a_media_type_and_byte_count_choice() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "aspeech", "gpt-4o-mini-tts", {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 48210} + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("audio/mpeg (48210 bytes)"),) + + +def test_speech_summary_without_a_media_type_is_the_byte_count_and_follows_the_capture_gate() -> None: + response: Final = {"object": "binary", "content_type": None, "num_bytes": 7} + shown: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aspeech", "gpt-4o-mini-tts", response), capture_content=True + ) + gated: Final = LLMCallSpanData.from_standard_logging_payload(_route_payload("aspeech", "gpt-4o-mini-tts", response)) + + assert shown.choices_out == (_assistant_choice("7 bytes"),) + assert gated.choices_out == () + + +def test_speech_response_without_a_byte_count_produces_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aspeech", "gpt-4o-mini-tts", {"object": "binary", "content_type": "audio/mpeg"}), + capture_content=True, + ) + + assert data.choices_out == () + + +def test_speech_binary_response_is_logged_as_its_summary_not_dropped() -> None: + import httpx + + from litellm.litellm_core_utils.litellm_logging import _extract_response_obj_and_hidden_params + from litellm.types.llms.openai import HttpxBinaryResponseContent + + raw: Final = httpx.Response(200, headers={"content-type": "audio/mpeg"}, content=b"\x00" * 1234) + response_obj, hidden_params = _extract_response_obj_and_hidden_params(HttpxBinaryResponseContent(raw), None) + + assert response_obj == {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 1234} + assert hidden_params is None + + +def test_speech_binary_response_still_streaming_reports_the_bytes_downloaded_so_far() -> None: + import httpx + + from litellm.types.llms.openai import HttpxBinaryResponseContent + + unread: Final = httpx.Response(200, stream=httpx.ByteStream(b"\x00" * 10)) + + assert HttpxBinaryResponseContent(unread).logging_summary() == { + "object": "binary", + "content_type": None, + "num_bytes": 0, + } + + def test_request_identity_prefers_canonical_team_keys(): from litellm.integrations.otel.model.payloads import RequestIdentity @@ -931,9 +1141,7 @@ def test_request_identity_prefers_canonical_team_keys(): def test_request_identity_falls_back_to_legacy_team_keys(): from litellm.integrations.otel.model.payloads import RequestIdentity - payload = _sample_payload( - metadata={"team_id": "legacy-team", "team_alias": "legacy"} - ) + payload = _sample_payload(metadata={"team_id": "legacy-team", "team_alias": "legacy"}) ident = RequestIdentity.from_payload(payload) assert ident.team_id == "legacy-team" assert ident.team_alias == "legacy" @@ -952,7 +1160,10 @@ def test_request_identity_falls_back_to_legacy_team_keys(): }, "from-header", ), - ({"proxy_server_request": {"headers": {"langfuse_trace_name": ""}}, "metadata": {"trace_name": "body"}}, "body"), + ( + {"proxy_server_request": {"headers": {"langfuse_trace_name": ""}}, "metadata": {"trace_name": "body"}}, + "body", + ), ({"proxy_server_request": {"headers": {}}, "metadata": {"user_api_key_team_id": "t1"}}, None), ({}, None), ], @@ -1002,7 +1213,15 @@ def test_caller_trace_name_prefers_the_langfuse_header_over_body_metadata(reques ), ({}, TraceControls()), ], - ids=["body", "headers-beat-body", "anthropic-body", "non-string-tags-dropped", "scalar-coercion", "mutation-controls-ignored", "empty"], + ids=[ + "body", + "headers-beat-body", + "anthropic-body", + "non-string-tags-dropped", + "scalar-coercion", + "mutation-controls-ignored", + "empty", + ], ) def test_caller_trace_controls_carry_user_session_and_tags(request_data, expected): assert caller_trace_controls({"litellm_params": request_data}) == expected @@ -1168,9 +1387,7 @@ def test_content_capture_opt_in_retains_bodies(): payload = _sample_payload( messages=[{"role": "user", "content": "secret prompt"}], ) - payload["response"]["choices"] = [ - {"finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}} - ] + payload["response"]["choices"] = [{"finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}] data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True) assert data.messages_in and data.messages_in[0]["content"] == "secret prompt" assert data.choices_out and data.choices_out[0]["message"]["content"] == "hi" @@ -1187,41 +1404,17 @@ def test_capture_span_content_resolves_modes(): # default (no_content) → off assert OpenTelemetryV2Config().capture_span_content is False + assert OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.SPAN_ONLY).capture_span_content is True assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.SPAN_ONLY - ).capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.SPAN_AND_EVENT - ).capture_span_content - is True + OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.SPAN_AND_EVENT).capture_span_content is True ) # event-only does not authorize span-attribute content - assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.EVENT_ONLY - ).capture_span_content - is False - ) + assert OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.EVENT_ONLY).capture_span_content is False # V1 accepted UPPER_SNAKE_CASE; the env value is case-insensitive so an # operator carrying ``SPAN_AND_EVENT`` forward still enables capture. - assert ( - OpenTelemetryV2Config( - capture_message_content="SPAN_AND_EVENT" - ).capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config(capture_message_content="SPAN_ONLY").capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config(capture_message_content="NO_CONTENT").capture_span_content - is False - ) + assert OpenTelemetryV2Config(capture_message_content="SPAN_AND_EVENT").capture_span_content is True + assert OpenTelemetryV2Config(capture_message_content="SPAN_ONLY").capture_span_content is True + assert OpenTelemetryV2Config(capture_message_content="NO_CONTENT").capture_span_content is False def test_capture_message_content_normalizer_only_touches_strings(): diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py index 9fa198c4ec5..1e2ae24a329 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py @@ -6,6 +6,8 @@ backends, so one trace lights up every configured destination. """ import json +from collections.abc import Mapping +from typing import Final import pytest @@ -249,6 +251,34 @@ def test_langfuse_mapper_renders_an_ocr_call_with_the_page_markdown_as_output(): assert attrs["langfuse.observation.type"] == "generation" +@pytest.mark.parametrize( + ("call_type", "response", "expected_content"), + [ + ("atext_completion", {"choices": [{"text": "Paris.", "finish_reason": "stop"}]}, "Paris."), + ("atranscription", {"text": "What is the weather like?"}, "What is the weather like?"), + ("amoderation", {"results": [{"flagged": True, "categories": {"violence": True}}]}, "flagged: violence"), + ("aimage_generation", {"data": [{"b64_json": "QUJDRA=="}]}, "b64_json image (4 bytes)"), + ("aspeech", {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 9}, "audio/mpeg (9 bytes)"), + ], +) +def test_langfuse_mapper_renders_every_non_chat_route_output_as_an_assistant_message( + call_type: str, response: Mapping[str, object], expected_content: str +) -> None: + payload: Final[dict[str, object]] = { + "call_type": call_type, + "custom_llm_provider": "openai", + "model": "m", + "messages": None, + "response": response, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True)) + + assert json.loads(attrs["langfuse.observation.output"]) == [ + {"role": "assistant", "content": expected_content, "refusal": None, "tool_calls": None} + ] + assert attrs["langfuse.observation.type"] == "generation" + + # --------------------------------------------------------------------------- # # Weave # --------------------------------------------------------------------------- # diff --git a/tests/test_litellm/litellm_core_utils/test_bug_report.py b/tests/test_litellm/litellm_core_utils/test_bug_report.py new file mode 100644 index 00000000000..f47361dda2e --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_bug_report.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import cast +from urllib.parse import parse_qs, unquote_plus, urlparse + +import httpx +import pytest + +import litellm +from litellm._version import version +from litellm.exceptions import APIConnectionError, BadRequestError, InternalServerError +from litellm.litellm_core_utils.bug_report import ( + DISABLE_ENV_VAR, + ISSUE_URL_BASE, + MAX_FRAMES, + MAX_URL_LENGTH, + allowlisted, + bug_report_enabled, + bug_report_issue_url, + bug_report_notice, + build_bug_report, + should_report_bug, + strip_bug_report_notice, +) +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + +def test_build_bug_report_keeps_only_litellm_frames(): + with pytest.raises(BadRequestError) as raised: + get_llm_provider(cast(str, None)) + report = build_bug_report(raised.value, surface="sdk") + + assert report.litellm_frames + assert all(frame.startswith("litellm/") for frame in report.litellm_frames) + assert all("test_bug_report.py" not in frame for frame in report.litellm_frames) + + +def test_issue_url_never_contains_the_exception_message(): + secret = "sk-abcdefghijklmnopqrstuvwxyz1234567890" + prompt = "my social security number is 123-45-6789" + report = build_bug_report(RuntimeError(f"{secret} {prompt}"), surface="sdk") + url = bug_report_issue_url(report) + query = parse_qs(urlparse(url).query) + + assert url.startswith(ISSUE_URL_BASE) + assert secret not in url and "123-45-6789" not in url and "social" not in url + assert query["title"] == ["[Bug]: RuntimeError in litellm"] + assert query["version"] == [version] + assert query["template"] == ["bug_report.yml"] + assert query["domain"] == ["Python SDK: the litellm package itself"] + assert query["deployment"] == ["pip / Python SDK"] + assert "Exception: `RuntimeError`" in query["description"][0] + assert "Python:" in query["description"][0] + + +def test_issue_url_drops_unknown_provider_and_call_type(): + report = build_bug_report( + ValueError("boom"), + surface="proxy", + custom_llm_provider="acme-internal-gateway", + ) + query = parse_qs(urlparse(bug_report_issue_url(report)).query) + + assert report.custom_llm_provider is None + assert "acme" not in bug_report_issue_url(report) + assert "Provider: unknown" in query["description"][0] + assert "Endpoint / call: unknown" in query["description"][0] + + +def test_allowlisted_only_passes_exact_members(): + allowed = frozenset({"/v1/chat/completions"}) + + assert allowlisted("/v1/chat/completions", allowed) == "/v1/chat/completions" + assert allowlisted("/v1/chat/completions/../../admin", allowed) is None + assert allowlisted(None, allowed) is None + assert allowlisted(["/v1/chat/completions"], allowed) is None + assert allowlisted({"provider": "openai"}, allowed) is None + + +def test_build_bug_report_survives_unhashable_provider_from_request_data(): + report = build_bug_report( + KeyError("missing"), + surface="proxy", + custom_llm_provider={"name": "openai"}, + ) + + assert report.custom_llm_provider is None + + +def test_frames_are_capped_and_url_is_bounded(): + namespace: dict[str, object] = {} + exec( + compile( + "def recurse(depth):\n if depth == 0:\n raise RuntimeError('deep')\n recurse(depth - 1)\n", + str(Path(litellm.__file__).with_name("fake_deep_module.py")), + "exec", + ), + namespace, + ) + recurse = cast(Callable[[int], None], namespace["recurse"]) + + with pytest.raises(RuntimeError) as raised: + recurse(200) + report = build_bug_report(raised.value, surface="proxy") + + assert len(report.litellm_frames) == MAX_FRAMES + assert all(frame.startswith("litellm/fake_deep_module.py:") for frame in report.litellm_frames) + assert len(bug_report_issue_url(report)) <= MAX_URL_LENGTH + + +def test_issue_url_builds_without_a_traceback(): + exc = RuntimeError("no traceback") + assert exc.__traceback__ is None + report = build_bug_report(exc, surface="proxy") + + assert report.litellm_frames == () + assert bug_report_issue_url(report).startswith(ISSUE_URL_BASE) + + +def test_bug_report_can_be_disabled(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(DISABLE_ENV_VAR, "true") + + assert bug_report_enabled() is False + assert should_report_bug(RuntimeError("boom")) is False + + +@pytest.mark.parametrize( + "exc", + [ + InternalServerError(message="upstream 500", llm_provider="openai", model="gpt-4"), + APIConnectionError( + message="connection reset", + llm_provider="openai", + model="gpt-4", + request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), + ), + BadRequestError(message="bad input", llm_provider="openai", model="gpt-4"), + "not an exception", + ], +) +def test_should_report_bug_skips_provider_and_network_errors(exc: object): + assert should_report_bug(exc) is False + + +def test_should_report_bug_accepts_plain_python_errors(): + assert should_report_bug(KeyError("missing")) is True + + +def test_proxy_known_provider_uses_translation_domain(): + report = build_bug_report( + RuntimeError("proxy failure"), + surface="proxy", + call_type="/v1/chat/completions", + custom_llm_provider="openai", + ) + query = parse_qs(urlparse(bug_report_issue_url(report)).query) + + assert report.custom_llm_provider == "openai" + assert query["domain"] == ["LLM translation: a specific provider's request or response"] + assert "Endpoint / call: /v1/chat/completions" in query["description"][0] + + +def test_strip_bug_report_notice(): + report = build_bug_report(RuntimeError("boom"), surface="sdk") + notice = bug_report_notice(report) + + assert strip_bug_report_notice(f"boom\n\n{notice}") == "boom\n" + assert strip_bug_report_notice("boom") == "boom" + + +def test_issue_description_renders_stream_and_config_block(): + report = build_bug_report( + RuntimeError("boom"), + surface="proxy", + stream=True, + config_lines=("router_settings.routing_strategy = least-busy", "litellm_settings.drop_params = true"), + ) + description = parse_qs(urlparse(bug_report_issue_url(report)).query)["description"][0] + + assert "Stream: true\n" in description + assert "```\nrouter_settings.routing_strategy = least-busy\nlitellm_settings.drop_params = true\n```" in description + + +@pytest.mark.parametrize("stream", [None, "true", 1]) +def test_issue_description_omits_stream_unless_it_is_a_bool(stream: object): + report = build_bug_report(RuntimeError("boom"), surface="proxy", stream=stream) + + assert report.stream is None + assert "Stream:" not in unquote_plus(bug_report_issue_url(report)) + + +def test_oversized_config_is_trimmed_from_the_end_before_any_frame(): + with pytest.raises(BadRequestError) as raised: + get_llm_provider(cast(str, None)) + config_lines = tuple(f"general_settings.flag_{index:04d} = true" for index in range(400)) + report = build_bug_report(raised.value, surface="proxy", config_lines=config_lines) + description = parse_qs(urlparse(url := bug_report_issue_url(report)).query)["description"][0] + + assert len(url) <= MAX_URL_LENGTH + assert all(frame in description for frame in report.litellm_frames) + assert "general_settings.flag_0000 = true" in description + assert "general_settings.flag_0399 = true" not in description diff --git a/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py b/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py index d1a5f78a859..dd2daef5484 100644 --- a/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py +++ b/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py @@ -1,23 +1,11 @@ -from tokenizers import AddedToken, Tokenizer -from tokenizers.models import WordLevel -from tokenizers.pre_tokenizers import Whitespace -from tokenizers.processors import TemplateProcessing - from litellm import decode, encode +from tokenizers import Tokenizer + +TOKENIZER_JSON = """{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":3,"content":"[BOS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[BOS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],"pair":[{"Sequence":{"id":"A","type_id":0}},{"Sequence":{"id":"B","type_id":1}}],"special_tokens":{"[BOS]":{"id":"[BOS]","ids":[3],"tokens":["[BOS]"]}}},"decoder":null,"model":{"type":"WordLevel","vocab":{"[UNK]":0,"Hello":1,"World":2},"unk_token":"[UNK]"}}""" def _create_custom_tokenizer(): - tokenizer = Tokenizer( - WordLevel({"[UNK]": 0, "Hello": 1, "World": 2}, unk_token="[UNK]") - ) - tokenizer.pre_tokenizer = Whitespace() - tokenizer.add_special_tokens([AddedToken("[BOS]", special=True)]) - bos_token_id = tokenizer.token_to_id("[BOS]") - assert bos_token_id is not None - tokenizer.post_processor = TemplateProcessing( - single="[BOS] $A", - special_tokens=[("[BOS]", bos_token_id)], - ) + tokenizer = Tokenizer.from_str(TOKENIZER_JSON) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index cfe7470fa76..5c5c2c9536b 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -3,8 +3,6 @@ import openai import pytest import litellm - - from litellm.litellm_core_utils.exception_mapping_utils import ( ExceptionCheckers, _get_body_error_code, @@ -974,6 +972,31 @@ def test_an_unmapped_exception_with_no_model_or_provider_is_a_connection_error(q assert "boom" in raised.value.message +def test_unmapped_sdk_exception_includes_bug_report_link(quiet_exception_mapping): + with pytest.raises(litellm.APIConnectionError) as raised: + exception_type( + model="my-model", + custom_llm_provider="minimax", + original_exception=ValueError("boom"), + ) + + assert "https://github.com/BerriAI/litellm/issues/new?" in str(raised.value) + assert "ValueError" in str(raised.value) + + +def test_unmapped_sdk_exception_bug_report_link_can_be_disabled(quiet_exception_mapping, monkeypatch): + monkeypatch.setenv("LITELLM_DISABLE_BUG_REPORT_LINK", "true") + + with pytest.raises(litellm.APIConnectionError) as raised: + exception_type( + model="my-model", + custom_llm_provider="minimax", + original_exception=ValueError("boom"), + ) + + assert "https://github.com/BerriAI/litellm/issues/new?" not in str(raised.value) + + def _raise_and_map(model: str | None, original_exception: Exception, custom_llm_provider: str | None) -> None: """Calls exception_type() from inside the except block, as litellm/main.py does, so traceback.format_exc() has a real stack.""" diff --git a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py index fe965f75f8f..521244c6ede 100644 --- a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py +++ b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py @@ -1,8 +1,12 @@ +from types import MappingProxyType +from typing import Final + import pytest - +from pydantic import TypeAdapter from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + inherit_message_logging_privacy, initialize_standard_callback_dynamic_params, iter_client_callback_metadata_dicts, ) @@ -189,6 +193,20 @@ def test_empty_kwargs_returns_empty_params(): assert dict(params) == {} +@pytest.mark.parametrize("child_privacy", (False, True)) +def test_inherited_privacy_only_strengthens_child_and_resets(child_privacy: bool) -> None: + kwargs: Final = TypeAdapter(dict[str, object]).validate_python( + MappingProxyType({"turn_off_message_logging": child_privacy}) + ) + with inherit_message_logging_privacy(False): + assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy + with inherit_message_logging_privacy(True), inherit_message_logging_privacy(False): + params: Final = initialize_standard_callback_dynamic_params(kwargs) + assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy + assert params["turn_off_message_logging"] is True + assert initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is None + + def test_newrelic_callback_params_are_not_extracted_from_request_kwargs(): kwargs = { "newrelic_api_key": "caller-key", diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 13b3775b7ce..f8c246dc28c 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -305,14 +305,15 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): class TestZeroCostDiagnostic: - DEPLOYMENT_ID: Final = "lit7898-per-second-priced-deployment" - MODEL_GROUP: Final = "per-second-priced-chat" + DEPLOYMENT_ID: Final = "lit7898-query-only-priced-deployment" + MODEL_GROUP: Final = "query-only-priced-chat" + QUERY_ONLY_PRICING: Final = {"input_cost_per_query": 0.00042} PER_SECOND_PRICING: Final = {"input_cost_per_second": 0.00042, "output_cost_per_second": 0.00042} FREE_PRICING: Final = {"input_cost_per_token": 0, "output_cost_per_token": 0} - @pytest.fixture(params=["per_second", "free"]) + @pytest.fixture(params=["query_only", "free"]) def deployment_pricing(self, request: pytest.FixtureRequest) -> Iterator[Mapping[str, float]]: - pricing: Final = self.PER_SECOND_PRICING if request.param == "per_second" else self.FREE_PRICING + pricing: Final = self.QUERY_ONLY_PRICING if request.param == "query_only" else self.FREE_PRICING litellm.register_model(model_cost={self.DEPLOYMENT_ID: pricing}, persist_across_reloads=False) try: yield pricing @@ -562,11 +563,11 @@ class TestZeroCostDiagnostic: priced_pricing: Final = {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06} usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) litellm.register_model( - model_cost={self.DEPLOYMENT_ID: self.PER_SECOND_PRICING, priced_id: priced_pricing}, + model_cost={self.DEPLOYMENT_ID: self.QUERY_ONLY_PRICING, priced_id: priced_pricing}, persist_across_reloads=False, ) try: - logging_obj: Final = self._logging_obj(self.PER_SECOND_PRICING) + logging_obj: Final = self._logging_obj(self.QUERY_ONLY_PRICING) with caplog.at_level(logging.WARNING, logger="LiteLLM"): assert logging_obj._response_cost_calculator(result=self._response(usage)) == 0.0 self._assert_flagged(logging_obj, caplog) @@ -575,7 +576,7 @@ class TestZeroCostDiagnostic: assert logging_obj._response_cost_calculator(result=self._response(usage)) == pytest.approx(5e-05) assert logging_obj.model_call_details["zero_cost_diagnostic"] is None - self._route_to_deployment(logging_obj, self.PER_SECOND_PRICING) + self._route_to_deployment(logging_obj, self.QUERY_ONLY_PRICING) assert logging_obj._response_cost_calculator(result=self._response(usage)) == 0.0 assert logging_obj.model_call_details["zero_cost_diagnostic"]["reason"] == "missing_pricing_key" @@ -590,7 +591,7 @@ class TestZeroCostDiagnostic: dated_model: Final = "lit7898-nano-2026-03-17" requested_model: Final = "lit7898-nano" usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) - cost_map_entry: Final = {"litellm_provider": "openai", "mode": "chat", **self.PER_SECOND_PRICING} + cost_map_entry: Final = {"litellm_provider": "openai", "mode": "chat", **self.QUERY_ONLY_PRICING} litellm.register_model( model_cost={dated_model: cost_map_entry, requested_model: cost_map_entry}, persist_across_reloads=False ) @@ -651,6 +652,24 @@ class TestZeroCostDiagnostic: assert logging_obj.model_call_details["zero_cost_diagnostic"] is None assert self._zero_cost_warnings(caplog) == [] + def test_per_second_priced_deployment_bills_the_call_duration_and_stays_silent( + self, caplog: pytest.LogCaptureFixture + ) -> None: + per_second_id: Final = "lit8315-per-second-priced-deployment" + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + litellm.register_model(model_cost={per_second_id: self.PER_SECOND_PRICING}, persist_across_reloads=False) + try: + logging_obj: Final = self._logging_obj(self.PER_SECOND_PRICING, deployment_id=per_second_id) + response: Final = self._response(usage) + response._response_ms = 1000.0 + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=response) == pytest.approx(0.00084) + + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + finally: + litellm.model_cost.pop(per_second_id, None) + @pytest.mark.parametrize("spilled_over", [True, False]) def test_ptu_deployment_is_judged_by_the_entry_the_calculator_priced_with( self, spilled_over: bool, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture @@ -668,7 +687,7 @@ class TestZeroCostDiagnostic: litellm.register_model( model_cost={ router_model_id: {**self.FREE_PRICING, "litellm_provider": "azure", "mode": "chat"}, - served_model: {**self.PER_SECOND_PRICING, "litellm_provider": "azure", "mode": "chat"}, + served_model: {**self.QUERY_ONLY_PRICING, "litellm_provider": "azure", "mode": "chat"}, }, persist_across_reloads=False, ) @@ -3509,6 +3528,20 @@ def _make_logging_obj(stream: bool) -> LitellmLogging: ) +def test_get_response_ms_measures_a_float_start_time_against_a_datetime_end_time(): + """The files paths construct the logging object with ``time.time()`` while the success + handler stamps a datetime end, and the per-second cost path reads this window.""" + logging_obj = _make_logging_obj(stream=False) + logging_obj.update_environment_variables( + model="openai/codex-mini-latest", user="", optional_params={}, litellm_params={} + ) + start_seconds = logging_obj.model_call_details["start_time"] + assert isinstance(start_seconds, float) + logging_obj.model_call_details["end_time"] = datetime.datetime.fromtimestamp(start_seconds + 1.5) + + assert logging_obj.get_response_ms() == pytest.approx(1500) + + def test_get_assembled_streaming_response_returns_none_for_non_streaming(): """Non-streaming requests should return None so the streaming block is skipped.""" import datetime diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index 276a67e0bd4..22298c00219 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -304,6 +304,26 @@ class TestPerformRedaction: assert delta["thinking_blocks"] is None assert delta["audio"] is None + def test_redacts_text_completion_choices_in_standard_logging_object(self): + details = { + "standard_logging_object": { + "response": { + "object": "text_completion", + "choices": [ + {"text": " Paris.", "finish_reason": "stop", "index": 0}, + {"text": "\n\nBlue", "finish_reason": "length", "index": 1}, + ], + } + } + } + + perform_redaction(details, None) + + assert details["standard_logging_object"]["response"]["choices"] == [ + {"text": "redacted-by-litellm", "finish_reason": "stop", "index": 0}, + {"text": "redacted-by-litellm", "finish_reason": "length", "index": 1}, + ] + def test_redacts_object_choices_inside_model_response_dict(self): result = { "choices": [ diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index f19a8891609..5ce6a4b1ce9 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -786,23 +786,23 @@ def test_token_counter(): import unittest -from litellm.utils import _select_tokenizer_helper, claude_json_str, encoding +from litellm.utils import _load_huggingface_tokenizer, _select_tokenizer_helper, claude_json_str, encoding # Clear the cache at module load to ensure clean state -_select_tokenizer_helper.cache_clear() +_load_huggingface_tokenizer.cache_clear() class TestTokenizerSelection(unittest.TestCase): def setUp(self): """Clear the LRU cache before each test method. - The _select_tokenizer_helper function is decorated with @lru_cache, - which can cause cache hits from previous tests when running with + The HuggingFace tokenizers behind _select_tokenizer_helper are cached with + @lru_cache, which can cause cache hits from previous tests when running with --dist=loadscope (tests from same file run on same worker). """ - _select_tokenizer_helper.cache_clear() + _load_huggingface_tokenizer.cache_clear() - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_llama3_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") @@ -817,7 +817,7 @@ class TestTokenizerSelection(unittest.TestCase): self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_cohere_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") @@ -837,10 +837,10 @@ class TestTokenizerSelection(unittest.TestCase): self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_str") - def test_claude_tokenizer_api_failure(self, mock_from_str): + @patch("litellm.utils.tokenizer_dispatch.anthropic") + def test_claude_tokenizer_api_failure(self, mock_anthropic): # Setup mock to raise an error - mock_from_str.side_effect = Exception("Failed to load tokenizer") + mock_anthropic.side_effect = Exception("Failed to load tokenizer") # Add Claude model to the list for testing litellm.anthropic_models = ["claude-2"] @@ -849,13 +849,13 @@ class TestTokenizerSelection(unittest.TestCase): result = _select_tokenizer_helper("claude-2") # Verify the attempt to load Claude tokenizer - mock_from_str.assert_called_once_with(claude_json_str) + mock_anthropic.assert_called_once_with() # Verify fallback to OpenAI tokenizer self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_llama2_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") diff --git a/tests/test_litellm/litellm_core_utils/test_tokenizer.py b/tests/test_litellm/litellm_core_utils/test_tokenizer.py new file mode 100644 index 00000000000..aa4a0fc6a1c --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_tokenizer.py @@ -0,0 +1,403 @@ +import copy +import os +import pickle +import subprocess +import sys +from pathlib import Path +from typing import Final, Literal + +import pytest +import tiktoken +from tokenizers import Tokenizer as ReferenceTokenizer + +import litellm +from litellm.caching._embedding_router import truncate_embedding_input +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding +from litellm.utils import claude_json_str +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + + +@pytest.mark.parametrize( + "name", ("cl100k_base", "o200k_base", "p50k_base", "p50k_edit", "r50k_base", "gpt2", "o200k_harmony") +) +@pytest.mark.parametrize( + "text", ("hello world", "café 漢字 🙂", "", "a\ud800b", "\ud83d\ude42", "🙂\ud83d\ude42\udfff", " " * 64) +) +def test_openai_encoding_matches_python_unicode_and_batches(name: str, text: str) -> None: + reference: Final = tiktoken.get_encoding(name) + encoding: Final = OpenAIEncoding.from_tiktoken(name) + expected: Final = reference.encode(text) + + assert encoding.encode(text) == expected + assert encoding.count(text) == len(expected) + assert encoding.encode_batch([text], num_threads=2) == reference.encode_batch([text], num_threads=2) + assert encoding.encode_ordinary_batch([text]) == reference.encode_ordinary_batch([text]) + assert encoding.decode_batch([expected]) == reference.decode_batch([expected]) + assert encoding.decode_bytes_batch([expected]) == reference.decode_bytes_batch([expected]) + + +@pytest.mark.parametrize("allowed", (frozenset(), frozenset({"<|endoftext|>"}), "all")) +@pytest.mark.parametrize("disallowed", (frozenset(), frozenset({"<|fim_prefix|>"}), "all")) +def test_openai_special_token_options_match_python( + allowed: frozenset[str] | Literal["all"], disallowed: frozenset[str] | Literal["all"] +) -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + encoding: Final = OpenAIEncoding.from_tiktoken(reference.name) + text: Final = "hello<|endoftext|><|fim_prefix|>world" + allowed_set: Final = reference.special_tokens_set if allowed == "all" else allowed + disallowed_set: Final = reference.special_tokens_set - allowed_set if disallowed == "all" else disallowed + if any(token in text for token in disallowed_set): + with pytest.raises(ValueError, match="disallowed special token"): + encoding.encode(text, allowed_special=allowed, disallowed_special=disallowed) + return + assert encoding.encode(text, allowed_special=allowed, disallowed_special=disallowed) == reference.encode( + text, allowed_special=allowed, disallowed_special=disallowed + ) + assert encoding.special_tokens_set == reference.special_tokens_set + assert encoding.eot_token == reference.eot_token + + +@pytest.mark.parametrize("errors", ("replace", "ignore", "backslashreplace", "strict")) +def test_openai_partial_token_decoding_preserves_error_policy(errors: str) -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + encoding: Final = OpenAIEncoding.from_tiktoken(reference.name) + tokens: Final = reference.encode("🙂")[:1] + assert encoding.decode_bytes(tokens) == reference.decode_bytes(tokens) + if errors == "strict": + with pytest.raises(UnicodeDecodeError): + encoding.decode(tokens, errors=errors) + return + assert encoding.decode(tokens, errors=errors) == reference.decode(tokens, errors=errors) + assert encoding.decode_tokens_bytes(tokens) == reference.decode_tokens_bytes(tokens) + + +def test_public_encoding_and_semantic_cache_preserve_truncated_unicode() -> None: + reference: Final = tiktoken.get_encoding(litellm.encoding.name) + text: Final = "🙂" + tokens: Final = reference.encode(text) + + assert litellm.encoding.encode(text, disallowed_special=()) == tokens + assert litellm.encoding.encode_batch([text]) == [tokens] + assert litellm.decode(tokens=tokens[:1]) == reference.decode(tokens[:1]) + assert truncate_embedding_input(text, "", 1) == reference.decode(tokens[:1]) + + +@pytest.mark.parametrize("add_special_tokens", (True, False)) +def test_huggingface_encoding_preserves_result_fields_and_serialization(add_special_tokens: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + tokenizer: Final = HuggingFaceTokenizer.from_str(TOKENIZER_JSON) + expected: Final = reference.encode("Hello World", add_special_tokens=add_special_tokens) + actual: Final = tokenizer.encode("Hello World", add_special_tokens=add_special_tokens) + + assert (actual.ids, actual.tokens, actual.type_ids, actual.offsets, actual.word_ids, actual.sequence_ids) == ( + expected.ids, + expected.tokens, + expected.type_ids, + expected.offsets, + expected.word_ids, + expected.sequence_ids, + ) + assert (actual.attention_mask, actual.special_tokens_mask, actual.n_sequences, len(actual)) == ( + expected.attention_mask, + expected.special_tokens_mask, + expected.n_sequences, + len(expected), + ) + assert copy.deepcopy(actual).ids == expected.ids + assert pickle.loads(pickle.dumps(actual)).offsets == expected.offsets + assert tokenizer.decode(actual.ids, skip_special_tokens=False) == reference.decode( + expected.ids, skip_special_tokens=False + ) + + +def test_huggingface_character_offsets_and_pretokenized_pairs_match_python() -> None: + reference: Final = ReferenceTokenizer.from_str(claude_json_str) + tokenizer: Final = HuggingFaceTokenizer.from_str(claude_json_str) + text: Final = "café 漢字 🙂" + actual: Final = tokenizer.encode(text) + expected: Final = reference.encode(text) + + assert actual.offsets == expected.offsets + assert actual.ids == expected.ids + assert ( + tokenizer.encode(["hello", "world"], ["again"], is_pretokenized=True).ids + == reference.encode(["hello", "world"], ["again"], is_pretokenized=True).ids + ) + + +def test_huggingface_batches_apply_padding_across_inputs() -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + reference.enable_padding(pad_id=0, pad_token="[UNK]") + tokenizer: Final = HuggingFaceTokenizer.from_str(reference.to_str()) + inputs: Final = ["Hello", ("Hello World", "World")] + expected: Final = reference.encode_batch(inputs) + actual: Final = tokenizer.encode_batch(inputs) + fast: Final = tokenizer.encode_batch_fast(inputs) + + assert [(item.ids, item.attention_mask, item.offsets) for item in actual] == [ + (item.ids, item.attention_mask, item.offsets) for item in expected + ] + assert [item.ids for item in fast] == [item.ids for item in expected] + assert tokenizer.decode_batch([item.ids for item in actual]) == reference.decode_batch( + [item.ids for item in expected] + ) + + +def test_caller_supplied_huggingface_tokenizer_preserves_public_encode_and_count() -> None: + tokenizer: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + custom: Final = {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + expected: Final = tokenizer.encode("Hello World").ids + + assert litellm.encode(text="Hello World", custom_tokenizer=custom) == expected + assert litellm.token_counter(text="Hello World", custom_tokenizer=custom) == len(expected) + assert litellm.decode(tokens=expected, custom_tokenizer=custom) == "Hello World" + + +def test_caller_supplied_tiktoken_treats_special_spellings_as_text() -> None: + tokenizer: Final = tiktoken.get_encoding("cl100k_base") + custom: Final = {"type": "openai_tokenizer", "tokenizer": tokenizer} + text: Final = "<|endoftext|>" + + assert litellm.encode(text=text, custom_tokenizer=custom) == tokenizer.encode(text, disallowed_special=()) + + +def test_public_tokenizer_objects_survive_pickle_and_deepcopy(tmp_path: Path) -> None: + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) + tokenizer: Final = custom["tokenizer"] + path: Final = tmp_path / "tokenizer.json" + tokenizer.save(str(path)) + + assert copy.deepcopy(custom)["tokenizer"].encode("Hello World").ids == tokenizer.encode("Hello World").ids + assert ( + pickle.loads(pickle.dumps(custom))["tokenizer"].encode("Hello World").ids == tokenizer.encode("Hello World").ids + ) + assert HuggingFaceTokenizer.from_file(str(path)).encode("Hello World").ids == tokenizer.encode("Hello World").ids + assert copy.deepcopy(litellm.encoding).encode("hello") == litellm.encoding.encode("hello") + assert pickle.loads(pickle.dumps(litellm.encoding)).encode("hello") == litellm.encoding.encode("hello") + + +@pytest.mark.parametrize("offline", ("0", "1")) +def test_hub_loader_preserves_environment_auth_cache_and_offline(tmp_path: Path, offline: str) -> None: + script: Final = """ +import json +import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import httpx +import huggingface_hub +from huggingface_hub.errors import LocalEntryNotFoundError +import litellm +payload = sys.argv[2].encode() +offline = sys.argv[3] == "1" +observed = [] +def handle(request): + assert not offline, "offline loading issued a request" + if request.url.path.endswith("/tokenizer.json"): + observed.append(request.headers.get("authorization")) + if request.headers.get("authorization") != "Bearer audit-fixture-token": + return httpx.Response(401) + return httpx.Response(200, headers={"content-length": str(len(payload)), "etag": '"fixture"', "x-repo-commit": "a" * 40}, content=payload if request.method == "GET" else b"") +if not offline: + huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) +try: + tokenizer = litellm.create_pretrained_tokenizer("test-fixture/tokenizer")["tokenizer"] +except LocalEntryNotFoundError: + assert offline + assert observed == [] +else: + assert not offline + assert "Bearer audit-fixture-token" in observed + assert tokenizer.decode(tokenizer.encode("Hello World").ids) == "Hello World" + assert tuple(Path(sys.argv[4]).rglob("tokenizer.json")) +print("compatible") +""" + result: Final = subprocess.run( + [ + sys.executable, + "-I", + "-c", + script, + str(Path(litellm.__file__).parent.parent), + TOKENIZER_JSON, + offline, + str(tmp_path / "cache"), + ], + capture_output=True, + text=True, + timeout=30, + env={ + **os.environ, + "HF_HOME": str(tmp_path / "home"), + "HF_HUB_CACHE": str(tmp_path / "cache"), + "HF_ENDPOINT": "http://127.0.0.1:9", + "HF_TOKEN": "audit-fixture-token", + "HF_HUB_OFFLINE": offline, + "HF_HUB_DISABLE_IMPLICIT_TOKEN": "0", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + + +@pytest.mark.parametrize("rust", (None, "0", "1")) +def test_tokenization_without_native_extension_stays_offline(tmp_path: Path, rust: str | None) -> None: + script: Final = """ +import importlib.abc +import sys +sys.path.insert(0, sys.argv[1]) +def reject_network(event, args): + if event == "socket.connect": + raise AssertionError("tokenizer attempted a network connection") +sys.addaudithook(reject_network) +class Block(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "litellm.rust_bridge._native": + raise ImportError("native extension is unavailable") +sys.meta_path.insert(0, Block()) +import litellm +from litellm.rust_bridge.tokenizer import get_encoding +import tiktoken +from tokenizers import Tokenizer +assert isinstance(litellm.encoding, tiktoken.Encoding) +for name in ("cl100k_base", "o200k_base", "o200k_harmony", "p50k_base", "p50k_edit"): + encoding = get_encoding(name) + text = "offline café 漢字 🙂" + " " * 64 + assert encoding.decode(encoding.encode(text)) == text +ids = litellm.encode(text="hello world") +assert litellm.decode(tokens=ids) == "hello world" +assert litellm.token_counter(model=None, text="hello world") == len(ids) +custom = litellm.create_tokenizer(sys.argv[2]) +assert isinstance(custom["tokenizer"], Tokenizer) +custom["tokenizer"].enable_padding(pad_id=0, pad_token="[UNK]") +assert litellm.decode(tokens=litellm.encode(text="Hello World", custom_tokenizer=custom), custom_tokenizer=custom) == "Hello World" +print("compatible") +""" + result: Final = subprocess.run( + [sys.executable, "-I", "-c", script, str(Path(litellm.__file__).parent.parent), TOKENIZER_JSON], + capture_output=True, + text=True, + timeout=30, + cwd=tmp_path, + env={ + **{key: value for key, value in os.environ.items() if key != "LITELLM_RUST"}, + **({"LITELLM_RUST": rust} if rust is not None else {}), + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "TIKTOKEN_CACHE_DIR": str(tmp_path / "unused-tokenizer-cache"), + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + assert not (tmp_path / "unused-tokenizer-cache").exists() + + +@pytest.mark.parametrize("is_pretokenized", (False, True)) +def test_huggingface_batch_sequence_containers_match_python(is_pretokenized: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + tokenizer: Final = HuggingFaceTokenizer.from_str(TOKENIZER_JSON) + inputs: Final = [["Hello", "World"], ("Hello", "World")] + actual: Final = tokenizer.encode_batch(inputs, is_pretokenized=is_pretokenized) + expected: Final = reference.encode_batch(inputs, is_pretokenized=is_pretokenized) + assert [(item.ids, item.type_ids, item.sequence_ids) for item in actual] == [ + (item.ids, item.type_ids, item.sequence_ids) for item in expected + ] + + +@pytest.mark.parametrize("name", ("cl100k_base", "o200k_base", "p50k_edit", "gpt2")) +def test_openai_encoding_exposes_the_tiktoken_vocabulary_surface(name: str) -> None: + reference: Final = tiktoken.get_encoding(name) + encoding: Final = OpenAIEncoding.from_tiktoken(name) + text: Final = "hello fanta" + + assert repr(encoding) == repr(reference) == f"" + assert (encoding.name, encoding.n_vocab, encoding.max_token_value) == ( + reference.name, + reference.n_vocab, + reference.max_token_value, + ) + assert encoding.token_byte_values() == reference.token_byte_values() + assert encoding.encode_single_token("hello") == reference.encode_single_token("hello") + assert encoding.encode_single_token(b"<|endoftext|>") == reference.eot_token + assert [encoding.is_special_token(token) for token in (0, reference.eot_token)] == [False, True] + assert encoding.decode_with_offsets(reference.encode(text)) == reference.decode_with_offsets(reference.encode(text)) + assert encoding.encode_to_numpy(text).tolist() == reference.encode_to_numpy(text).tolist() + stable, completions = encoding.encode_with_unstable(text) + expected_stable, expected_completions = reference.encode_with_unstable(text) + assert (stable, sorted(completions)) == (expected_stable, sorted(expected_completions)) + with pytest.raises(KeyError): + encoding.encode_single_token("<|not-a-token|>") + + +def test_huggingface_tokenizer_exposes_the_tokenizers_vocabulary_surface() -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + reference.enable_padding(pad_id=0, pad_token="[UNK]", length=4) + reference.enable_truncation(max_length=3, stride=1, strategy="only_first", direction="left") + tokenizer: Final = HuggingFaceTokenizer.from_str(reference.to_str()) + + assert tokenizer.token_to_id("Hello") == reference.token_to_id("Hello") == 1 + assert tokenizer.id_to_token(3) == reference.id_to_token(3) == "[BOS]" + assert tokenizer.id_to_token(99) is None + assert tokenizer.get_vocab() == reference.get_vocab() + assert tokenizer.get_vocab(with_added_tokens=False) == reference.get_vocab(with_added_tokens=False) + assert tokenizer.get_vocab_size() == reference.get_vocab_size() == 4 + assert tokenizer.get_vocab_size(with_added_tokens=False) == reference.get_vocab_size(with_added_tokens=False) + added: Final = tokenizer.get_added_tokens_decoder() + expected_added: Final = reference.get_added_tokens_decoder() + assert {token_id: str(token) for token_id, token in added.items()} == { + token_id: str(token) for token_id, token in expected_added.items() + } + assert added[3].special == expected_added[3].special + assert tokenizer.num_special_tokens_to_add(False) == reference.num_special_tokens_to_add(False) == 1 + assert tokenizer.num_special_tokens_to_add(True) == reference.num_special_tokens_to_add(True) == 0 + assert tokenizer.padding == reference.padding + assert tokenizer.truncation == reference.truncation + assert tokenizer.encode_special_tokens == reference.encode_special_tokens is False + assert HuggingFaceTokenizer.from_buffer(TOKENIZER_JSON.encode()).encode("Hello").ids == [3, 1] + assert HuggingFaceTokenizer.from_str(TOKENIZER_JSON).padding is None + assert HuggingFaceTokenizer.from_str(TOKENIZER_JSON).truncation is None + + +def test_huggingface_encoding_exposes_the_tokenizers_lookup_and_mutation_surface() -> None: + reference: Final = ReferenceTokenizer.from_str(claude_json_str) + tokenizer: Final = HuggingFaceTokenizer.from_str(claude_json_str) + text: Final = "hello wide world" + actual: Final = tokenizer.encode(text, "again") + expected: Final = reference.encode(text, "again") + + lookups: Final = ( + lambda encoding: [encoding.token_to_chars(index) for index in range(len(encoding))], + lambda encoding: [encoding.token_to_word(index) for index in range(len(encoding))], + lambda encoding: [encoding.token_to_sequence(index) for index in range(len(encoding))], + lambda encoding: [encoding.char_to_token(position) for position in range(len(text))], + lambda encoding: [encoding.char_to_word(position) for position in range(len(text))], + lambda encoding: [encoding.char_to_token(position, 1) for position in range(5)], + lambda encoding: [encoding.word_to_tokens(word) for word in range(3)], + lambda encoding: [encoding.word_to_chars(word) for word in range(3)], + lambda encoding: [encoding.word_to_tokens(0, 1), encoding.word_to_chars(0, 1)], + ) + for lookup in lookups: + assert lookup(actual) == lookup(expected) + assert repr(actual) == repr(expected) + + actual.truncate(4, stride=1, direction="left") + expected.truncate(4, stride=1, direction="left") + assert (actual.ids, [item.ids for item in actual.overflowing]) == ( + expected.ids, + [item.ids for item in expected.overflowing], + ) + actual.pad(6, direction="left", pad_id=7, pad_type_id=1, pad_token="") + expected.pad(6, direction="left", pad_id=7, pad_type_id=1, pad_token="") + assert (actual.ids, actual.attention_mask, actual.type_ids, actual.tokens) == ( + expected.ids, + expected.attention_mask, + expected.type_ids, + expected.tokens, + ) + actual.set_sequence_id(3) + expected.set_sequence_id(3) + assert actual.sequence_ids == expected.sequence_ids + merged: Final = type(actual).merge([actual, tokenizer.encode("more")]) + assert merged.ids == type(expected).merge([expected, reference.encode("more")]).ids + assert merged.offsets == type(expected).merge([expected, reference.encode("more")]).offsets + with pytest.raises(ValueError, match="direction"): + actual.pad(8, direction="sideways") diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 633dd1d9460..7167a67d80d 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,8 +1,12 @@ -import pytest - +import json +from typing import Final from unittest.mock import MagicMock, patch +import httpx +import pytest +import respx + import litellm from litellm.constants import ( ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, @@ -3771,6 +3775,70 @@ def test_multiple_compaction_blocks(): assert compaction_blocks[1]["content"] == "Second summary..." +@pytest.mark.parametrize("messages_api,gateway,native_endpoint", [ + (False, False, False), (True, False, False), (False, True, False), (True, True, False), (True, True, True), +]) +async def test_native_compaction_wire_roundtrip( + messages_api: bool, gateway: bool, native_endpoint: bool, + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True") + monkeypatch.setattr(litellm.anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None) + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False) + block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"} + operation: Final = {"type": "summarize", "instructions": "Keep identifiers"} + usage: Final = {"input_tokens": 0, "output_tokens": 0, + "iterations": [{"type": "compaction", "input_tokens": 103, "output_tokens": 165}]} + chat_wire: Final = gateway and not native_endpoint + base: Final = "https://gateway.test/v1" if gateway else "https://api.anthropic.com/v1" + route: Final = respx_mock.post(f"{base}/{'chat/completions' if chat_wire else 'messages'}") + + def respond(request: httpx.Request) -> httpx.Response: + payload: Final = json.loads(request.content) + assert len(request.headers.get_list("anthropic-beta")) == 1 + assert {value.strip() for value in request.headers["anthropic-beta"].split(",")} == { + "compact-2026-09-04", "interleaved-thinking-2025-05-14", + } + if "compaction" in payload: + assert payload["compaction"] == operation + else: + assert payload["messages"][0] == {"role": "assistant", "content": [block]} + body: Final = ( + {"id": "chatcmpl_compact", "object": "chat.completion", "created": 1, "model": "claude-sonnet-5", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "", + "provider_specific_fields": {"compaction_blocks": [block]}}}], + "usage": {"prompt_tokens": 103, "completion_tokens": 165, "total_tokens": 268}} + if chat_wire else + {"id": "msg_compact", "type": "message", "role": "assistant", "model": "claude-sonnet-5", + "content": [block], "stop_reason": "compaction", "usage": usage} + ) + return httpx.Response(200, json=body) + + route.mock(side_effect=respond) + call: Final = litellm.anthropic.messages.acreate if messages_api else litellm.acompletion + params: Final = dict( + model=f"{'openai/' if gateway else ''}anthropic/claude-sonnet-5", api_key="test", max_tokens=512, + api_base=base if gateway else "https://api.anthropic.com", + extra_headers={"Anthropic-Beta": f"interleaved-thinking-2025-05-14{',compact-2026-09-04' if gateway else ''}"}, + model_info={"supported_endpoints": ["/v1/messages"]} if native_endpoint else {}, + ) + response: Final = await call( + messages=[{"role": "user", "content": "Remember identifiers"}], compaction=operation, **params + ) + message: Final = response if messages_api else response.choices[0].message.model_dump() + blocks: Final = message["content"] if messages_api else message["provider_specific_fields"]["compaction_blocks"] + assert blocks == [block] + if messages_api: + assert response["stop_reason"] == "compaction" + if not chat_wire: + assert response["usage"] == usage + if not gateway: + replay: Final = {"role": "assistant", "content": blocks} if messages_api else message + await call(messages=[replay, {"role": "user", "content": "Continue"}], **params) + assert route.call_count == (1 if gateway else 2) + + def test_compaction_block_request_transformation(): """ Test that compaction blocks from provider_specific_fields are correctly diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index b9a82e3fc68..d03174bc2c6 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -60,6 +60,20 @@ def test_translate_openai_response_to_anthropic_empty_choices() -> None: assert result["usage"]["input_tokens"] == 10 +@pytest.mark.parametrize("text,count,expected_stop", [ + ("", 1, "compaction"), (None, 1, "compaction"), ("Answer", 1, "max_tokens"), + (" ", 1, "max_tokens"), ("", 2, "max_tokens"), ("", 0, "max_tokens"), +]) +def test_native_compaction_response_roundtrip(text: str | None, count: int, expected_stop: str) -> None: + block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"} + message: Final = Message(content=text, provider_specific_fields={"compaction_blocks": [block] * count}) + response: Final = ModelResponse(choices=[Choices(message=message, finish_reason="length")], usage=Usage()) + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + expected_text: Final = [{"type": "text", "text": text}] if text is not None and (text != "" or not count) else [] + assert result["content"] == [*([block] * count), *expected_text] + assert result["stop_reason"] == expected_stop + + def test_translate_chat_refusal_to_anthropic_response(): response = ModelResponse( id="chatcmpl-refusal", diff --git a/tests/test_litellm/llms/custom_httpx/test_asgi_handler.py b/tests/test_litellm/llms/custom_httpx/test_asgi_handler.py new file mode 100644 index 00000000000..dbfa066a02d --- /dev/null +++ b/tests/test_litellm/llms/custom_httpx/test_asgi_handler.py @@ -0,0 +1,53 @@ +import asyncio +from collections.abc import Mapping +from typing import Final + +import pytest +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse, RedirectResponse +from starlette.routing import Route + +from litellm.llms.custom_httpx.asgi_handler import get_async_asgi_client + + +@pytest.mark.asyncio +async def test_cached_client_isolates_concurrent_apps_and_request_credentials() -> None: + ready: Final = (asyncio.Event(), asyncio.Event()) + + async def call(index: int) -> Mapping[str, object]: + async def endpoint(request: Request) -> JSONResponse: + ready[index].set() + await ready[1 - index].wait() + assert request.scope["root_path"] == f"/gateway-{index}" + assert request.client == (f"192.0.2.{index + 1}", 4321) + assert request.headers["authorization"] == f"Bearer key-{index}" + return JSONResponse({"app": index}, headers={"set-cookie": f"session=app-{index}; Path=/"}) + + app: Final = Starlette(routes=[Route("/child", endpoint, methods=["POST"])]) + with get_async_asgi_client(app, f"/gateway-{index}", (f"192.0.2.{index + 1}", 4321)) as client: + response: Final = await client.post( + f"https://proxy.test/gateway-{index}/child", headers={"authorization": f"Bearer key-{index}"}, + ) + assert response.status_code == 200 + assert not client.cookies + with get_async_asgi_client(app) as reused: + assert reused is client + return response.json() + + results: Final = await asyncio.wait_for(asyncio.gather(call(0), call(1)), timeout=5) + assert results == [{"app": 0}, {"app": 1}] + + +@pytest.mark.asyncio +async def test_internal_client_does_not_follow_redirects_or_environment_proxies(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HTTPS_PROXY", "http://unreachable.invalid:8080") + + async def endpoint(request: Request) -> RedirectResponse: + return RedirectResponse("https://external.invalid/credentials") + + app: Final = Starlette(routes=[Route("/redirect", endpoint, methods=["POST"])]) + with get_async_asgi_client(app) as client: + response: Final = await client.post("https://proxy.test/redirect", headers={"authorization": "Bearer fixture"}) + assert response.status_code == 307 + assert not response.history diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 9f42a523350..7725aca1948 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -14470,6 +14470,21 @@ async def test_request_selected_during_guardrail_runs_concurrently_with_tool(mon assert result.content[0].text == "executed" +@pytest.mark.asyncio +@pytest.mark.parametrize("in_config,in_db,expected", [(True, False, True), (False, True, False), (True, True, False)]) +async def test_server_response_identifies_read_only_config(in_config, in_db, expected): + manager = MCPServerManager() + server = MCPServer(server_id="source-server", name="source_server", transport=MCPTransport.http) + manager.config_mcp_servers = {server.server_id: server} if in_config else {} + manager.registry = {server.server_id: server} if in_db else {} + + listed = await manager.get_all_mcp_servers_unfiltered() + + assert len(listed) == 1 + assert listed[0].model_dump().get("is_config") is expected + assert manager._build_mcp_server_table(server).model_dump().get("is_config") is expected + + @pytest.mark.asyncio @pytest.mark.parametrize("with_caller,legacy_factory", [(True, False), (False, False), (True, True)]) async def test_client_sampling_does_not_fill_explicit_context_from_another_ambient_caller(with_caller, legacy_factory): diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 33614d2eeca..89e72debbf5 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -4,6 +4,7 @@ Tests PII detection and masking for different message formats """ import asyncio +import json from contextlib import asynccontextmanager from unittest.mock import MagicMock, patch @@ -18,7 +19,7 @@ from litellm.proxy.guardrails.guardrail_hooks.presidio import ( ) from litellm.exceptions import GuardrailRaisedException from litellm.types.guardrails import LitellmParams, PiiAction, PiiEntityType -from litellm.types.utils import Choices, Message, ModelResponse +from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices from litellm.exceptions import BlockedPiiEntityError @@ -2331,47 +2332,320 @@ async def test_apply_guardrail_masks_on_request(): assert "John Smith" not in result["texts"][0] +def _anthropic_sse(event_type: str, payload: dict) -> bytes: + return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode() + + +def _anthropic_text_deltas(chunks: list[bytes]) -> list[tuple[int, str]]: + deltas = [] + for line in b"".join(chunks).decode().split("\n"): + if not line.startswith("data: "): + continue + event = json.loads(line[6:]) + if event.get("type") == "content_block_delta" and event["delta"].get("type") == "text_delta": + deltas.append((event["index"], event["delta"]["text"])) + return deltas + + +def _chat_delta_chunk(text: str, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-out-mask", + choices=[StreamingChoices(index=0, delta=Delta(content=text, role="assistant"), finish_reason=finish_reason)], + created=1, + model="gpt-4", + object="chat.completion.chunk", + ) + + @pytest.mark.asyncio -async def test_apply_to_output_streaming_bytes_only_logs_warning(): +async def test_apply_to_output_streaming_chat_chunks_are_masked_as_one_response(): """ - Regression test: when apply_to_output=True and the stream contains only - bytes chunks (Anthropic native SSE), output masking is skipped. - A warning must be logged so operators are aware. + Structured chat completion chunks are buffered, assembled and masked as a + whole, so a card number split across deltas cannot reach the caller. """ guardrail = _OPTIONAL_PresidioPIIMasking( mock_testing=True, apply_to_output=True, + mock_redacted_text={"text": "my card is "}, + ) + + async def mock_stream(): + yield _chat_delta_chunk("my card is 4111") + yield _chat_delta_chunk(" 1111 1111 1111") + yield _chat_delta_chunk("", finish_reason="stop") + + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={"messages": [{"role": "user", "content": "what is my card"}]}, + ): + collected.append(chunk) + + assert all(isinstance(chunk, ModelResponseStream) for chunk in collected) + joined = "".join(chunk.choices[0].delta.content or "" for chunk in collected) + assert joined == "my card is " + assert collected[-1].choices[0].finish_reason == "stop" + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_bytes_after_chat_chunks_are_passed_through_in_order(): + """ + Once structured chunks have been buffered, a trailing bytes frame belongs to + the same stream and must be forwarded rather than treated as a new SSE stream. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": "hello"}, + ) + trailer = b"data: [DONE]\n\n" + + async def mock_stream(): + yield _chat_delta_chunk("hello", finish_reason="stop") + yield trailer + + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert collected[0] == trailer + assert len(collected) == 2 + assert isinstance(collected[1], ModelResponseStream) + assert collected[1].choices[0].delta.content == "hello" + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_masks_text_split_across_deltas(): + """ + Anthropic native /v1/messages streams reach the post_call hook as raw SSE + bytes. Output masking must run over the whole content block so a card + number split across text_delta events cannot reach the caller. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": ""}, ) byte_chunks = [ - b'data: {"type":"content_block_delta","delta":{"text":"Hello"}}\n\n', - b'data: {"type":"content_block_delta","delta":{"text":" world"}}\n\n', + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "4111"}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " 1111 1111 1111"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {}}), + _anthropic_sse("message_stop", {"type": "message_stop"}), ] async def mock_stream(): for b in byte_chunks: yield b - mock_user_api_key = UserAPIKeyAuth(api_key="test-key") + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert all(isinstance(chunk, bytes) for chunk in collected) + joined = b"".join(collected).decode() + assert "4111" not in joined + assert "".join(text for _, text in _anthropic_text_deltas(collected)) == "" + assert joined.count("event: message_start") == 1 + assert joined.count("event: message_stop") == 1 + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_without_pii_are_forwarded_unchanged(): + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": "Hello world"}, + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello"}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " world"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {}}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b collected = [] - with patch("litellm.proxy.guardrails.guardrail_hooks.presidio.verbose_proxy_logger") as mock_logger: + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert collected == byte_chunks + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_fail_closed_when_presidio_is_unreachable(): + """ + The raw SSE stream is fully drained before masking, so a Presidio outage + must surface as an error to the caller: replaying the unscanned frames + would hand over whatever PII the model generated. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + presidio_analyzer_api_base="http://127.0.0.1:9", + presidio_anonymizer_api_base="http://127.0.0.1:9", + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello world"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b + + collected = [] + + async def collect_masked_stream(): async for chunk in guardrail.async_post_call_streaming_iterator_hook( - user_api_key_dict=mock_user_api_key, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), response=mock_stream(), request_data={}, ): collected.append(chunk) - # All bytes should be yielded through - assert len(collected) == len(byte_chunks) - for original, received in zip(byte_chunks, collected): - assert original == received + with pytest.raises(Exception, match="Presidio PII analysis failed"): + await collect_masked_stream() - # Warning must be logged about skipped masking - mock_logger.warning.assert_called_once() - warning_msg = mock_logger.warning.call_args[0][0] - assert "Output PII masking was skipped" in warning_msg + assert collected == [] + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_block_action_raises_instead_of_replaying(): + """ + A BLOCK on generated PII must refuse the streaming /v1/messages response the + same way it refuses the non streaming one, not replay the raw frames. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + apply_to_output=True, + mock_testing=False, + presidio_analyzer_api_base="http://test-analyzer/", + presidio_anonymizer_api_base="http://test-anonymizer/", + pii_entities_config={PiiEntityType.CREDIT_CARD: PiiAction.BLOCK}, + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "4111 1111 1111 1111"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b + + analyzer_hit = [{"entity_type": "CREDIT_CARD", "score": 0.99, "start": 0, "end": 19}] + collected = [] + + async def collect_masked_stream(): + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + with patch.object(guardrail, "_get_session_iterator", _make_mock_session_iterator(analyzer_hit)): + with pytest.raises(BlockedPiiEntityError): + await collect_masked_stream() + + assert collected == [] + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_propagates_upstream_error_when_nothing_was_buffered(): + """ + An upstream guardrail that rejects the stream before the first chunk must + surface as an error to the caller, not as an empty 200 stream. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": ""}, + ) + + async def failing_stream(): + raise RuntimeError("upstream guardrail rejected the stream") + yield b"" + + with pytest.raises(RuntimeError, match="upstream guardrail rejected the stream"): + async for _ in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=failing_stream(), + request_data={}, + ): + pass @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 4907b4ea054..0f19675edb9 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -6183,10 +6183,10 @@ async def test_configured_estimate_blocks_the_overrun_the_static_floor_admits(mo assert await admitted({"default_estimated_output_tokens": 3000}) == 2 -def test_internal_call_origin_success_ops_are_skipped(): - """Internal sub-calls (auto-router classifier, shadow eval shadow/judge) bill spend - to the caller's key but must not consume its TPM counters: the same kwargs charge - ops without the origin stamp and none with it.""" +@pytest.mark.parametrize("origin", ["shadow_eval_judge", "autorouter_compaction"]) +@pytest.mark.parametrize("rate_limit_type", ["input", "output", "total"]) +def test_internal_call_origin_success_ops_are_skipped(origin, rate_limit_type): + """Foreground compaction charges the same scopes as ordinary caller traffic.""" handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(DualCache()) ) @@ -6202,23 +6202,27 @@ def test_internal_call_origin_success_ops_are_skipped(): def _kwargs(metadata: Dict[str, Any]) -> Dict[str, Any]: return { "standard_logging_object": { - "metadata": {"user_api_key_hash": hash_token("sk-internal-origin")} + "metadata": { + "user_api_key_hash": hash_token("sk-internal-origin"), + "user_api_key_team_id": "compaction-team", + "user_api_key_project_id": "compaction-project", + } }, "litellm_params": {"metadata": metadata}, "model": "gpt-4o-mini", } charged = handler._build_success_event_pipeline_operations( - kwargs=_kwargs({}), response_obj=response, rate_limit_type="output" + kwargs=_kwargs({}), response_obj=response, rate_limit_type=rate_limit_type ) skipped = handler._build_success_event_pipeline_operations( - kwargs=_kwargs({INTERNAL_CALL_ORIGIN_METADATA_KEY: "shadow_eval_judge"}), + kwargs=_kwargs({INTERNAL_CALL_ORIGIN_METADATA_KEY: origin}), response_obj=response, - rate_limit_type="output", + rate_limit_type=rate_limit_type, ) assert charged - assert skipped == [] + assert skipped == (charged if origin == "autorouter_compaction" else []) def _conflicting_budget_bodies() -> Dict[str, Dict[str, object]]: diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 75545a574e3..f684e2040bd 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -1,8 +1,10 @@ import hashlib import json +import logging from datetime import datetime, timezone from types import SimpleNamespace from typing import Final +from unittest.mock import AsyncMock, MagicMock import httpx import pytest @@ -17,12 +19,15 @@ from litellm.proxy._types import ( LiteLLM_UserTableFiltered, LitellmUserRoles, NewUserRequest, + ProxyErrorTypes, ProxyException, UpdateUserRequest, UserAPIKeyAuth, ) from litellm.proxy.management_endpoints.internal_user_endpoints import ( LiteLLM_UserTableWithKeyCount, + _authorize_user_list_request, + _resolve_org_filter_for_user_search, _resolve_user_email_metadata, _update_internal_user_params, get_user_key_counts, @@ -4653,3 +4658,124 @@ async def test_delete_user_evicts_cached_user_rows(mocker: MockerFixture) -> Non assert await cache.async_get_cache(key=deleted.user_id, model_type=LiteLLM_UserTable) is None assert await cache.async_get_cache(key=survivor.user_id, model_type=LiteLLM_UserTable) == survivor broadcast.assert_awaited_once_with(cache_key=deleted.user_id) + + +_DB_OUTAGE_503_BODY: Final = { + "error": { + "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + "type": "no_db_connection", + "param": "None", + "code": "503", + } +} + + +def _user_read_raising(mocker: MockerFixture, error: Exception) -> tuple[MagicMock, MagicMock]: + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock(side_effect=error) + cache = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + cache.async_set_cache = AsyncMock() + mocker.patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True) + return prisma_client, cache + + +def _db_unavailable_fallback_identity(route: str) -> UserAPIKeyAuth: + from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID + + return UserAPIKeyAuth( + key_name="failed-to-connect-to-db", + token="failed-to-connect-to-db", + user_id=DB_UNAVAILABLE_FALLBACK_USER_ID, + user_role=LitellmUserRoles.INTERNAL_USER, + request_route=route, + ) + + +@pytest.mark.asyncio +async def test_authorize_user_list_request_propagates_a_db_outage_instead_of_answering_403(mocker): + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + + with pytest.raises(httpx.ConnectError): + await _authorize_user_list_request( + user_api_key_dict=_db_unavailable_fallback_identity("/user/list"), + organization_ids=None, + prisma_client=prisma_client, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +@pytest.mark.asyncio +async def test_resolve_org_filter_for_user_search_propagates_a_db_outage_instead_of_answering_403(mocker): + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + mocker.patch( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_ui_settings_cached", + return_value={"scope_user_search_to_org": True}, + ) + + with pytest.raises(httpx.ConnectError): + await _resolve_org_filter_for_user_search( + user_api_key_dict=_db_unavailable_fallback_identity("/user/filter/ui"), + team_id=None, + prisma_client=prisma_client, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +@pytest.mark.asyncio +async def test_ui_view_users_answers_a_db_outage_as_503_no_db_connection_not_as_its_own_500(mocker, caplog): + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + mocker.patch( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_ui_settings_cached", + return_value={"scope_user_search_to_org": True}, + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.service_logging_obj.async_service_failure_hook = AsyncMock() + mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma_client) + mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) + mocker.patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"), pytest.raises(ProxyException) as raised: + await ui_view_users( + user_api_key_dict=_db_unavailable_fallback_identity("/user/filter/ui"), + user_id=None, + user_email="lit", + team_id=None, + page=1, + page_size=50, + ) + + assert raised.value.code == "503" + assert raised.value.type == ProxyErrorTypes.no_db_connection + assert isinstance(raised.value.__cause__, httpx.ConnectError) + outage_logs: Final = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and "ConnectError" in r.getMessage()] + assert outage_logs == ["Database unavailable during user search: ConnectError"] + + +@pytest.mark.parametrize( + ("route", "params"), + [("/user/list", {}), ("/user/filter/ui", {"user_email": "lit"})], + ids=["user_list", "user_filter_ui"], +) +def test_user_routes_answer_503_no_db_connection_when_the_callers_user_read_hits_a_db_outage( + mocker, route: str, params: dict[str, str] +): + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + mocker.patch( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_ui_settings_cached", + return_value={"scope_user_search_to_org": True}, + ) + mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma_client) + mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) + app.dependency_overrides[user_api_key_auth] = lambda: _db_unavailable_fallback_identity(route) + try: + response = TestClient(app, raise_server_exceptions=False).get(route, params=params) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 503, response.text + assert response.json() == _DB_OUTAGE_503_BODY diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 47540eb5d6d..53645e62034 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -7900,3 +7900,43 @@ class TestGetMcpToolsWireShape: assert tool["outputSchema"] == {"type": "integer"} assert "_meta" in tool assert not {"input_schema", "output_schema", "meta"} & tool.keys() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role,expected_status", [ + (LitellmUserRoles.PROXY_ADMIN, 404), + (LitellmUserRoles.INTERNAL_USER, 403), +]) +async def test_config_server_edit_preserves_api_contract_without_creating_rows(role, expected_status): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + + manager = MCPServerManager() + server = generate_mock_mcp_server_config_record(server_id="read-only-config") + manager.config_mcp_servers = {server.server_id: server} + original = server.model_dump() + prisma = MagicMock() + prisma.db.litellm_mcpservertable.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_mcpservertable.update = AsyncMock(return_value=None) + with ( + patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), + patch.object(mgmt_endpoints, "get_prisma_client_or_throw", return_value=prisma), + ): + with pytest.raises(HTTPException) as exc: + await mgmt_endpoints.edit_mcp_server( + payload=UpdateMCPServerRequest(server_id=server.server_id, description="UI edit"), + user_api_key_dict=UserAPIKeyAuth(user_id="actor", user_role=role), + ) + + assert exc.value.status_code == expected_status + if role == LitellmUserRoles.PROXY_ADMIN: + assert exc.value.detail == { + "error": f"MCP Server not found, passed server_id={server.server_id}" + } + prisma.db.litellm_mcpservertable.update.assert_awaited_once() + else: + prisma.db.litellm_mcpservertable.update.assert_not_awaited() + prisma.db.litellm_mcpservertable.create.assert_not_called() + prisma.db.litellm_mcpservertable.create_many.assert_not_called() + prisma.tx.assert_not_called() + assert server.model_dump() == original + assert manager.registry == {} diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index e95359ace8a..8bf9c598b1a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -7,6 +7,7 @@ from collections.abc import Sequence from typing import Final, Optional, cast from unittest.mock import AsyncMock, MagicMock, PropertyMock, call, patch +import httpx import pytest from fastapi import HTTPException from fastapi.testclient import TestClient @@ -42,6 +43,8 @@ from litellm.proxy.management_endpoints.team_endpoints import ( _STRIP_DELETED_TEAM_FROM_USERS_SQL, GetTeamMemberPermissionsResponse, UpdateTeamMemberPermissionsRequest, + _build_team_list_where_conditions, + _get_org_admin_org_ids, _persist_deleted_team_records, _save_deleted_team_records, _transform_teams_to_deleted_records, @@ -16569,3 +16572,81 @@ def test_team_member_update_request_rejects_unusable_temp_budget_increase(increa TeamMemberUpdateRequest( team_id="team-1", user_id="user-1", temp_budget_increase=increase, temp_budget_expiry="2030-01-01T00:00:00Z" ) + + +_DB_OUTAGE_503_BODY: Final = { + "error": { + "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + "type": "no_db_connection", + "param": "None", + "code": "503", + } +} + + +def _user_read_raising(error: Exception) -> tuple[MagicMock, MagicMock]: + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock(side_effect=error) + cache = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + cache.async_set_cache = AsyncMock() + return prisma_client, cache + + +def _db_unavailable_fallback_identity(route: str) -> UserAPIKeyAuth: + from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID + + return UserAPIKeyAuth( + key_name="failed-to-connect-to-db", + token="failed-to-connect-to-db", + user_id=DB_UNAVAILABLE_FALLBACK_USER_ID, + user_role=LitellmUserRoles.INTERNAL_USER, + request_route=route, + ) + + +@pytest.mark.asyncio +async def test_get_org_admin_org_ids_propagates_a_db_outage_instead_of_answering_not_an_org_admin(): + prisma_client, cache = _user_read_raising(httpx.ConnectError("All connection attempts failed")) + + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): + with pytest.raises(httpx.ConnectError): + await _get_org_admin_org_ids( + user_id="outage-probe-user", + prisma_client=prisma_client, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +@pytest.mark.asyncio +async def test_build_team_list_where_conditions_propagates_a_db_outage_instead_of_answering_user_not_found(): + prisma_client, cache = _user_read_raising(httpx.ConnectError("All connection attempts failed")) + + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): + with pytest.raises(httpx.ConnectError): + await _build_team_list_where_conditions( + prisma_client=prisma_client, + team_id=None, + team_alias=None, + organization_id=None, + user_id="outage-probe-user", + use_deleted_table=False, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +def test_list_team_v2_answers_503_no_db_connection_when_the_callers_user_read_hits_a_db_outage(monkeypatch): + prisma_client, cache = _user_read_raising(httpx.ConnectError("All connection attempts failed")) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + app.dependency_overrides[user_api_key_auth] = lambda: _db_unavailable_fallback_identity("/v2/team/list") + try: + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): + response = TestClient(app, raise_server_exceptions=False).get("/v2/team/list") + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 503, response.text + assert response.json() == _DB_OUTAGE_503_BODY diff --git a/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py b/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py index b3028ae71dd..089c2d57594 100644 --- a/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py +++ b/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py @@ -11,8 +11,10 @@ from __future__ import annotations import json from types import SimpleNamespace +from typing import Final from unittest.mock import MagicMock +import httpx import pytest from fastapi import HTTPException from fastapi.exceptions import RequestValidationError @@ -364,6 +366,39 @@ async def test_otel_unhandled_exception_handler_returns_500_generic_payload(): } +_DB_OUTAGE_503_BODY: Final = { + "error": { + "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + "type": "no_db_connection", + "param": "None", + "code": "503", + } +} + + +def _raised_from(outer: Exception, cause: Exception) -> Exception: + try: + raise outer from cause + except Exception as chained: + return chained + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "exc", + [ + httpx.ConnectError("All connection attempts failed"), + _raised_from(RuntimeError("user read failed"), httpx.ConnectError("All connection attempts failed")), + ], + ids=["raw_connect_error", "connect_error_as_cause"], +) +async def test_otel_unhandled_exception_handler_answers_a_db_outage_with_503_no_db_connection(exc): + response = await otel_unhandled_exception_handler(request=_make_request(path="/v2/team/list"), exc=exc) + + assert response.status_code == 503 + assert json.loads(response.body) == _DB_OUTAGE_503_BODY + + @pytest.mark.asyncio async def test_otel_unhandled_exception_handler_reraises_proxy_exception_error(): """ProxyException / HTTPException / RequestValidationError are re-raised diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index e9695685ca5..9954351fa2e 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -818,6 +818,35 @@ async def test_proxy_startup_event_prunes_dead_workers_live_gauges(tmp_path): assert counter.exists() +@pytest.mark.asyncio +@pytest.mark.parametrize(("disable_model_info_refresh", "job_scheduled"), [(True, False), (False, True)]) +async def test_proxy_startup_event_honors_disable_model_info_refresh( + disable_model_info_refresh: bool, job_scheduled: bool +) -> None: + """``general_settings.disable_model_info_refresh: true`` keeps the proxy from polling every + OpenAI-compatible deployment's ``/v1/models`` in the background, so a proxy fronting a replay + fixture (or a metered upstream) makes only the calls its clients asked for.""" + scheduler = AsyncIOScheduler() + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} | { + "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true" + } + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.object(ps, "scheduler", scheduler), + patch.dict(ps.general_settings, {"disable_model_info_refresh": disable_model_info_refresh}), + ): + try: + async with proxy_startup_event(app=None): + job = scheduler.get_job("refresh_model_info") + finally: + if scheduler.running: + scheduler.shutdown(wait=False) + + assert (job is not None) is job_scheduled, ( + f"disable_model_info_refresh={disable_model_info_refresh} but refresh_model_info job is {job}" + ) + + def test_otel_global_provider_published_after_callback_init(): """The OTel V2 global-provider publish must run after callback initialization in ``proxy_startup_event``. diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py index 214b5cde7da..13488106df4 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -4,7 +4,7 @@ import json import math from datetime import datetime, timedelta, timezone from types import MappingProxyType -from typing import Final +from typing import Final, cast import pytest @@ -33,6 +33,7 @@ from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.rust_bridge import bindings, configuration from litellm.rust_bridge import token_counter as rust_token_counter +from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo TOKEN_COUNTING_ROUTES: Final = ( @@ -239,6 +240,22 @@ class _FakeUpstream(Exception): pass +class _FakeTokenizer: + """Stands in for one shared native `Tokenizer`; only its name identifies it.""" + + def __init__(self, name: str, json: str | None = None) -> None: + self.name = name + self.json = json + + +def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: + """Point the counter's tokenizer lookups at fakes; the codec path keeps falling back to Python.""" + fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} + anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) + monkeypatch.setattr(tokenizer_dispatch, "native_encoding", fakes.__getitem__) + monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic) + + class _FakeNative: RustBridgeDeclined = _FakeDeclined RustUpstreamError = _FakeUpstream @@ -257,19 +274,13 @@ class _RecordingCounter: class _RecordingFactory: - """Stands in for the native `TokenCounter` class: called with tokenizer JSON, or `from_*_ranks`.""" + """Stands in for the native `TokenCounter` class, built over a loaded `Tokenizer`.""" def __init__(self) -> None: self.calls: list[tuple[rust_token_counter.RustTokenizer, bytes]] = [] - def __call__(self, tokenizer_json: str) -> _RecordingCounter: - return _RecordingCounter(self, "anthropic") - - def from_cl100k_ranks(self, rank_file: str) -> _RecordingCounter: - return _RecordingCounter(self, "cl100k_base") - - def from_o200k_ranks(self, rank_file: str) -> _RecordingCounter: - return _RecordingCounter(self, "o200k_base") + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: + return _RecordingCounter(self, cast(rust_token_counter.RustTokenizer, tokenizer.name)) class _DecliningCounter: @@ -278,19 +289,14 @@ class _DecliningCounter: class _DecliningFactory: - def __call__(self, tokenizer_json: str) -> _DecliningCounter: - return _DecliningCounter() - - def from_cl100k_ranks(self, rank_file: str) -> _DecliningCounter: - return _DecliningCounter() - - def from_o200k_ranks(self, rank_file: str) -> _DecliningCounter: + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _DecliningCounter: return _DecliningCounter() @pytest.fixture def rust_counter(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) + _fake_native_tokenizers(monkeypatch) rust_token_counter._counter.cache_clear() configuration.reset_rust_configuration() yield diff --git a/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py b/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py new file mode 100644 index 00000000000..49bbe148386 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py @@ -0,0 +1,191 @@ +"""Tests for input-token counting shared across the reservation path's models.""" + +from __future__ import annotations + +import json +from types import MappingProxyType +from typing import Final, cast + +import pytest + +import litellm +from litellm.proxy.spend_tracking.input_tokens import ( + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, + count_input_tokens, + count_input_tokens_for_model, +) +from litellm.rust_bridge import bindings, configuration, token_counter +from litellm.rust_bridge import tokenizer as tokenizer_dispatch +from litellm.rust_bridge.token_counter import RustTokenizer + +ANTHROPIC_MODEL: Final = "claude-sonnet-4-5-20250929" +CL100K_MODEL: Final = "gpt-4" +O200K_MODEL: Final = "gpt-4o" +PYTHON_ONLY_MODEL: Final = "replicate/meta/llama-2-70b-chat" +MESSAGES: Final = [{"role": "user", "content": "hello"}] +RUST_TOKENS: Final = 777 + + +class _FakeDeclined(Exception): + pass + + +class _FakeUpstream(Exception): + pass + + +class _FakeTokenizer: + """Stands in for one shared native `Tokenizer`; only its name identifies it.""" + + def __init__(self, name: str, json: str | None = None) -> None: + self.name = name + self.json = json + + +def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: + """Point the counter's tokenizer lookups at fakes; the codec path keeps falling back to Python.""" + fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} + anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) + monkeypatch.setattr(tokenizer_dispatch, "native_encoding", fakes.__getitem__) + monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic) + + +class _FakeNative: + RustBridgeDeclined = _FakeDeclined + RustUpstreamError = _FakeUpstream + + +class _RecordingCounter: + def __init__(self, factory: _RecordingFactory, tokenizer: RustTokenizer) -> None: + self.factory = factory + self.tokenizer = tokenizer + + async def acount_request(self, body: bytes) -> object: + self.factory.calls.append((self.tokenizer, body)) + return {"model": "", "input_tokens": RUST_TOKENS} + + +class _RecordingFactory: + def __init__(self) -> None: + self.calls: list[tuple[RustTokenizer, bytes]] = [] + + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: + return _RecordingCounter(self, cast(RustTokenizer, tokenizer.name)) + + +class _DecliningCounter: + async def acount_request(self, body: bytes) -> object: + raise _FakeDeclined("unsupported request shape") + + +class _DecliningFactory: + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _DecliningCounter: + return _DecliningCounter() + + +@pytest.fixture(autouse=True) +def _reset_bridge(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) + _fake_native_tokenizers(monkeypatch) + token_counter.TOKEN_COUNTER.reset() + token_counter._counter.cache_clear() + configuration.reset_rust_configuration() + yield + token_counter.TOKEN_COUNTER.reset() + token_counter._counter.cache_clear() + configuration.reset_rust_configuration() + + +def _body(model: object) -> tuple[dict[str, object], bytes]: + body: Final = {"model": model, "messages": MESSAGES} + return body, json.dumps(body).encode() + + +@pytest.mark.asyncio +async def test_models_sharing_a_tokenizer_are_counted_once_and_merged() -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(factory) + request_body, raw_body = _body([ANTHROPIC_MODEL, CL100K_MODEL, O200K_MODEL, "gpt-5", PYTHON_ONLY_MODEL]) + + counts: Final = await count_input_tokens( + request_body=request_body, + raw_body=raw_body, + models=(ANTHROPIC_MODEL, CL100K_MODEL, O200K_MODEL, "gpt-5", PYTHON_ONLY_MODEL), + ) + + assert factory.calls == [("anthropic", raw_body), ("cl100k_base", raw_body), ("o200k_base", raw_body)] + assert dict(counts) == { + ANTHROPIC_MODEL: RUST_TOKENS, + CL100K_MODEL: RUST_TOKENS, + O200K_MODEL: RUST_TOKENS, + "gpt-5": RUST_TOKENS, + PYTHON_ONLY_MODEL: count_input_tokens_for_model(request_body=request_body, model=PYTHON_ONLY_MODEL), + } + + +@pytest.mark.asyncio +async def test_rust_disabled_counts_everything_in_python() -> None: + factory: Final = _RecordingFactory() + litellm.rust(False) + token_counter.TOKEN_COUNTER.override(factory) + request_body, raw_body = _body([ANTHROPIC_MODEL, CL100K_MODEL]) + + counts: Final = await count_input_tokens( + request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL, CL100K_MODEL) + ) + + assert factory.calls == [] + assert dict(counts) == { + model: count_input_tokens_for_model(request_body=request_body, model=model) + for model in (ANTHROPIC_MODEL, CL100K_MODEL) + } + + +@pytest.mark.asyncio +async def test_missing_raw_body_counts_in_python() -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(factory) + request_body, _ = _body(ANTHROPIC_MODEL) + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=None, models=(ANTHROPIC_MODEL,)) + + assert factory.calls == [] + assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) + + +@pytest.mark.asyncio +async def test_missing_binding_counts_in_python() -> None: + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(None) + request_body, raw_body = _body(ANTHROPIC_MODEL) + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL,)) + + assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) + + +@pytest.mark.asyncio +async def test_declined_request_counts_in_python() -> None: + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(_DecliningFactory()) + request_body, raw_body = _body(ANTHROPIC_MODEL) + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL,)) + + assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) + assert counts[ANTHROPIC_MODEL] != RUST_TOKENS + + +@pytest.mark.asyncio +async def test_large_input_is_still_counted() -> None: + request_body: Final = { + "model": CL100K_MODEL, + "messages": [{"role": "user", "content": "x" * (TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS + 1)}], + } + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=None, models=(CL100K_MODEL,)) + + assert counts[CL100K_MODEL] == count_input_tokens_for_model(request_body=request_body, model=CL100K_MODEL) + assert isinstance(counts, MappingProxyType) diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 86bf896188f..b3913079bb2 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -39,8 +39,6 @@ from litellm.proxy.common_utils.user_api_key_cache import ( model_access_group_spend_counter_key, ) from litellm.proxy.spend_tracking.budget_reservation import ( - TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, - _approximate_input_size, _get_model_access_group_budget_counters, estimate_request_max_cost, get_budget_window_start, @@ -49,6 +47,10 @@ from litellm.proxy.spend_tracking.budget_reservation import ( release_budget_reservation_on_cancel, reserve_budget_for_request, ) +from litellm.proxy.spend_tracking.input_tokens import ( + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, + _approximate_input_size, +) from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget diff --git a/tests/test_litellm/proxy/test_bug_report_config.py b/tests/test_litellm/proxy/test_bug_report_config.py new file mode 100644 index 00000000000..ce4baa946f9 --- /dev/null +++ b/tests/test_litellm/proxy/test_bug_report_config.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from collections.abc import Iterator, Mapping + +import pytest + +from litellm.proxy import proxy_server +from litellm.proxy.bug_report_config import build_proxy_bug_report, safe_config_lines + +CUSTOMER_STRINGS = ( + "acme", + "sk-live-secret", + "hunter2", + "postgres://", + "10.0.0.7", +) + +CUSTOMER_CONFIG: Mapping[str, object] = { + "model_list": [ + { + "model_name": "acme-prod-gpt4", + "litellm_params": { + "model": "azure/acme-gpt4o-deployment", + "api_base": "https://acme-eastus.openai.azure.com", + "api_key": "sk-live-secret-1", + "rpm": 600, + "acme_extra_param": "acme", + }, + }, + { + "model_name": "acme-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-live-secret-2"}, + }, + { + "model_name": "acme-backup", + "litellm_params": {"model": "azure/acme-backup-deployment", "api_key": "sk-live-secret-3"}, + }, + {"model_name": "acme-bare", "litellm_params": {"model": "acme-custom-model"}}, + ], + "litellm_settings": { + "callbacks": ["langfuse", "acme_hooks.audit_logger"], + "drop_params": True, + "num_retries": 3, + "acme_internal_flag": True, + "cache": True, + "cache_params": { + "type": "redis", + "host": "10.0.0.7", + "port": 6379, + "password": "hunter2", + "acme_cache_option": "acme", + }, + }, + "router_settings": { + "routing_strategy": "latency-based-routing", + "redis_host": "10.0.0.7", + "acme_router_option": True, + }, + "guardrails": [ + {"guardrail_name": "acme-pii-mask", "litellm_params": {"guardrail": "presidio", "mode": "pre_call"}}, + { + "guardrail_name": "acme-policy", + "litellm_params": {"guardrail": "acme_guardrails.PolicyCheck", "api_key": "sk-live-secret-4"}, + }, + ], + "environment_variables": {"ACME_PROD_OPENAI_KEY": "sk-live-secret-5", "ACME_TENANT": "acme"}, +} + +CUSTOMER_GENERAL_SETTINGS: Mapping[str, object] = { + "master_key": "sk-live-secret-master", + "database_url": "postgres://user:hunter2@10.0.0.7/litellm", + "key_management_system": "aws_secret_manager", + "store_model_in_db": True, + "health_check_interval": 300, + "acme_sso_tenant": "acme-prod", +} + + +def test_safe_config_lines_keep_only_flags_and_litellm_defined_values(): + lines = safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS) + + assert lines == ( + "general_settings.key_management_system = aws_secret_manager", + "general_settings.store_model_in_db = true", + "litellm_settings.callbacks = [langfuse]", + "litellm_settings.drop_params = true", + "litellm_settings.cache = true", + "litellm_settings.cache_params.type = redis", + "router_settings.routing_strategy = latency-based-routing", + "guardrails[0].litellm_params.guardrail = presidio", + "guardrails[0].litellm_params.mode = pre_call", + "model_list[*].provider = [azure, openai]", + ) + assert not any(customer_string in "\n".join(lines) for customer_string in CUSTOMER_STRINGS) + + +@pytest.mark.parametrize( + ("config", "expected_lines"), + [ + ({"router_settings": {"routing_strategy": "acme-strategy"}}, ()), + ({"litellm_settings": {"cache_params": {"type": "acme-cache"}}}, ()), + ( + {"litellm_settings": {"success_callback": ["acme_logger", "langsmith"]}}, + ("litellm_settings.success_callback = [langsmith]",), + ), + ({"litellm_settings": {"callbacks": ["acme_hooks.audit_logger"]}}, ()), + ], +) +def test_string_values_show_only_when_litellm_defines_them( + config: Mapping[str, object], expected_lines: tuple[str, ...] +): + assert safe_config_lines(config, {}) == expected_lines + + +def test_secrets_numbers_and_unknown_values_leave_no_line(): + general_settings: Mapping[str, object] = { + "master_key": "sk-live-secret-master", + "health_check_interval": 300, + "store_model_in_db": object(), + "alerting": {"acme": "webhook"}, + "background_health_checks": False, + } + + assert safe_config_lines({}, general_settings) == ("general_settings.background_health_checks = false",) + + +def test_malformed_sections_produce_no_lines(): + config: Mapping[str, object] = { + "litellm_settings": "acme", + "router_settings": ["acme"], + "guardrails": {"acme": {"litellm_params": {"guardrail": "presidio"}}}, + "model_list": "acme", + "environment_variables": None, + } + + assert safe_config_lines(config, {}) == () + + +@pytest.fixture +def loaded_proxy_config(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + previous_config = proxy_server.proxy_config.get_config_state() + proxy_server.proxy_config.update_config_state(config=CUSTOMER_CONFIG) + monkeypatch.setattr(proxy_server, "general_settings", dict(CUSTOMER_GENERAL_SETTINGS)) + yield + proxy_server.proxy_config.update_config_state(config=previous_config) + + +@pytest.mark.usefixtures("loaded_proxy_config") +def test_build_proxy_bug_report_reads_the_loaded_proxy_config(): + report = build_proxy_bug_report(RuntimeError("boom"), stream=False) + + assert report.surface == "proxy" + assert report.stream is False + assert report.config_lines == safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 218d8246715..f96142def06 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -4,6 +4,7 @@ import datetime import json from types import MappingProxyType, SimpleNamespace from typing import AsyncGenerator, Callable, Final, Iterator, Optional, Sequence +from urllib.parse import unquote_plus from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -13,6 +14,12 @@ from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid +from litellm.litellm_core_utils.bug_report import ( + DISABLE_ENV_VAR, + ISSUE_URL_BASE, + bug_report_notice, + build_bug_report, +) from litellm.constants import ( CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_LITELLM_CALL_ID_LENGTH, @@ -4290,6 +4297,21 @@ class TestHandleLLMApiExceptionRetryAfter: proxy_exc = await self._invoke(ValueError("some other failure")) assert "retry-after" not in proxy_exc.headers + async def test_handle_llm_api_exception_strips_bug_report_notice_from_client_message(self, caplog): + report = build_bug_report(RuntimeError("boom"), surface="sdk") + notice = bug_report_notice(report) + exc = litellm.APIConnectionError( + message=f"boom\n{notice}", + model="gpt-4o", + llm_provider="openai", + ) + + with caplog.at_level("ERROR"): + proxy_exc = await self._invoke(exc) + + assert ISSUE_URL_BASE not in proxy_exc.message + assert ISSUE_URL_BASE in caplog.text + async def test_handle_llm_api_exception_retry_after_survives_callback_headers(self): from litellm.types.router import RouterRateLimitError @@ -9374,6 +9396,94 @@ async def test_handle_llm_api_exception_forwards_litellm_response_headers_when_r assert exc_info.value.headers["llm_provider-x-request-id"] == "req_openai_400" +@pytest.mark.asyncio +async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + monkeypatch.delenv(DISABLE_ENV_VAR, raising=False) + processor = ProxyBaseLLMRequestProcessing( + data={ + "proxy_server_request": {"url": "https://example.test/v1/chat/completions?debug=true"}, + "model": "acme-prod-gpt4", + "custom_llm_provider": "openai", + "stream": True, + } + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + with pytest.raises(ProxyException): + await processor._handle_llm_api_exception( + e=RuntimeError("unmapped for user@example.com"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + issue_url = next(word for word in caplog.text.split() if word.startswith(ISSUE_URL_BASE)) + assert "Endpoint / call: /v1/chat/completions" in unquote_plus(issue_url) + assert "Provider: openai" in unquote_plus(issue_url) + assert "Stream: true" in unquote_plus(issue_url) + assert "acme-prod-gpt4" not in unquote_plus(issue_url) + assert "user@example.com" not in unquote_plus(issue_url) + + +@pytest.mark.asyncio +async def test_handle_llm_api_exception_bug_report_drops_unknown_route( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + monkeypatch.delenv(DISABLE_ENV_VAR, raising=False) + processor = ProxyBaseLLMRequestProcessing( + data={"proxy_server_request": {"url": "https://example.test/v1/files/file-customer-123/content"}} + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + with pytest.raises(ProxyException): + await processor._handle_llm_api_exception( + e=RuntimeError("unmapped"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + issue_url = next(word for word in caplog.text.split() if word.startswith(ISSUE_URL_BASE)) + assert "Endpoint / call: unknown" in unquote_plus(issue_url) + assert "file-customer-123" not in unquote_plus(issue_url) + + +@pytest.mark.asyncio +async def test_handle_llm_api_exception_skips_bug_report_for_provider_status( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + monkeypatch.delenv(DISABLE_ENV_VAR, raising=False) + + class ProviderRateLimitError(Exception): + def __init__(self, message: str): + super().__init__(message) + self.status_code = 429 + + processor = ProxyBaseLLMRequestProcessing(data={}) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + with pytest.raises(ProxyException): + await processor._handle_llm_api_exception( + e=ProviderRateLimitError("rate limited"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + assert ISSUE_URL_BASE not in caplog.text + + class TestBackgroundResponseRetrievalGovernance: """LIT-7175: retrieving a background Response attaches the model's post_call policy pipelines.""" diff --git a/tests/test_litellm/proxy/test_native_compaction.py b/tests/test_litellm/proxy/test_native_compaction.py new file mode 100644 index 00000000000..d24624aacc5 --- /dev/null +++ b/tests/test_litellm/proxy/test_native_compaction.py @@ -0,0 +1,163 @@ +import asyncio +from collections.abc import Awaitable, Mapping +from types import MappingProxyType +from typing import Final, Literal + +import pytest +from fastapi import FastAPI, Request +from pydantic import TypeAdapter + +from litellm.caching.caching import DualCache +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.redact_messages import should_redact_message_logging +from litellm.proxy import common_request_processing, proxy_server +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import can_key_call_model +from litellm.proxy._types import ProxyException +from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash, get_request_stash +from litellm.proxy.native_compaction import with_proxy_compaction_executor +from litellm.router import Router +from litellm.router_strategy.complexity_router.context_compaction import compaction_executor, reject_recursive_compactor +from litellm.types.utils import ModelResponse + +_HEADERS: Final = ( + (b"authorization", b"Bearer sk-compaction-fixture"), (b"cookie", b"session=fixture"), + (b"content-length", b"99999"), (b"x-litellm-call-id", b"parent"), + (b"litellm-disable-message-redaction", b"true"), (b"x-litellm-num-retries", b"8"), + (b"X-LiteLLM-Timeout", b"600"), (b"x-litellm-stream-timeout", b"500"), +) + + +async def _child( + protocol: Literal["chat", "messages"] = "chat", forged: bool = False, parent_model: str | None = None +) -> Mapping[str, object]: + executor: Final = compaction_executor.get() + assert executor is not None + payload: Final = TypeAdapter(Mapping[str, object]).validate_json( + b'{"model":"compactor","messages":[{"role":"user","content":"history"}],' + b'"num_retries":0,"timeout":7,"stream_timeout":7,"disable_fallbacks":true,"stream":false,' + b'"metadata":{"turn_off_message_logging":true}}' + ) + return await executor(protocol, MappingProxyType({ + "litellm_metadata" if protocol == "messages" and key == "metadata" else key: value + for key, value in payload.items() if forged or key != "metadata" + }), parent_model) + + +def _request(app: FastAPI) -> Request: + return Request(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({ + "type": "http", "app": app, "scheme": "https", "server": ("proxy.test", 443), + "path": "/gateway/parent", "root_path": "/gateway", "query_string": b"parent=1", + "client": ("192.0.2.1", 4321), "headers": _HEADERS, + }))) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("protocol", ("chat", "messages")) +async def test_child_preserves_credentials_and_isolates_context(protocol: Literal["chat", "messages"]) -> None: + app: Final = FastAPI() + stash: Final = get_or_create_request_stash() + + @app.post("/v1/chat/completions" if protocol == "chat" else "/v1/messages") + async def endpoint(request: Request) -> Mapping[str, object]: + assert get_request_stash() is None and compaction_executor.get() is None + assert request.client == ("192.0.2.1", 4321) and request.url.scheme == "https" + assert request.scope["root_path"] == "/gateway" and request.cookies["session"] == "fixture" + assert request.headers["authorization"] == "Bearer sk-compaction-fixture" and not request.query_params + assert "x-litellm-call-id" not in request.headers + assert "litellm-disable-message-redaction" not in request.headers + assert int(request.headers["content-length"]) == len(await request.body()) + with pytest.raises(BadRequestError, match="regular model group"): + reject_recursive_compactor("auto-router") + return MappingProxyType({"summary": "compacted"}) + + with inherit_message_logging_privacy(True): + assert (await with_proxy_compaction_executor(_child(protocol), _request(app)))["summary"] == "compacted" + assert get_request_stash() is stash and compaction_executor.get() is None + reject_recursive_compactor("auto-router") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("protocol", ("chat", "messages")) +@pytest.mark.parametrize("policy", ("allowed", "denied", "forged", "router_alias", "unrelated_alias")) +async def test_real_proxy_child_auth_privacy_and_body_policy( + monkeypatch: pytest.MonkeyPatch, protocol: Literal["chat", "messages"], policy: str, +) -> None: + cache: Final = DualCache() + token: Final = proxy_server.hash_token("sk-compaction-fixture") + models: Final = {"denied": ("answer",), "router_alias": ("auto",), "unrelated_alias": ("other-auto",)}.get(policy, ("compactor",)) + auth: Final = UserAPIKeyAuth.model_validate(MappingProxyType({"token": token, "models": models})) + await cache.async_set_cache(key=token, value=auth) + dispatched: Final = asyncio.Event() + allowed: Final = policy in ("allowed", "router_alias") + + async def route( + data: Mapping[str, object], llm_router: Router | None, user_model: str | None, + route_type: str, user_api_key_dict: UserAPIKeyAuth | None, + ) -> Awaitable[ModelResponse]: + dispatched.set() + assert allowed + if policy == "router_alias": + with pytest.raises(ProxyException): + await can_key_call_model("unrelated-compactor", None, auth, None) + assert (data["num_retries"], data["timeout"], data["stream_timeout"]) == (0, 7, 7) + assert data["disable_fallbacks"] is True and data["stream"] is False + logging: Final = data["litellm_logging_obj"] + assert isinstance(logging, Logging) + assert logging.standard_callback_dynamic_params.get("turn_off_message_logging") is True + assert should_redact_message_logging(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({ + "litellm_params": data, "standard_callback_dynamic_params": logging.standard_callback_dynamic_params, + }))) + return asyncio.sleep(0, result=ModelResponse(id="private-summary", model="compactor")) + + monkeypatch.setattr(proxy_server.app, "dependency_overrides", {}) + monkeypatch.setattr(proxy_server, "master_key", "sk-master-fixture") + monkeypatch.setattr(proxy_server, "prisma_client", object()) + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(common_request_processing, "route_request", route) + with inherit_message_logging_privacy(True): + call: Final = with_proxy_compaction_executor( + _child(protocol, policy == "forged", "auto" if policy.endswith("alias") else None), _request(proxy_server.app) + ) + if allowed: + assert (await call)["id"] == "private-summary" + else: + status: Final = 401 if policy == "forged" else 403 + with pytest.raises(BadRequestError, match=rf"child request failed \(HTTP {status}\)"): + await call + assert dispatched.is_set() is allowed + assert compaction_executor.get() is None + if policy.endswith("alias"): + with pytest.raises(ProxyException): + await can_key_call_model("compactor", None, auth, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("timeout", [False, True]) +async def test_cancelling_parent_cancels_and_drains_child(timeout: bool) -> None: + app: Final = FastAPI() + started: Final = asyncio.Event() + stopped: Final = asyncio.Event() + + @app.post("/v1/chat/completions") + async def endpoint() -> None: + started.set() + try: + await asyncio.Event().wait() + finally: + stopped.set() + + parent: Final = asyncio.create_task(with_proxy_compaction_executor(_child(), _request(app))) + await asyncio.wait_for(started.wait(), timeout=5) + if timeout: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(parent, timeout=0) + else: + parent.cancel() + with pytest.raises(asyncio.CancelledError): + await parent + assert stopped.is_set() and compaction_executor.get() is None diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index dd330d32ce6..f45715953f9 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -489,6 +489,68 @@ async def test_post_call_stream_masking_guardrail_keeps_own_iterator_on_anthropi assert delivered == chunks +@pytest.mark.asyncio +async def test_post_call_stream_presidio_output_masking_masks_anthropic_messages_stream(monkeypatch): + """Regression: the presidio output-masking callback built by initialize_presidio + was rerouted onto the unified scan-only path on /v1/messages, so a card number + the analyzer flagged still streamed to the caller unmasked.""" + import json + + from litellm.caching.caching import DualCache + from litellm.proxy.guardrails.guardrail_registry import InMemoryGuardrailHandler + from litellm.types.guardrails import SupportedGuardrailIntegrations + + handler = InMemoryGuardrailHandler() + result = handler.initialize_guardrail( + guardrail={ + "guardrail_name": "presidio-card-mask", + "litellm_params": { + "guardrail": SupportedGuardrailIntegrations.PRESIDIO.value, + "mode": ["pre_call", "post_call"], + "default_on": True, + "presidio_analyzer_api_base": "https://fakelink.com/v1/presidio/analyze", + "presidio_anonymizer_api_base": "https://fakelink.com/v1/presidio/anonymize", + "pii_entities_config": {"CREDIT_CARD": "MASK"}, + "mock_redacted_text": {"text": "", "items": []}, + }, + } + ) + guardrail_id = result["guardrail_id"] + callbacks = [ + handler.guardrail_id_to_custom_guardrail[guardrail_id], + *handler.guardrail_id_to_sibling_callbacks[guardrail_id], + ] + monkeypatch.setattr(litellm, "callbacks", callbacks) + + chunks = _anthropic_stream_chunks(["4111", " 1111 1111 1111"]) + + async def fake_stream(): + for chunk in chunks: + yield chunk + + delivered = [] + async for chunk in ProxyLogging(user_api_key_cache=DualCache()).async_post_call_streaming_iterator_hook( + response=fake_stream(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/messages"), + request_data={ + "model": "claude-sonnet-5", + "litellm_logging_obj": _streaming_logging_obj(), + "metadata": {}, + }, + ): + delivered.append(chunk) + + wire = b"".join(delivered).decode() + text_deltas = [ + json.loads(line[6:])["delta"]["text"] + for line in wire.split("\n") + if line.startswith("data: ") and json.loads(line[6:]).get("delta", {}).get("type") == "text_delta" + ] + assert "4111" not in wire, wire + assert "".join(text_deltas) == "", wire + assert wire.count("event: message_stop") == 1, wire + + class _AppliesGuardrail(CustomGuardrail): """Implements the unified interface only, so the proxy routes it to unified_guardrail.""" diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 2792e176e0e..26bfd5c52bd 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -14877,7 +14877,7 @@ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_coun async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeypatch): - from tokenizers import Tokenizer + from litellm.rust_bridge._native import Tokenizer from litellm import Router from tests.test_litellm.litellm_core_utils.event_loop_lag import assert_loop_stayed_free, timed_with_loop_lags @@ -14890,7 +14890,7 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp time.sleep(0.3) return claude_tokenizer - monkeypatch.setattr(litellm.utils, "Tokenizer", SlowHubTokenizer) + monkeypatch.setattr("litellm.rust_bridge.tokenizer.from_pretrained", SlowHubTokenizer.from_pretrained) monkeypatch.setattr( "litellm.proxy.proxy_server.llm_router", Router( @@ -14914,7 +14914,7 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revision_and_token(monkeypatch): - from tokenizers import Tokenizer + from litellm.rust_bridge._native import Tokenizer from litellm import Router from litellm.types.router import DeploymentTypedDict @@ -14931,7 +14931,7 @@ async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revisi }, } - monkeypatch.setattr(litellm.utils, "Tokenizer", MagicMock(from_pretrained=from_pretrained)) + monkeypatch.setattr("litellm.rust_bridge.tokenizer.from_pretrained", from_pretrained) monkeypatch.setattr( "litellm.proxy.proxy_server.llm_router", Router( diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 734408d9b61..0fc7295a717 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -6,10 +6,12 @@ import pytest from fastapi import HTTPException from litellm.caching.caching import DualCache +from litellm.exceptions import InternalServerError from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.bug_report import ISSUE_URL_BASE from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth -from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy from litellm.types.guardrails import GuardrailEventHooks @@ -2418,3 +2420,16 @@ def test_mcp_auth_policy_uses_original_request_model(monkeypatch, model, expecte synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs) assert ("model-rule" in synthetic["metadata"]["guardrails"]) is expected assert "request-rule" in synthetic["metadata"]["guardrails"] + + +def test_handle_exception_on_proxy_logs_bug_report_only_for_unmapped_500(caplog): + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + provider_result = handle_exception_on_proxy( + InternalServerError(message="upstream 500", llm_provider="openai", model="gpt-4") + ) + assert ISSUE_URL_BASE not in caplog.text + internal_result = handle_exception_on_proxy(KeyError("missing")) + + assert provider_result.code == internal_result.code == "500" + assert ISSUE_URL_BASE in caplog.text + assert ISSUE_URL_BASE not in internal_result.message diff --git a/tests/test_litellm/router_strategy/complexity_router/test_context_compaction.py b/tests/test_litellm/router_strategy/complexity_router/test_context_compaction.py new file mode 100644 index 00000000000..c6242f775c8 --- /dev/null +++ b/tests/test_litellm/router_strategy/complexity_router/test_context_compaction.py @@ -0,0 +1,503 @@ +import asyncio +import json +from collections.abc import Iterator, Mapping +from copy import deepcopy +from functools import partial +from typing import Final, Literal + +import httpx +import pytest +import respx + +import litellm +from litellm.llms import compaction as native +from litellm.router_strategy.complexity_router.config import ContextCompactionConfig +from litellm.router_strategy.complexity_router.context_compaction import ( + CompactionState, + Surface, + arm_compaction, + compact_to_fit, + compaction_executor, +) +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES +from litellm.types.router import Deployment + +pytestmark: Final = [pytest.mark.asyncio, pytest.mark.usefixtures("local_model_cost_map")] +SCHEMA: Final = {"type": "object", "properties": {"code": {"type": "string"}}} + + +@pytest.fixture(autouse=True) +def native_catalog(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setenv("LITELLM_LICENSE", "") + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False) + monkeypatch.setitem(litellm.model_cost, "summary-fixture", { + "litellm_provider": "anthropic", "mode": "chat", "max_input_tokens": 32000, + "max_output_tokens": 4096, "supports_anthropic_compaction": True, + }) + + +def make_router( + window: int | None = 512, settings: Mapping[str, object] | None = None, *, compactor_window: int = 32000, + conflict: bool = False, output: int | None = 64, + answer_defaults: Mapping[str, object] | None = None, + context_fallback: bool = False, +) -> litellm.Router: + config: Final = { + "tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "large", "REASONING": "large"}, + "keyword_tier_rules": [{"keywords": ["answer", "tail result"], "tier": "SIMPLE"}], + "enable_context_window_escalation": False, "max_tokens_from_tier_model": False, + **(settings or {}), + } + return litellm.Router(model_list=[ + {"model_name": "auto", "litellm_params": { + "model": "auto_router/complexity_router", "complexity_router_config": config, + }}, + {"model_name": "small", "litellm_params": { + "model": "openai/arbitrary-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test", "max_retries": 0, + **(answer_defaults or {}), + }, "model_info": {"id": "pinned-answer", "max_input_tokens": window, "max_output_tokens": output}}, + {"model_name": "large", "litellm_params": { + "model": "anthropic/summary-fixture", "api_base": "https://compact.test", "api_key": "compact-test", + **({"stop": ["deployment policy"]} if conflict else {}), + }, "model_info": {"id": "native-compactor", "max_input_tokens": compactor_window, "max_output_tokens": 4096}}, + {"model_name": "backup", "litellm_params": { + "model": "anthropic/summary-fixture", "api_base": "https://compact.test", "api_key": "backup-test", + }, "model_info": {"id": "backup-compactor", "max_input_tokens": 32000, "max_output_tokens": 4096}}, + ], enable_pre_call_checks=True, num_retries=0, disable_cooldowns=True, + retry_policy={"InternalServerErrorRetries": 1}, + context_window_fallbacks=[{"auto": ["large"]}] if context_fallback else []) + + +def exchange(surface: Surface, phase: str) -> list[dict[str, object]]: + identifier: Final = f"{phase}-call" + result: Final = f"{phase} result" + if surface == "responses": + return [ + {"type": "function_call", "call_id": identifier, "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "call_id": identifier, "output": result}, + ] + if surface == "messages": + return [ + {"role": "assistant", "content": [{"type": "tool_use", "id": identifier, "name": "lookup", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": identifier, "content": result}]}, + ] + return [ + {"role": "assistant", "tool_calls": [ + {"id": identifier, "type": "function", "function": {"name": "lookup", "arguments": "{}"}}, + ]}, + {"role": "tool", "tool_call_id": identifier, "content": result}, + ] + + +def history(surface: Surface) -> dict[str, object]: + conversation: Final = [ + {"role": "user", "content": "Project code MAPLE-47. Background detail. " * 150}, + {"role": "assistant", "content": "Recorded"}, + *exchange(surface, "prefix"), + {"role": "user", "content": "Answer with the project code"}, + *exchange(surface, "tail"), + ] + function: Final = {"name": "lookup", "parameters": SCHEMA} + if surface == "responses": + return {"instructions": "Keep the code exact", "tools": [{"type": "function", **function}], "input": [ + {"role": "developer", "content": "Retain the original spelling"}, *conversation, + ]} + if surface == "messages": + return {"system": "Keep the code exact", "tools": [{"name": "lookup", "input_schema": SCHEMA}], "messages": [ + *conversation, + ]} + return {"tools": [{"type": "function", "function": function}], "messages": [ + {"role": "system", "content": "Keep the code exact"}, *conversation, + ]} + + +def native_reply(summary: str = "Project code MAPLE-47", signed: bool = True, truncated: bool = False) -> httpx.Response: + return httpx.Response(200, json={ + "id": "msg_compact", "type": "message", "role": "assistant", "model": "summary-fixture", + "content": [{"type": "compaction", "content": summary, **({"signature": "native-signature"} if signed else {})}], + "stop_reason": "max_tokens" if truncated else "compaction", "usage": {"input_tokens": 0, "output_tokens": 0, "iterations": [ + {"type": "compaction", "input_tokens": 1200, "output_tokens": 20}, + ]}, + }) + + +def answer_reply(request: httpx.Request, expected_model: str = "arbitrary-answer") -> httpx.Response: + payload: Final = json.loads(request.content) + assert payload["model"] == expected_model + assert request.headers["authorization"] == "Bearer answer-test" + if request.url.path.endswith("responses"): + return httpx.Response(200, json={ + "id": "resp_answer", "object": "response", "created_at": 0, "status": "completed", + "model": payload["model"], "output": [{"id": "msg_answer", "type": "message", "role": "assistant", + "status": "completed", "content": [{"type": "output_text", "text": "MAPLE-47", "annotations": []}]}], + "usage": {"input_tokens": 60, "output_tokens": 8, "total_tokens": 68}, + }) + return httpx.Response(200, json={ + "id": "answer", "object": "chat.completion", "created": 0, "model": payload["model"], + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "MAPLE-47"}}], + "usage": {"prompt_tokens": 60, "completion_tokens": 8, "total_tokens": 68}, + }) + + +@pytest.fixture +def wire() -> Iterator[tuple[respx.Route, respx.Route]]: + with respx.mock(assert_all_called=False) as transport: + compactor: Final = transport.post("https://compact.test/v1/messages").mock(return_value=native_reply()) + answer: Final = transport.route(method="POST", host="answer.test").mock(side_effect=answer_reply) + yield compactor, answer + + +async def invoke(router: litellm.Router, surface: Surface, payload: Mapping[str, object], retries: int = 0) -> object: + if surface == "responses": + return await router.aresponses(model="auto", max_output_tokens=64, num_retries=retries, **payload) + if surface == "messages": + return await router.aanthropic_messages(model="auto", max_tokens=64, num_retries=retries, **payload) + return await router.acompletion(model="auto", max_tokens=64, num_retries=retries, **payload) + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("near", [False, True]) +@pytest.mark.parametrize("configured", [False, True]) +async def test_all_surfaces_compact_and_keep_selected_answerer( + wire: tuple[respx.Route, respx.Route], surface: Surface, near: bool, configured: bool, +) -> None: + payload: Final = history(surface) + original: Final = deepcopy(payload) + counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload) + window: Final = int((counted + 32) / ContextCompactionConfig().trigger_ratio) + 1 if near else 512 + assert (counted < window) is near + settings: Final = {"enable_context_window_escalation": True, + **({"context_compaction": {"model": "large", "max_tokens": 512}} if configured else {})} + router: Final = make_router(window, settings) + captured: Final = asyncio.Queue[Mapping[str, object]]() + compactor, answer = wire + retry: Final = near and configured + + def answer_after_retry(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, json={"error": {"message": "retry answer"}}) if answer.call_count == 0 else answer_reply(request) + + if retry: + answer.mock(side_effect=answer_after_retry) + + async def execute( + protocol: native.CompactionProtocol, request: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: + assert parent_model == "auto" + result: Final = await native.dispatch(router, protocol, request) + captured.put_nowait(result) + return result + + token: Final = compaction_executor.set(execute) + try: + response: Final = await invoke(router, surface, payload, retries=int(retry)) + finally: + compaction_executor.reset(token) + assert compactor.call_count == captured.qsize() == 1 + assert answer.call_count == router.total_calls["openai/arbitrary-answer"] == 1 + int(retry) + compact_request: Final = compactor.calls[0].request + compact_body: Final = json.loads(compact_request.content) + answer_body: Final = json.loads(answer.calls[0].request.content) + assert answer_body == json.loads(answer.calls[-1].request.content) + assert compact_body["model"] == "summary-fixture" and compact_body["compaction"] == {"type": "summarize"} + assert ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value in compact_request.headers["anthropic-beta"].split(",") + assert compact_body["max_tokens"] == (512 if configured else ContextCompactionConfig().max_tokens) + assert "Background detail" in str(compact_body) and "tail-call" not in str(compact_body) + assert "prefix-call" in str(compact_body) and "prefix result" in str(compact_body) + assert "Keep the code exact" in str(compact_body["system"]) + assert compact_body["tools"][0]["input_schema"] == SCHEMA + assert "MAPLE-47" in str(answer_body) and "Background detail" not in str(answer_body) + assert "prefix-call" not in str(answer_body) + assert "native-signature" not in str(answer_body) and "compaction" not in answer_body + assert "tail-call" in str(answer_body) and "tail result" in str(answer_body) + assert "MAPLE-47" in str(response) + usage: Final = captured.get_nowait()["usage"] + if surface == "messages": + assert usage["iterations"][0]["input_tokens"] == 1200 and usage["iterations"][0]["output_tokens"] == 20 + else: + assert usage["prompt_tokens"] == 1200 and usage["completion_tokens"] == 20 + if surface == "responses": + assert answer_body["input"][-3:] == original["input"][-3:] + assert answer_body["input"][0] == original["input"][0] + assert answer_body["instructions"] == original["instructions"] and answer_body["tools"] == original["tools"] + assert payload == original + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("mode", ["fitting", "false", "null"]) +async def test_fitting_and_disabled_requests_do_not_compact( + wire: tuple[respx.Route, respx.Route], surface: Surface, mode: str, +) -> None: + settings: Final = {} if mode == "fitting" else {"context_compaction": False if mode == "false" else None} + payload: Final = history(surface) + original: Final = deepcopy(payload) + counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload) + await invoke(make_router(20000 if mode == "fitting" else counted + 64, settings), surface, payload) + compactor, answer = wire + assert compactor.call_count == 0 and answer.call_count == 1 + assert "Background detail" in answer.calls[0].request.content.decode() + assert payload == original + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("reason", ["single", "unclosed", "no_compactor"]) +async def test_fitting_request_survives_unavailable_compaction( + wire: tuple[respx.Route, respx.Route], surface: Surface, reason: str, +) -> None: + key: Final = "input" if surface == "responses" else "messages" + items: Final = [{"role": "user", "content": "Answer with MAPLE-47. Detail. " * 80}] + payload: Final = history(surface) if reason == "no_compactor" else {key: ( + items if reason == "single" else [*items, *exchange(surface, "open")[:1], {"role": "user", "content": "Answer"}] + )} + counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload) + settings: Final = {"tiers": {"SIMPLE": "small", "MEDIUM": "small", "COMPLEX": "small", "REASONING": "small"}} if reason == "no_compactor" else {} + await invoke(make_router(counted + 1, settings), surface, payload) + compactor, answer = wire + assert compactor.call_count == 0 and answer.call_count == 1 + assert "Detail" in str(answer.calls[0].request.content) or "Background detail" in str(answer.calls[0].request.content) + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +async def test_uncompactable_overflow_uses_explicit_context_fallback( + wire: tuple[respx.Route, respx.Route], surface: Surface, +) -> None: + compactor, answer = wire + compactor.mock(return_value=httpx.Response(200, json={**native_reply().json(), + "content": [{"type": "text", "text": "MAPLE-47"}], "stop_reason": "end_turn"})) + payload: Final = {"input" if surface == "responses" else "messages": [ + {"role": "user", "content": "Answer with MAPLE-47. Detail. " * 150}, + ]} + await invoke(make_router(context_fallback=True), surface, payload) + assert answer.call_count == 0 and compactor.call_count == 1 + assert "compaction" not in json.loads(compactor.calls[0].request.content) + + +@pytest.mark.parametrize("escalate", [False, True]) +async def test_no_native_compactor_respects_explicit_escalation( + wire: tuple[respx.Route, respx.Route], monkeypatch: pytest.MonkeyPatch, escalate: bool, +) -> None: + monkeypatch.setitem(litellm.model_cost["summary-fixture"], "supports_anthropic_compaction", False) + router: Final = make_router(settings={"enable_context_window_escalation": escalate}) + compactor, answer = wire + compactor.mock(return_value=httpx.Response(200, json={**native_reply().json(), + "content": [{"type": "text", "text": "MAPLE-47"}], "stop_reason": "end_turn"})) + if not escalate: + with pytest.raises(litellm.ContextWindowExceededError, match="No configured compactor"): + await invoke(router, "chat", history("chat")) + assert compactor.call_count == 0 + else: + await invoke(router, "chat", history("chat")) + assert compactor.call_count == 1 + assert "compaction" not in json.loads(compactor.calls[0].request.content) + assert answer.call_count == 0 + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +async def test_undersized_native_compactor_does_not_block_explicit_escalation( + wire: tuple[respx.Route, respx.Route], surface: Surface, +) -> None: + router: Final = make_router(compactor_window=512, settings={ + "enable_context_window_escalation": True, + "tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "wide", "REASONING": "wide"}, + }) + router.add_deployment(Deployment( + model_name="wide", litellm_params={ + "model": "openai/wide-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test", + }, model_info={"id": "wide-answer", "max_input_tokens": 32000, "max_output_tokens": 64}, + )) + compactor, answer = wire + answer.mock(side_effect=partial(answer_reply, expected_model="wide-answer")) + await invoke(router, surface, history(surface)) + assert compactor.call_count == 0 and answer.call_count == 1 + assert json.loads(answer.calls[0].request.content)["model"] == "wide-answer" + + +@pytest.mark.parametrize( + ("surface", "failure"), + [(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")] + + [("messages", "truncated")], +) +async def test_bad_native_result_never_reaches_answerer( + wire: tuple[respx.Route, respx.Route], surface: Surface, failure: str, +) -> None: + compactor, answer = wire + reply: Final = httpx.Response(500, json={"error": {"type": "api_error", "message": "failed"}}) if failure == "provider" else native_reply( + "too large " * 2000 if failure == "oversized" else "MAPLE-47", signed=failure != "unsigned", + truncated=failure == "truncated", + ) + compactor.mock(return_value=reply) + with pytest.raises((litellm.BadRequestError, litellm.InternalServerError)): + await invoke(make_router(), surface, history(surface)) + assert compactor.call_count == 1 and answer.call_count == 0 + + +@pytest.mark.parametrize("failure", ["item", "content", "tools", "unclosed", "missing", "duplicate", "instructions", "retained"]) +@pytest.mark.parametrize("needed", [False, True]) +async def test_unsafe_responses_reject_only_when_compaction_needed( + wire: tuple[respx.Route, respx.Route], failure: str, needed: bool, +) -> None: + payload: Final = history("responses") + extra: Final = { + "item": [{"type": "computer_call", "call_id": "opaque-tool"}], + "content": [{"role": "assistant", "content": [{"type": "refusal", "refusal": "cannot"}]}], + "unclosed": [{"type": "function_call", "call_id": "unclosed", "name": "lookup", "arguments": "{}"}], + "missing": [{"type": "function_call", "name": "lookup", "arguments": "{}"}], + "duplicate": exchange("responses", "prefix"), + "instructions": [{"role": "developer", "content": "Changed instructions"}], + } + request: Final = { + **payload, "input": [*payload["input"][:3], *extra.get(failure, []), *payload["input"][3:]], + **({"tools": [{"type": "computer_use_preview", "display_width": 800, "display_height": 600}]} if failure == "tools" else {}), + **({"instructions": "Keep every instruction " * 600} if failure == "retained" else {}), + } + if needed: + with pytest.raises(litellm.BadRequestError, match="Context compaction"): + await invoke(make_router(), "responses", request) + assert all(route.call_count == 0 for route in wire) + else: + await invoke(make_router(20000), "responses", request) + compactor, answer = wire + assert compactor.call_count == 0 and answer.call_count == 1 + + +@pytest.mark.parametrize("owned", [ + {"previous_response_id": "resp_parent"}, {"conversation": "conv_parent"}, + {"context_management": [{"type": "compaction", "compact_threshold": 1000}]}, {"compaction": {"type": "summarize"}}, + {"input": [{"type": "reasoning", "encrypted_content": "opaque"}]}, + {"input": [{"type": "reasoning", "summary": [{"type": "summary_text", "text": "prior reasoning"}]}]}, + {"input": [{"type": "compaction", "encrypted_content": "opaque"}]}, + {"input": [{"type": "item_reference", "id": "item_parent"}]}, + {"input": [{"role": "assistant", "content": "visible", "encrypted_content": "opaque"}]}, + {"input": [{"role": "user", "content": [{"type": "encrypted_content", "encrypted_content": "opaque"}]}]}, +]) +@pytest.mark.parametrize("arm_first", [False, True]) +async def test_client_owned_history_bypasses_compaction( + wire: tuple[respx.Route, respx.Route], owned: Mapping[str, object], arm_first: bool, +) -> None: + router: Final = make_router() + deployment: Final = router.get_deployment(model_id="pinned-answer") + assert deployment is not None + state: Final = CompactionState() + request: Final = {**history("responses"), **owned, "model": "small", "max_tokens": 64, "_context_compaction_state": state} + original: Final = deepcopy({key: value for key, value in request.items() if key != "_context_compaction_state"}) + before_defaults: Final = {"_context_compaction_state": state} if arm_first else request + await arm_compaction(before_defaults, ContextCompactionConfig(), ("large",)) + counted: Final = router._count_pre_call_check_tokens(None, request["input"], request) + if arm_first and counted > 512: + with pytest.raises(litellm.ContextWindowExceededError): + await compact_to_fit(router, deployment.model_dump(), request, "responses") + else: + result: Final = await compact_to_fit(router, deployment.model_dump(), request, "responses") + assert result is request + assert {key: value for key, value in request.items() if key != "_context_compaction_state"} == original + assert all(route.call_count == 0 for route in wire) + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("source", ["request", "deployment"]) +async def test_client_managed_overflow_keeps_context_window_admission( + wire: tuple[respx.Route, respx.Route], surface: Surface, source: str, +) -> None: + managed: Final = {"context_management": {"edits": []}} + router: Final = make_router(answer_defaults=managed if source == "deployment" else None) + payload: Final = {**history(surface), **(managed if source == "request" else {})} + compactor, answer = wire + if source == "deployment": + with pytest.raises(litellm.ContextWindowExceededError): + await invoke(router, surface, payload) + assert compactor.call_count == 0 + else: + await invoke(router, surface, payload) + assert compactor.call_count == 1 + assert "compaction" not in json.loads(compactor.calls[0].request.content) + assert answer.call_count == 0 + + +@pytest.mark.parametrize("case", ["conflicting_defaults", "small_window", "capability_false", "capability_missing"]) +async def test_automatic_compactor_skips_conflicts_and_requires_capacity_and_capability( + wire: tuple[respx.Route, respx.Route], monkeypatch: pytest.MonkeyPatch, case: str, +) -> None: + if case.startswith("capability"): + metadata: Final = {key: value for key, value in litellm.model_cost["summary-fixture"].items() + if key != "supports_anthropic_compaction"} + monkeypatch.setitem(litellm.model_cost, "summary-fixture", { + **metadata, **({"supports_anthropic_compaction": False} if case == "capability_false" else {}), + }) + conflict: Final = case == "conflicting_defaults" + settings: Final = {"tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "backup", "REASONING": "backup"}} if conflict else {} + router: Final = make_router(settings=settings, conflict=conflict, compactor_window=512 if case == "small_window" else 32000) + if conflict: + await invoke(router, "chat", history("chat")) + compactor, answer = wire + assert compactor.call_count == answer.call_count == 1 + assert compactor.calls[0].request.headers["x-api-key"] == "backup-test" + assert "stop_sequences" not in json.loads(compactor.calls[0].request.content) + else: + with pytest.raises(litellm.BadRequestError, match="No configured compactor"): + await invoke(router, "chat", history("chat")) + assert all(route.call_count == 0 for route in wire) + + +@pytest.mark.parametrize("unknown_output", [False, True]) +@pytest.mark.parametrize("overflow", [False, True]) +async def test_unusable_output_budget_still_enforces_known_input_window( + wire: tuple[respx.Route, respx.Route], unknown_output: bool, overflow: bool, +) -> None: + payload: Final = history("chat") + counted: Final = make_router(output=None)._count_pre_call_check_tokens(payload["messages"], None, payload) + window: Final = 512 if overflow else counted + 64 + output: Final = None if unknown_output else window + router: Final = make_router(window, output=output) + deployment: Final = router.get_deployment(model_id="pinned-answer") + assert deployment is not None + state: Final = CompactionState(config=ContextCompactionConfig(), candidates=("large",)) + request: Final = {**payload, "model": "small", "max_tokens": output, "_context_compaction_state": state} + if overflow: + with pytest.raises(litellm.BadRequestError, match="known input window"): + await compact_to_fit(router, deployment.model_dump(), request, "chat") + else: + assert await compact_to_fit(router, deployment.model_dump(), request, "chat") is request + assert all(route.call_count == 0 for route in wire) + + +@pytest.mark.parametrize("outcome", ["success", "timeout", "cancel"]) +async def test_retry_reuses_summary_or_terminal_cancellation(outcome: Literal["success", "timeout", "cancel"]) -> None: + router: Final = make_router() + deployment: Final = router.get_deployment(model_id="pinned-answer") + assert deployment is not None + state: Final = CompactionState(config=ContextCompactionConfig(model="large", max_tokens=512, timeout_seconds=0.02)) + request: Final = {**history("messages"), "model": "small", "max_tokens": 64, "_context_compaction_state": state} + calls: Final = asyncio.Queue[None]() + started: Final = asyncio.Event() + stopped: Final = asyncio.Event() + + async def execute( + protocol: native.CompactionProtocol, payload: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: + calls.put_nowait(None) + started.set() + try: + return native_reply().json() if outcome == "success" else await asyncio.Future[Mapping[str, object]]() + finally: + stopped.set() + + token: Final = compaction_executor.set(execute) + try: + first: Final = asyncio.create_task(compact_to_fit(router, deployment.model_dump(), request, "messages")) + await asyncio.wait_for(started.wait(), timeout=2) + if outcome == "cancel": + first.cancel() + if outcome == "success": + assert await first == await compact_to_fit(router, deployment.model_dump(), request, "messages") + changed: Final = {**request, "messages": [{"role": "user", "content": "new history"}, *request["messages"]]} + with pytest.raises(litellm.BadRequestError, match="History changed"): + await compact_to_fit(router, deployment.model_dump(), changed, "messages") + else: + error: Final = asyncio.CancelledError if outcome == "cancel" else asyncio.TimeoutError + with pytest.raises(error): + await first + with pytest.raises(error): + await compact_to_fit(router, deployment.model_dump(), request, "messages") + assert calls.qsize() == 1 and stopped.is_set() + finally: + compaction_executor.reset(token) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index baa15ac1568..4e146b59b61 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -15317,6 +15317,7 @@ class TestHealthFallbackDispatch: router: Final = self._router( config={ + "context_compaction": False, "tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}, "enable_context_window_escalation": True, } @@ -15399,6 +15400,7 @@ class TestHealthFallbackDispatch: async def test_modality_default_must_also_fit_context(self, default_fits: bool) -> None: router: Final = self._router( config={ + "context_compaction": False, "modality_routing": True, "tiers": {"SIMPLE": "primary"}, "enable_context_window_escalation": True, diff --git a/tests/test_litellm/rust_bridge/test_catalog.py b/tests/test_litellm/rust_bridge/test_catalog.py index 3fa5f3de7ab..82e3766e8a2 100644 --- a/tests/test_litellm/rust_bridge/test_catalog.py +++ b/tests/test_litellm/rust_bridge/test_catalog.py @@ -53,7 +53,7 @@ def test_shipped_decisions( enabled: Final = environment == "1" if environment is not None else process is not False assert catalog.rollout(context) is Rollout.RUST_OPT_OUT assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if enabled else Decision.PYTHON) - elif route is Route.MESSAGES: + elif route in (Route.MESSAGES, Route.TOKEN_COUNTER, Route.TOKENIZER): enabled: Final = environment == "1" if environment is not None else process is True assert catalog.rollout(context) is Rollout.RUST_OPT_IN assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if enabled else Decision.PYTHON) diff --git a/tests/test_litellm/rust_bridge/test_token_counter.py b/tests/test_litellm/rust_bridge/test_token_counter.py index 71aa79cc4bb..3da291c898d 100644 --- a/tests/test_litellm/rust_bridge/test_token_counter.py +++ b/tests/test_litellm/rust_bridge/test_token_counter.py @@ -12,25 +12,32 @@ from types import MappingProxyType from typing import Final import pytest -import tiktoken -from tokenizers import Tokenizer import litellm from litellm.constants import TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding -from litellm.proxy.spend_tracking.budget_reservation import _count_input_tokens +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model from litellm.rust_bridge import bindings, configuration from litellm.rust_bridge import token_counter as bridge +from litellm.rust_bridge import tokenizer as tokenizer_dispatch +from litellm.rust_bridge._native import Tokenizer from litellm.utils import claude_json_str MODEL: Final = "claude-sonnet-4-5-20250929" CL100K_MODEL: Final = "gpt-4" O200K_MODEL: Final = "gpt-4o" +MODEL_BY_TOKENIZER: Final[MappingProxyType[bridge.RustTokenizer, str]] = MappingProxyType( + {"anthropic": MODEL, "cl100k_base": CL100K_MODEL, "o200k_base": O200K_MODEL} +) TOKENIZERS: Final[tuple[bridge.RustTokenizer, ...]] = ("anthropic", "cl100k_base", "o200k_base") -RANK_FILE_LINES: Final = MappingProxyType({"cl100k_base": 100_256, "o200k_base": 199_998}) BODY: Final = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": "hello"}]}).encode() +def _counted(body: dict[str, object], model: str) -> tuple[bytes, dict[str, object]]: + raw: Final = json.dumps({**body, "model": model}).encode() + return raw, json.loads(raw) + + class _FakeDeclined(Exception): pass @@ -39,14 +46,41 @@ class _FakeUpstream(Exception): pass +class _FakeTokenizer: + """Stands in for one shared native `Tokenizer`; only its name identifies it.""" + + def __init__(self, name: str, json: str | None = None) -> None: + self.name = name + self.json = json + + +def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: + """Point the counter's tokenizer lookups at fakes while the bridge is faked; the codec path + keeps falling back to Python. Parity tests that restore the real extension get the real + lookups back.""" + fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} + anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) + real_encoding: Final = tokenizer_dispatch.native_encoding + real_anthropic: Final = tokenizer_dispatch.native_anthropic + + def faked() -> bool: + return isinstance(bindings.get_native_bridge(), _FakeNative) + + monkeypatch.setattr( + tokenizer_dispatch, "native_encoding", lambda name: fakes[name] if faked() else real_encoding(name) + ) + monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic if faked() else real_anthropic()) + + class _FakeNative: RustBridgeDeclined = _FakeDeclined RustUpstreamError = _FakeUpstream class _RecordingCounter: - def __init__(self, tokenizer_json: str) -> None: - self.tokenizer_json = tokenizer_json + def __init__(self, tokenizer: _FakeTokenizer, fast: bool) -> None: + self.tokenizer = tokenizer + self.fast = fast self.bodies: list[bytes] = [] async def acount_request(self, body: bytes) -> object: @@ -55,25 +89,16 @@ class _RecordingCounter: class _RecordingFactory: - """Stands in for the native `TokenCounter` class: callable for tokenizer JSON, `from_*_ranks` for rank files.""" + """Stands in for the native `TokenCounter` class, built over a loaded `Tokenizer`.""" def __init__(self) -> None: self.counters: list[_RecordingCounter] = [] - self.rank_files: list[str] = [] - def __call__(self, tokenizer_json: str) -> _RecordingCounter: - counter = _RecordingCounter(tokenizer_json) + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: + counter = _RecordingCounter(tokenizer, fast) self.counters.append(counter) return counter - def from_cl100k_ranks(self, rank_file: str) -> _RecordingCounter: - self.rank_files.append(rank_file) - return self("cl100k_base") - - def from_o200k_ranks(self, rank_file: str) -> _RecordingCounter: - self.rank_files.append(rank_file) - return self("o200k_base") - class _RaisingCounter: def __init__(self, error: Exception) -> None: @@ -89,13 +114,7 @@ class _RaisingFactory: def __init__(self, error: Exception) -> None: self.error = error - def __call__(self, tokenizer_json: str) -> _RaisingCounter: - return _RaisingCounter(self.error) - - def from_cl100k_ranks(self, rank_file: str) -> _RaisingCounter: - return _RaisingCounter(self.error) - - def from_o200k_ranks(self, rank_file: str) -> _RaisingCounter: + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RaisingCounter: return _RaisingCounter(self.error) @@ -105,6 +124,7 @@ def _reset_bridge(monkeypatch: pytest.MonkeyPatch): bridge._counter.cache_clear() configuration.reset_rust_configuration() monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) + _fake_native_tokenizers(monkeypatch, anthropic_json=claude_json_str) yield bridge.TOKEN_COUNTER.reset() bridge._counter.cache_clear() @@ -117,8 +137,12 @@ async def test_disabled_bridge_never_constructs_a_counter(tokenizer: bridge.Rust factory: Final = _RecordingFactory() litellm.rust(False) bridge.TOKEN_COUNTER.override(factory) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - assert await bridge.count_input_tokens(BODY, tokenizer) is None + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) + + assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) assert factory.counters == [] @@ -128,31 +152,33 @@ async def test_enabled_bridge_returns_typed_count_and_reuses_one_counter() -> No litellm.rust(True) bridge.TOKEN_COUNTER.override(factory) - first: Final = await bridge.count_input_tokens(BODY, "anthropic") - second: Final = await bridge.count_input_tokens(BODY, "anthropic") + first: Final = await bridge.native_count(factory, "anthropic", BODY) + second: Final = await bridge.native_count(factory, "anthropic", BODY) assert first == bridge.InputTokenCount(model=MODEL, input_tokens=42) assert second == first assert len(factory.counters) == 1 assert factory.counters[0].bodies == [BODY, BODY] - assert json.loads(factory.counters[0].tokenizer_json)["model"]["type"] == "BPE" + assert factory.counters[0].fast is False + assert factory.counters[0].tokenizer is tokenizer_dispatch.native_anthropic() + assert json.loads(factory.counters[0].tokenizer.json or "")["model"]["type"] == "BPE" @pytest.mark.asyncio @pytest.mark.parametrize("tokenizer", ("cl100k_base", "o200k_base")) -async def test_tiktoken_counter_is_built_from_the_vendored_rank_file_once(tokenizer: bridge.RustTokenizer) -> None: +async def test_tiktoken_counter_is_built_over_the_shared_encoding_once(tokenizer: bridge.RustTokenizer) -> None: factory: Final = _RecordingFactory() litellm.rust(True) bridge.TOKEN_COUNTER.override(factory) - first: Final = await bridge.count_input_tokens(BODY, tokenizer) - second: Final = await bridge.count_input_tokens(BODY, tokenizer) + first: Final = await bridge.native_count(factory, tokenizer, BODY) + second: Final = await bridge.native_count(factory, tokenizer, BODY) assert first == second == bridge.InputTokenCount(model=MODEL, input_tokens=42) - assert len(factory.rank_files) == 1 - assert factory.rank_files[0].startswith("IQ== 0\n") - assert factory.rank_files[0].count("\n") == RANK_FILE_LINES[tokenizer] - assert factory.counters[0].tokenizer_json == tokenizer + assert len(factory.counters) == 1 + assert factory.counters[0].tokenizer.name == tokenizer + assert factory.counters[0].tokenizer is tokenizer_dispatch.native_encoding(tokenizer) + assert factory.counters[0].fast is False assert factory.counters[0].bodies == [BODY, BODY] @@ -162,13 +188,13 @@ async def test_each_tokenizer_gets_its_own_cached_counter() -> None: litellm.rust(True) bridge.TOKEN_COUNTER.override(factory) - await bridge.count_input_tokens(BODY, "anthropic") - await bridge.count_input_tokens(BODY, "cl100k_base") - await bridge.count_input_tokens(BODY, "o200k_base") - await bridge.count_input_tokens(BODY, "anthropic") - await bridge.count_input_tokens(BODY, "o200k_base") + await bridge.native_count(factory, "anthropic", BODY) + await bridge.native_count(factory, "cl100k_base", BODY) + await bridge.native_count(factory, "o200k_base", BODY) + await bridge.native_count(factory, "anthropic", BODY) + await bridge.native_count(factory, "o200k_base", BODY) - assert [counter.tokenizer_json for counter in factory.counters][1:] == ["cl100k_base", "o200k_base"] + assert [counter.tokenizer.name for counter in factory.counters] == ["anthropic", "cl100k_base", "o200k_base"] assert [len(counter.bodies) for counter in factory.counters] == [2, 1, 2] @@ -176,8 +202,11 @@ async def test_each_tokenizer_gets_its_own_cached_counter() -> None: async def test_missing_native_module_falls_back(monkeypatch: pytest.MonkeyPatch) -> None: litellm.rust(True) monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, MODEL) - assert [await bridge.count_input_tokens(BODY, tokenizer) for tokenizer in TOKENIZERS] == [None, None, None] + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(MODEL,)) + + assert counts[MODEL] == count_input_tokens_for_model(request_body=request_body, model=MODEL) @pytest.mark.asyncio @@ -185,8 +214,12 @@ async def test_missing_native_module_falls_back(monkeypatch: pytest.MonkeyPatch) async def test_declined_request_falls_back(tokenizer: bridge.RustTokenizer) -> None: litellm.rust(True) bridge.TOKEN_COUNTER.override(_RaisingFactory(_FakeDeclined("request has no messages"))) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - assert await bridge.count_input_tokens(BODY, tokenizer) is None + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) + + assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) @pytest.mark.asyncio @@ -194,8 +227,12 @@ async def test_declined_request_falls_back(tokenizer: bridge.RustTokenizer) -> N async def test_runtime_failure_falls_back(tokenizer: bridge.RustTokenizer) -> None: litellm.rust(True) bridge.TOKEN_COUNTER.override(_RaisingFactory(RuntimeError("encode failed"))) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - assert await bridge.count_input_tokens(BODY, tokenizer) is None + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) + + assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) @pytest.mark.parametrize( @@ -273,8 +310,8 @@ def test_rust_tokenizer_names_the_encoding_python_actually_counts_with(model: st "Hello, world! camelCase ABCdef \u00e9\u00e8 12345 \u3053\u3093\u306b\u3061\u306f <|endoftext|>\r\n" * 9 ) python_count: Final = litellm.token_counter(model=model, text=text) - cl100k_count: Final = len(tiktoken.get_encoding("cl100k_base").encode(text, disallowed_special=())) - o200k_count: Final = len(tiktoken.get_encoding("o200k_base").encode(text, disallowed_special=())) + cl100k_count: Final = Tokenizer.from_tiktoken("cl100k_base").count(text) + o200k_count: Final = Tokenizer.from_tiktoken("o200k_base").count(text) assert cl100k_count != o200k_count match bridge.rust_tokenizer(model): case "cl100k_base": @@ -282,7 +319,7 @@ def test_rust_tokenizer_names_the_encoding_python_actually_counts_with(model: st case "o200k_base": assert python_count == o200k_count case "anthropic": - assert python_count == len(Tokenizer.from_str(claude_json_str).encode(text).ids) + assert python_count == Tokenizer.from_json(claude_json_str).count(text) assert python_count not in {cl100k_count, o200k_count} case None: pytest.fail(f"{model} must have a Rust tokenizer") @@ -386,12 +423,11 @@ async def test_native_count_matches_python_budget_counter( litellm.rust(True) body: Final = json.dumps(request_body).replace(MODEL, model) - rust_count: Final = await bridge.count_input_tokens(body.encode(), tokenizer) - python_count: Final = _count_input_tokens(request_body=json.loads(body), model=model) + request_body_parsed: Final = json.loads(body) + counts: Final = await count_input_tokens(request_body=request_body_parsed, raw_body=body.encode(), models=(model,)) + python_count: Final = count_input_tokens_for_model(request_body=request_body_parsed, model=model) - assert rust_count is not None - assert rust_count.model == json.loads(body).get("model") - assert rust_count.input_tokens == python_count + assert counts[model] == python_count @pytest.mark.asyncio @@ -405,15 +441,14 @@ async def test_tiktoken_counts_long_text_exactly_where_python_chunks( litellm.rust(True) text: Final = "x " * 20_000 body: Final = {"model": model, "messages": [{"role": "user", "content": text}]} - encoding: Final = tiktoken.get_encoding(tokenizer) - exact: Final = 3 + len(encoding.encode("user")) + len(encoding.encode(text)) + 3 + encoding: Final = Tokenizer.from_tiktoken(tokenizer) + exact: Final = 3 + encoding.count("user") + encoding.count(text) + 3 chunks: Final = -(-len(text) // TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS) - rust_count: Final = await bridge.count_input_tokens(json.dumps(body).encode(), tokenizer) - python_count: Final = _count_input_tokens(request_body=body, model=model) + counts: Final = await count_input_tokens(request_body=body, raw_body=json.dumps(body).encode(), models=(model,)) + python_count: Final = count_input_tokens_for_model(request_body=body, model=model) - assert rust_count is not None - assert rust_count.input_tokens == exact + assert counts[model] == exact assert python_count is not None assert exact < python_count <= exact + chunks @@ -440,5 +475,9 @@ async def test_native_declines_shapes_python_prices_differently( native: Final = pytest.importorskip("litellm.rust_bridge._native") monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) litellm.rust(True) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, parsed = _counted(request_body, model) - assert await bridge.count_input_tokens(json.dumps(request_body).encode(), tokenizer) is None + counts: Final = await count_input_tokens(request_body=parsed, raw_body=raw, models=(model,)) + + assert counts.get(model) == count_input_tokens_for_model(request_body=parsed, model=model) diff --git a/tests/test_litellm/rust_bridge/test_tokenizer.py b/tests/test_litellm/rust_bridge/test_tokenizer.py new file mode 100644 index 00000000000..0de7ad50b1e --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_tokenizer.py @@ -0,0 +1,134 @@ +from collections.abc import Generator +from typing import Final + +import pytest +import tiktoken +from tokenizers import Tokenizer + +import litellm +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding +from litellm.rust_bridge import configuration, tokenizer +from litellm.utils import _select_tokenizer +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + + +@pytest.fixture(autouse=True) +def isolated_configuration(monkeypatch: pytest.MonkeyPatch) -> Generator[None]: + monkeypatch.delenv("LITELLM_RUST", raising=False) + configuration.reset_rust_configuration() + yield + tokenizer.TOKENIZER.reset() + configuration.reset_rust_configuration() + + +@pytest.mark.parametrize("environment", (None, "0", "1")) +@pytest.mark.parametrize("process", (None, False, True)) +def test_tokenizer_factories_follow_rollout( + monkeypatch: pytest.MonkeyPatch, environment: str | None, process: bool | None +) -> None: + configuration.rust(process) + if environment is not None: + monkeypatch.setenv("LITELLM_RUST", environment) + enabled: Final = environment == "1" if environment is not None else process is True + encoding: Final = tokenizer.get_encoding("cl100k_base") + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) + reference: Final = Tokenizer.from_str(TOKENIZER_JSON) + + assert isinstance(encoding, OpenAIEncoding if enabled else tiktoken.Encoding) + assert isinstance(custom["tokenizer"], HuggingFaceTokenizer if enabled else Tokenizer) + assert encoding.encode("café 漢字 🙂") == tiktoken.get_encoding(encoding.name).encode("café 漢字 🙂") + assert litellm.encode(text="Hello World", custom_tokenizer=custom) == reference.encode("Hello World").ids + assert litellm.token_counter(text="Hello World", custom_tokenizer=custom) == len(reference.encode("Hello World")) + + +def test_missing_native_binding_keeps_python_tokenizer_api() -> None: + configuration.rust(True) + tokenizer.TOKENIZER.override(None) + encoding: Final = tokenizer.get_encoding("cl100k_base") + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON)["tokenizer"] + + assert isinstance(encoding, tiktoken.Encoding) + assert isinstance(custom, Tokenizer) + custom.enable_padding(pad_id=0, pad_token="[UNK]") + assert [item.ids for item in custom.encode_batch(["Hello", "Hello World"])] == [[3, 1, 0], [3, 1, 2]] + + +def test_cached_selection_follows_backend_changes(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", True) + configuration.rust(True) + native: Final = _select_tokenizer("dispatch-fixture")["tokenizer"] + configuration.rust(False) + python: Final = _select_tokenizer("dispatch-fixture")["tokenizer"] + + assert isinstance(native, OpenAIEncoding) + assert isinstance(python, tiktoken.Encoding) + assert native.encode("hello") == python.encode("hello") + + +def test_declined_native_factory_falls_back_before_tokenizing() -> None: + from litellm.rust_bridge._native import RustBridgeDeclined + + class UnavailableTokenizer: + @staticmethod + def from_json(json: str) -> None: + raise RustBridgeDeclined("huggingface feature is disabled") + + configuration.rust(True) + binding: Final = tokenizer._as_factory(UnavailableTokenizer) + tokenizer.TOKENIZER.override(binding) + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) + + assert isinstance(custom["tokenizer"], Tokenizer) + assert ( + litellm.decode(tokens=litellm.encode(text="Hello World", custom_tokenizer=custom), custom_tokenizer=custom) + == "Hello World" + ) + + +@pytest.mark.parametrize( + ("model", "text"), + ( + ("gpt-4o", "hello <|endoftext|> world"), + ("gpt-3.5-turbo", "café 漢字 🙂"), + ("text-davinci-003", " def f():\n return 1\n"), + ("tokenizer-parity-fixture", "hello again"), + ), +) +def test_public_token_api_is_identical_across_backends(monkeypatch: pytest.MonkeyPatch, model: str, text: str) -> None: + """`litellm.token_counter`, `encode` and `decode` return the same values whichever backend + the catalog picks; only the object types differ.""" + monkeypatch.setattr(litellm, "anthropic_models", {*litellm.anthropic_models, "tokenizer-parity-fixture"}) + messages: Final = [{"role": "user", "content": text}, {"role": "assistant", "content": "ok"}] + + def observe() -> tuple[int, int, list[int], str]: + ids: Final = litellm.encode(model=model, text=text) + return ( + litellm.token_counter(model=model, text=text), + litellm.token_counter(model=model, messages=messages), + ids, + litellm.decode(model=model, tokens=ids), + ) + + configuration.rust(False) + python: Final = observe() + configuration.rust(True) + rust: Final = observe() + + assert rust == python + + +def test_cached_huggingface_tokenizers_follow_backend_changes(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer as RustHuggingFaceTokenizer + from litellm.utils import _load_huggingface_tokenizer + + monkeypatch.setattr(litellm, "anthropic_models", {*litellm.anthropic_models, "tokenizer-cache-fixture"}) + _load_huggingface_tokenizer.cache_clear() + configuration.rust(True) + native: Final = _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] + configuration.rust(False) + python: Final = _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] + configuration.rust(True) + + assert isinstance(native, RustHuggingFaceTokenizer) + assert isinstance(python, Tokenizer) + assert _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] is native diff --git a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py index e449d4392d8..0eae041f535 100644 --- a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -56,10 +56,11 @@ def _write_wheel( metadata_tags: tuple[str, ...] | None = (_EXPECTED_TAG,), dist_info: str = _DIST_INFO, duplicate_wheel: bool = False, + native_bytes: bytes = b"synthetic native extension", ) -> Path: wheel: Final = tmp_path / f"litellm-1.100.0-{filename_tag}.whl" with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive: - archive.writestr(_NATIVE_MEMBER, b"synthetic native extension") + archive.writestr(_NATIVE_MEMBER, native_bytes) archive.writestr( f"{dist_info}/METADATA", "Metadata-Version: 2.1\nName: litellm\nVersion: 1.100.0\n", @@ -195,3 +196,17 @@ def test_rejects_production_module_exposing_panic_hook(tmp_path: Path) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) assert _run_verifier(wheel, exposes_panic=True) == 1 + + +@pytest.mark.parametrize("embedded", (False, True)) +def test_vocabulary_is_packaged_once(tmp_path: Path, embedded: bool) -> None: + ranks: Final = b"AA== 0\nAQ== 1\nAg== 2\n" + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + native_bytes=b"native engine" + (ranks if embedded else b""), + ) + with zipfile.ZipFile(wheel, "a") as archive: + archive.writestr("litellm/litellm_core_utils/tokenizers/" + "a" * 40, ranks) + + assert _run_verifier(wheel) == (1 if embedded else 0) diff --git a/tests/test_litellm/test_anthropic_beta_headers_filtering.py b/tests/test_litellm/test_anthropic_beta_headers_filtering.py index 19b26120672..8656a7564d2 100644 --- a/tests/test_litellm/test_anthropic_beta_headers_filtering.py +++ b/tests/test_litellm/test_anthropic_beta_headers_filtering.py @@ -83,6 +83,7 @@ class TestAnthropicBetaHeadersFiltering: filtered = filter_and_transform_beta_headers( beta_headers=all_headers, provider=provider ) + assert ("compact-2026-09-04" in filtered) is (provider == "anthropic") for header in unsupported_headers: assert ( diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 84c60e029cb..bdeedb5f38c 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,3 +1,4 @@ +import datetime import time from typing import Final @@ -29,6 +30,7 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, Usage, ) +from litellm.types.videos.main import VideoObject @pytest.fixture @@ -3038,21 +3040,23 @@ def test_completion_cost_logs_cache_and_reasoning_breakdown_for_custom_pricing() assert total == pytest.approx(100 * 1e-6 + 800 * 1e-7 + 100 * 1.25e-6 + 500 * 2e-6) -def test_cost_per_token_per_second_pricing(monkeypatch): +@pytest.mark.parametrize("custom_llm_provider", ["together_ai", "openai", "anthropic", "bedrock", "azure"]) +def test_cost_per_token_per_second_pricing(monkeypatch, custom_llm_provider: str): """ Models priced by duration (input/output_cost_per_second) with no per-token rates - must be billed as cost_per_second * response_time_ms / 1000 in cost_per_token. + must be billed as cost_per_second * response_time_ms / 1000 in cost_per_token, + whether or not the provider has its own cost calculator. """ monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - model = "test-per-second-pricing-model" + model = f"test-per-second-pricing-{custom_llm_provider}" litellm.register_model( model_cost={ model: { "input_cost_per_second": 0.02, "output_cost_per_second": 0.04, - "litellm_provider": "together_ai", + "litellm_provider": custom_llm_provider, "mode": "chat", } } @@ -3060,7 +3064,7 @@ def test_cost_per_token_per_second_pricing(monkeypatch): prompt_cost, completion_cost_value = cost_per_token( model=model, - custom_llm_provider="together_ai", + custom_llm_provider=custom_llm_provider, prompt_tokens=10, completion_tokens=20, response_time_ms=1500.0, @@ -3070,6 +3074,143 @@ def test_cost_per_token_per_second_pricing(monkeypatch): assert completion_cost_value == pytest.approx(0.04 * 1.5) +def test_cost_per_token_keeps_token_pricing_when_per_second_rates_are_also_set(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + model = "test-token-and-per-second-pricing-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + + prompt_cost, completion_cost_value = cost_per_token( + model=model, + custom_llm_provider="openai", + prompt_tokens=10, + completion_tokens=20, + response_time_ms=1500.0, + ) + + assert prompt_cost == pytest.approx(10 * 1e-6) + assert completion_cost_value == pytest.approx(20 * 2e-6) + + +def _logging_obj_with_call_window(duration_ms: float) -> Logging: + start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0) + logging_obj: Final = Logging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="per-second-call-window", + function_id="f", + ) + logging_obj.model_call_details["start_time"] = start_time + logging_obj.model_call_details["end_time"] = start_time + datetime.timedelta(milliseconds=duration_ms) + return logging_obj + + +@pytest.mark.parametrize( + ("stamped_response_ms", "total_time", "logged_duration_ms", "expected_seconds"), + [(None, 0.0, 1500.0, 1.5), (3000.0, 0.0, 1500.0, 3.0), (None, 2500.0, 1500.0, 2.5), (3000.0, 2500.0, 1500.0, 3.0)], +) +def test_completion_cost_per_second_deployment_bills_the_call_duration( + monkeypatch, + stamped_response_ms: float | None, + total_time: float, + logged_duration_ms: float, + expected_seconds: float, +): + """ + A deployment priced only per second bills the stamped ``_response_ms`` when there is one, + then the caller's explicit ``total_time``, and the logging object's start/end window otherwise + (a streamed response is never stamped). + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + deployment_id = "per-second-openai-deployment" + litellm.register_model( + model_cost={ + deployment_id: { + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + response = ModelResponse( + model="gpt-5.4-nano", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + response._response_ms = stamped_response_ms + + cost = completion_cost( + completion_response=response, + model="openai/gpt-5.4-nano", + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + total_time=total_time, + litellm_logging_obj=_logging_obj_with_call_window(logged_duration_ms), + ) + + assert cost == pytest.approx((0.02 + 0.04) * expected_seconds) + + +@pytest.mark.parametrize("mode", ["audio_transcription", "audio_speech", "video_generation", "realtime"]) +def test_cost_per_token_leaves_media_second_rates_to_their_dedicated_paths(monkeypatch, mode: str): + """ + A media-mode entry's per-second rates price audio or video seconds, which the dedicated + transcription, speech, video, and realtime paths bill from the media itself, so a call that + reaches the generic path with only a wall-clock duration must not bill them. + """ + model = f"test-media-per-second-{mode}" + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_second": 0.02, "output_cost_per_second": 0.4, "litellm_provider": "openai", "mode": mode}, + ) + + assert cost_per_token(model=model, custom_llm_provider="openai", response_time_ms=2000.0) == (0.0, 0.0) + + +def test_completion_cost_video_status_poll_bills_nothing_on_a_per_second_video_model(monkeypatch): + """ + Polling a video job returns a ``VideoObject`` with no stamped duration, so the cost path falls + back to the logging object's call window; on a video model priced per output second that + window must not be billed, or every status poll would charge for the seconds it took to answer. + """ + model = "test-veo-per-second-poll" + monkeypatch.setitem( + litellm.model_cost, + model, + {"output_cost_per_second": 0.4, "litellm_provider": "vertex_ai", "mode": "video_generation"}, + ) + video = VideoObject(id="video_1", object="video", status="completed", model=model, progress=100) + + cost = completion_cost( + completion_response=video, + model=model, + custom_llm_provider="vertex_ai", + call_type=CallTypes.video_retrieve.value, + litellm_logging_obj=_logging_obj_with_call_window(2000.0), + ) + + assert cost == 0.0 + + def _batch_cache_usage() -> Usage: return Usage( prompt_tokens=11000, diff --git a/tests/test_litellm/test_cost_map_guard.py b/tests/test_litellm/test_cost_map_guard.py index 1b4330ed62c..1595bd9864f 100644 --- a/tests/test_litellm/test_cost_map_guard.py +++ b/tests/test_litellm/test_cost_map_guard.py @@ -114,6 +114,40 @@ def test_unclassified_entry_key_is_reported() -> None: assert "Unclassified keys" in failure and "weird_thing" in failure +STALE_HEAD: Final = _snapshot(BASE_MAP, backup=_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)}), schema="{}") +CODE_ONLY: Final = ( + "litellm/utils.py", + "tests/test_litellm/test_utils.py", + "docs/model_prices_and_context_window.json", +) + + +def test_human_pr_that_leaves_the_cost_map_alone_skips_the_file_checks() -> None: + unparseable: Final = guard.Snapshot(cost_map="{not json", backup="", schema="") + assert _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=False) == () + assert _failures(STALE_HEAD, changed_files=(), bot=False) == () + assert _failures(unparseable, changed_files=CODE_ONLY, bot=False) == () + + +@pytest.mark.parametrize("guarded_path", guard.GUARDED_PATHS) +def test_touching_any_cost_map_file_keeps_the_file_checks(guarded_path: str) -> None: + failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, guarded_path), bot=False) + assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] + assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] + + +def test_bot_pr_always_gets_the_file_checks() -> None: + failures: Final = _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=True) + assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] + assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] + + +def test_contract_names_the_skip() -> None: + assert guard.contract_for(False, CODE_ONLY) == "human PR, cost map untouched" + assert guard.contract_for(False, (*CODE_ONLY, guard.SCHEMA_PATH)) == "human PR, file checks only" + assert guard.contract_for(True, CODE_ONLY) == "bot contract enforced" + + def test_bot_may_only_touch_the_cost_map_files() -> None: changed = (*guard.GUARDED_PATHS, "litellm/utils.py", ".github/workflows/cost-map-guard.yml") assert _failures(BASE, changed_files=changed, bot=False) == () @@ -147,6 +181,16 @@ def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str: (repo / guard.BACKUP_PATH).parent.mkdir(exist_ok=True) (repo / guard.BACKUP_PATH).write_text(text) (repo / guard.SCHEMA_PATH).write_text(schema_module.render(schema_module.build_schema(cost_map))) + return _git_commit(repo, message) + + +def _commit_code_only(repo: Path, message: str) -> str: + (repo / "litellm").mkdir(exist_ok=True) + (repo / "litellm" / "utils.py").write_text(f"print('{message}')\n") + return _git_commit(repo, message) + + +def _git_commit(repo: Path, message: str) -> str: subprocess.run(("git", "add", "-A"), cwd=repo, check=True) subprocess.run( ("git", "-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-q", "-m", message), @@ -186,6 +230,40 @@ def test_main_reads_both_revisions_from_git( assert expected_line in result.stdout.splitlines() +def test_main_skips_the_file_checks_on_a_stale_base_the_pr_never_touched(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + _commit(tmp_path, BASE_MAP, "base") + (tmp_path / guard.BACKUP_PATH).write_text(_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)})) + (tmp_path / guard.SCHEMA_PATH).write_text("{}") + stale_base: Final = _commit_code_only(tmp_path, "stale base with drifted backup and schema") + head: Final = _commit_code_only(tmp_path, "code change on the stale base") + human: Final = _run_guard(tmp_path, stale_base, head, "litellm_fix_pricing") + assert human.returncode == 0, human.stdout + human.stderr + assert "cost map guard passed (human PR, cost map untouched)" in human.stdout.splitlines() + bot: Final = _run_guard(tmp_path, stale_base, head, BOT_REF) + assert bot.returncode == 1 + backup_failure: Final = f"- {guard.BACKUP_PATH} differs from {guard.COST_MAP_PATH}; copy the root file over it" + assert backup_failure in bot.stdout.splitlines() + + +def test_main_keeps_the_file_checks_when_a_cost_map_file_is_renamed(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, BASE_MAP, "base") + subprocess.run(("git", "mv", guard.COST_MAP_PATH, "renamed.json"), cwd=tmp_path, check=True) + head: Final = _git_commit(tmp_path, "rename the cost map") + result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing") + assert result.returncode == 1, result.stdout + result.stderr + assert "cost map guard failed (human PR, file checks only):" in result.stdout.splitlines() + + +def test_main_fails_when_the_changed_files_cannot_be_read(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + head: Final = _commit(tmp_path, BASE_MAP, "head") + result: Final = _run_guard(tmp_path, "0" * 40, head, "litellm_fix_pricing") + assert result.returncode == 1, result.stdout + result.stderr + assert result.stdout.startswith("cost map guard failed: git diff ") + + def test_main_rejects_a_bot_pr_that_edits_code(tmp_path: Path) -> None: subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) base = _commit(tmp_path, BASE_MAP, "base") diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index cf61a6d9f65..38aad3ace1c 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -178,6 +178,14 @@ def test_potential_model_names_keeps_provider_prefixed_candidate(): assert bare["provider_prefixed_model_name"] == bare["combined_model_name"] == "perplexity/glm-5.2" +@pytest.mark.parametrize("capability", [True, False, None]) +def test_get_model_info_anthropic_compaction( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, capability: bool | None +) -> None: + monkeypatch.setitem(litellm.model_cost["claude-sonnet-5"], "supports_anthropic_compaction", capability) + assert litellm.get_model_info("claude-sonnet-5")["supports_anthropic_compaction"] is capability + + def test_get_model_info_strips_openai_finetune_ids_without_a_custom_suffix(local_model_cost_map): info = litellm.get_model_info(model="ft:gpt-4o-2024-08-06:my-org::abc123", custom_llm_provider="openai") assert info["key"] == "ft:gpt-4o-2024-08-06" @@ -861,6 +869,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "source": {"type": "string"}, "comment": {"type": "string"}, "supports_assistant_prefill": {"type": "boolean"}, + "supports_anthropic_compaction": {"type": "boolean"}, "supports_audio_input": {"type": "boolean"}, "supports_audio_output": {"type": "boolean"}, "gemini_native_audio": {"type": "boolean"}, diff --git a/tests/test_litellm_rust/test_fork_guard.py b/tests/test_litellm_rust/test_fork_guard.py index 086397bab5c..2a8fb6f9fca 100644 --- a/tests/test_litellm_rust/test_fork_guard.py +++ b/tests/test_litellm_rust/test_fork_guard.py @@ -1,5 +1,6 @@ import os import textwrap +from typing import Final import pytest @@ -144,3 +145,76 @@ def test_sdk_call_in_a_child_forked_after_native_use_raises_instead_of_hanging() result = run_child_interpreter(_SDK_CONTRACT, env=env, timeout=120) assert result.returncode == 0, result.stderr + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="fork only") +@pytest.mark.parametrize("warm_fast_counter", (False, True)) +def test_tokenizers_share_the_native_process_guard(warm_fast_counter: bool) -> None: + script: Final = """ +import asyncio +import os +import litellm +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens +from litellm.rust_bridge import _native +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer +from litellm.utils import claude_json_str + +litellm.anthropic_models = {*litellm.anthropic_models, "tokenizer-fork-fixture"} +_native.reserve_process_for_forking() +for create in ( + lambda: _native.Tokenizer.from_tiktoken("cl100k_base"), + lambda: _native.Tokenizer.from_json(claude_json_str), + lambda: litellm.token_counter(model="tokenizer-fork-fixture", text="hello"), +): + try: + create() + except _native.ProcessReservedForForking: + pass + else: + raise AssertionError("reserved parent ran a native tokenizer") +assert not _native.process_state_started() + +pid = os.fork() +if pid == 0: + tokenizer = HuggingFaceTokenizer.from_str(claude_json_str) + encoding = _native.Tokenizer.from_tiktoken("cl100k_base") + if os.environ["WARM_FAST_COUNTER"] == "True": + _native.TokenCounter.from_tokenizer(encoding, fast=True) + expected = [item.ids for item in tokenizer.encode_batch(["hello", "world"])] + assert _native.process_state_started() + grandchild = os.fork() + if grandchild == 0: + for call in ( + lambda: tokenizer.encode_batch(["hello", "world"]), + lambda: tokenizer.encode("hello"), + lambda: encoding.count("hello"), + lambda: encoding.count("hello", fast=True), + lambda: _native.TokenCounter.from_tokenizer(encoding), + lambda: _native.TokenCounter.from_tokenizer(encoding, fast=True), + lambda: _native.Tokenizer.from_tiktoken("cl100k_base"), + lambda: asyncio.run(count_input_tokens({"prompt": "hello"}, b'{"prompt": "hello"}', ("counter-fork-fixture",))), + ): + try: + call() + except _native.ForkedAfterNativeRuntimeStarted: + pass + else: + os._exit(1) + os._exit(0) + assert os.waitpid(grandchild, 0)[1] == 0 + assert [item.ids for item in tokenizer.encode_batch(["hello", "world"])] == expected + os._exit(0) +assert os.waitpid(pid, 0)[1] == 0 +""" + result: Final = run_child_interpreter( + script, + env={ + **os.environ, + "OBJC_DISABLE_INITIALIZE_FORK_SAFETY": "YES", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "WARM_FAST_COUNTER": str(warm_fast_counter), + }, + timeout=30, + ) + + assert result.returncode == 0, result.stderr diff --git a/tests/test_litellm_rust/test_tokenizer.py b/tests/test_litellm_rust/test_tokenizer.py new file mode 100644 index 00000000000..98d5259b652 --- /dev/null +++ b/tests/test_litellm_rust/test_tokenizer.py @@ -0,0 +1,130 @@ +import json +from typing import Final + +import pytest +import tiktoken +from tokenizers import Tokenizer as ReferenceTokenizer + +from litellm.rust_bridge import _native +from litellm.utils import claude_json_str +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + +pytestmark = pytest.mark.requires_rust_extension + + +def test_tiktoken_codec_round_trips_and_counts() -> None: + tokenizer: Final = _native.Tokenizer.from_tiktoken("cl100k_base") + encoded: Final = tokenizer.encode("hello world") + + assert tokenizer.name == "cl100k_base" + assert tokenizer.count("hello world") == len(encoded) + assert tokenizer.decode(encoded) == "hello world" + + +def test_huggingface_codec_skips_special_tokens() -> None: + tokenizer: Final = _native.Tokenizer.from_json(claude_json_str) + encoded: Final = tokenizer.encode("hello") + + assert "" in tokenizer.decode(encoded, skip_special_tokens=False) + assert tokenizer.decode(encoded, skip_special_tokens=True) == "hello" + + +def test_tiktoken_codec_keeps_the_requested_encoding_name() -> None: + assert _native.Tokenizer.from_tiktoken("gpt2").name == "gpt2" + assert _native.Tokenizer.from_tiktoken("r50k_base").name == "r50k_base" + assert _native.Tokenizer.from_tiktoken("gpt2").encode("hi") == _native.Tokenizer.from_tiktoken("r50k_base").encode( + "hi" + ) + + +def test_tiktoken_codec_exposes_its_vocabulary() -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + tokenizer: Final = _native.Tokenizer.from_tiktoken("cl100k_base") + + assert tokenizer.special_tokens() == reference._special_tokens + assert tokenizer.max_token_value() == reference.max_token_value + assert tokenizer.token_byte_values() == reference.token_byte_values() + assert tokenizer.encode_single_token(b"hello") == reference.encode_single_token("hello") + assert tokenizer.is_special_token(reference.eot_token) and not tokenizer.is_special_token(0) + with pytest.raises(KeyError): + tokenizer.encode_single_token(b"<|not-a-token|>") + + +def test_huggingface_codec_rejects_tiktoken_only_calls() -> None: + tokenizer: Final = _native.Tokenizer.from_json(claude_json_str) + with pytest.raises(ValueError, match="requires a tiktoken encoding"): + tokenizer.token_byte_values() + with pytest.raises(ValueError, match="requires a Hugging Face tokenizer"): + _native.Tokenizer.from_tiktoken("cl100k_base").get_vocab() + + +def test_unknown_tiktoken_encoding_raises_value_error() -> None: + with pytest.raises(ValueError, match="unsupported tokenizer"): + _native.Tokenizer.from_tiktoken("unknown-encoding") + + +def test_tiktoken_codec_decodes_truncated_unicode_like_python() -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + tokenizer: Final = _native.Tokenizer.from_tiktoken(reference.name) + encoded: Final = reference.encode("🙂漢字") + + assert tuple(tokenizer.decode(encoded[:end]) for end in range(1, len(encoded) + 1)) == tuple( + reference.decode(encoded[:end]) for end in range(1, len(encoded) + 1) + ) + + +FAST_TEXTS: Final = ( + "", + "hello world <|endoftext|>", + "café 漢字 ع 🙂 line\r\n indented 123456789", + "x a\u0301 fi", +) + + +def test_fast_counting_is_an_opt_in_over_the_same_loaded_tokenizer() -> None: + for tokenizer in ( + _native.Tokenizer.from_tiktoken("cl100k_base"), + _native.Tokenizer.from_tiktoken("o200k_base"), + _native.Tokenizer.from_json(claude_json_str), + ): + assert [tokenizer.count(text, fast=True) for text in FAST_TEXTS] == [ + tokenizer.count(text) for text in FAST_TEXTS + ] + + +@pytest.mark.parametrize( + "name", ("cl100k_base", "o200k_base", "o200k_harmony", "p50k_base", "p50k_edit", "r50k_base", "gpt2") +) +@pytest.mark.asyncio +async def test_token_counter_counts_over_a_shared_tokenizer(name: str) -> None: + messages: Final = [{"role": "user", "content": "hello wide world"}, {"role": "assistant", "content": "ok"}] + body: Final = json.dumps({"model": "gpt-4", "messages": messages}).encode() + tokenizer: Final = _native.Tokenizer.from_tiktoken(name) + reference: Final = tiktoken.get_encoding(name) + for text in FAST_TEXTS: + assert tokenizer.count(text, fast=True) == tokenizer.count(text) == len(reference.encode_ordinary(text)) + + exact: Final = await _native.TokenCounter.from_tokenizer(tokenizer).acount_request(body) + fast: Final = await _native.TokenCounter.from_tokenizer(tokenizer, fast=True).acount_request(body) + + assert exact == fast + assert exact["input_tokens"] == 3 + sum( + 3 + len(reference.encode_ordinary(message["role"])) + len(reference.encode_ordinary(message["content"])) + for message in messages + ) + + +@pytest.mark.parametrize("configured", (False, True)) +@pytest.mark.asyncio +async def test_fast_count_preserves_huggingface_configuration(configured: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + if configured: + reference.enable_truncation(max_length=3) + reference.enable_padding(pad_id=0, pad_token="[UNK]", length=5) + tokenizer: Final = _native.Tokenizer.from_json(reference.to_str()) + counter: Final = _native.TokenCounter.from_tokenizer(tokenizer, fast=True) + for text in ("", "Hello", "Hello World Hello World", "[BOS] Hello"): + expected: Final = len(reference.encode(text)) + assert tokenizer.count(text, fast=True) == tokenizer.count(text) == expected + result: Final = await counter.acount_request(json.dumps({"prompt": text}).encode()) + assert result["input_tokens"] == expected diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py index 50409b2ea2c..37201e8155b 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -7,10 +7,12 @@ through _hidden_params to the x-litellm-callback-duration-ms response header. import asyncio import datetime +from typing import Final from unittest.mock import MagicMock import pytest +import litellm import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod import litellm.proxy.common_request_processing as common_request_processing_mod from litellm.litellm_core_utils.litellm_logging import Logging @@ -22,7 +24,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.utils import ModelResponse +from litellm.types.utils import ModelResponse, Usage class TestCallbackDurationMs: @@ -583,3 +585,57 @@ class TestLoggingInitCallbackDuration: # Should still be set (deep copy of None is essentially a no-op) assert hasattr(obj, "callback_duration_ms") assert obj.callback_duration_ms >= 0 + + +def test_update_response_metadata_prices_per_second_deployment_from_its_stamped_duration(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + deployment_id: Final = "per-second-deployment-response-metadata" + litellm.register_model( + model_cost={ + deployment_id: { + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0) + logging_obj: Final = Logging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="per-second-response-metadata", + function_id="f", + ) + logging_obj.update_environment_variables( + model="gpt-5.4-nano", + litellm_params={ + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "metadata": {"model_info": {"id": deployment_id}}, + }, + optional_params={}, + custom_llm_provider="openai", + ) + logging_obj.model_call_details["end_time"] = start_time + datetime.timedelta(seconds=10) + result: Final = ModelResponse( + model="gpt-5.4-nano", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + + update_response_metadata( + result=result, + logging_obj=logging_obj, + model="gpt-5.4-nano", + kwargs={"model_info": {"id": deployment_id}}, + start_time=start_time, + end_time=start_time + datetime.timedelta(seconds=2), + ) + + assert result._response_ms == pytest.approx(2000) + assert result._hidden_params["response_cost"] == pytest.approx((0.02 + 0.04) * 2) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx index da564f23de5..58e1e9bcce9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx @@ -4,6 +4,8 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { MCPServerView } from "./mcp_server_view"; import * as networking from "@/components/networking"; +import { setSecureItem } from "@/utils/secureStorage"; +import { EDIT_OAUTH_UI_STATE_KEY } from "./mcp_server_edit"; import type { MCPServer } from "@/components/mcp_tools/types"; vi.mock(".", () => ({ @@ -68,6 +70,7 @@ const openUserCredentials = async (props: Record) => { describe("MCPServerView", () => { beforeEach(() => { vi.clearAllMocks(); + sessionStorage.clear(); }); // Name, alias and description each label the header and a Settings row, so @@ -146,6 +149,37 @@ describe("MCPServerView", () => { expect(screen.queryByRole("button", { name: "Edit Settings" })).not.toBeInTheDocument(); }); + it.each([false, true])("keeps config settings read-only with isEditing=%s", async (isEditing) => { + renderView({ is_config: true }, { isEditing }); + + await userEvent.click(screen.getByRole("tab", { name: "Settings" })); + + expect(screen.getByRole("button", { name: "Edit Settings" })).toBeDisabled(); + expect(screen.getByText("Defined in config. Edit your YAML configuration to make changes")).toBeVisible(); + expect(screen.queryByText("edit form")).not.toBeInTheDocument(); + }); + + it.each([true, false])("honors config read-only state on OAuth return: %s", async (isConfig) => { + setSecureItem(EDIT_OAUTH_UI_STATE_KEY, JSON.stringify({ serverId: "srv-1" })); + renderView({ is_config: isConfig }); + + await userEvent.click(screen.getByRole("tab", { name: "Settings" })); + + if (isConfig) { + expect(screen.queryByText("edit form")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Edit Settings" })).toBeDisabled(); + } else { + expect(screen.getByText("edit form")).toBeVisible(); + } + }); + + it("does not open the editor for a view-only admin", async () => { + renderView({}, { isViewOnly: true, isEditing: true }); + await userEvent.click(screen.getByRole("tab", { name: "Settings" })); + expect(screen.getByRole("button", { name: "Edit Settings" })).toBeDisabled(); + expect(screen.queryByText("edit form")).not.toBeInTheDocument(); + }); + it("opens on the tab named by initialTabIndex", async () => { renderView({}, { initialTabIndex: 1 }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx index a346c7d986b..6045be4607d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx @@ -62,7 +62,8 @@ export const MCPServerView: React.FC = ({ }) => { // Open the editing Settings tab on first render when returning from the edit OAuth // redirect, so the "token fetched" feedback shows where the user left off (Settings=2). - const returningFromEditOAuth = isReturningFromEditOAuth(isProxyAdmin, mcpServer.server_id); + const canEdit = isProxyAdmin && !isViewOnly && !mcpServer.is_config; + const returningFromEditOAuth = isReturningFromEditOAuth(canEdit, mcpServer.server_id); const [editing, setEditing] = useState(isEditing || returningFromEditOAuth); const [showFullUrl, setShowFullUrl] = useState(false); const [copiedStates, setCopiedStates] = useState>({}); @@ -224,13 +225,18 @@ export const MCPServerView: React.FC = ({

MCP Server Settings

- {editing ? null : ( - )}
- {editing ? ( + {mcpServer.is_config && ( +

+ Defined in config. Edit your YAML configuration to make changes +

+ )} + {editing && canEdit ? (