From 0132f34356907d312273991deba2bba487d9db72 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:40:25 -0700 Subject: [PATCH 01/28] chore(prices): sync OpenRouter prices: 2 models, 2 new (#42407) openrouter/nex-agi/nex-n2.5-mini: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_pdf_input, supports_reasoning, supports_web_search, supports_audio_input, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/nex-agi/nex-n2.5-pro: max_tokens, supports_vision, max_input_tokens, max_output_tokens, supports_pdf_input, supports_reasoning, supports_web_search, supports_audio_input, supports_tool_choice, supports_prompt_caching, supports_response_schema, supports_function_calling, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 40 +++++++++++++++++++ model_prices_and_context_window.json | 40 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 08cb2ddb8f7..77cada25d25 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -77528,5 +77528,45 @@ "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true + }, + "openrouter/nex-agi/nex-n2.5-mini": { + "cache_read_input_token_cost": 2.5e-09, + "input_cost_per_token": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false + }, + "openrouter/nex-agi/nex-n2.5-pro": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 08cb2ddb8f7..77cada25d25 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -77528,5 +77528,45 @@ "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true + }, + "openrouter/nex-agi/nex-n2.5-mini": { + "cache_read_input_token_cost": 2.5e-09, + "input_cost_per_token": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false + }, + "openrouter/nex-agi/nex-n2.5-pro": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false } } From a550b95d70a7c6e538aac9cdc7ebfb949c38051e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:40:48 -0700 Subject: [PATCH 02/28] ci: skip cost map file checks on PRs that leave the cost map untouched (#42406) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- ci_cd/cost_map_guard.py | 41 +++++++++--- tests/test_litellm/test_cost_map_guard.py | 78 +++++++++++++++++++++++ 2 files changed, 110 insertions(+), 9 deletions(-) diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 50aa40ba220..5842cf6f1ac 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -1,8 +1,11 @@ """Guard the cost map on pull requests. -Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file, -and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named -litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models. +Every pull request whose diff against its merge base touches one of the three cost map files gets the file +checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the +map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies +and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot +(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three +files and may only add or update models. """ from __future__ import annotations @@ -108,20 +111,37 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str ) +def touches_cost_map(changed_files: Sequence[str]) -> bool: + return any(path in GUARDED_PATHS for path in changed_files) + + +def contract_for(bot: bool, changed_files: Sequence[str]) -> str: + if bot: + return "bot contract enforced" + return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched" + + def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]: + if not bot and not touches_cost_map(changed_files): + return () head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH) if isinstance(head_map, str): return (head_map,) return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ())) -def _git(*args: str) -> str: +def _git(*args: str) -> str | None: result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True) - return result.stdout if result.returncode == 0 else "" + return result.stdout if result.returncode == 0 else None def snapshot(revision: str) -> Snapshot: - return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS)) + return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS)) + + +def changed_files(base: str, head: str) -> tuple[str, ...] | None: + diff: Final = _git("diff", "--name-only", "--no-renames", base, head) + return None if diff is None else tuple(diff.splitlines()) def main(argv: Sequence[str]) -> int: @@ -131,9 +151,12 @@ def main(argv: Sequence[str]) -> int: parser.add_argument("--head-ref", required=True, help="head branch name of the pull request") args: Final = parser.parse_args(argv) bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX) - changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines()) - failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot) - contract: Final = "bot contract enforced" if bot else "human PR, file checks only" + changed: Final = changed_files(args.base, args.head) + if changed is None: + print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown") + return 1 + failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot) + contract: Final = contract_for(bot, changed) if failures: print(f"cost map guard failed ({contract}):") print("\n".join(f"- {failure}" for failure in failures)) diff --git a/tests/test_litellm/test_cost_map_guard.py b/tests/test_litellm/test_cost_map_guard.py index 1b4330ed62c..1595bd9864f 100644 --- a/tests/test_litellm/test_cost_map_guard.py +++ b/tests/test_litellm/test_cost_map_guard.py @@ -114,6 +114,40 @@ def test_unclassified_entry_key_is_reported() -> None: assert "Unclassified keys" in failure and "weird_thing" in failure +STALE_HEAD: Final = _snapshot(BASE_MAP, backup=_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)}), schema="{}") +CODE_ONLY: Final = ( + "litellm/utils.py", + "tests/test_litellm/test_utils.py", + "docs/model_prices_and_context_window.json", +) + + +def test_human_pr_that_leaves_the_cost_map_alone_skips_the_file_checks() -> None: + unparseable: Final = guard.Snapshot(cost_map="{not json", backup="", schema="") + assert _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=False) == () + assert _failures(STALE_HEAD, changed_files=(), bot=False) == () + assert _failures(unparseable, changed_files=CODE_ONLY, bot=False) == () + + +@pytest.mark.parametrize("guarded_path", guard.GUARDED_PATHS) +def test_touching_any_cost_map_file_keeps_the_file_checks(guarded_path: str) -> None: + failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, guarded_path), bot=False) + assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] + assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] + + +def test_bot_pr_always_gets_the_file_checks() -> None: + failures: Final = _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=True) + assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] + assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] + + +def test_contract_names_the_skip() -> None: + assert guard.contract_for(False, CODE_ONLY) == "human PR, cost map untouched" + assert guard.contract_for(False, (*CODE_ONLY, guard.SCHEMA_PATH)) == "human PR, file checks only" + assert guard.contract_for(True, CODE_ONLY) == "bot contract enforced" + + def test_bot_may_only_touch_the_cost_map_files() -> None: changed = (*guard.GUARDED_PATHS, "litellm/utils.py", ".github/workflows/cost-map-guard.yml") assert _failures(BASE, changed_files=changed, bot=False) == () @@ -147,6 +181,16 @@ def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str: (repo / guard.BACKUP_PATH).parent.mkdir(exist_ok=True) (repo / guard.BACKUP_PATH).write_text(text) (repo / guard.SCHEMA_PATH).write_text(schema_module.render(schema_module.build_schema(cost_map))) + return _git_commit(repo, message) + + +def _commit_code_only(repo: Path, message: str) -> str: + (repo / "litellm").mkdir(exist_ok=True) + (repo / "litellm" / "utils.py").write_text(f"print('{message}')\n") + return _git_commit(repo, message) + + +def _git_commit(repo: Path, message: str) -> str: subprocess.run(("git", "add", "-A"), cwd=repo, check=True) subprocess.run( ("git", "-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-q", "-m", message), @@ -186,6 +230,40 @@ def test_main_reads_both_revisions_from_git( assert expected_line in result.stdout.splitlines() +def test_main_skips_the_file_checks_on_a_stale_base_the_pr_never_touched(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + _commit(tmp_path, BASE_MAP, "base") + (tmp_path / guard.BACKUP_PATH).write_text(_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)})) + (tmp_path / guard.SCHEMA_PATH).write_text("{}") + stale_base: Final = _commit_code_only(tmp_path, "stale base with drifted backup and schema") + head: Final = _commit_code_only(tmp_path, "code change on the stale base") + human: Final = _run_guard(tmp_path, stale_base, head, "litellm_fix_pricing") + assert human.returncode == 0, human.stdout + human.stderr + assert "cost map guard passed (human PR, cost map untouched)" in human.stdout.splitlines() + bot: Final = _run_guard(tmp_path, stale_base, head, BOT_REF) + assert bot.returncode == 1 + backup_failure: Final = f"- {guard.BACKUP_PATH} differs from {guard.COST_MAP_PATH}; copy the root file over it" + assert backup_failure in bot.stdout.splitlines() + + +def test_main_keeps_the_file_checks_when_a_cost_map_file_is_renamed(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, BASE_MAP, "base") + subprocess.run(("git", "mv", guard.COST_MAP_PATH, "renamed.json"), cwd=tmp_path, check=True) + head: Final = _git_commit(tmp_path, "rename the cost map") + result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing") + assert result.returncode == 1, result.stdout + result.stderr + assert "cost map guard failed (human PR, file checks only):" in result.stdout.splitlines() + + +def test_main_fails_when_the_changed_files_cannot_be_read(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + head: Final = _commit(tmp_path, BASE_MAP, "head") + result: Final = _run_guard(tmp_path, "0" * 40, head, "litellm_fix_pricing") + assert result.returncode == 1, result.stdout + result.stderr + assert result.stdout.startswith("cost map guard failed: git diff ") + + def test_main_rejects_a_bot_pr_that_edits_code(tmp_path: Path) -> None: subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) base = _commit(tmp_path, BASE_MAP, "base") From 0abd9267c106c8b6a276b1e68824cc7688e19435 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 04:41:11 +0000 Subject: [PATCH 03/28] feat(tokenizer): preserve Python defaults with opt-in Rust dispatch (#42174) * ci: benchmark and gate an installed release wheel Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: simplify installed-wheel benchmark check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(rust): add native tokenizer codec Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(tokenizer): route Python tokenization through the Rust extension Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(lint): format tokenizer call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(packaging): restore runtime dependencies and native images Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tokenizer): preserve Python SDK behavior with Rust tokenizers * fix(tokenizer): restore compatibility paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(tokenizer): count custom tokenizers directly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tokenizer): preserve caller-supplied Python tokenizer counts * fix(tokenizer): reuse packaged vocabularies in the native wheel * refactor(rust_bridge): route token counting through the catalog as RUST_OPT_IN Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend_tracking): compare tokenizer groups by value Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(deps): re-resolve filelock under the <4.0 pin Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(llms): align transformation override signatures with base configs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * build(rust): use fat LTO to keep the native wheel under the 35 MB limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(tokenizer): preserve Python defaults with opt-in Rust dispatch * test(proxy): tolerate missing litellm.utils.Tokenizer when patching it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): patch the tokenizer dispatch function instead of the removed alias Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(tokenizer): give the Rust wrappers the tiktoken and tokenizers surface Callers of litellm.encoding and litellm.create_tokenizer must see the same read-only API whichever backend the catalog selects. - OpenAIEncoding mirrors tiktoken.Encoding: n_vocab, max_token_value, token_byte_values, encode_single_token, encode_with_unstable, encode_to_numpy, decode_with_offsets, is_special_token, repr; the Rust tiktoken crate keeps a Vocabulary beside each CoreBPE and reports the requested encoding name (gpt2 stays gpt2). - HuggingFaceTokenizer mirrors the read-only tokenizers.Tokenizer surface (token_to_id, id_to_token, get_vocab, get_vocab_size, get_added_tokens_decoder, num_special_tokens_to_add, padding, truncation, encode_special_tokens, from_buffer); HuggingFaceEncoding gains the char/word/token lookups, pad, truncate, set_sequence_id and merge. Mutators stay on the Python tokenizer. - from_json/from_pretrained claim the fork gate only when the huggingface feature is compiled in; the surrogate fallback matches on the Codec. - Tokenizer caching is keyed on the same catalog Context the dispatch runs on; rust_tokenizer reads the encoding name without loading an encoding; LITELLM_RUST parsing is cached. - Drop the unused tiktoken_encoding_for_model export and Error::Download. Co-Authored-By: Claude Fable 5.1 * fix(tokenizer): close the exhaustive matches with assert_never CodeQL reads a `match` over a Literal with no default arm as an implicit `None` return. `assert_never` makes the exhaustiveness explicit for both the HuggingFace tokenizer loader and the Rust token-counter factory. Co-Authored-By: Claude Fable 5.1 * feat(tokenizer): derive the fast counter from the shared tokenizer The count-only counter (`fast` feature) and the codec each parsed the same artifact: TokenCounter took the Anthropic JSON and the tiktoken rank files from Python while Tokenizer loaded them again. One parse now serves both. - FastTokenizer builds from a model another loader holds: `from_shared` takes the Arc the HF codec keeps, and `from_*_pairs` take the ranks the tiktoken vocabulary already parsed. - `FastCounter::fast_counter` in the core crate derives it from either codec; encodings the fast scanner does not reproduce are refused. - Native `Tokenizer.count(text, fast=False)` opts into that counter, built once per tokenizer on first use; `TokenCounter.from_tokenizer(tokenizer, fast=False)` replaces the JSON and rank-file constructors. - The Python route counts over the native tokenizers the codec path shares (`native_encoding`, `native_anthropic`) and no longer reads rank files; the packaged Anthropic tokenizer has one loader, `tokenizer_dispatch.anthropic`. - Public wrappers gain `count(text, fast=False)`. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Yujong Lee Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 --- .github/actions/cache-cargo-build/action.yml | 10 +- .github/scripts/verify_linux_native_wheel.py | 14 +- .github/workflows/codspeed.yml | 36 +- backend/Dockerfile | 2 + litellm-rust/Cargo.lock | 4 + litellm-rust/Cargo.toml | 2 +- .../crates/host-python/src/execution.rs | 11 +- litellm-rust/crates/host-python/src/lib.rs | 2 +- litellm-rust/crates/python-bridge/Cargo.toml | 2 +- litellm-rust/crates/python-bridge/src/lib.rs | 12 +- .../crates/python-bridge/src/routes/mod.rs | 1 + .../python-bridge/src/routes/token_counter.rs | 87 +++ .../crates/python-bridge/src/token_counter.rs | 158 ---- .../crates/python-bridge/src/tokenizer.rs | 713 ++++++++++++++++++ .../crates/token-counter-fast/src/lib.rs | 41 +- .../crates/token-counter-fast/src/scanner.rs | 13 +- .../crates/token-counter-fast/src/tiktoken.rs | 24 +- .../token-counter-huggingface/Cargo.toml | 1 + .../token-counter-huggingface/src/error.rs | 2 + .../token-counter-huggingface/src/lib.rs | 231 +++++- .../crates/token-counter-tiktoken/Cargo.toml | 3 + .../crates/token-counter-tiktoken/src/lib.rs | 188 ++++- .../token-counter-tiktoken/src/ranks.rs | 340 +++++++++ litellm-rust/crates/token-counter/README.md | 10 +- .../crates/token-counter/src/error.rs | 2 + litellm-rust/crates/token-counter/src/fast.rs | 88 ++- .../crates/token-counter/src/huggingface.rs | 23 +- litellm-rust/crates/token-counter/src/lib.rs | 2 +- .../crates/token-counter/src/tiktoken.rs | 29 +- .../crates/token-counter/src/tokenizer.rs | 6 + litellm/_lazy_imports.py | 24 +- litellm/litellm_core_utils/README.md | 3 +- .../litellm_core_utils/default_encoding.py | 15 - litellm/litellm_core_utils/token_counter.py | 33 +- litellm/litellm_core_utils/tokenizer.py | 402 ++++++++++ litellm/llms/a2a/chat/transformation.py | 5 +- .../aiml/image_generation/transformation.py | 5 +- .../aiohttp_openai/chat/transformation.py | 5 +- .../llms/amazon_nova/chat/transformation.py | 4 +- .../llms/anthropic/batches/transformation.py | 5 +- litellm/llms/anthropic/chat/transformation.py | 5 +- .../anthropic/completion/transformation.py | 4 +- litellm/llms/azure/chat/gpt_transformation.py | 5 +- .../llms/azure_ai/agents/transformation.py | 5 +- .../azure_model_router/transformation.py | 4 +- litellm/llms/azure_ai/chat/transformation.py | 4 +- .../image_generation/mai_transformation.py | 5 +- .../audio_transcription/transformation.py | 5 +- .../bridges/completion_transformation.py | 4 +- litellm/llms/base_llm/chat/transformation.py | 5 +- .../base_llm/completion/transformation.py | 5 +- .../llms/base_llm/embedding/transformation.py | 5 +- litellm/llms/base_llm/files/transformation.py | 5 +- .../image_generation/transformation.py | 5 +- .../image_variations/transformation.py | 9 +- .../bedrock/chat/agentcore/transformation.py | 5 +- .../bedrock/chat/converse_transformation.py | 4 +- .../chat/invoke_agent/transformation.py | 5 +- .../amazon_deepseek_transformation.py | 4 +- .../amazon_moonshot_transformation.py | 5 +- .../amazon_nova_transformation.py | 4 +- .../amazon_qwen2_transformation.py | 4 +- .../amazon_qwen3_transformation.py | 4 +- ...mazon_twelvelabs_pegasus_transformation.py | 5 +- .../anthropic_claude3_transformation.py | 5 +- .../base_invoke_transformation.py | 5 +- .../image_generation/transformation.py | 5 +- litellm/llms/brave/search/__init__.py | 14 +- litellm/llms/bytez/chat/transformation.py | 5 +- litellm/llms/clarifai/chat/transformation.py | 5 +- litellm/llms/cohere/chat/transformation.py | 5 +- litellm/llms/cohere/chat/v2_transformation.py | 5 +- litellm/llms/cohere/embed/handler.py | 6 +- .../image_generation/transformation.py | 5 +- .../llms/compactifai/chat/transformation.py | 5 +- litellm/llms/custom_httpx/aiohttp_handler.py | 5 +- litellm/llms/custom_httpx/llm_http_handler.py | 6 +- .../image_generation/transformation.py | 5 +- .../llms/databricks/chat/transformation.py | 7 +- .../llms/deprecated_providers/aleph_alpha.py | 7 +- litellm/llms/edenai/chat/transformation.py | 5 +- .../edenai/image_generation/transformation.py | 5 +- .../image_generation/bria_transformation.py | 5 +- .../flux_pro_v11_ultra_transformation.py | 5 +- .../ideogram_v3_transformation.py | 5 +- .../imagen4_transformation.py | 5 +- .../recraft_v3_transformation.py | 5 +- .../stable_diffusion_transformation.py | 5 +- .../fal_ai/image_generation/transformation.py | 5 +- .../llms/fireworks_ai/chat/transformation.py | 4 +- .../gemini/image_generation/transformation.py | 5 +- litellm/llms/gigachat/chat/transformation.py | 5 +- litellm/llms/groq/chat/transformation.py | 4 +- litellm/llms/huggingface/embedding/handler.py | 5 +- .../huggingface/embedding/transformation.py | 5 +- litellm/llms/langflow/chat/transformation.py | 5 +- litellm/llms/langgraph/chat/transformation.py | 5 +- litellm/llms/lemonade/chat/transformation.py | 4 +- litellm/llms/mistral/chat/transformation.py | 4 +- litellm/llms/nlp_cloud/chat/transformation.py | 5 +- litellm/llms/oci/chat/transformation.py | 5 +- litellm/llms/ollama/chat/transformation.py | 5 +- .../llms/ollama/completion/transformation.py | 5 +- litellm/llms/oobabooga/chat/transformation.py | 5 +- .../llms/openai/chat/gpt_transformation.py | 5 +- .../dall_e_2_transformation.py | 5 +- .../dall_e_3_transformation.py | 5 +- .../image_generation/gpt_transformation.py | 5 +- .../openai/image_variations/transformation.py | 6 +- litellm/llms/openai/openai.py | 5 +- .../llms/openai_like/chat/transformation.py | 5 +- .../llms/openrouter/chat/transformation.py | 5 +- .../image_generation/transformation.py | 5 +- .../llms/perplexity/chat/transformation.py | 4 +- .../llms/petals/completion/transformation.py | 4 +- litellm/llms/predibase/chat/transformation.py | 5 +- .../image_generation/transformation.py | 5 +- litellm/llms/replicate/chat/transformation.py | 5 +- .../image_generation/transformation.py | 7 +- .../sagemaker/completion/transformation.py | 5 +- litellm/llms/sap/chat/transformation.py | 9 +- .../image_generation/transformation.py | 5 +- .../topaz/image_variations/transformation.py | 6 +- .../llms/triton/completion/transformation.py | 8 +- .../vertex_ai/agent_engine/transformation.py | 5 +- .../vertex_gemini_transformation.py | 5 +- .../vertex_imagen_transformation.py | 5 +- .../anthropic/transformation.py | 4 +- .../llama3/transformation.py | 4 +- .../vertex_gemma_models/transformation.py | 7 +- .../llms/watsonx/completion/transformation.py | 5 +- litellm/main.py | 21 +- .../spend_tracking/budget_reservation.py | 127 +--- litellm/proxy/spend_tracking/input_tokens.py | 173 +++++ litellm/rust_bridge/_native.pyi | 107 ++- litellm/rust_bridge/catalog.py | 4 + litellm/rust_bridge/configuration.py | 3 + litellm/rust_bridge/token_counter.py | 70 +- litellm/rust_bridge/tokenizer.py | 108 +++ litellm/types/utils.py | 4 +- litellm/utils.py | 92 +-- migrations/Dockerfile | 2 + pyproject.toml | 13 +- tests/benchmarks/conftest.py | 18 + .../test_custom_tokenizer_bug.py | 8 +- .../test_decode_special_tokens.py | 20 +- .../litellm_core_utils/test_token_counter.py | 24 +- .../litellm_core_utils/test_tokenizer.py | 403 ++++++++++ .../spend_tracking/test_budget_reservation.py | 40 +- .../proxy/spend_tracking/test_input_tokens.py | 191 +++++ .../proxy/test_budget_reservation.py | 6 +- tests/test_litellm/proxy/test_proxy_server.py | 8 +- .../test_litellm/rust_bridge/test_catalog.py | 2 +- .../rust_bridge/test_token_counter.py | 159 ++-- .../rust_bridge/test_tokenizer.py | 134 ++++ .../test_verify_linux_native_wheel.py | 17 +- tests/test_litellm_rust/test_fork_guard.py | 74 ++ tests/test_litellm_rust/test_tokenizer.py | 130 ++++ uv.lock | 119 ++- 159 files changed, 4151 insertions(+), 954 deletions(-) create mode 100644 litellm-rust/crates/python-bridge/src/routes/token_counter.rs delete mode 100644 litellm-rust/crates/python-bridge/src/token_counter.rs create mode 100644 litellm-rust/crates/python-bridge/src/tokenizer.rs create mode 100644 litellm-rust/crates/token-counter-tiktoken/src/ranks.rs create mode 100644 litellm/litellm_core_utils/tokenizer.py create mode 100644 litellm/proxy/spend_tracking/input_tokens.py create mode 100644 litellm/rust_bridge/tokenizer.py create mode 100644 tests/test_litellm/litellm_core_utils/test_tokenizer.py create mode 100644 tests/test_litellm/proxy/spend_tracking/test_input_tokens.py create mode 100644 tests/test_litellm/rust_bridge/test_tokenizer.py create mode 100644 tests/test_litellm_rust/test_tokenizer.py diff --git a/.github/actions/cache-cargo-build/action.yml b/.github/actions/cache-cargo-build/action.yml index c3b8ce22c68..222fad637fb 100644 --- a/.github/actions/cache-cargo-build/action.yml +++ b/.github/actions/cache-cargo-build/action.yml @@ -15,6 +15,12 @@ description: >- cache the same directory for different workloads, and a shared key would let whichever ran first deny the others a save. +inputs: + profile: + description: "Cargo profile the build uses (dev or release)" + required: false + default: "dev" + runs: using: composite steps: @@ -25,6 +31,6 @@ runs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-maturin-dev- + ${{ runner.os }}-maturin-${{ inputs.profile }}- diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index ea6d2401084..f2b82f86b47 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -134,7 +134,16 @@ def main( uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members) native_path: Final = wheel.parent / "native" / Path(native_member.filename).name native_path.parent.mkdir(parents=True, exist_ok=True) - native_path.write_bytes(archive.read(native_member)) + native_bytes: Final = archive.read(native_member) + native_path.write_bytes(native_bytes) + duplicated_vocabularies: Final = tuple( + member.filename + for member in wheel_members + if member.filename.startswith("litellm/litellm_core_utils/tokenizers/") + and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name) + and member.file_size > 0 + and archive.read(member) in native_bytes + ) wheel_metadata_tags_match: Final = ( len(wheel_metadata_tags) == len(expanded_filename_tags) @@ -205,7 +214,7 @@ def main( native_module: Final = load_native_module(native_path) native_module_loads: Final = native_module is not None panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") - native_size_limit: Final = 40_000_000 + native_size_limit: Final = 35_000_000 native_size_within_limit: Final = native_member.file_size <= native_size_limit validations: Final = ( (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), @@ -223,6 +232,7 @@ def main( ("Native module loads", native_module_loads), ("Production module omits the panic test hook", panic_test_hook_absent), (f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit), + ("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies), ("Wheel contents are valid", not unexpected_members), ) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index fd7513a3937..ec7e211faa1 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -13,6 +13,7 @@ on: - ".github/workflows/codspeed.yml" - ".github/actions/setup-uv-with-retries/**" - ".github/actions/cache-cargo-build/**" + - ".github/scripts/uv_sync_with_retries.sh" pull_request: branches: - main @@ -25,6 +26,7 @@ on: - ".github/workflows/codspeed.yml" - ".github/actions/setup-uv-with-retries/**" - ".github/actions/cache-cargo-build/**" + - ".github/scripts/uv_sync_with_retries.sh" # Allow CodSpeed to trigger backtest performance analysis # in order to generate initial data workflow_dispatch: @@ -59,19 +61,27 @@ jobs: - name: Cache the Rust build uses: ./.github/actions/cache-cargo-build + with: + profile: release # Build the wheel and resolve every dependency outside the CodSpeed # runner: the same maturin build took 42 minutes inside `codspeed run` # versus under 3 minutes as a plain step (LIT-6183) - - name: Build environment + - name: Build the release wheel + run: uv build --wheel --out-dir dist + + - name: Install the wheel into the benchmark environment + run: | + UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12 + uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl + + - name: Collect benchmarks + env: + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + LITELLM_REQUIRE_INSTALLED_WHEEL: "1" run: > - env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 - uv run --frozen --no-default-groups - --with pytest==8.3.5 - --with pytest-codspeed==4.3.0 - --with "mcp>=2.2.0,<3.0" - --with "a2a-sdk>=1.1.0,<2.0" - pytest + "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest + --import-mode=importlib -p pytest_codspeed.plugin tests/benchmarks/ --codspeed @@ -82,13 +92,9 @@ jobs: with: mode: simulation run: > - env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 - uv run --frozen --no-default-groups - --with pytest==8.3.5 - --with pytest-codspeed==4.3.0 - --with "mcp>=2.2.0,<3.0" - --with "a2a-sdk>=1.1.0,<2.0" - pytest + env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1 + "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest + --import-mode=importlib -p pytest_codspeed.plugin tests/benchmarks/ --codspeed diff --git a/backend/Dockerfile b/backend/Dockerfile index 622fedcd70d..57e0a43a98d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra saml \ --python python3.13 +RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ + RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ prisma generate --schema=./schema.prisma diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 03e0dabbc17..37384cbfa53 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3253,6 +3253,7 @@ dependencies = [ name = "litellm-token-counter-huggingface" version = "0.1.0" dependencies = [ + "serde_json", "thiserror 2.0.19", "tokenizers", ] @@ -3261,6 +3262,9 @@ dependencies = [ name = "litellm-token-counter-tiktoken" version = "0.1.0" dependencies = [ + "base64 0.22.1", + "once_cell", + "rustc-hash", "thiserror 2.0.19", "tiktoken-rs", ] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 0e8941cb8e6..813d0713128 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -89,7 +89,7 @@ veil = "0.3.0" [profile.release] opt-level = 3 -lto = "thin" +lto = "fat" codegen-units = 1 panic = "unwind" debug = false diff --git a/litellm-rust/crates/host-python/src/execution.rs b/litellm-rust/crates/host-python/src/execution.rs index 083c184e37e..b435bf241d2 100644 --- a/litellm-rust/crates/host-python/src/execution.rs +++ b/litellm-rust/crates/host-python/src/execution.rs @@ -29,7 +29,7 @@ pyo3::create_exception!( static FORK_GATE: ForkGate = ForkGate::new(); -/// Whether this process has started the Tokio runtime. +/// Whether this process has entered process-bound native execution. pub fn runtime_started() -> bool { FORK_GATE.started(std::process::id()) } @@ -40,9 +40,8 @@ pub fn reserve_process_for_forking() -> Result<(), RuntimeAlreadyStarted> { FORK_GATE.reserve(std::process::id()) } -/// The only door to the Tokio runtime: every route reaches it through this module, which is -/// what lets the gate speak for the whole extension. `clippy.toml` disallows going around it. -fn enter_runtime() -> PyResult<()> { +/// Claims process-bound native state before runtime startup or tokenizer execution. +pub fn enter_native() -> PyResult<()> { FORK_GATE .enter(std::process::id()) .map_err(|refused| match refused { @@ -60,7 +59,7 @@ fn enter_runtime() -> PyResult<()> { #[expect(clippy::disallowed_methods, reason = "this is the gated door")] fn runtime() -> PyResult<&'static Runtime> { - enter_runtime()?; + enter_native()?; Ok(pyo3_async_runtimes::tokio::get_runtime()) } @@ -70,7 +69,7 @@ where F: Future> + Send + 'static, T: for<'py> IntoPyObject<'py> + Send + 'static, { - enter_runtime()?; + enter_native()?; pyo3_async_runtimes::tokio::future_into_py(py, future) } diff --git a/litellm-rust/crates/host-python/src/lib.rs b/litellm-rust/crates/host-python/src/lib.rs index 7d164ab7535..4a33975a918 100644 --- a/litellm-rust/crates/host-python/src/lib.rs +++ b/litellm-rust/crates/host-python/src/lib.rs @@ -20,7 +20,7 @@ pub use argument::lookup; pub use callable::wrap_failure; pub use driver::run_call; pub use execution::{ - ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value, + ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, enter_native, poll_async_value, reserve_process_for_forking, run_async, run_async_value, run_sync, run_sync_value, runtime_started, }; diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 4e6c510d104..7846beef28a 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -10,7 +10,7 @@ name = "_native" crate-type = ["cdylib"] [features] -default = ["abi3", "fast"] +default = ["abi3", "fast", "huggingface", "tiktoken"] abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] panic-test = [] diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index ed9bc90f650..b1fc5244d6f 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -12,7 +12,7 @@ mod routes; reason = "secret-manager foundations await rollout activation" )] mod secrets; -mod token_counter; +mod tokenizer; #[pymodule(gil_used = true)] mod _native { @@ -37,7 +37,12 @@ mod _native { #[pymodule_export] use crate::routes::responses::ResponsesWebSocketConnection; #[pymodule_export] - use crate::token_counter::TokenCounter; + use crate::routes::token_counter::TokenCounter; + #[cfg(feature = "huggingface")] + #[pymodule_export] + use crate::tokenizer::HuggingFaceEncoding; + #[pymodule_export] + use crate::tokenizer::Tokenizer; #[pymodule_export] use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking}; use pyo3::{prelude::*, types::PyModule}; @@ -83,10 +88,13 @@ mod tests { "achat_completions", "ResponsesWebSocketConnection", "TokenCounter", + "Tokenizer", "gil_stats", "process_state_started", "reserve_process_for_forking", ]; + #[cfg(feature = "huggingface")] + expected.push("HuggingFaceEncoding"); expected.sort_unstable(); let mut public_names: Vec = native_module(py) diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index 2d6b849a6b1..8a78a26423d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod chat_completions; pub(crate) mod messages; pub(crate) mod ocr; pub(crate) mod responses; +pub(crate) mod token_counter; #[cfg(test)] mod tests { diff --git a/litellm-rust/crates/python-bridge/src/routes/token_counter.rs b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs new file mode 100644 index 00000000000..168c4883b0b --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs @@ -0,0 +1,87 @@ +use std::sync::Arc; +use std::{num::NonZero, thread::available_parallelism}; + +use litellm_host_python::{enter_native, run_async}; +use litellm_token_counter::{ + CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, +}; +use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, + types::PyAny, +}; +use tokio::sync::Semaphore; + +use crate::errors::RustBridgeDeclined; +use crate::tokenizer::Tokenizer; + +/// Counts the input tokens of a raw request body off the Python event loop with +/// the GIL released. Python owns which requests get here and what to do with +/// the count. At most one encode per core runs at a time; the rest wait in the +/// async task, where a cancelled Python awaiter drops them before any blocking +/// work is scheduled. +#[pyclass(frozen)] +pub(crate) struct TokenCounter { + inner: Arc, + encode_slots: Arc, +} + +#[pymethods] +impl TokenCounter { + #[staticmethod] + #[pyo3(signature = (tokenizer, fast = false))] + fn from_tokenizer(py: Python<'_>, tokenizer: &Tokenizer, fast: bool) -> PyResult { + enter_native()?; + let inner = CoreTokenCounter::new(tokenizer.counter(py, fast)); + Ok(Self { + inner: Arc::new(inner), + encode_slots: Arc::new(Semaphore::new(encode_parallelism())), + }) + } + + fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult> { + let counter = Arc::clone(&self.inner); + let encode_slots = Arc::clone(&self.encode_slots); + let body = body.to_vec(); + run_async( + py, + async move { + let _slot = encode_slots + .acquire_owned() + .await + .map_err(|error| Error::Task(error.to_string()))?; + tokio::task::spawn_blocking(move || count_body(&counter, &body)) + .await + .map_err(|error| Error::Task(error.to_string()))? + }, + token_count_error_to_pyerr, + ) + } +} + +fn encode_parallelism() -> usize { + available_parallelism().map_or(1, NonZero::get) +} + +fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result { + let request = CountableRequest::parse(body)?; + counter.count_request(&request) +} + +pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr { + let message = error.to_string(); + match error { + Error::Load(_) + | Error::Ranks(_) + | Error::UnicodeClasses + | Error::UnsupportedTokenizer(_) => PyValueError::new_err(message), + Error::RequestParse(_) + | Error::MissingInput + | Error::FloatText + | Error::ContentBlock + | Error::ArrayItems + | Error::JsonSerialization(_) + | Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message), + Error::Encode(_) | Error::Decode(_) | Error::Task(_) => PyRuntimeError::new_err(message), + } +} diff --git a/litellm-rust/crates/python-bridge/src/token_counter.rs b/litellm-rust/crates/python-bridge/src/token_counter.rs deleted file mode 100644 index 244401e6696..00000000000 --- a/litellm-rust/crates/python-bridge/src/token_counter.rs +++ /dev/null @@ -1,158 +0,0 @@ -use std::sync::Arc; - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -use std::{num::NonZero, thread::available_parallelism}; - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -use litellm_host_python::release_gil; -use litellm_host_python::run_async; -use litellm_token_counter::{ - CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, -}; -use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, - prelude::*, - types::PyAny, -}; -use tokio::sync::Semaphore; - -use crate::errors::RustBridgeDeclined; - -/// Counts the input tokens of a raw request body off the Python event loop with -/// the GIL released. Python owns which requests get here and what to do with -/// the count. At most one encode per core runs at a time; the rest wait in the -/// async task, where a cancelled Python awaiter drops them before any blocking -/// work is scheduled. -#[pyclass(frozen)] -pub(crate) struct TokenCounter { - inner: Arc, - encode_slots: Arc, -} - -#[pymethods] -impl TokenCounter { - #[new] - fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_json_fast(tokenizer_json)) - } - #[cfg(all(not(feature = "fast"), feature = "huggingface"))] - { - Self::load(py, || CoreTokenCounter::from_json(tokenizer_json)) - } - #[cfg(not(any(feature = "fast", feature = "huggingface")))] - { - let _ = (py, tokenizer_json); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast or huggingface feature", - )) - } - } - - #[staticmethod] - fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file)) - } - #[cfg(not(feature = "fast"))] - { - let _ = (py, rank_file); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast feature", - )) - } - } - - #[staticmethod] - fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file)) - } - #[cfg(not(feature = "fast"))] - { - let _ = (py, rank_file); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast feature", - )) - } - } - - #[staticmethod] - fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { - #[cfg(feature = "tiktoken")] - { - Self::load(py, || CoreTokenCounter::from_tiktoken(encoding)) - } - #[cfg(not(feature = "tiktoken"))] - { - let _ = (py, encoding); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the tiktoken feature", - )) - } - } - - fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult> { - let counter = Arc::clone(&self.inner); - let encode_slots = Arc::clone(&self.encode_slots); - let body = body.to_vec(); - run_async( - py, - async move { - let _slot = encode_slots - .acquire_owned() - .await - .map_err(|error| Error::Task(error.to_string()))?; - tokio::task::spawn_blocking(move || count_body(&counter, &body)) - .await - .map_err(|error| Error::Task(error.to_string()))? - }, - token_count_error_to_pyerr, - ) - } -} - -impl TokenCounter { - #[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] - fn load( - py: Python<'_>, - load: impl FnOnce() -> Result + Send, - ) -> PyResult { - let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?; - Ok(Self { - inner: Arc::new(inner), - encode_slots: Arc::new(Semaphore::new(encode_parallelism())), - }) - } -} - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -fn encode_parallelism() -> usize { - available_parallelism().map_or(1, NonZero::get) -} - -fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result { - let request = CountableRequest::parse(body)?; - counter.count_request(&request) -} - -fn token_count_error_to_pyerr(error: Error) -> PyErr { - let message = error.to_string(); - match error { - Error::Load(_) - | Error::Ranks(_) - | Error::UnicodeClasses - | Error::UnsupportedTokenizer(_) => PyValueError::new_err(message), - Error::RequestParse(_) - | Error::MissingInput - | Error::FloatText - | Error::ContentBlock - | Error::ArrayItems - | Error::JsonSerialization(_) - | Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message), - Error::Encode(_) | Error::Task(_) => PyRuntimeError::new_err(message), - } -} diff --git a/litellm-rust/crates/python-bridge/src/tokenizer.rs b/litellm-rust/crates/python-bridge/src/tokenizer.rs new file mode 100644 index 00000000000..df219d55eb6 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/tokenizer.rs @@ -0,0 +1,713 @@ +//! The Python face of the text codecs: one `Tokenizer` class over the tiktoken and Hugging +//! Face backends, carrying the read-only surface of `tiktoken.Encoding` and +//! `tokenizers.Tokenizer` that `litellm/litellm_core_utils/tokenizer.py` wraps. +use std::borrow::Cow; +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +use std::collections::HashMap; +use std::sync::Arc; +#[cfg(feature = "fast")] +use std::sync::OnceLock; + +use litellm_host_python::{enter_native, release_gil}; +#[cfg(feature = "fast")] +use litellm_token_counter::fast::{FastCounter, FastTokenizer}; +use litellm_token_counter::{Error, TextCodec}; +use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString}; + +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +use pyo3::exceptions::PyValueError; +#[cfg(feature = "huggingface")] +use pyo3::{exceptions::PyIOError, types::PyDict}; +#[cfg(feature = "tiktoken")] +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError}, + types::PyBytes, +}; + +#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))] +use crate::errors::RustBridgeDeclined; +use crate::routes::token_counter::token_count_error_to_pyerr; + +#[cfg(feature = "huggingface")] +use litellm_token_counter::huggingface::{ + EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, PaddingStrategy, + TruncationDirection, encoding_from_json, encoding_to_json, +}; +#[cfg(feature = "tiktoken")] +use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary}; + +#[cfg(feature = "tiktoken")] +pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { + enter_native()?; + let resource: std::path::PathBuf = + PyModule::import(py, "litellm.litellm_core_utils.tokenizers")? + .getattr("__file__")? + .extract()?; + release_gil(py, || { + TiktokenTokenizer::from_cached_ranks(encoding, |file| { + std::fs::read_to_string(resource.with_file_name(file)) + }) + }) + .map_err(|error| token_count_error_to_pyerr(error.into())) +} + +pub(crate) enum Codec { + #[cfg(feature = "tiktoken")] + Tiktoken(TiktokenTokenizer), + #[cfg(feature = "huggingface")] + HuggingFace(HuggingFaceTokenizer), +} + +impl Codec { + pub(crate) fn codec(&self) -> &dyn TextCodec { + match *self { + #[cfg(feature = "tiktoken")] + Self::Tiktoken(ref tokenizer) => tokenizer, + #[cfg(feature = "huggingface")] + Self::HuggingFace(ref tokenizer) => tokenizer, + } + } + + #[cfg(feature = "fast")] + fn fast_counter(&self) -> Option { + match *self { + #[cfg(feature = "tiktoken")] + Self::Tiktoken(ref tokenizer) => tokenizer.fast_counter(), + #[cfg(feature = "huggingface")] + Self::HuggingFace(ref tokenizer) => tokenizer.fast_counter(), + } + } +} + +/// The loaded model is shared: `TokenCounter::from_tokenizer` counts with the same parse, +/// and the opt-in count-only counter is derived from it once, on first use. +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Tokenizer { + inner: Arc, + #[cfg(feature = "fast")] + fast: OnceLock>>, +} + +#[pymethods] +impl Tokenizer { + #[staticmethod] + fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { + #[cfg(feature = "tiktoken")] + { + let tokenizer = load_tiktoken(py, encoding)?; + Ok(Self::new(Codec::Tiktoken(tokenizer))) + } + #[cfg(not(feature = "tiktoken"))] + { + let _ = (py, encoding); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the tiktoken feature", + )) + } + } + + #[staticmethod] + fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult { + #[cfg(feature = "huggingface")] + { + enter_native()?; + let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json)) + .map_err(|error| token_count_error_to_pyerr(error.into()))?; + Ok(Self::new(Codec::HuggingFace(tokenizer))) + } + #[cfg(not(feature = "huggingface"))] + { + let _ = (py, tokenizer_json); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the huggingface feature", + )) + } + } + + #[staticmethod] + #[pyo3(signature = (identifier, revision = "main", token = None))] + fn from_pretrained( + py: Python<'_>, + identifier: &str, + revision: &str, + token: Option<&str>, + ) -> PyResult { + #[cfg(feature = "huggingface")] + { + enter_native()?; + let kwargs = PyDict::new(py); + kwargs.set_item("repo_id", identifier)?; + kwargs.set_item("filename", "tokenizer.json")?; + kwargs.set_item("revision", revision)?; + kwargs.set_item("token", token)?; + let path: String = PyModule::import(py, "huggingface_hub")? + .getattr("hf_hub_download")? + .call((), Some(&kwargs))? + .extract()?; + let json = + release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?; + Self::from_json(py, &json) + } + #[cfg(not(feature = "huggingface"))] + { + let _ = (py, identifier, revision, token); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the huggingface feature", + )) + } + } + + fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult> { + enter_native()?; + let text = self.text(text)?; + release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr) + } + + #[pyo3(signature = (ids, skip_special_tokens = true))] + fn decode(&self, py: Python<'_>, ids: Vec, skip_special_tokens: bool) -> PyResult { + enter_native()?; + release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens)) + .map_err(token_count_error_to_pyerr) + } + + #[pyo3(signature = (text, fast = false))] + fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>, fast: bool) -> PyResult { + enter_native()?; + let text = self.text(text)?; + let counter = self.counter(py, fast); + release_gil(py, || { + litellm_token_counter::Tokenizer::count_tokens(&counter, &text) + }) + .map_err(token_count_error_to_pyerr) + } + + #[getter] + fn name(&self) -> &str { + self.inner.codec().name() + } + + // ---- tiktoken: the `tiktoken.Encoding` surface ------------------------------------------ + + #[cfg(feature = "tiktoken")] + fn encode_special( + &self, + py: Python<'_>, + text: &Bound<'_, PyString>, + allowed: Vec, + ) -> PyResult> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let text = self.text(text)?; + release_gil(py, || tokenizer.encode_special(&text, &allowed)) + .map_err(PyRuntimeError::new_err) + } + + /// tiktoken's `encode_with_unstable`: `(stable_tokens, completions)`. + #[cfg(feature = "tiktoken")] + fn encode_with_unstable( + &self, + py: Python<'_>, + text: &Bound<'_, PyString>, + allowed: Vec, + ) -> PyResult<(Vec, Vec>)> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let text = self.text(text)?; + Ok(release_gil(py, || { + tokenizer.encode_with_unstable(&text, &allowed) + })) + } + + /// The special tokens by text: tiktoken's `_special_tokens`. + #[cfg(feature = "tiktoken")] + fn special_tokens(&self) -> PyResult> { + Ok(self + .vocabulary()? + .special_tokens() + .map(|(token, rank)| (token.to_owned(), rank)) + .collect()) + } + + #[cfg(feature = "tiktoken")] + fn max_token_value(&self) -> PyResult { + Ok(self.vocabulary()?.max_token_value()) + } + + #[cfg(feature = "tiktoken")] + fn is_special_token(&self, token: u32) -> PyResult { + Ok(self.vocabulary()?.is_special_token(token)) + } + + /// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`. + #[cfg(feature = "tiktoken")] + fn token_byte_values<'py>(&self, py: Python<'py>) -> PyResult>> { + let vocabulary = self.vocabulary()?; + let values = release_gil(py, || vocabulary.token_byte_values()); + Ok(values.iter().map(|value| PyBytes::new(py, value)).collect()) + } + + /// The token of one whole piece; `KeyError` when it is not in the vocabulary. + #[cfg(feature = "tiktoken")] + fn encode_single_token(&self, py: Python<'_>, piece: Vec) -> PyResult { + self.vocabulary()? + .encode_single_token(&piece) + .ok_or_else(|| PyKeyError::new_err(PyBytes::new(py, &piece).unbind())) + } + + #[cfg(feature = "tiktoken")] + fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec) -> PyResult> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let bytes = + release_gil(py, || tokenizer.decode_bytes(&ids)).map_err(PyKeyError::new_err)?; + Ok(PyBytes::new(py, &bytes)) + } + + // ---- Hugging Face: the `tokenizers.Tokenizer` surface ----------------------------------- + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))] + fn encode_huggingface( + &self, + py: Python<'_>, + sequence: Sequence, + pair: Option, + is_pretokenized: bool, + add_special_tokens: bool, + fast: bool, + ) -> PyResult { + enter_native()?; + let tokenizer = self.huggingface()?; + let sequence = sequence.input(is_pretokenized)?; + let input = match pair { + Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?), + None => EncodeInput::Single(sequence), + }; + release_gil(py, || { + tokenizer.encode_result(input, add_special_tokens, fast) + }) + .map(|inner| HuggingFaceEncoding { inner }) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))] + fn encode_batch_huggingface( + &self, + py: Python<'_>, + inputs: Vec<(Sequence, Option)>, + is_pretokenized: bool, + add_special_tokens: bool, + fast: bool, + ) -> PyResult> { + enter_native()?; + let tokenizer = self.huggingface()?; + let inputs = inputs + .into_iter() + .map(|(sequence, pair)| { + let sequence = sequence.input(is_pretokenized)?; + match pair { + Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)), + None => Ok(EncodeInput::Single(sequence)), + } + }) + .collect::>>()?; + release_gil(py, || { + tokenizer.encode_batch_result(inputs, add_special_tokens, fast) + }) + .map(|encodings| { + encodings + .into_iter() + .map(|inner| HuggingFaceEncoding { inner }) + .collect() + }) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (pretty = false))] + fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult { + enter_native()?; + let tokenizer = self.huggingface()?; + release_gil(py, || tokenizer.to_json(pretty)) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + fn token_to_id(&self, token: &str) -> PyResult> { + Ok(self.huggingface()?.token_to_id(token)) + } + + #[cfg(feature = "huggingface")] + fn id_to_token(&self, id: u32) -> PyResult> { + Ok(self.huggingface()?.id_to_token(id)) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (with_added_tokens = true))] + fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult> { + let tokenizer = self.huggingface()?; + Ok(release_gil(py, || tokenizer.vocab(with_added_tokens))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (with_added_tokens = true))] + fn get_vocab_size(&self, with_added_tokens: bool) -> PyResult { + Ok(self.huggingface()?.vocab_size(with_added_tokens)) + } + + /// The added tokens by id as `(id, (content, single_word, lstrip, rstrip, normalized, + /// special))`, for Python to rebuild as `tokenizers.AddedToken`. + #[cfg(feature = "huggingface")] + fn added_tokens_decoder(&self) -> PyResult> { + Ok(self + .huggingface()? + .added_tokens_decoder() + .into_iter() + .map(|(id, token)| { + ( + id, + ( + token.content, + token.single_word, + token.lstrip, + token.rstrip, + token.normalized, + token.special, + ), + ) + }) + .collect()) + } + + /// The padding parameters as `tokenizers.Tokenizer.padding` reports them. + #[cfg(feature = "huggingface")] + fn padding<'py>(&self, py: Python<'py>) -> PyResult>> { + let Some(params) = self.huggingface()?.padding() else { + return Ok(None); + }; + let padding = PyDict::new(py); + padding.set_item( + "length", + match params.strategy { + PaddingStrategy::BatchLongest => None, + PaddingStrategy::Fixed(length) => Some(length), + }, + )?; + padding.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?; + padding.set_item("pad_id", params.pad_id)?; + padding.set_item("pad_type_id", params.pad_type_id)?; + padding.set_item("pad_token", ¶ms.pad_token)?; + padding.set_item("direction", params.direction.as_ref())?; + Ok(Some(padding)) + } + + /// The truncation parameters as `tokenizers.Tokenizer.truncation` reports them. + #[cfg(feature = "huggingface")] + fn truncation<'py>(&self, py: Python<'py>) -> PyResult>> { + let Some(params) = self.huggingface()?.truncation() else { + return Ok(None); + }; + let truncation = PyDict::new(py); + truncation.set_item("max_length", params.max_length)?; + truncation.set_item("stride", params.stride)?; + truncation.set_item("strategy", params.strategy.as_ref())?; + truncation.set_item("direction", params.direction.as_ref())?; + Ok(Some(truncation)) + } + + #[cfg(feature = "huggingface")] + fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult { + Ok(self.huggingface()?.num_special_tokens_to_add(is_pair)) + } + + #[cfg(feature = "huggingface")] + fn encode_special_tokens(&self) -> PyResult { + Ok(self.huggingface()?.encode_special_tokens()) + } +} + +#[cfg(feature = "huggingface")] +type AddedTokenFields = (String, bool, bool, bool, bool, bool); + +impl Tokenizer { + fn new(inner: Codec) -> Self { + Self { + inner: Arc::new(inner), + #[cfg(feature = "fast")] + fast: OnceLock::new(), + } + } + + pub(crate) fn counter(&self, py: Python<'_>, fast: bool) -> SharedCounter { + #[cfg(feature = "fast")] + if fast { + let counter = self.fast.get().unwrap_or_else(|| { + release_gil(py, || { + self.fast + .get_or_init(|| self.inner.fast_counter().map(Arc::new)) + }) + }); + if let Some(counter) = counter { + return SharedCounter::Fast(Arc::clone(counter)); + } + } + #[cfg(not(feature = "fast"))] + let _ = (py, fast); + SharedCounter::Codec(Arc::clone(&self.inner)) + } + + /// A Python `str` as UTF-8. tiktoken replaces lone surrogates the way its Python `encode` + /// does; `tokenizers` rejects them, so that backend keeps the encode error. + fn text<'a>(&self, text: &'a Bound<'_, PyString>) -> PyResult> { + match text.to_cow() { + Ok(text) => Ok(text), + Err(error) => match *self.inner { + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(_) if error.is_instance_of::(text.py()) => { + text.call_method1("encode", ("utf-16", "surrogatepass"))? + .call_method1("decode", ("utf-16", "replace"))? + .extract::() + .map(Cow::Owned) + } + _ => Err(error), + }, + } + } + + #[cfg(feature = "tiktoken")] + fn tiktoken(&self) -> PyResult<&TiktokenTokenizer> { + match *self.inner { + Codec::Tiktoken(ref tokenizer) => Ok(tokenizer), + #[cfg(feature = "huggingface")] + Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")), + } + } + + #[cfg(feature = "tiktoken")] + fn vocabulary(&self) -> PyResult<&Vocabulary> { + self.tiktoken()?.vocabulary().ok_or_else(|| { + PyRuntimeError::new_err("this encoding was built without its vocabulary") + }) + } + + #[cfg(feature = "huggingface")] + fn huggingface(&self) -> PyResult<&HuggingFaceTokenizer> { + match *self.inner { + Codec::HuggingFace(ref tokenizer) => Ok(tokenizer), + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")), + } + } +} + +pub(crate) enum SharedCounter { + Codec(Arc), + #[cfg(feature = "fast")] + Fast(Arc), +} + +impl litellm_token_counter::Tokenizer for SharedCounter { + fn count_tokens(&self, text: &str) -> Result { + match self { + Self::Codec(codec) => codec.codec().count_tokens(text), + #[cfg(feature = "fast")] + Self::Fast(counter) => counter.count_tokens(text).map_err(Error::from), + } + } +} + +#[cfg(feature = "huggingface")] +#[derive(FromPyObject)] +pub(crate) enum Sequence { + Text(String), + Words(Vec), +} + +#[cfg(feature = "huggingface")] +impl Sequence { + fn input(self, is_pretokenized: bool) -> PyResult> { + match (self, is_pretokenized) { + (Self::Text(text), false) => Ok(text.into()), + (Self::Words(words), true) => Ok(words.into()), + _ => Err(pyo3::exceptions::PyTypeError::new_err( + "input must match is_pretokenized", + )), + } + } +} + +#[cfg(feature = "huggingface")] +fn direction(value: &str, left: T, right: T, what: &str) -> PyResult { + match value { + "left" => Ok(left), + "right" => Ok(right), + other => Err(PyValueError::new_err(format!( + "invalid {what} direction {other:?}: expected 'left' or 'right'" + ))), + } +} + +/// `tokenizers.Encoding`, mutable like the original: `pad`, `truncate` and `set_sequence_id` +/// change it in place. +#[cfg(feature = "huggingface")] +#[pyclass(module = "litellm.rust_bridge._native")] +pub(crate) struct HuggingFaceEncoding { + inner: Encoding, +} + +#[cfg(feature = "huggingface")] +#[pymethods] +impl HuggingFaceEncoding { + #[new] + #[pyo3(signature = (json = None))] + fn new(json: Option<&str>) -> PyResult { + let inner = match json { + Some(json) => encoding_from_json(json) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + None => Encoding::default(), + }; + Ok(Self { inner }) + } + + #[staticmethod] + #[pyo3(signature = (encodings, growing_offsets = true))] + fn merge(encodings: Vec>, growing_offsets: bool) -> Self { + Self { + inner: Encoding::merge( + encodings.iter().map(|encoding| encoding.inner.clone()), + growing_offsets, + ), + } + } + + fn __reduce__<'py>( + &self, + py: Python<'py>, + ) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> { + let json = encoding_to_json(&self.inner) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok((py.get_type::(), (json,))) + } + + fn __repr__(&self) -> String { + format!( + "Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \ + attention_mask, special_tokens_mask, overflowing])", + self.inner.len() + ) + } + + fn __len__(&self) -> usize { + self.inner.len() + } + #[getter] + fn ids(&self) -> Vec { + self.inner.get_ids().to_vec() + } + #[getter] + fn tokens(&self) -> Vec { + self.inner.get_tokens().to_vec() + } + #[getter] + fn offsets(&self) -> Vec<(usize, usize)> { + self.inner.get_offsets().to_vec() + } + #[getter] + fn type_ids(&self) -> Vec { + self.inner.get_type_ids().to_vec() + } + #[getter] + fn attention_mask(&self) -> Vec { + self.inner.get_attention_mask().to_vec() + } + #[getter] + fn special_tokens_mask(&self) -> Vec { + self.inner.get_special_tokens_mask().to_vec() + } + #[getter] + fn word_ids(&self) -> Vec> { + self.inner.get_word_ids().to_vec() + } + #[getter] + fn sequence_ids(&self) -> Vec> { + self.inner.get_sequence_ids() + } + #[getter] + fn overflowing(&self) -> Vec { + self.inner + .get_overflowing() + .iter() + .cloned() + .map(|inner| Self { inner }) + .collect() + } + #[getter] + fn n_sequences(&self) -> usize { + self.inner.n_sequences() + } + + #[pyo3(signature = (word_index, sequence_index = 0))] + fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> { + self.inner.word_to_tokens(word_index, sequence_index) + } + #[pyo3(signature = (word_index, sequence_index = 0))] + fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> { + self.inner.word_to_chars(word_index, sequence_index) + } + fn token_to_sequence(&self, token_index: usize) -> Option { + self.inner.token_to_sequence(token_index) + } + fn token_to_chars(&self, token_index: usize) -> Option<(usize, usize)> { + self.inner + .token_to_chars(token_index) + .map(|(_, offsets)| offsets) + } + fn token_to_word(&self, token_index: usize) -> Option { + self.inner.token_to_word(token_index).map(|(_, word)| word) + } + #[pyo3(signature = (char_pos, sequence_index = 0))] + fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option { + self.inner.char_to_token(char_pos, sequence_index) + } + #[pyo3(signature = (char_pos, sequence_index = 0))] + fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option { + self.inner.char_to_word(char_pos, sequence_index) + } + + fn set_sequence_id(&mut self, sequence_id: usize) { + self.inner.set_sequence_id(sequence_id); + } + + #[pyo3(signature = (length, direction = "right", pad_id = 0, pad_type_id = 0, pad_token = "[PAD]"))] + fn pad( + &mut self, + length: usize, + direction: &str, + pad_id: u32, + pad_type_id: u32, + pad_token: &str, + ) -> PyResult<()> { + let direction = self::direction( + direction, + PaddingDirection::Left, + PaddingDirection::Right, + "padding", + )?; + self.inner + .pad(length, pad_id, pad_type_id, pad_token, direction); + Ok(()) + } + + #[pyo3(signature = (max_length, stride = 0, direction = "right"))] + fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> { + let direction = self::direction( + direction, + TruncationDirection::Left, + TruncationDirection::Right, + "truncation", + )?; + self.inner.truncate(max_length, stride, direction); + Ok(()) + } +} diff --git a/litellm-rust/crates/token-counter-fast/src/lib.rs b/litellm-rust/crates/token-counter-fast/src/lib.rs index ce91af642ea..157ee847658 100644 --- a/litellm-rust/crates/token-counter-fast/src/lib.rs +++ b/litellm-rust/crates/token-counter-fast/src/lib.rs @@ -8,6 +8,8 @@ mod scanner; mod tiktoken; mod unicode_classes; +use std::sync::Arc; + use byte_level::ByteLevelCounter; use scanner::{SplitPattern, TiktokenCounter}; @@ -15,22 +17,29 @@ pub use error::Error; enum Encoder { HuggingFace { - tokenizer: Box, + tokenizer: Arc, byte_level: Option, }, Tiktoken(TiktokenCounter), } +/// A count-only tokenizer. Its model tables are immutable, so one built from an already +/// loaded model (`from_shared`, `from_*_pairs`) adds only the count-specific tables. pub struct FastTokenizer(Encoder); impl FastTokenizer { pub fn from_json(json: &str) -> Result { let tokenizer = json.parse::().map_err(Error::Load)?; + Ok(Self::from_shared(Arc::new(tokenizer))) + } + + /// Counts with a Hugging Face model another codec already holds; nothing is re-parsed. + pub fn from_shared(tokenizer: Arc) -> Self { let byte_level = ByteLevelCounter::detect(&tokenizer); - Ok(Self(Encoder::HuggingFace { - tokenizer: Box::new(tokenizer), + Self(Encoder::HuggingFace { + tokenizer, byte_level, - })) + }) } pub fn from_cl100k_ranks(ranks: &str) -> Result { @@ -41,12 +50,36 @@ impl FastTokenizer { Self::from_ranks(SplitPattern::O200k, ranks) } + /// `cl100k_base` from ranks another loader already parsed. + pub fn from_cl100k_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_pairs(SplitPattern::Cl100k, pairs) + } + + /// `o200k_base` (and `o200k_harmony`, whose ordinary tokens are the same) from ranks + /// another loader already parsed. + pub fn from_o200k_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_pairs(SplitPattern::O200k, pairs) + } + fn from_ranks(split: SplitPattern, ranks: &str) -> Result { TiktokenCounter::from_ranks(split, ranks) .map(Encoder::Tiktoken) .map(Self) } + fn from_pairs<'a>( + split: SplitPattern, + pairs: impl IntoIterator, + ) -> Result { + TiktokenCounter::from_pairs(split, pairs) + .map(Encoder::Tiktoken) + .map(Self) + } + pub fn count_tokens(&self, text: &str) -> Result { match &self.0 { Encoder::Tiktoken(counter) => Ok(counter.count(text)), diff --git a/litellm-rust/crates/token-counter-fast/src/scanner.rs b/litellm-rust/crates/token-counter-fast/src/scanner.rs index c2c3057aeeb..882ea91db81 100644 --- a/litellm-rust/crates/token-counter-fast/src/scanner.rs +++ b/litellm-rust/crates/token-counter-fast/src/scanner.rs @@ -39,8 +39,19 @@ pub(super) struct TiktokenCounter { impl TiktokenCounter { pub(super) fn from_ranks(split: SplitPattern, rank_file: &str) -> Result { + Self::new(split, MergeRanks::parse(rank_file)?) + } + + pub(super) fn from_pairs<'a>( + split: SplitPattern, + pairs: impl IntoIterator, + ) -> Result { + Self::new(split, MergeRanks::from_pairs(pairs)?) + } + + fn new(split: SplitPattern, ranks: MergeRanks) -> Result { Ok(Self { - ranks: MergeRanks::parse(rank_file)?, + ranks, piece_len: split.piece_len(), unicode_classes: UnicodeClasses::get().ok_or(Error::UnicodeClasses)?, }) diff --git a/litellm-rust/crates/token-counter-fast/src/tiktoken.rs b/litellm-rust/crates/token-counter-fast/src/tiktoken.rs index 16172b7a688..68f09b14a25 100644 --- a/litellm-rust/crates/token-counter-fast/src/tiktoken.rs +++ b/litellm-rust/crates/token-counter-fast/src/tiktoken.rs @@ -22,11 +22,25 @@ pub(super) struct MergeRanks(FxHashMap, Rank>); impl MergeRanks { pub(super) fn parse(text: &str) -> Result { - let ranks = text - .lines() - .filter(|line| !line.is_empty()) - .map(parse_line) - .collect::, _>>()?; + Self::from_entries(text.lines().filter(|line| !line.is_empty()).map(parse_line)) + } + + /// The same table from ranks another loader already parsed. + pub(super) fn from_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_entries(pairs.into_iter().map(|(bytes, rank)| { + if rank == NO_RANK { + return Err(Error::Ranks(format!("rank {rank} is reserved"))); + } + Ok((Box::from(bytes), rank)) + })) + } + + fn from_entries( + entries: impl Iterator, Rank), Error>>, + ) -> Result { + let ranks = entries.collect::, _>>()?; if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) { return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token"))); } diff --git a/litellm-rust/crates/token-counter-huggingface/Cargo.toml b/litellm-rust/crates/token-counter-huggingface/Cargo.toml index 6d8cb85e524..a5c2b1bb160 100644 --- a/litellm-rust/crates/token-counter-huggingface/Cargo.toml +++ b/litellm-rust/crates/token-counter-huggingface/Cargo.toml @@ -6,5 +6,6 @@ license.workspace = true repository.workspace = true [dependencies] +serde_json.workspace = true thiserror.workspace = true tokenizers.workspace = true diff --git a/litellm-rust/crates/token-counter-huggingface/src/error.rs b/litellm-rust/crates/token-counter-huggingface/src/error.rs index adc4551886f..e7f6260321b 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/error.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/error.rs @@ -6,4 +6,6 @@ pub enum Error { Load(#[source] tokenizers::Error), #[error("tokenization failed: {0}")] Encode(#[source] tokenizers::Error), + #[error("token decoding failed: {0}")] + Decode(#[source] tokenizers::Error), } diff --git a/litellm-rust/crates/token-counter-huggingface/src/lib.rs b/litellm-rust/crates/token-counter-huggingface/src/lib.rs index 8e05c2cca46..170a36aea05 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/lib.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/lib.rs @@ -2,22 +2,243 @@ mod error; -pub use error::Error; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; -pub struct HuggingFaceTokenizer(Box); +pub use error::Error; +use tokenizers::PostProcessor; +pub use tokenizers::{ + AddedToken, EncodeInput, Encoding, InputSequence, PaddingDirection, PaddingParams, + PaddingStrategy, TruncationDirection, TruncationParams, +}; + +pub fn encoding_from_json(json: &str) -> Result { + serde_json::from_str(json).map_err(|error| Error::Load(error.into())) +} + +pub fn encoding_to_json(encoding: &Encoding) -> Result { + serde_json::to_string(encoding).map_err(|error| Error::Load(error.into())) +} + +pub struct HuggingFaceTokenizer { + tokenizer: Arc, + special_token_ids: HashSet, +} impl HuggingFaceTokenizer { pub fn from_json(json: &str) -> Result { json.parse::() - .map(Box::new) - .map(Self) + .map(Self::new) .map_err(Error::Load) } + fn new(tokenizer: tokenizers::Tokenizer) -> Self { + let special_token_ids: HashSet = tokenizer + .get_added_tokens_decoder() + .into_iter() + .filter_map(|(id, token)| token.special.then_some(id)) + .collect(); + Self { + tokenizer: Arc::new(tokenizer), + special_token_ids, + } + } + + /// The parsed model, for a count-only counter to share instead of parsing it again. + pub fn shared(&self) -> Arc { + Arc::clone(&self.tokenizer) + } + pub fn count_tokens(&self, text: &str) -> Result { - self.0 + self.tokenizer .encode_fast(text, true) .map(|encoding| encoding.len()) .map_err(Error::Encode) } + + pub fn encode(&self, text: &str) -> Result, Error> { + self.tokenizer + .encode_fast(text, true) + .map(|encoding| encoding.get_ids().to_vec()) + .map_err(Error::Encode) + } + + pub fn encode_result<'a>( + &self, + input: EncodeInput<'a>, + add_special_tokens: bool, + fast: bool, + ) -> Result { + if fast { + return self + .tokenizer + .encode_fast(input, add_special_tokens) + .map_err(Error::Encode); + } + self.tokenizer + .encode_char_offsets(input, add_special_tokens) + .map_err(Error::Encode) + } + + pub fn encode_batch_result<'a>( + &self, + inputs: Vec>, + add_special_tokens: bool, + fast: bool, + ) -> Result, Error> { + if fast { + return self + .tokenizer + .encode_batch_fast(inputs, add_special_tokens) + .map_err(Error::Encode); + } + self.tokenizer + .encode_batch_char_offsets(inputs, add_special_tokens) + .map_err(Error::Encode) + } + + pub fn to_json(&self, pretty: bool) -> Result { + self.tokenizer.to_string(pretty).map_err(Error::Load) + } + + pub fn token_to_id(&self, token: &str) -> Option { + self.tokenizer.token_to_id(token) + } + + pub fn id_to_token(&self, id: u32) -> Option { + self.tokenizer.id_to_token(id) + } + + pub fn vocab(&self, with_added_tokens: bool) -> HashMap { + self.tokenizer.get_vocab(with_added_tokens) + } + + pub fn vocab_size(&self, with_added_tokens: bool) -> usize { + self.tokenizer.get_vocab_size(with_added_tokens) + } + + /// The added tokens by id, in id order. + pub fn added_tokens_decoder(&self) -> Vec<(u32, AddedToken)> { + let mut added: Vec<(u32, AddedToken)> = self + .tokenizer + .get_added_tokens_decoder() + .into_iter() + .collect(); + added.sort_unstable_by_key(|(id, _)| *id); + added + } + + pub fn padding(&self) -> Option<&PaddingParams> { + self.tokenizer.get_padding() + } + + pub fn truncation(&self) -> Option<&TruncationParams> { + self.tokenizer.get_truncation() + } + + /// How many special tokens the post-processor adds to a single sequence or a pair. + pub fn num_special_tokens_to_add(&self, is_pair: bool) -> usize { + self.tokenizer + .get_post_processor() + .map_or(0, |processor| processor.added_tokens(is_pair)) + } + + pub fn encode_special_tokens(&self) -> bool { + self.tokenizer.get_encode_special_tokens() + } + + pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result { + if !skip_special_tokens { + return self.tokenizer.decode(ids, false).map_err(Error::Decode); + } + let filtered_ids: Vec = ids + .iter() + .copied() + .filter(|id| !self.special_token_ids.contains(id)) + .collect(); + self.tokenizer + .decode(&filtered_ids, true) + .map_err(Error::Decode) + } + + pub fn name(&self) -> &str { + "huggingface" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codecs_round_trip_and_skip_special_tokens() { + let json = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json" + )); + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + let ids = tokenizer.encode("hello").unwrap(); + + assert!(tokenizer.decode(&ids, false).unwrap().contains("")); + assert_eq!(tokenizer.decode(&ids, true).unwrap(), "hello"); + } + + #[test] + fn decode_filters_special_added_tokens() { + let json = r#"{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 1, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": {"type": "Whitespace"}, + "post_processor": null, + "decoder": null, + "model": { + "type": "WordLevel", + "vocab": {"": 0, "": 1, "hello": 2}, + "unk_token": "" + } + }"#; + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + + assert!(!tokenizer.decode(&[1, 2], true).unwrap().contains("")); + assert!(tokenizer.decode(&[1, 2], false).unwrap().contains("")); + } + + #[test] + fn vocabulary_lookups_mirror_the_tokenizers_api() { + let json = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json" + )); + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + let ids = tokenizer.encode("hello").unwrap(); + + let token = tokenizer.id_to_token(ids[0]).unwrap(); + assert_eq!(tokenizer.token_to_id(&token), Some(ids[0])); + assert_eq!(tokenizer.id_to_token(u32::MAX), None); + assert_eq!(tokenizer.vocab(true).len(), tokenizer.vocab_size(true)); + assert!(tokenizer.vocab_size(true) >= tokenizer.vocab_size(false)); + let added = tokenizer.added_tokens_decoder(); + assert!(added.windows(2).all(|pair| pair[0].0 < pair[1].0)); + assert!(added.iter().any(|(_, token)| token.special)); + assert!(tokenizer.padding().is_none()); + assert!(tokenizer.truncation().is_none()); + assert!(!tokenizer.encode_special_tokens()); + assert_eq!( + tokenizer.num_special_tokens_to_add(false), + tokenizer.encode("").unwrap().len() + ); + } } diff --git a/litellm-rust/crates/token-counter-tiktoken/Cargo.toml b/litellm-rust/crates/token-counter-tiktoken/Cargo.toml index 494a9233e69..2fb3103e0c8 100644 --- a/litellm-rust/crates/token-counter-tiktoken/Cargo.toml +++ b/litellm-rust/crates/token-counter-tiktoken/Cargo.toml @@ -6,5 +6,8 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true +once_cell = "1.21.3" +rustc-hash = "2.1.3" thiserror.workspace = true tiktoken-rs.workspace = true diff --git a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs index ecdb3946eee..f049e90a1cb 100644 --- a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs +++ b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs @@ -1,27 +1,125 @@ #![forbid(unsafe_code)] mod error; +mod ranks; + +use std::collections::HashSet; pub use error::UnsupportedTokenizer; +pub use ranks::{LoadError, Vocabulary}; -pub struct TiktokenTokenizer(&'static tiktoken_rs::CoreBPE); +pub struct TiktokenTokenizer { + encoder: &'static tiktoken_rs::CoreBPE, + /// Present for encodings built from a rank file; the embedded tiktoken-rs singletons + /// behind [`from_name`](Self::from_name) keep their ranks private. + vocabulary: Option<&'static Vocabulary>, + name: &'static str, +} impl TiktokenTokenizer { + /// Builds `name` from its packaged rank file (read through `load`), once per process. + /// The tokenizer reports the requested name, so `gpt2` stays `gpt2` like tiktoken does. + pub fn from_cached_ranks( + name: &str, + load: impl FnOnce(&str) -> std::io::Result, + ) -> Result { + let (loaded, name) = ranks::load(name, load)?; + Ok(Self { + encoder: &loaded.bpe, + vocabulary: Some(&loaded.vocabulary), + name, + }) + } + + /// The encodings tiktoken-rs embeds, for hosts without the packaged rank files. pub fn from_name(name: &str) -> Result { - let tokenizer = match name { - "cl100k_base" => tiktoken_rs::cl100k_base_singleton(), - "o200k_base" => tiktoken_rs::o200k_base_singleton(), - "o200k_harmony" => tiktoken_rs::o200k_harmony_singleton(), - "p50k_base" => tiktoken_rs::p50k_base_singleton(), - "p50k_edit" => tiktoken_rs::p50k_edit_singleton(), - "r50k_base" | "gpt2" => tiktoken_rs::r50k_base_singleton(), + let (encoder, name) = match name { + "cl100k_base" => (tiktoken_rs::cl100k_base_singleton(), "cl100k_base"), + "o200k_base" => (tiktoken_rs::o200k_base_singleton(), "o200k_base"), + "o200k_harmony" => (tiktoken_rs::o200k_harmony_singleton(), "o200k_harmony"), + "p50k_base" => (tiktoken_rs::p50k_base_singleton(), "p50k_base"), + "p50k_edit" => (tiktoken_rs::p50k_edit_singleton(), "p50k_edit"), + "r50k_base" => (tiktoken_rs::r50k_base_singleton(), "r50k_base"), + "gpt2" => (tiktoken_rs::r50k_base_singleton(), "gpt2"), _ => return Err(UnsupportedTokenizer(name.to_owned())), }; - Ok(Self(tokenizer)) + Ok(Self { + encoder, + vocabulary: None, + name, + }) + } + + pub fn vocabulary(&self) -> Option<&Vocabulary> { + self.vocabulary } pub fn count_tokens(&self, text: &str) -> usize { - self.0.count_ordinary(text) + self.encoder.count_ordinary(text) + } + + pub fn encode(&self, text: &str) -> Vec { + self.encoder.encode_ordinary(text) + } + + pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result, String> { + let allowed = allowed.iter().map(String::as_str).collect(); + self.encoder + .encode(text, &allowed) + .map(|(ids, _)| ids) + .map_err(|error| error.to_string()) + } + + pub fn special_tokens(&self) -> HashSet { + self.encoder + .special_tokens() + .into_iter() + .map(str::to_owned) + .collect() + } + + /// tiktoken's `encode_with_unstable`: the stable prefix of `text`'s tokens and every + /// token sequence the unstable tail could still become, sorted for a stable order. + pub fn encode_with_unstable( + &self, + text: &str, + allowed: &[String], + ) -> (Vec, Vec>) { + let allowed = allowed.iter().map(String::as_str).collect(); + let (stable, completions) = self.encoder._encode_unstable_native(text, &allowed); + let mut completions: Vec> = completions.into_iter().collect(); + completions.sort_unstable(); + (stable, completions) + } + + pub fn decode_bytes(&self, ids: &[u32]) -> Result, String> { + self.encoder + .decode_bytes(ids) + .map_err(|error| error.to_string()) + } + + pub fn decode(&self, ids: &[u32]) -> Result { + self.encoder + .decode_bytes(ids) + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .map_err(|error| error.to_string()) + } + + pub fn name(&self) -> &str { + self.name + } +} + +pub fn encoding_for_model(model: &str) -> Option<&'static str> { + match tiktoken_rs::tokenizer::get_tokenizer(model)? { + tiktoken_rs::tokenizer::Tokenizer::Cl100kBase => Some("cl100k_base"), + tiktoken_rs::tokenizer::Tokenizer::O200kBase => Some("o200k_base"), + tiktoken_rs::tokenizer::Tokenizer::O200kHarmony => Some("o200k_harmony"), + tiktoken_rs::tokenizer::Tokenizer::P50kBase => Some("p50k_base"), + tiktoken_rs::tokenizer::Tokenizer::P50kEdit => Some("p50k_edit"), + tiktoken_rs::tokenizer::Tokenizer::R50kBase | tiktoken_rs::tokenizer::Tokenizer::Gpt2 => { + Some("r50k_base") + } } } @@ -66,5 +164,75 @@ mod tests { panic!("unknown encoding must be rejected"); }; assert_eq!(name, "unknown-encoding"); + assert_eq!(TiktokenTokenizer::from_name("gpt2").unwrap().name(), "gpt2"); + } + + #[test] + fn codecs_round_trip_named_encodings() { + let encodings = [ + "cl100k_base", + "o200k_base", + "o200k_harmony", + "p50k_base", + "p50k_edit", + "r50k_base", + "gpt2", + ]; + let texts = ["hello world", "café 漢字 مرحبا 🙂", "line one\nline two"]; + for name in encodings { + let tokenizer = TiktokenTokenizer::from_name(name).unwrap(); + for text in texts { + assert_eq!( + tokenizer.decode(&tokenizer.encode(text)).unwrap(), + text, + "{name}: {text:?}", + ); + } + } + } + + #[test] + fn decoding_token_prefixes_replaces_incomplete_utf8() { + let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap(); + let reference = tiktoken_rs::cl100k_base_singleton(); + let ids = tokenizer.encode("🙂漢字"); + for end in 1..=ids.len() { + let bytes = reference.decode_bytes(&ids[..end]).unwrap(); + assert_eq!( + tokenizer.decode(&ids[..end]).unwrap(), + String::from_utf8_lossy(&bytes), + ); + } + assert!(tokenizer.decode(&[u32::MAX]).is_err()); + } + + #[test] + fn unstable_encoding_prefixes_stay_consistent_with_full_encoding() { + let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap(); + let text = "hello fanta"; + let (stable, completions) = tokenizer.encode_with_unstable(text, &[]); + assert!( + text.as_bytes() + .starts_with(&tokenizer.decode_bytes(&stable).unwrap()) + ); + assert!(!completions.is_empty()); + for completion in &completions { + let mut ids = stable.clone(); + ids.extend(completion); + assert!( + tokenizer + .decode_bytes(&ids) + .unwrap() + .starts_with(text.as_bytes()) + ); + } + assert!(completions.windows(2).all(|pair| pair[0] < pair[1])); + } + + #[test] + fn encoding_for_model_maps_known_models() { + assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base")); + assert_eq!(encoding_for_model("text-davinci-003"), Some("p50k_base")); + assert_eq!(encoding_for_model("unknown-model"), None); } } diff --git a/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs b/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs new file mode 100644 index 00000000000..1f5e5262de5 --- /dev/null +++ b/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs @@ -0,0 +1,340 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; +use once_cell::sync::OnceCell; +use rustc_hash::FxHashMap; +use thiserror::Error; +use tiktoken_rs::{CoreBPE, O200K_BASE_PAT_STR, Rank}; + +use crate::UnsupportedTokenizer; + +const CL100K: &str = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"; +const O200K: &str = "fb374d419588a4632f3f557e76b4b70aebbca790"; +const P50K: &str = "ec7223a39ce59f226a68acc30dc1af2788490e15"; +const LEGACY_PATTERN: &str = + r"'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s"; +const CL100K_PATTERN: &str = r"'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s"; + +static CL100K_ENCODER: OnceCell = OnceCell::new(); +static O200K_ENCODER: OnceCell = OnceCell::new(); +static HARMONY_ENCODER: OnceCell = OnceCell::new(); +static P50K_ENCODER: OnceCell = OnceCell::new(); +static EDIT_ENCODER: OnceCell = OnceCell::new(); +static R50K_ENCODER: OnceCell = OnceCell::new(); + +/// One encoding built from a rank file: the BPE engine plus the vocabulary it was built +/// from, kept because `CoreBPE` does not expose its ranks and tiktoken's Python API does +/// (`token_byte_values`, `encode_single_token`, `max_token_value`, `_special_tokens`). +pub(super) struct Loaded { + pub(super) bpe: CoreBPE, + pub(super) vocabulary: Vocabulary, +} + +/// The byte-level vocabulary of a tiktoken encoding. +pub struct Vocabulary { + ranks: FxHashMap, Rank>, + special_tokens: FxHashMap, + max_token_value: Rank, +} + +impl Vocabulary { + /// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`. + pub fn token_byte_values(&self) -> Vec> { + let mut values: Vec> = self.ranks.keys().cloned().collect(); + values.sort_unstable(); + values + } + + /// The rank of one whole token: a mergeable piece first, then a special token's text. + pub fn encode_single_token(&self, piece: &[u8]) -> Option { + if let Some(rank) = self.ranks.get(piece) { + return Some(*rank); + } + std::str::from_utf8(piece) + .ok() + .and_then(|text| self.special_tokens.get(text).copied()) + } + + pub fn max_token_value(&self) -> Rank { + self.max_token_value + } + + /// Every mergeable token with its rank, for building other tables from one parse. + pub fn ranks(&self) -> impl Iterator + '_ { + self.ranks + .iter() + .map(|(bytes, rank)| (bytes.as_slice(), *rank)) + } + + /// The special tokens with their ranks, tiktoken's `_special_tokens`. + pub fn special_tokens(&self) -> impl Iterator + '_ { + self.special_tokens + .iter() + .map(|(token, rank)| (token.as_str(), *rank)) + } + + pub fn is_special_token(&self, rank: Rank) -> bool { + self.special_tokens.values().any(|special| *special == rank) + } +} + +#[derive(Debug, Error)] +pub enum LoadError { + #[error(transparent)] + Unsupported(#[from] UnsupportedTokenizer), + #[error("failed to load tiktoken ranks: {0}")] + Ranks(String), +} + +/// Loads `name` once per process. The returned name is the one requested (`gpt2` stays +/// `gpt2`, as `tiktoken.get_encoding("gpt2").name` does), while `gpt2` and `r50k_base` share +/// one cached encoder. +pub(super) fn load( + name: &str, + load_file: impl FnOnce(&str) -> std::io::Result, +) -> Result<(&'static Loaded, &'static str), LoadError> { + let (requested, canonical, file, cache) = match name { + "cl100k_base" => ("cl100k_base", "cl100k_base", CL100K, &CL100K_ENCODER), + "o200k_base" => ("o200k_base", "o200k_base", O200K, &O200K_ENCODER), + "o200k_harmony" => ("o200k_harmony", "o200k_harmony", O200K, &HARMONY_ENCODER), + "p50k_base" => ("p50k_base", "p50k_base", P50K, &P50K_ENCODER), + "p50k_edit" => ("p50k_edit", "p50k_edit", P50K, &EDIT_ENCODER), + "r50k_base" => ("r50k_base", "r50k_base", P50K, &R50K_ENCODER), + "gpt2" => ("gpt2", "r50k_base", P50K, &R50K_ENCODER), + _ => return Err(UnsupportedTokenizer(name.to_owned()).into()), + }; + let loaded = cache.get_or_try_init(|| { + let ranks = load_file(file).map_err(|error| LoadError::Ranks(error.to_string()))?; + build(canonical, &ranks) + })?; + Ok((loaded, requested)) +} + +fn build(name: &str, ranks: &str) -> Result { + let parsed = ranks + .lines() + .map(parse_rank) + .collect::, _>>()?; + let encoder: FxHashMap<_, _> = parsed + .into_iter() + .filter(|(_, rank)| name != "r50k_base" || *rank < 50256) + .collect(); + if encoder + .values() + .collect::>() + .len() + != encoder.len() + || (0..=u8::MAX).any(|byte| !encoder.contains_key(&[byte][..])) + { + return Err(LoadError::Ranks("invalid vocabulary ranks".into())); + } + let (pattern, specials): (&str, &[(&str, Rank)]) = match name { + "cl100k_base" => ( + CL100K_PATTERN, + &[ + ("<|endoftext|>", 100257), + ("<|fim_prefix|>", 100258), + ("<|fim_middle|>", 100259), + ("<|fim_suffix|>", 100260), + ("<|endofprompt|>", 100276), + ], + ), + "o200k_base" => ( + O200K_BASE_PAT_STR, + &[("<|endoftext|>", 199999), ("<|endofprompt|>", 200018)], + ), + "o200k_harmony" => ( + O200K_BASE_PAT_STR, + &[ + ("<|startoftext|>", 199998), + ("<|endoftext|>", 199999), + ("<|reserved_200000|>", 200000), + ("<|reserved_200001|>", 200001), + ("<|return|>", 200002), + ("<|constrain|>", 200003), + ("<|reserved_200004|>", 200004), + ("<|channel|>", 200005), + ("<|start|>", 200006), + ("<|end|>", 200007), + ("<|message|>", 200008), + ("<|reserved_200009|>", 200009), + ("<|reserved_200010|>", 200010), + ("<|reserved_200011|>", 200011), + ("<|call|>", 200012), + ], + ), + "p50k_edit" => ( + LEGACY_PATTERN, + &[ + ("<|endoftext|>", 50256), + ("<|fim_prefix|>", 50281), + ("<|fim_middle|>", 50282), + ("<|fim_suffix|>", 50283), + ], + ), + _ => (LEGACY_PATTERN, &[("<|endoftext|>", 50256)]), + }; + let reserved = (200013..=201087) + .filter(|_| name == "o200k_harmony") + .map(|rank| (format!("<|reserved_{rank}|>"), rank)); + let special_tokens: FxHashMap = specials + .iter() + .map(|(token, rank)| ((*token).to_owned(), *rank)) + .chain(reserved) + .collect(); + let max_token_value = encoder + .values() + .chain(special_tokens.values()) + .copied() + .max() + .ok_or_else(|| LoadError::Ranks("empty vocabulary".into()))?; + let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern) + .map_err(|error| LoadError::Ranks(error.to_string()))?; + Ok(Loaded { + bpe, + vocabulary: Vocabulary { + ranks: encoder, + special_tokens, + max_token_value, + }, + }) +} + +fn parse_rank(line: &str) -> Result<(Vec, Rank), LoadError> { + let (token, rank) = line + .split_once(' ') + .ok_or_else(|| LoadError::Ranks("missing rank".into()))?; + let bytes = STANDARD + .decode(token) + .map_err(|error| LoadError::Ranks(error.to_string()))?; + let rank = rank + .parse() + .map_err(|error: std::num::ParseIntError| LoadError::Ranks(error.to_string()))?; + Ok((bytes, rank)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TiktokenTokenizer; + + fn read_packaged_ranks(file: &str) -> std::io::Result { + std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../litellm/litellm_core_utils/tokenizers") + .join(file), + ) + } + + #[test] + fn packaged_encodings_match_embedded_encodings_and_reuse_successful_loads() { + for name in [ + "cl100k_base", + "o200k_base", + "o200k_harmony", + "p50k_base", + "p50k_edit", + "r50k_base", + "gpt2", + ] { + if name != "gpt2" { + assert!( + TiktokenTokenizer::from_cached_ranks(name, |_| { + Err(std::io::Error::other("unreadable vocabulary")) + }) + .is_err() + ); + } + let loads = std::sync::atomic::AtomicUsize::new(0); + let barrier = std::sync::Barrier::new(4); + let encoders = std::thread::scope(|scope| { + let tasks: Vec<_> = (0..4) + .map(|_| { + scope.spawn(|| { + barrier.wait(); + TiktokenTokenizer::from_cached_ranks(name, |file| { + loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + read_packaged_ranks(file) + }) + .unwrap() + }) + }) + .collect(); + tasks + .into_iter() + .map(|task| task.join().unwrap()) + .collect::>() + }); + assert_eq!(loads.into_inner(), usize::from(name != "gpt2")); + let actual = &encoders[0]; + let expected = TiktokenTokenizer::from_name(name).unwrap(); + assert_eq!(actual.special_tokens(), expected.special_tokens()); + let specials: Vec<_> = expected.special_tokens().into_iter().collect(); + let special_text = specials.join(" "); + assert_eq!( + actual.encode_special(&special_text, &specials).unwrap(), + expected.encode_special(&special_text, &specials).unwrap() + ); + for text in [ + "", + "café 漢字 ع 🙂", + "a\r\nb\t ", + " hello 123456789", + &special_text, + ] { + let ids = expected.encode(text); + assert_eq!(actual.encode(text), ids, "{name}: {text:?}"); + assert_eq!(actual.count_tokens(text), ids.len(), "{name}: {text:?}"); + assert_eq!( + actual.decode_bytes(&ids).unwrap(), + expected.decode_bytes(&ids).unwrap() + ); + } + let cached = TiktokenTokenizer::from_cached_ranks(name, |_| { + panic!("reloaded cached vocabulary") + }) + .unwrap(); + assert_eq!(cached.encode("cached"), expected.encode("cached")); + assert_eq!(cached.name(), name); + assert!(expected.vocabulary().is_none()); + assert_vocabulary_lookups(name, actual); + } + } + + /// The token-level lookups tiktoken's Python `Encoding` exposes, checked against the + /// encoder itself and against the known vocabulary sizes. + fn assert_vocabulary_lookups(name: &str, tokenizer: &TiktokenTokenizer) { + let max_token_value = match name { + "cl100k_base" => 100_276, + "o200k_base" => 200_018, + "o200k_harmony" => 201_087, + "p50k_base" => 50_280, + "p50k_edit" => 50_283, + "r50k_base" | "gpt2" => 50_256, + _ => unreachable!("{name}"), + }; + let vocabulary = tokenizer.vocabulary().unwrap(); + assert_eq!(vocabulary.max_token_value(), max_token_value, "{name}"); + let values = vocabulary.token_byte_values(); + assert!(values.windows(2).all(|pair| pair[0] < pair[1]), "{name}"); + for piece in values.iter().step_by(997) { + let rank = vocabulary.encode_single_token(piece).unwrap(); + assert_eq!(tokenizer.decode_bytes(&[rank]).unwrap(), *piece, "{name}"); + assert!(!vocabulary.is_special_token(rank), "{name}"); + } + for (token, rank) in vocabulary.special_tokens() { + assert_eq!(vocabulary.encode_single_token(token.as_bytes()), Some(rank)); + assert!(vocabulary.is_special_token(rank), "{name}: {token}"); + } + assert_eq!(vocabulary.encode_single_token(b"<|not-a-token|>"), None); + } + + #[test] + fn malformed_ranks_return_errors_instead_of_panicking() { + for ranks in ["", "IQ==", "IQ== x", "!!! 1", "IQ== 1"] { + assert!(build("cl100k_base", ranks).is_err()); + } + let repeated_rank = (0..=u8::MAX) + .map(|byte| format!("{} 0\n", STANDARD.encode([byte]))) + .collect::(); + assert!(build("cl100k_base", &repeated_rank).is_err()); + } +} diff --git a/litellm-rust/crates/token-counter/README.md b/litellm-rust/crates/token-counter/README.md index a6b2b50aac0..3c6381fcea7 100644 --- a/litellm-rust/crates/token-counter/README.md +++ b/litellm-rust/crates/token-counter/README.md @@ -1,6 +1,10 @@ # Token counting -`Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface +`Tokenizer` is the text-counting interface. `TextCodec` adds encoding, decoding, and a name. `TokenCounter` applies LiteLLM request, message, and tool accounting using any `Tokenizer` + +Counts follow the codec: tiktoken treats special-token spellings as ordinary text, while Hugging Face applies its added tokens, post-processing, padding, and truncation. `fast=True` preserves those semantics and requests acceleration where available. Unsupported configurations use the normal codec, including tiktoken encodings without a scanner and builds without the `fast` feature. Invalid input and process-guard errors still propagate. Runtime request counting currently uses the normal codec; the custom accelerator is retained for explicit use and testing + +`FastCounter: TextCodec` exposes an optional accelerator over a loaded codec. `None` means callers should use that codec. The Python bridge caches this selection per immutable tokenizer, shares it with request counters, and initializes it with the GIL released. Hugging Face can also choose the full encoder per input when added tokens require it The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation @@ -8,7 +12,9 @@ The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through t The `tiktoken` feature provides `tiktoken::TiktokenTokenizer` through `tiktoken-rs`. Select an encoding with `TokenCounter::from_tiktoken`. The supported names are `cl100k_base`, `o200k_base`, `o200k_harmony`, `p50k_base`, `p50k_edit`, `r50k_base`, and `gpt2` -All three backends are enabled by default. The Python extension builds with `fast` only, which keeps the wheel at the size it had before the split. With `default-features = false`, callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend +All three backends are enabled by default in this crate and the Python extension. With `default-features = false`, Rust callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend + +Python `tiktoken` and `tokenizers` remain runtime dependencies and the default implementations. The catalog independently selects the tokenizer and request-counting routes. Enabling Rust changes factory dispatch; existing tokenizer objects keep their backend. Native Hugging Face wrappers provide an immutable encoding and decoding API, while training and mutable configuration remain available through the Python backend Budget checks, cost calculation, and the `max_tokens` adjustment policy belong to `litellm-core-utils`. The counter does not own prices, budgets, or request limits diff --git a/litellm-rust/crates/token-counter/src/error.rs b/litellm-rust/crates/token-counter/src/error.rs index b05ce007e46..6a94b0e39b8 100644 --- a/litellm-rust/crates/token-counter/src/error.rs +++ b/litellm-rust/crates/token-counter/src/error.rs @@ -32,6 +32,8 @@ pub enum Error { JsonUtf8(#[source] FromUtf8Error), #[error("tokenization failed: {0}")] Encode(#[source] Box), + #[error("token decoding failed: {0}")] + Decode(String), #[error("token counting task failed: {0}")] Task(String), } diff --git a/litellm-rust/crates/token-counter/src/fast.rs b/litellm-rust/crates/token-counter/src/fast.rs index de3f86abd68..f601c3cbcca 100644 --- a/litellm-rust/crates/token-counter/src/fast.rs +++ b/litellm-rust/crates/token-counter/src/fast.rs @@ -1,7 +1,31 @@ use litellm_token_counter_fast::Error as BackendError; pub use litellm_token_counter_fast::FastTokenizer; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; + +pub trait FastCounter: TextCodec { + fn fast_counter(&self) -> Option; +} + +#[cfg(feature = "huggingface")] +impl FastCounter for crate::huggingface::HuggingFaceTokenizer { + fn fast_counter(&self) -> Option { + Some(FastTokenizer::from_shared(self.shared())) + } +} + +#[cfg(feature = "tiktoken")] +impl FastCounter for crate::tiktoken::TiktokenTokenizer { + fn fast_counter(&self) -> Option { + let vocabulary = self.vocabulary()?; + match self.name() { + "cl100k_base" => FastTokenizer::from_cl100k_pairs(vocabulary.ranks()), + "o200k_base" | "o200k_harmony" => FastTokenizer::from_o200k_pairs(vocabulary.ranks()), + _ => return None, + } + .ok() + } +} impl TokenCounter { pub fn from_json_fast(tokenizer_json: &str) -> Result { @@ -39,3 +63,65 @@ impl From for Error { } } } + +#[cfg(all(test, feature = "huggingface", feature = "tiktoken"))] +mod tests { + use super::*; + use crate::huggingface::HuggingFaceTokenizer; + use crate::tiktoken::TiktokenTokenizer; + + const TEXTS: [&str; 4] = [ + "", + "hello world <|endoftext|>", + "café 漢字 ع 🙂 line\r\n indented 123456789", + "system a\u{301} fi", + ]; + + fn packaged(file: &str) -> String { + std::fs::read_to_string(format!( + "{}/../../../litellm/litellm_core_utils/tokenizers/{file}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() + } + + #[test] + fn fast_counters_derived_from_codecs_count_like_the_codecs() { + let huggingface = + HuggingFaceTokenizer::from_json(&packaged("anthropic_tokenizer.json")).unwrap(); + let fast = huggingface.fast_counter().unwrap(); + for text in TEXTS { + assert_eq!( + fast.count_tokens(text).unwrap(), + Tokenizer::count_tokens(&huggingface, text).unwrap(), + "{text:?}" + ); + } + + for name in ["cl100k_base", "o200k_base", "o200k_harmony"] { + let tiktoken = + TiktokenTokenizer::from_cached_ranks(name, |file| Ok(packaged(file))).unwrap(); + let fast = tiktoken.fast_counter().unwrap(); + for text in TEXTS { + assert_eq!( + fast.count_tokens(text).unwrap(), + tiktoken.count_tokens(text), + "{name}: {text:?}" + ); + } + } + } + + #[test] + fn encodings_without_a_fast_scanner_keep_the_codec() { + let tiktoken = + TiktokenTokenizer::from_cached_ranks("p50k_base", |file| Ok(packaged(file))).unwrap(); + assert!(tiktoken.fast_counter().is_none()); + assert!( + TiktokenTokenizer::from_name("cl100k_base") + .unwrap() + .fast_counter() + .is_none() + ); + } +} diff --git a/litellm-rust/crates/token-counter/src/huggingface.rs b/litellm-rust/crates/token-counter/src/huggingface.rs index fb7683b373e..43613fac8ea 100644 --- a/litellm-rust/crates/token-counter/src/huggingface.rs +++ b/litellm-rust/crates/token-counter/src/huggingface.rs @@ -1,7 +1,11 @@ use litellm_token_counter_huggingface::Error as BackendError; -pub use litellm_token_counter_huggingface::HuggingFaceTokenizer; +pub use litellm_token_counter_huggingface::{ + AddedToken, EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, + PaddingParams, PaddingStrategy, TruncationDirection, TruncationParams, encoding_from_json, + encoding_to_json, +}; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; impl TokenCounter { pub fn from_json(tokenizer_json: &str) -> Result { @@ -17,11 +21,26 @@ impl Tokenizer for HuggingFaceTokenizer { } } +impl TextCodec for HuggingFaceTokenizer { + fn encode(&self, text: &str) -> Result, Error> { + HuggingFaceTokenizer::encode(self, text).map_err(Error::from) + } + + fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result { + HuggingFaceTokenizer::decode(self, ids, skip_special_tokens).map_err(Error::from) + } + + fn name(&self) -> &str { + HuggingFaceTokenizer::name(self) + } +} + impl From for Error { fn from(error: BackendError) -> Self { match error { BackendError::Load(source) => Self::Load(source), BackendError::Encode(source) => Self::Encode(source), + BackendError::Decode(source) => Self::Decode(source.to_string()), } } } diff --git a/litellm-rust/crates/token-counter/src/lib.rs b/litellm-rust/crates/token-counter/src/lib.rs index 446c91049de..84bf35ca2ba 100644 --- a/litellm-rust/crates/token-counter/src/lib.rs +++ b/litellm-rust/crates/token-counter/src/lib.rs @@ -20,5 +20,5 @@ pub mod tiktoken; pub use counter::{InputTokenCount, TokenCounter}; pub use error::Error; -pub use tokenizer::Tokenizer; +pub use tokenizer::{TextCodec, Tokenizer}; pub use types::CountableRequest; diff --git a/litellm-rust/crates/token-counter/src/tiktoken.rs b/litellm-rust/crates/token-counter/src/tiktoken.rs index 07c1c9f5b73..5883a629ea5 100644 --- a/litellm-rust/crates/token-counter/src/tiktoken.rs +++ b/litellm-rust/crates/token-counter/src/tiktoken.rs @@ -1,7 +1,7 @@ -pub use litellm_token_counter_tiktoken::TiktokenTokenizer; -use litellm_token_counter_tiktoken::UnsupportedTokenizer; +use litellm_token_counter_tiktoken::{LoadError, UnsupportedTokenizer}; +pub use litellm_token_counter_tiktoken::{TiktokenTokenizer, Vocabulary, encoding_for_model}; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; impl TokenCounter { pub fn from_tiktoken(encoding: &str) -> Result { @@ -17,8 +17,31 @@ impl Tokenizer for TiktokenTokenizer { } } +impl TextCodec for TiktokenTokenizer { + fn encode(&self, text: &str) -> Result, Error> { + Ok(TiktokenTokenizer::encode(self, text)) + } + + fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result { + TiktokenTokenizer::decode(self, ids).map_err(|error| Error::Decode(error.to_string())) + } + + fn name(&self) -> &str { + TiktokenTokenizer::name(self) + } +} + impl From for Error { fn from(error: UnsupportedTokenizer) -> Self { Self::UnsupportedTokenizer(error.0) } } + +impl From for Error { + fn from(error: LoadError) -> Self { + match error { + LoadError::Unsupported(error) => error.into(), + LoadError::Ranks(message) => Self::Ranks(message), + } + } +} diff --git a/litellm-rust/crates/token-counter/src/tokenizer.rs b/litellm-rust/crates/token-counter/src/tokenizer.rs index 88c29c672a7..146ac5b4d0d 100644 --- a/litellm-rust/crates/token-counter/src/tokenizer.rs +++ b/litellm-rust/crates/token-counter/src/tokenizer.rs @@ -4,6 +4,12 @@ pub trait Tokenizer: Send + Sync { fn count_tokens(&self, text: &str) -> Result; } +pub trait TextCodec: Tokenizer { + fn encode(&self, text: &str) -> Result, Error>; + fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result; + fn name(&self) -> &str; +} + #[cfg(test)] mod tests { use super::*; diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index fe3c7c264ee..29fb46fa125 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -58,7 +58,8 @@ from ._lazy_imports_registry import ( if TYPE_CHECKING: import httpx - from tiktoken import Encoding + + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def get_litellm_globals() -> dict[str, object]: @@ -89,26 +90,11 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo # These are special lazy loaders for things that are used internally # They're separate from the main lazy import system because they have specific use cases -# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup -_default_encoding: "Encoding | None" = None +def _get_default_encoding() -> "Tokenizer": + from litellm.rust_bridge.tokenizer import get_encoding -def _get_default_encoding() -> "Encoding": - """ - Lazily load and cache the default OpenAI encoding. - - This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken) - at `litellm` import time. The encoding is cached after the first import. - - This is used internally by utils.py functions that need the encoding but shouldn't - trigger its import during module load. - """ - global _default_encoding - if _default_encoding is None: - from litellm.litellm_core_utils.default_encoding import encoding - - _default_encoding = encoding - return _default_encoding + return get_encoding("cl100k_base") # Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time diff --git a/litellm/litellm_core_utils/README.md b/litellm/litellm_core_utils/README.md index b61c8982762..a5f5e8326b9 100644 --- a/litellm/litellm_core_utils/README.md +++ b/litellm/litellm_core_utils/README.md @@ -6,8 +6,9 @@ Core files: - `streaming_handler.py`: The core streaming logic + streaming related helper utils - `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`. - `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types. -- `default_encoding.py`: code for loading the default encoding (tiktoken) +- `default_encoding.py`: code for loading the default Python tokenizer and bundled cache - `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name. - `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s" - `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion]) +Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer. diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index 71b30614d8d..c3b6a008411 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -1,5 +1,4 @@ import os -from pathlib import Path from typing import Final import litellm @@ -15,20 +14,6 @@ except (ImportError, AttributeError): filename = pkg_resources.resource_filename(__name__, "litellm_core_utils/tokenizers") -CL100K_BASE_RANK_FILE: Final = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4" -O200K_BASE_RANK_FILE: Final = "fb374d419588a4632f3f557e76b4b70aebbca790" - - -def cl100k_base_rank_file() -> str: - """The vendored tiktoken `cl100k_base` rank file (`base64(token) rank` lines).""" - return Path(filename, CL100K_BASE_RANK_FILE).read_text(encoding="ascii") - - -def o200k_base_rank_file() -> str: - """The vendored tiktoken `o200k_base` rank file (`base64(token) rank` lines).""" - return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii") - - # Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory # unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR. # This keeps tiktoken fully offline-capable by default (see #1071). diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index bf37b1be2e4..5d7956059e4 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -10,7 +10,6 @@ import anyio import anyio.lowlevel import httpx import tiktoken -from tokenizers import Tokenizer from typing_extensions import ParamSpec, TypeVar import litellm @@ -30,8 +29,10 @@ from litellm.constants import ( TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import _get_httpx_client +from litellm.rust_bridge.tokenizer import get_encoding from litellm.types.llms.anthropic import ( AnthropicContentParamSource, AnthropicContentParamSourceFileId, @@ -622,9 +623,11 @@ def _get_exact_count_function( if model is not None or custom_tokenizer is not None: tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model) if tokenizer_json["type"] == "huggingface_tokenizer": - tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"] + tokenizer: Final[HuggingFace] = tokenizer_json["tokenizer"] def count_tokens(text: str) -> int: + if isinstance(tokenizer, HuggingFaceTokenizer): + return tokenizer.count(text) return len(tokenizer.encode_batch_fast([text])[0]) return count_tokens @@ -632,31 +635,43 @@ def _get_exact_count_function( encoding: Final = openai_tokenizer_encoding(model) def encode_length(text: str) -> int: - return len(encoding.encode(text, disallowed_special=())) + return _encoding_count(encoding, text) return _get_tiktoken_count_function(encode_length) else: raise ValueError("Unsupported tokenizer type") else: + default_encoding: Final = _get_default_encoding() def encode_length(text: str) -> int: - return len(_get_default_encoding().encode(text, disallowed_special=())) + return _encoding_count(default_encoding, text) return _get_tiktoken_count_function(encode_length) -def openai_tokenizer_encoding(model: str) -> tiktoken.Encoding: - """The tiktoken encoding `token_counter` uses for a model on the `openai_tokenizer` path.""" +def _encoding_count(encoding: Encoding, text: str) -> int: + if isinstance(encoding, OpenAIEncoding): + return encoding.count(text) + return len(encoding.encode(text, disallowed_special=())) + + +def openai_tokenizer_encoding(model: str) -> Encoding: + """The encoding `token_counter` uses for a model on the `openai_tokenizer` path.""" + return get_encoding(openai_tokenizer_encoding_name(model)) + + +def openai_tokenizer_encoding_name(model: str) -> str: + """The tiktoken encoding name for `model`, without loading the encoding.""" from litellm.utils import print_verbose model_to_use: Final = _fix_model_name(model) if "gpt-4o" in model_to_use: - return tiktoken.get_encoding("o200k_base") + return "o200k_base" try: - return tiktoken.encoding_for_model(model_to_use) + return tiktoken.encoding_name_for_model(model_to_use) except KeyError: print_verbose("Warning: model not found. Using cl100k_base encoding.") - return tiktoken.get_encoding("cl100k_base") + return "cl100k_base" def uses_legacy_message_accounting(model: str) -> bool: diff --git a/litellm/litellm_core_utils/tokenizer.py b/litellm/litellm_core_utils/tokenizer.py new file mode 100644 index 00000000000..aea187fa08e --- /dev/null +++ b/litellm/litellm_core_utils/tokenizer.py @@ -0,0 +1,402 @@ +"""Python faces of the Rust text codecs. + +``OpenAIEncoding`` mirrors ``tiktoken.Encoding`` and ``HuggingFaceTokenizer`` mirrors +``tokenizers.Tokenizer``, so a caller holding ``litellm.encoding`` or the object returned by +``litellm.create_tokenizer`` sees the same read-only surface whichever backend the Rust catalog +selected. Both wrappers are immutable: ``tokenizers`` mutators (``enable_padding``, +``enable_truncation``, ``add_tokens``) stay on the Python tokenizer. +""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping, Sequence, Set +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable + +import tiktoken +from tokenizers import AddedToken +from tokenizers import Tokenizer as PythonHuggingFaceTokenizer + +if TYPE_CHECKING: + import numpy as np + import numpy.typing as npt + + from litellm.rust_bridge._native import HuggingFaceEncoding + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + +SpecialTokens: TypeAlias = Literal["all"] | Collection[str] +AllowedSpecial: TypeAlias = Literal["all"] | Set[str] +HuggingFaceInput: TypeAlias = str | list[str] | tuple[str, ...] +HuggingFaceBatchInput: TypeAlias = HuggingFaceInput | tuple[HuggingFaceInput, HuggingFaceInput] | list[HuggingFaceInput] + + +@dataclass(frozen=True, slots=True) +class OpenAIEncoding: + """``tiktoken.Encoding`` over the Rust tiktoken codec.""" + + _native: NativeTokenizer + _special_tokens: Mapping[str, int] + + @staticmethod + def wrap(native: NativeTokenizer) -> OpenAIEncoding: + return OpenAIEncoding(native, MappingProxyType(native.special_tokens())) + + @staticmethod + def from_tiktoken(encoding: str) -> OpenAIEncoding: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return OpenAIEncoding.wrap(NativeTokenizer.from_tiktoken(encoding)) + + def __repr__(self) -> str: + return f"" + + @property + def name(self) -> str: + return self._native.name + + @property + def max_token_value(self) -> int: + return self._native.max_token_value() + + @property + def n_vocab(self) -> int: + """For backwards compatibility. Prefer to use `enc.max_token_value + 1`.""" + return self.max_token_value + 1 + + @property + def eot_token(self) -> int: + return self._special_tokens["<|endoftext|>"] + + @property + def special_tokens_set(self) -> set[str]: # mutable-ok: [LIT001, LIT002] SDK return type + return set(self._special_tokens) + + def is_special_token(self, token: int) -> bool: + return self._native.is_special_token(token) + + # ---- encoding ------------------------------------------------------------------------- + + def encode_ordinary(self, text: str) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.encode(text) + + def encode( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type + allowed: Final = self._allowed(text, allowed_special, disallowed_special) + if not allowed: + return self.encode_ordinary(text) + return self._native.encode_special(text, tuple(allowed)) + + def encode_to_numpy( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> npt.NDArray[np.uint32]: + import numpy + + return numpy.asarray( + self.encode(text, allowed_special=allowed_special, disallowed_special=disallowed_special), + dtype=numpy.uint32, + ) + + def encode_ordinary_batch( + self, text: Sequence[str], *, num_threads: int = 8 + ) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(self.encode_ordinary, text) + ) + + def encode_batch( + self, + text: Sequence[str], + *, + num_threads: int = 8, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + encode: Final = partial(self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special) + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(encode, text) + ) + + def encode_with_unstable( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> tuple[list[int], list[list[int]]]: # mutable-ok: [LIT001, LIT002] SDK return type + """The stable tokens of `text` and every completion its unstable tail could become. + + Completions come back sorted; tiktoken returns them in hash order.""" + allowed: Final = self._allowed(text, allowed_special, disallowed_special) + return self._native.encode_with_unstable(text, tuple(allowed)) + + def encode_single_token(self, text_or_bytes: str | bytes) -> int: + """The token of one whole piece, special tokens included. Raises `KeyError` otherwise.""" + piece: Final = text_or_bytes.encode("utf-8") if isinstance(text_or_bytes, str) else text_or_bytes + return self._native.encode_single_token(piece) + + def count(self, text: str, fast: bool = False) -> int: + """Count ordinary text; `fast` accelerates supported encodings and otherwise counts normally.""" + return self._native.count(text, fast) + + # ---- decoding ------------------------------------------------------------------------- + + def decode_bytes(self, tokens: Sequence[int]) -> bytes: + return self._native.decode_bytes(tokens) + + def decode(self, tokens: Sequence[int], errors: str = "replace") -> str: + return self.decode_bytes(tokens).decode("utf-8", errors=errors) + + def decode_single_token_bytes(self, token: int) -> bytes: + return self.decode_bytes((token,)) + + def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + return [ # mutable-ok: [LIT002] SDK returns a list + self.decode_single_token_bytes(token) for token in tokens + ] + + def decode_with_offsets( + self, tokens: Sequence[int] + ) -> tuple[str, list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + """The decoded text and, per token, the index of the first character holding its bytes. + + Like tiktoken, raises `UnicodeDecodeError` when the tokens do not decode to valid UTF-8.""" + token_bytes: Final = self.decode_tokens_bytes(tokens) + text_len = 0 + offsets: Final[list[int]] = [] # mutable-ok: [LIT001] local accumulator + for token in token_bytes: + offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0))) + text_len += sum(1 for c in token if not 0x80 <= c < 0xC0) + return b"".join(token_bytes).decode("utf-8", errors="strict"), offsets + + def decode_batch( + self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8 + ) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(partial(self.decode, errors=errors), batch) + ) + + def decode_bytes_batch( + self, batch: Sequence[Sequence[int]], *, num_threads: int = 8 + ) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(self.decode_bytes, batch) + ) + + def token_byte_values(self) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.token_byte_values() + + def __reduce__(self) -> tuple[Callable[[str], OpenAIEncoding], tuple[str]]: + return (OpenAIEncoding.from_tiktoken, (self.name,)) + + # ---- private -------------------------------------------------------------------------- + + def _allowed(self, text: str, allowed_special: AllowedSpecial, disallowed_special: SpecialTokens) -> frozenset[str]: + """tiktoken's special-token policy: which specials `text` may encode, after rejecting + any it must not contain.""" + allowed: Final = frozenset(self._special_tokens) if allowed_special == "all" else frozenset(allowed_special) + disallowed: Final = ( + frozenset(self._special_tokens) - allowed if disallowed_special == "all" else frozenset(disallowed_special) + ) + for token in disallowed: + if token in text: + raise ValueError( + f"Encountered text corresponding to disallowed special token {token!r}.\n" + "If you want this text to be encoded as a special token, " + f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n" + "If you want this text to be encoded as normal text, disable the check for this token " + f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n" + "To disable this check for all special tokens, pass `disallowed_special=()`.\n" + ) + return allowed + + +@dataclass(frozen=True, slots=True) +class HuggingFaceTokenizer: + """The read-only ``tokenizers.Tokenizer`` surface over the Rust Hugging Face codec.""" + + _native: NativeTokenizer + + @staticmethod + def from_str(json: str) -> HuggingFaceTokenizer: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return HuggingFaceTokenizer(NativeTokenizer.from_json(json)) + + from_json = from_str + + @staticmethod + def from_buffer(buffer: bytes) -> HuggingFaceTokenizer: + return HuggingFaceTokenizer.from_str(buffer.decode("utf-8")) + + @staticmethod + def from_file(path: str) -> HuggingFaceTokenizer: + return HuggingFaceTokenizer.from_str(Path(path).read_text(encoding="utf-8")) + + @staticmethod + def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFaceTokenizer: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return HuggingFaceTokenizer(NativeTokenizer.from_pretrained(identifier, revision=revision, token=token)) + + def to_str(self, pretty: bool = False) -> str: + return self._native.to_json(pretty) + + def save(self, path: str, pretty: bool = True) -> None: + Path(path).write_text(self.to_str(pretty), encoding="utf-8") + + @property + def name(self) -> str: + return self._native.name + + # ---- vocabulary ----------------------------------------------------------------------- + + def token_to_id(self, token: str) -> int | None: + return self._native.token_to_id(token) + + def id_to_token(self, id: int) -> str | None: + return self._native.id_to_token(id) + + def get_vocab( + self, with_added_tokens: bool = True + ) -> dict[str, int]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.get_vocab(with_added_tokens) + + def get_vocab_size(self, with_added_tokens: bool = True) -> int: + return self._native.get_vocab_size(with_added_tokens) + + def get_added_tokens_decoder(self) -> dict[int, AddedToken]: # mutable-ok: [LIT001, LIT002] SDK return type + return { # mutable-ok: [LIT002] SDK returns a dict + token_id: AddedToken( + content, single_word=single_word, lstrip=lstrip, rstrip=rstrip, normalized=normalized, special=special + ) + for token_id, ( + content, + single_word, + lstrip, + rstrip, + normalized, + special, + ) in self._native.added_tokens_decoder() + } + + def num_special_tokens_to_add(self, is_pair: bool) -> int: + return self._native.num_special_tokens_to_add(is_pair) + + @property + def padding(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.padding() + + @property + def truncation(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.truncation() + + @property + def encode_special_tokens(self) -> bool: + return self._native.encode_special_tokens() + + # ---- encoding and decoding ------------------------------------------------------------ + + def encode( + self, + sequence: HuggingFaceInput, + pair: HuggingFaceInput | None = None, + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> HuggingFaceEncoding: + return self._native.encode_huggingface(sequence, pair, is_pretokenized, add_special_tokens) + + def encode_batch( + self, + input: Sequence[HuggingFaceBatchInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=False) + + def encode_batch_fast( + self, + input: Sequence[HuggingFaceBatchInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=True) + + def _encode_batch( + self, input: Sequence[HuggingFaceBatchInput], is_pretokenized: bool, add_special_tokens: bool, fast: bool + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + sequences: Final = tuple(_batch_input(item, is_pretokenized) for item in input) + return self._native.encode_batch_huggingface(sequences, is_pretokenized, add_special_tokens, fast) + + def count(self, text: str, fast: bool = False) -> int: + """Count with this tokenizer's configuration; `fast` uses acceleration where supported.""" + return self._native.count(text, fast) + + def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str: + return self._native.decode(ids, skip_special_tokens=skip_special_tokens) + + def decode_batch( + self, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True + ) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type + return [ # mutable-ok: [LIT002] SDK returns a list + self.decode(ids, skip_special_tokens=skip_special_tokens) for ids in sequences + ] + + def __reduce__(self) -> tuple[Callable[[str], HuggingFaceTokenizer], tuple[str]]: + return (HuggingFaceTokenizer.from_str, (self.to_str(),)) + + +def _batch_input( + item: HuggingFaceBatchInput, is_pretokenized: bool +) -> tuple[HuggingFaceInput, HuggingFaceInput | None]: + if isinstance(item, str): + return (item, None) + if is_pretokenized and all(isinstance(word, str) for word in item): + return (tuple(word for word in item if isinstance(word, str)), None) + if len(item) != 2: + raise TypeError("batch input must be a sequence or a pair of sequences") + return (item[0], item[1]) + + +Encoding: TypeAlias = tiktoken.Encoding | OpenAIEncoding +HuggingFace: TypeAlias = PythonHuggingFaceTokenizer | HuggingFaceTokenizer +Tokenizer: TypeAlias = Encoding | HuggingFace + + +class _AddedToken(Protocol): + @property + def special(self) -> bool: ... + + +@runtime_checkable +class _AddedTokenDecoder(Protocol): + def get_added_tokens_decoder(self) -> Mapping[int, _AddedToken]: ... + + +def strip_special_tokens(tokenizer: object, tokens: Sequence[int]) -> Sequence[int]: + """Drop the special added tokens before a Python `tokenizers` decode; the Rust codec's + `decode(skip_special_tokens=True)` already does this itself.""" + if isinstance(tokenizer, HuggingFaceTokenizer) or not isinstance(tokenizer, _AddedTokenDecoder): + return tokens + try: + added: Final = tokenizer.get_added_tokens_decoder() + except Exception: # noqa: BLE001 # optional metadata failures historically fall back to decoding + return tokens + special_ids: Final = frozenset(token_id for token_id, token in added.items() if token.special) + return tuple(token for token in tokens if token not in special_ids) diff --git a/litellm/llms/a2a/chat/transformation.py b/litellm/llms/a2a/chat/transformation.py index 77f26b65de0..c5a71daaba5 100644 --- a/litellm/llms/a2a/chat/transformation.py +++ b/litellm/llms/a2a/chat/transformation.py @@ -23,9 +23,8 @@ from ..common_utils import ( from .streaming_iterator import A2AModelResponseIterator if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer _REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = ( @@ -292,7 +291,7 @@ class A2AConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/aiml/image_generation/transformation.py b/litellm/llms/aiml/image_generation/transformation.py index 4f4cd074165..b585d35a2ac 100644 --- a/litellm/llms/aiml/image_generation/transformation.py +++ b/litellm/llms/aiml/image_generation/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -171,7 +170,7 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/aiohttp_openai/chat/transformation.py b/litellm/llms/aiohttp_openai/chat/transformation.py index 530896bf9b0..a06c670e3f1 100644 --- a/litellm/llms/aiohttp_openai/chat/transformation.py +++ b/litellm/llms/aiohttp_openai/chat/transformation.py @@ -16,9 +16,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -68,7 +67,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/amazon_nova/chat/transformation.py b/litellm/llms/amazon_nova/chat/transformation.py index 7551fb28c21..a93fbf1e933 100644 --- a/litellm/llms/amazon_nova/chat/transformation.py +++ b/litellm/llms/amazon_nova/chat/transformation.py @@ -17,7 +17,7 @@ from litellm.types.utils import ModelResponse from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonNovaChatConfig(OpenAILikeChatConfig): @@ -86,7 +86,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/batches/transformation.py b/litellm/llms/anthropic/batches/transformation.py index 4f4d39f09b0..7dee7513538 100644 --- a/litellm/llms/anthropic/batches/transformation.py +++ b/litellm/llms/anthropic/batches/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest from litellm.types.utils import LiteLLMBatch, LlmProviders, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -290,7 +289,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 545e920156e..c221e9f1505 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -100,9 +100,8 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -2688,7 +2687,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/completion/transformation.py b/litellm/llms/anthropic/completion/transformation.py index b15b0159bd9..46ab27ab0c7 100644 --- a/litellm/llms/anthropic/completion/transformation.py +++ b/litellm/llms/anthropic/completion/transformation.py @@ -33,7 +33,7 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AnthropicTextError(BaseLLMException): @@ -185,7 +185,7 @@ class AnthropicTextConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 424422612db..355714c0daf 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -29,9 +29,8 @@ from ...base_llm.chat.transformation import BaseConfig from ..common_utils import AzureOpenAIError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -304,7 +303,7 @@ class AzureOpenAIConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/agents/transformation.py b/litellm/llms/azure_ai/agents/transformation.py index 60ce81a23c7..baba3149963 100644 --- a/litellm/llms/azure_ai/agents/transformation.py +++ b/litellm/llms/azure_ai/agents/transformation.py @@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -297,7 +296,7 @@ class AzureAIAgentsConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/azure_model_router/transformation.py b/litellm/llms/azure_ai/azure_model_router/transformation.py index 9e35e396e15..0e4c8ca0d15 100644 --- a/litellm/llms/azure_ai/azure_model_router/transformation.py +++ b/litellm/llms/azure_ai/azure_model_router/transformation.py @@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AzureModelRouterConfig(AzureAIStudioConfig): @@ -59,7 +59,7 @@ class AzureModelRouterConfig(AzureAIStudioConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 00e1c1e25ba..779a86629e2 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -30,7 +30,7 @@ from litellm.types.utils import ModelResponse, ProviderField from litellm.utils import _add_path_to_api_base, supports_tool_choice if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AzureFoundryErrorStrings(str, enum.Enum): @@ -305,7 +305,7 @@ class AzureAIStudioConfig(OpenAIConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 67b1a8bcab3..18b4b6f456a 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -12,9 +12,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): """Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5).""" @@ -245,7 +246,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 2296909cfe1..e4bf148abf3 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -12,9 +12,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -121,7 +120,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/bridges/completion_transformation.py b/litellm/llms/base_llm/bridges/completion_transformation.py index 87b55152d09..a5c03705088 100644 --- a/litellm/llms/base_llm/bridges/completion_transformation.py +++ b/litellm/llms/base_llm/bridges/completion_transformation.py @@ -7,10 +7,10 @@ from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Union if TYPE_CHECKING: - import tiktoken from pydantic import BaseModel from litellm import LiteLLMLoggingObj, ModelResponse + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.types.llms.openai import AllMessageValues @@ -39,7 +39,7 @@ class CompletionTransformationBridge(ABC): messages: list["AllMessageValues"], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 7bfc87a30d6..7decf1b4186 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -21,9 +21,8 @@ from litellm.types.llms.openai import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.types.utils import ModelResponse from ..base_utils import ( @@ -344,7 +343,7 @@ class BaseConfig(ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/base_llm/completion/transformation.py b/litellm/llms/base_llm/completion/transformation.py index fb472dfa63b..b8ebbfed12f 100644 --- a/litellm/llms/base_llm/completion/transformation.py +++ b/litellm/llms/base_llm/completion/transformation.py @@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -68,7 +67,7 @@ class BaseTextCompletionConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/embedding/transformation.py b/litellm/llms/base_llm/embedding/transformation.py index da87dcc7f98..46ac3ccf433 100644 --- a/litellm/llms/base_llm/embedding/transformation.py +++ b/litellm/llms/base_llm/embedding/transformation.py @@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -80,7 +79,7 @@ class BaseEmbeddingConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/files/transformation.py b/litellm/llms/base_llm/files/transformation.py index 254995c028f..5ecf033fb2c 100644 --- a/litellm/llms/base_llm/files/transformation.py +++ b/litellm/llms/base_llm/files/transformation.py @@ -21,9 +21,8 @@ from litellm.types.utils import LlmProviders, ModelResponse from ..chat.transformation import BaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.router import Router as _Router from litellm.types.llms.openai import HttpxBinaryResponseContent @@ -231,7 +230,7 @@ class BaseFilesConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/image_generation/transformation.py b/litellm/llms/base_llm/image_generation/transformation.py index 4616441133e..7ac440c6f48 100644 --- a/litellm/llms/base_llm/image_generation/transformation.py +++ b/litellm/llms/base_llm/image_generation/transformation.py @@ -11,9 +11,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -93,7 +92,7 @@ class BaseImageGenerationConfig(ABC): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/base_llm/image_variations/transformation.py b/litellm/llms/base_llm/image_variations/transformation.py index d3e02139e0e..15a4e0f243c 100644 --- a/litellm/llms/base_llm/image_variations/transformation.py +++ b/litellm/llms/base_llm/image_variations/transformation.py @@ -17,9 +17,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -82,7 +81,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: pass @@ -98,7 +97,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: pass @@ -125,7 +124,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index e1a9a807abc..29133bcfaf9 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -40,9 +40,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -990,7 +989,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 4f9b1f56a5b..21830eb0d8e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -99,7 +99,7 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer # Computer use tool prefixes supported by Bedrock BEDROCK_COMPUTER_USE_TOOLS: Final = [ @@ -1920,7 +1920,7 @@ class AmazonConverseConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/litellm/llms/bedrock/chat/invoke_agent/transformation.py index d489e47c3b5..6877af74494 100644 --- a/litellm/llms/bedrock/chat/invoke_agent/transformation.py +++ b/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -37,9 +37,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -438,7 +437,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py index 5a3f4f17b8b..5699f94d084 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -25,7 +25,7 @@ from litellm.types.utils import ( from .amazon_llama_transformation import AmazonLlamaConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonDeepSeekR1Config(AmazonLlamaConfig): @@ -39,7 +39,7 @@ class AmazonDeepSeekR1Config(AmazonLlamaConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py index 5d39b68d9d5..f8b730b6cd0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py @@ -21,9 +21,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.types.utils import ModelResponse LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -198,7 +197,7 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py index bc97551d57a..8d1ff1d2bd9 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py @@ -28,7 +28,7 @@ from ..converse_transformation import AmazonConverseConfig from .base_invoke_transformation import AmazonInvokeConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer _CachePointCarrier = TypeVar("_CachePointCarrier", SystemContentBlock, ContentBlock) _INJECTION_POINTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) @@ -128,7 +128,7 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py index c78375c37bb..67364ccfda0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py @@ -21,7 +21,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonQwen2Config(AmazonQwen3Config): @@ -44,7 +44,7 @@ class AmazonQwen2Config(AmazonQwen3Config): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py index e251fb15725..2e19dbf77af 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py @@ -19,7 +19,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): @@ -170,7 +170,7 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py index 39cded4ed64..fe287111fdd 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, Usage from litellm.utils import get_base64_str if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -190,7 +189,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 1326dc22ca0..a8b94fb5703 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -359,7 +358,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 90a2692f68a..dcc5e249d8a 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -34,9 +34,8 @@ from litellm.types.utils import ModelResponse, Usage from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -288,7 +287,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/black_forest_labs/image_generation/transformation.py b/litellm/llms/black_forest_labs/image_generation/transformation.py index 119ffff1c34..e5c2bdf59e9 100644 --- a/litellm/llms/black_forest_labs/image_generation/transformation.py +++ b/litellm/llms/black_forest_labs/image_generation/transformation.py @@ -29,9 +29,8 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -258,7 +257,7 @@ class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/brave/search/__init__.py b/litellm/llms/brave/search/__init__.py index cc1168d7ef8..de70c62e040 100644 --- a/litellm/llms/brave/search/__init__.py +++ b/litellm/llms/brave/search/__init__.py @@ -1,7 +1,7 @@ -""" -Brave Search API module. -""" - -from litellm.llms.brave.search.transformation import BraveSearchConfig - -__all__ = ["BraveSearchConfig"] +""" +Brave Search API module. +""" + +from litellm.llms.brave.search.transformation import BraveSearchConfig + +__all__ = ["BraveSearchConfig"] diff --git a/litellm/llms/bytez/chat/transformation.py b/litellm/llms/bytez/chat/transformation.py index 7977db0f056..e622761dd7f 100644 --- a/litellm/llms/bytez/chat/transformation.py +++ b/litellm/llms/bytez/chat/transformation.py @@ -23,9 +23,8 @@ from litellm.utils import CustomStreamWrapper, ModelResponse, Usage from ..common_utils import API_BASE, BytezError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -187,7 +186,7 @@ class BytezChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/clarifai/chat/transformation.py b/litellm/llms/clarifai/chat/transformation.py index 76d35467497..a0946254de0 100644 --- a/litellm/llms/clarifai/chat/transformation.py +++ b/litellm/llms/clarifai/chat/transformation.py @@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -87,7 +86,7 @@ class ClarifaiConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/chat/transformation.py b/litellm/llms/cohere/chat/transformation.py index fa46bd7f6cf..a26cdc81695 100644 --- a/litellm/llms/cohere/chat/transformation.py +++ b/litellm/llms/cohere/chat/transformation.py @@ -15,9 +15,8 @@ from ..common_utils import ModelResponseIterator as CohereModelResponseIterator from ..common_utils import validate_environment as cohere_validate_environment if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -227,7 +226,7 @@ class CohereChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/chat/v2_transformation.py b/litellm/llms/cohere/chat/v2_transformation.py index 4252e7d02e9..37c53640d18 100644 --- a/litellm/llms/cohere/chat/v2_transformation.py +++ b/litellm/llms/cohere/chat/v2_transformation.py @@ -20,9 +20,8 @@ from ..common_utils import CohereError, CohereV2ModelResponseIterator from ..common_utils import validate_environment as cohere_validate_environment if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -191,7 +190,7 @@ class CohereV2ChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/embed/handler.py b/litellm/llms/cohere/embed/handler.py index 3cebf6b9a90..6d9543d7c30 100644 --- a/litellm/llms/cohere/embed/handler.py +++ b/litellm/llms/cohere/embed/handler.py @@ -20,7 +20,7 @@ from litellm.types.utils import EmbeddingResponse from .v1_transformation import CohereEmbeddingConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def validate_environment(api_key, headers: dict): @@ -60,7 +60,7 @@ async def async_embedding( api_base: str, api_key: str | None, headers: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", client: AsyncHTTPHandler | None = None, ): ## LOGGING @@ -122,7 +122,7 @@ def embedding( logging_obj: LiteLLMLoggingObj, optional_params: dict, headers: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", data: dict | CohereEmbeddingRequest | None = None, complete_api_base: str | None = None, api_key: str | None = None, diff --git a/litellm/llms/cometapi/image_generation/transformation.py b/litellm/llms/cometapi/image_generation/transformation.py index 03c820de198..4432c151a64 100644 --- a/litellm/llms/cometapi/image_generation/transformation.py +++ b/litellm/llms/cometapi/image_generation/transformation.py @@ -13,9 +13,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -132,7 +131,7 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/compactifai/chat/transformation.py b/litellm/llms/compactifai/chat/transformation.py index 3c5a889ce63..a02db2338d8 100644 --- a/litellm/llms/compactifai/chat/transformation.py +++ b/litellm/llms/compactifai/chat/transformation.py @@ -17,9 +17,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -66,7 +65,7 @@ class CompactifAIChatConfig(OpenAIGPTConfig): messages: Sequence[AllMessageValues], optional_params: Mapping[str, object], litellm_params: Mapping[str, object], - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/custom_httpx/aiohttp_handler.py b/litellm/llms/custom_httpx/aiohttp_handler.py index 0809ef5274f..034c9514092 100644 --- a/litellm/llms/custom_httpx/aiohttp_handler.py +++ b/litellm/llms/custom_httpx/aiohttp_handler.py @@ -26,9 +26,8 @@ from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProv from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -268,7 +267,7 @@ class BaseLLMAIOHTTPHandler: messages: list, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, client: ClientSession | None = None, ): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 18d4c27fec1..333ce523e34 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -192,12 +192,12 @@ def _rust_responses_websocket_enabled( from .http_handler import get_shared_realtime_ssl_context if TYPE_CHECKING: - import tiktoken from aiohttp import ClientSession from websockets.asyncio.client import ClientConnection from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) @@ -493,7 +493,7 @@ class BaseLLMHTTPHandler: messages: list, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, client: AsyncHTTPHandler | None = None, json_mode: bool = False, @@ -559,7 +559,7 @@ class BaseLLMHTTPHandler: api_base: str | None, custom_llm_provider: str, model_response: ModelResponse, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", logging_obj: LiteLLMLoggingObj, optional_params: dict, timeout: float | httpx.Timeout, diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index c0e278a96ef..ffa60a3d9bc 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -38,9 +38,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -165,7 +164,7 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 30144d29f51..538904b34e6 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -149,9 +149,8 @@ def _split_parallel_tool_calls(messages: list[AllMessageValues]) -> list[AllMess if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -189,7 +188,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): return "databricks" @classmethod - def get_config(cls): + def get_config(cls, *, model: str | None = None): return super().get_config() def get_required_params(self) -> list[ProviderField]: @@ -651,7 +650,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/deprecated_providers/aleph_alpha.py b/litellm/llms/deprecated_providers/aleph_alpha.py index 2ad9ce4edc8..8f5c35f32f2 100644 --- a/litellm/llms/deprecated_providers/aleph_alpha.py +++ b/litellm/llms/deprecated_providers/aleph_alpha.py @@ -277,12 +277,7 @@ def completion( ## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here. prompt_tokens: Final = len(encoding.encode(prompt)) - completion_tokens: Final = len( - encoding.encode( - model_response["choices"][0]["message"]["content"], - disallowed_special=(), - ) - ) + completion_tokens: Final = len(encoding.encode(model_response["choices"][0]["message"]["content"])) model_response.created = int(time.time()) model_response.model = model diff --git a/litellm/llms/edenai/chat/transformation.py b/litellm/llms/edenai/chat/transformation.py index 4fd5a9d550b..67d308d9e38 100644 --- a/litellm/llms/edenai/chat/transformation.py +++ b/litellm/llms/edenai/chat/transformation.py @@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, Usage from ..common_utils import EdenAIException, reported_cost, resolve_api_base, resolve_api_key if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding _OPTIONAL_MAPPING: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) @@ -97,7 +96,7 @@ class EdenAIChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], # mutable-ok: inherited contract optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract - encoding: "tiktoken.Encoding | None", + encoding: "Encoding | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/edenai/image_generation/transformation.py b/litellm/llms/edenai/image_generation/transformation.py index 2965c4041de..7f729cd7fbb 100644 --- a/litellm/llms/edenai/image_generation/transformation.py +++ b/litellm/llms/edenai/image_generation/transformation.py @@ -19,9 +19,8 @@ from litellm.utils import convert_to_model_response_object from ..common_utils import EdenAIException, endpoint_url, json_headers, pick, reported_cost if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding _SUPPORTED_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = ( "background", @@ -94,7 +93,7 @@ class EdenAIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict[str, object], # mutable-ok: inherited contract optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract - encoding: "tiktoken.Encoding | None", + encoding: "Encoding | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/bria_transformation.py b/litellm/llms/fal_ai/image_generation/bria_transformation.py index c528550811a..53da48e62ca 100644 --- a/litellm/llms/fal_ai/image_generation/bria_transformation.py +++ b/litellm/llms/fal_ai/image_generation/bria_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -187,7 +186,7 @@ class FalAIBriaConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py index 6b8558b8124..7c63e1077f1 100644 --- a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py +++ b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageResponse from .transformation import FalAIBaseConfig, fal_images_to_image_objects if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -194,7 +193,7 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py index 04b4f426878..ad1852a622b 100644 --- a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -150,7 +149,7 @@ class FalAIIdeogramV3Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py index 8a6665b2585..3624b76a4a3 100644 --- a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py +++ b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -182,7 +181,7 @@ class FalAIImagen4Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py index 4880dfec7e3..934ce420d53 100644 --- a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -172,7 +171,7 @@ class FalAIRecraftV3Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py index bc3a4d07282..79d8800773b 100644 --- a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py +++ b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -208,7 +207,7 @@ class FalAIStableDiffusionConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/transformation.py b/litellm/llms/fal_ai/image_generation/transformation.py index fd8e280da1c..8f081c7228d 100644 --- a/litellm/llms/fal_ai/image_generation/transformation.py +++ b/litellm/llms/fal_ai/image_generation/transformation.py @@ -16,9 +16,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -117,7 +116,7 @@ class FalAIBaseConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 28ebb39a303..196022b3558 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -46,7 +46,7 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def _map_reasoning_effort(value: object) -> object: @@ -708,7 +708,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index d009fe4cd72..bb8d7455031 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -24,9 +24,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -173,7 +172,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index d9250ea8836..c047dc0c881 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -26,9 +26,8 @@ from ..authenticator import get_access_token from ..file_handler import upload_file_sync if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -416,7 +415,7 @@ class GigaChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: tiktoken.Encoding | None, + encoding: Tokenizer | None, api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 41a2df17c6f..1da7ad0a7b5 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -27,7 +27,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, ServerToolUs from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer GROQ_COMPOUND_MODELS: Final = frozenset({"compound", "compound-mini"}) @@ -286,7 +286,7 @@ class GroqChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index 57d1357ee46..60917c68221 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -1,6 +1,5 @@ import json import os -from collections.abc import Sequence from typing import Final, Literal, Protocol, get_args import httpx @@ -32,7 +31,7 @@ hf_tasks_embeddings: Final = ( class _SupportsTokenEncode(Protocol): """Token encoder handle. Only ``encode`` is ever called on it here.""" - def encode(self, text: str, *, disallowed_special: tuple[str, ...]) -> Sequence[int]: ... + def encode(self, text: str) -> list[int]: ... def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> str | None: @@ -214,7 +213,7 @@ class HuggingFaceEmbedding(BaseLLM): model_response.model = model input_tokens = 0 for text in input: - input_tokens += len(encoding.encode(text, disallowed_special=())) + input_tokens += len(encoding.encode_ordinary(text)) setattr( model_response, diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index 33b0e21e326..3fdd4abda73 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -25,9 +25,8 @@ from litellm.utils import token_counter from ..common_utils import HuggingFaceError, hf_task_list, hf_tasks, output_parser if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -479,7 +478,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/langflow/chat/transformation.py b/litellm/llms/langflow/chat/transformation.py index 17ae7017cf6..a53887b36af 100644 --- a/litellm/llms/langflow/chat/transformation.py +++ b/litellm/llms/langflow/chat/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -225,7 +224,7 @@ class LangFlowConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index 84d79e6bd31..c9388ee472f 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -23,9 +23,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -415,7 +414,7 @@ class LangGraphConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index c01ad2a0edd..341e8dd2e12 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -19,7 +19,7 @@ from litellm.types.utils import ModelResponse from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class LemonadeChatConfig(OpenAILikeChatConfig): @@ -231,7 +231,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index f77e828b59a..33b567e9710 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -32,7 +32,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: str) -> str: @@ -580,7 +580,7 @@ class MistralConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/nlp_cloud/chat/transformation.py b/litellm/llms/nlp_cloud/chat/transformation.py index 17c547618d3..2ff48894619 100644 --- a/litellm/llms/nlp_cloud/chat/transformation.py +++ b/litellm/llms/nlp_cloud/chat/transformation.py @@ -14,9 +14,8 @@ from litellm.utils import ModelResponse, Usage from ..common_utils import NLPCloudError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -175,7 +174,7 @@ class NLPCloudConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 8e4e41b4ac1..ecff823a18d 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -65,9 +65,8 @@ from litellm.types.utils import ( from litellm.utils import supports_reasoning if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -603,7 +602,7 @@ class OCIChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index bcde8a041a6..cb3080e6534 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -31,9 +31,8 @@ from litellm.types.utils import ModelResponse, ModelResponseStream from ..common_utils import OllamaError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -321,7 +320,7 @@ class OllamaChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 3eb2c833094..0fc1cd926b8 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -35,9 +35,8 @@ from litellm.types.utils import ( from ..common_utils import OllamaError, OllamaModelInfo, _convert_image if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -252,7 +251,7 @@ class OllamaConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/oobabooga/chat/transformation.py b/litellm/llms/oobabooga/chat/transformation.py index 43d627102b6..05383a35389 100644 --- a/litellm/llms/oobabooga/chat/transformation.py +++ b/litellm/llms/oobabooga/chat/transformation.py @@ -11,9 +11,8 @@ from litellm.types.utils import ModelResponse, Usage from ..common_utils import OobaboogaError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -39,7 +38,7 @@ class OobaboogaConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 9dbcf0cc089..b63684db782 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -60,9 +60,8 @@ from litellm.utils import convert_to_model_response_object from ..common_utils import OpenAIError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.types.llms.openai import ChatCompletionToolParam @@ -671,7 +670,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai/image_generation/dall_e_2_transformation.py b/litellm/llms/openai/image_generation/dall_e_2_transformation.py index 74936cf1895..ffc6f1d5fe9 100644 --- a/litellm/llms/openai/image_generation/dall_e_2_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_2_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class DallE2ImageGenerationConfig(BaseImageGenerationConfig): """ @@ -52,7 +53,7 @@ class DallE2ImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_generation/dall_e_3_transformation.py b/litellm/llms/openai/image_generation/dall_e_3_transformation.py index 5c561d011a9..90b7eaedf2f 100644 --- a/litellm/llms/openai/image_generation/dall_e_3_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_3_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class DallE3ImageGenerationConfig(BaseImageGenerationConfig): """ @@ -52,7 +53,7 @@ class DallE3ImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py index 8dc4d8953ea..c3a826616ed 100644 --- a/litellm/llms/openai/image_generation/gpt_transformation.py +++ b/litellm/llms/openai/image_generation/gpt_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class GPTImageGenerationConfig(BaseImageGenerationConfig): """ @@ -61,7 +62,7 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_variations/transformation.py b/litellm/llms/openai/image_variations/transformation.py index afd2909b697..73b44d5ea6a 100644 --- a/litellm/llms/openai/image_variations/transformation.py +++ b/litellm/llms/openai/image_variations/transformation.py @@ -12,7 +12,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ..common_utils import OpenAIError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class OpenAIImageVariationConfig(BaseImageVariationConfig): @@ -53,7 +53,7 @@ class OpenAIImageVariationConfig(BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: return model_response @@ -68,7 +68,7 @@ class OpenAIImageVariationConfig(BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: return model_response diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 869ad387c5a..7ac0d988074 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -6,9 +6,10 @@ from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx if TYPE_CHECKING: - import tiktoken from aiohttp import ClientSession + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + import openai from openai import AsyncOpenAI, OpenAI from openai._base_client import make_request_options @@ -277,7 +278,7 @@ class OpenAIConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai_like/chat/transformation.py b/litellm/llms/openai_like/chat/transformation.py index 030710c8b2d..e5d6cbb7e5e 100644 --- a/litellm/llms/openai_like/chat/transformation.py +++ b/litellm/llms/openai_like/chat/transformation.py @@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -131,7 +130,7 @@ class OpenAILikeChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 77a902149d9..08d43c169f4 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -23,9 +23,8 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig from ..common_utils import OpenRouterException if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class CacheControlSupportedModels(str, Enum): @@ -182,7 +181,7 @@ class OpenrouterConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openrouter/image_generation/transformation.py b/litellm/llms/openrouter/image_generation/transformation.py index 6bbda324336..67d90d027ec 100644 --- a/litellm/llms/openrouter/image_generation/transformation.py +++ b/litellm/llms/openrouter/image_generation/transformation.py @@ -50,9 +50,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer else: LiteLLMLoggingObj = Any @@ -319,7 +318,7 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/perplexity/chat/transformation.py b/litellm/llms/perplexity/chat/transformation.py index 354f7692fd5..dca2f9857b8 100644 --- a/litellm/llms/perplexity/chat/transformation.py +++ b/litellm/llms/perplexity/chat/transformation.py @@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues, ChatCompletionAnnotation from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class PerplexityChatConfig(OpenAIGPTConfig): @@ -75,7 +75,7 @@ class PerplexityChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/petals/completion/transformation.py b/litellm/llms/petals/completion/transformation.py index 3e0de14a7b2..ee20c2b12d5 100644 --- a/litellm/llms/petals/completion/transformation.py +++ b/litellm/llms/petals/completion/transformation.py @@ -14,7 +14,7 @@ from litellm.types.utils import ModelResponse from ..common_utils import PetalsError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class PetalsConfig(BaseConfig): @@ -112,7 +112,7 @@ class PetalsConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/predibase/chat/transformation.py b/litellm/llms/predibase/chat/transformation.py index 2a63c489395..69924396a1e 100644 --- a/litellm/llms/predibase/chat/transformation.py +++ b/litellm/llms/predibase/chat/transformation.py @@ -18,9 +18,8 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage from ..common_utils import PredibaseError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -150,7 +149,7 @@ class PredibaseConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/recraft/image_generation/transformation.py b/litellm/llms/recraft/image_generation/transformation.py index 3a04e0a62b4..f65bf1e7292 100644 --- a/litellm/llms/recraft/image_generation/transformation.py +++ b/litellm/llms/recraft/image_generation/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.recraft import RecraftImageGenerationRequestParams from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -122,7 +121,7 @@ class RecraftImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/replicate/chat/transformation.py b/litellm/llms/replicate/chat/transformation.py index 769160c6ced..f7e09b7bec0 100644 --- a/litellm/llms/replicate/chat/transformation.py +++ b/litellm/llms/replicate/chat/transformation.py @@ -19,9 +19,8 @@ from litellm.utils import token_counter from ..common_utils import ReplicateError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -237,7 +236,7 @@ class ReplicateConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/runwayml/image_generation/transformation.py b/litellm/llms/runwayml/image_generation/transformation.py index 5913709c8a0..e5e988328d8 100644 --- a/litellm/llms/runwayml/image_generation/transformation.py +++ b/litellm/llms/runwayml/image_generation/transformation.py @@ -22,9 +22,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -308,7 +307,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: @@ -383,7 +382,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/sagemaker/completion/transformation.py b/litellm/llms/sagemaker/completion/transformation.py index 576018f0046..e1bc496a82d 100644 --- a/litellm/llms/sagemaker/completion/transformation.py +++ b/litellm/llms/sagemaker/completion/transformation.py @@ -24,9 +24,8 @@ from litellm.utils import token_counter from ..common_utils import SagemakerError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -198,7 +197,7 @@ class SagemakerConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index d64d7a57281..4c73ccacc16 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -15,9 +15,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -57,7 +56,7 @@ def validate_dict(data: dict, model) -> dict: return model(**data).model_dump(by_alias=True, exclude_unset=True) -def _messages_to_sap_template(messages: list[dict[str, str]]) -> list: +def _messages_to_sap_template(messages: list[AllMessageValues]) -> list: template: Final = [] for message in messages: if message["role"] == "user": @@ -311,7 +310,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): def transform_request( self, model: str, - messages: list[dict[str, str]], + messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, headers: dict, @@ -383,7 +382,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/stability/image_generation/transformation.py b/litellm/llms/stability/image_generation/transformation.py index cf3576a9404..656ffe395c8 100644 --- a/litellm/llms/stability/image_generation/transformation.py +++ b/litellm/llms/stability/image_generation/transformation.py @@ -26,9 +26,8 @@ from litellm.types.llms.stability import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -207,7 +206,7 @@ class StabilityImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/topaz/image_variations/transformation.py b/litellm/llms/topaz/image_variations/transformation.py index f4753c8ba17..94f60d29cb9 100644 --- a/litellm/llms/topaz/image_variations/transformation.py +++ b/litellm/llms/topaz/image_variations/transformation.py @@ -23,7 +23,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ..common_utils import TopazException, TopazModelInfo if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): @@ -139,7 +139,7 @@ class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: image_content: Final = await raw_response.read() @@ -158,7 +158,7 @@ class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: image_content: Final = raw_response.content diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 3c868b3a96f..b37bbd78f2b 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -29,7 +29,7 @@ from litellm.types.utils import ( from ..common_utils import TritonError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class TritonConfig(BaseConfig): @@ -95,7 +95,7 @@ class TritonConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: @@ -215,7 +215,7 @@ class TritonGenerateConfig(TritonConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: @@ -280,7 +280,7 @@ class TritonInferConfig(TritonConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index c5ca9f38144..b37bf473731 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -29,9 +29,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -285,7 +284,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index b2c52c53580..17ccf16837b 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -24,9 +24,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -284,7 +283,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py index 8faf7b0d484..ae6e08611ae 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py @@ -20,9 +20,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -214,7 +213,7 @@ class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 508f68b3eca..2fab6f438f6 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -14,7 +14,7 @@ from ....anthropic.chat.transformation import AnthropicConfig from .output_params_utils import sanitize_vertex_anthropic_output_params if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class VertexAIError(Exception): @@ -197,7 +197,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py index 89a5b8a570e..f2d2c0896d2 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py @@ -21,7 +21,7 @@ from litellm.types.utils import ( from ...common_utils import VertexAIError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class VertexAILlama3Config(OpenAIGPTConfig): @@ -112,7 +112,7 @@ class VertexAILlama3Config(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py index 67b01c2dc43..2ae8b4cd188 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py @@ -24,9 +24,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_model_iterator import MockResponseIterator @@ -276,7 +275,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): litellm_params: dict, client: HTTPHandler | httpx.Client | None = None, timeout: float | httpx.Timeout | None = None, - encoding: "tiktoken.Encoding | None" = None, + encoding: "Tokenizer | None" = None, ): """Synchronous completion request""" from litellm.utils import convert_to_model_response_object @@ -366,7 +365,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): litellm_params: dict, client: AsyncHTTPHandler | httpx.AsyncClient | None = None, timeout: float | httpx.Timeout | None = None, - encoding: "tiktoken.Encoding | None" = None, + encoding: "Tokenizer | None" = None, ): """Asynchronous completion request""" from litellm.utils import convert_to_model_response_object diff --git a/litellm/llms/watsonx/completion/transformation.py b/litellm/llms/watsonx/completion/transformation.py index 2be007336b4..2fec8485cf9 100644 --- a/litellm/llms/watsonx/completion/transformation.py +++ b/litellm/llms/watsonx/completion/transformation.py @@ -21,9 +21,8 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -280,7 +279,7 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/main.py b/litellm/main.py index 2570b93455f..7d231a1bf7a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -37,7 +37,6 @@ if TYPE_CHECKING: import dotenv import httpx import openai -import tiktoken from pydantic import BaseModel from typing_extensions import overload @@ -100,6 +99,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) +from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, foundry_chat_rejects_function_tools_while_reasoning, @@ -7489,7 +7489,9 @@ def text_completion( if isinstance(prompt, list): import concurrent.futures - tokenizer: Final = tiktoken.encoding_for_model("text-davinci-003") + from litellm.rust_bridge.tokenizer import get_encoding + + tokenizer: Final = get_encoding("p50k_base") ## if it's a 2d list - each element in the list is a text_completion() request if len(prompt) > 0 and isinstance(prompt[0], list): responses: Final = [None for x in prompt] # init responses @@ -9259,7 +9261,7 @@ async def acount_tokens( except Exception as e: verbose_logger.debug("Provider token counting failed for model=%s, falling back to local: %s", model, e) - # Fallback to local tiktoken-based token counting + # Fallback to local token counting fallback_messages = messages or [] if system and fallback_messages: fallback_messages = [{"role": "system", "content": system}] + fallback_messages @@ -9278,16 +9280,16 @@ async def acount_tokens( # Cache for encoding to avoid repeated __getattr__ calls -_encoding_cache: tiktoken.Encoding | None = None +_encoding_cache: Tokenizer | None = None -def _load_module_encoding() -> tiktoken.Encoding: +def _load_module_encoding() -> Tokenizer: import sys return sys.modules[__name__].encoding -def _get_encoding() -> tiktoken.Encoding: +def _get_encoding() -> Tokenizer: """Get encoding, loading it lazily if needed.""" global _encoding_cache if _encoding_cache is None: @@ -9296,18 +9298,15 @@ def _get_encoding() -> tiktoken.Encoding: return _encoding_cache -def _load_default_encoding() -> tiktoken.Encoding: +def _load_default_encoding() -> Tokenizer: from litellm._lazy_imports import _get_default_encoding return _get_default_encoding() -def __getattr__(name: str) -> tiktoken.Encoding: +def __getattr__(name: str) -> Tokenizer: """Lazy import handler for main module""" if name == "encoding": - # Use _get_default_encoding which properly sets TIKTOKEN_CACHE_DIR - # before loading tiktoken, ensuring the local cache is used - # instead of downloading from the internet _encoding: Final = _load_default_encoding() # Cache it in the module's __dict__ for subsequent accesses import sys diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index ac9b07a55f7..e28fa2c06a4 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -36,10 +36,10 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router -from litellm.rust_bridge.token_counter import RustTokenizer, count_input_tokens, rust_tokenizer from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict @@ -1489,9 +1489,6 @@ def _get_request_models( return (model,) if isinstance(model, str) else tuple(model) -TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 - - async def count_request_input_tokens( request_body: dict, route: str, @@ -1500,105 +1497,11 @@ async def count_request_input_tokens( ) -> Mapping[str, int]: """Input-token count per candidate model, counted once per request. - Tokenizing is the reservation path's dominant CPU cost and is O(prompt), so - counting a large prompt inline stalls every other request on the worker. - Models whose tokenizer the Rust bridge ports (Anthropic, tiktoken cl100k_base - and o200k_base) are counted from the raw body by the bridge when it is enabled, once per - distinct tokenizer, which parses and tokenizes with the GIL released. - Everything it declines is counted in Python, large prompts in a worker - thread. The counts are reused by both the max-cost and the input-cost - estimate. - """ + The counts are reused by both the max-cost and the input-cost estimate.""" models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - tokenizers: Final[Mapping[str, RustTokenizer | None]] = MappingProxyType( - {model: rust_tokenizer(model) for model in models} - ) - distinct_tokenizers: Final[tuple[RustTokenizer, ...]] = tuple( - dict.fromkeys(tokenizer for tokenizer in tokenizers.values() if tokenizer is not None) - ) - rust_counts_by_tokenizer: Final[Mapping[RustTokenizer, int]] = MappingProxyType( - { - tokenizer: count.input_tokens - for tokenizer in distinct_tokenizers - if raw_body is not None and (count := await count_input_tokens(raw_body, tokenizer)) is not None - } - ) - rust_counts: Final = MappingProxyType( - { - model: rust_counts_by_tokenizer[tokenizer] - for model, tokenizer in tokenizers.items() - if tokenizer is not None and tokenizer in rust_counts_by_tokenizer - } - ) - python_models: Final = tuple(model for model in models if model not in rust_counts) - python_counts: Final = ( - MappingProxyType({}) - if not python_models - else _count_input_tokens_for_models(request_body=request_body, models=python_models) - if _approximate_input_size(request_body) < TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS - else await asyncio.to_thread( - _count_input_tokens_for_models, - request_body=request_body, - models=python_models, - ) - ) - verbose_proxy_logger.debug("input token counts: rust=%s python=%s", dict(rust_counts), dict(python_counts)) - return MappingProxyType({**rust_counts, **python_counts}) - - -def _count_input_tokens_for_models( - request_body: dict, - models: Sequence[str], -) -> Mapping[str, int]: - return MappingProxyType( - { - model: tokens - for model in models - if (tokens := _count_input_tokens(request_body=request_body, model=model)) is not None - } - ) - - -_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") - - -def _approximate_input_size(request_body: Mapping[str, object]) -> int: - """Length of the request's input text, a cheap stand-in for tokenizing cost. - - Every field _count_input_tokens hands the tokenizer is sized here, and - rendering rather than walking keeps mapping keys in the total, which a tool - schema's property names are.""" - return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) - - -def _count_input_tokens(request_body: dict, model: str) -> int | None: - try: - if "messages" in request_body: - try: - return litellm.token_counter( - model=model, - messages=request_body.get("messages") or (), - tools=request_body.get("tools"), - tool_choice=request_body.get("tool_choice"), - ) - except ValueError: - return _count_text_tokens(model=model, text=request_body.get("messages")) - if "prompt" in request_body: - return _count_text_tokens(model=model, text=request_body.get("prompt")) - if "input" in request_body: - return _count_text_tokens(model=model, text=request_body.get("input")) - if "query" in request_body or "documents" in request_body: - query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query")) - document_tokens: Final = _count_text_tokens( - model=model, - text=request_body.get("documents"), - ) - return query_tokens + document_tokens - except Exception: - verbose_proxy_logger.debug("Unable to count input tokens for budget reservation", exc_info=True) - return None + return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models) def _estimate_input_tokens( @@ -1609,7 +1512,9 @@ def _estimate_input_tokens( input_tokens: int | None = None, ) -> int | None: counted: Final = ( - input_tokens if input_tokens is not None else _count_input_tokens(request_body=request_body, model=model) + input_tokens + if input_tokens is not None + else count_input_tokens_for_model(request_body=request_body, model=model) ) if counted is not None: return counted @@ -1658,26 +1563,6 @@ def _requested_output_tokens(request_body: Mapping[str, object]) -> int | None: return next((tokens for tokens in map(_to_int, candidates) if tokens is not None), None) -def _count_text_tokens(model: str, text: object) -> int: - if text is None: - return 0 - - token_count = 0 - stack: Final = [text] - while stack: - item = stack.pop() - if item is None: - continue - if isinstance(item, list): - stack.extend(item) - continue - if isinstance(item, dict): - token_count += litellm.token_counter(model=model, text=json.dumps(item)) - continue - token_count += litellm.token_counter(model=model, text=str(item)) - return token_count - - def _get_output_multiplier(request_body: dict) -> int: output_multiplier = 1 for key in ("n", "best_of"): diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py new file mode 100644 index 00000000000..6c7083fb6db --- /dev/null +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -0,0 +1,173 @@ +"""Input-token counting for the budget reservation path. + +Tokenizing is the reservation path's dominant CPU cost and is O(prompt), so +counting a large prompt inline stalls every other request on the worker. +Models whose tokenizer the Rust bridge ports (Anthropic, tiktoken cl100k_base +and o200k_base) are counted from the raw body by the bridge, once per distinct +tokenizer, which parses and tokenizes with the GIL released. Everything it +declines, and every model with no Rust tokenizer, is counted in Python, large +prompts in a worker thread. +""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.rust_bridge import runtime +from litellm.rust_bridge.catalog import Route, RouteContext +from litellm.rust_bridge.token_counter import ( + TOKEN_COUNTER, + RustTokenCounterFactory, + RustTokenizer, + native_count, + rust_tokenizer, +) + +TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 + +_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") + + +def _approximate_input_size(request_body: Mapping[str, object]) -> int: + """Length of the request's input text, a cheap stand-in for tokenizing cost. + + Every field count_input_tokens_for_model hands the tokenizer is sized here, + and rendering rather than walking keeps mapping keys in the total, which a + tool schema's property names are.""" + return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) + + +async def count_input_tokens( + request_body: dict, + raw_body: bytes | None, + models: Sequence[str], +) -> Mapping[str, int]: + """Input-token count per model, sharing one native count across models that + select the same tokenizer.""" + tokenizers: Final[tuple[tuple[str, RustTokenizer | None], ...]] = tuple( + (model, rust_tokenizer(model)) for model in models + ) + groups: Final[tuple[RustTokenizer | None, ...]] = tuple(dict.fromkeys(tokenizer for _, tokenizer in tokenizers)) + group_counts: Final = [ + await _count_group( + request_body=request_body, + raw_body=raw_body, + tokenizer=tokenizer, + models=tuple(model for model, selected in tokenizers if selected == tokenizer), + ) + for tokenizer in groups + ] + counts: Final = MappingProxyType({model: tokens for group in group_counts for model, tokens in group.items()}) + verbose_proxy_logger.debug("input token counts: %s", dict(counts)) + return counts + + +async def _count_group( + request_body: dict, + raw_body: bytes | None, + tokenizer: RustTokenizer | None, + models: tuple[str, ...], +) -> Mapping[str, int]: + async def python() -> Mapping[str, int]: + if _approximate_input_size(request_body) < TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: + return _count_input_tokens_for_models(request_body=request_body, models=models) + return await asyncio.to_thread( + _count_input_tokens_for_models, + request_body=request_body, + models=models, + ) + + if tokenizer is None or raw_body is None: + return await python() + try: + return await runtime.arun( + RouteContext(Route.TOKEN_COUNTER, provider=tokenizer), + binding=TOKEN_COUNTER, + native=lambda factory: _native_counts(factory, tokenizer, raw_body, models), + python=python, + ) + except (RuntimeError, ValueError) as error: + from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking + + if isinstance(error, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): + raise + verbose_proxy_logger.debug("Rust token counter (%s) failed, counting in Python: %s", tokenizer, error) + return await python() + + +async def _native_counts( + factory: RustTokenCounterFactory, + tokenizer: RustTokenizer, + raw_body: bytes, + models: tuple[str, ...], +) -> Mapping[str, int]: + count: Final = await native_count(factory, tokenizer, raw_body) + verbose_proxy_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, count.input_tokens) + return MappingProxyType({model: count.input_tokens for model in models}) + + +def _count_input_tokens_for_models( + request_body: dict, + models: Sequence[str], +) -> Mapping[str, int]: + return MappingProxyType( + { + model: tokens + for model in models + if (tokens := count_input_tokens_for_model(request_body=request_body, model=model)) is not None + } + ) + + +def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: + try: + if "messages" in request_body: + try: + return litellm.token_counter( + model=model, + messages=request_body.get("messages") or (), + tools=request_body.get("tools"), + tool_choice=request_body.get("tool_choice"), + ) + except ValueError: + return _count_text_tokens(model=model, text=request_body.get("messages")) + if "prompt" in request_body: + return _count_text_tokens(model=model, text=request_body.get("prompt")) + if "input" in request_body: + return _count_text_tokens(model=model, text=request_body.get("input")) + if "query" in request_body or "documents" in request_body: + query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query")) + document_tokens: Final = _count_text_tokens( + model=model, + text=request_body.get("documents"), + ) + return query_tokens + document_tokens + except Exception: + verbose_proxy_logger.debug("Unable to count input tokens for budget reservation", exc_info=True) + return None + + +def _count_text_tokens(model: str, text: object) -> int: + if text is None: + return 0 + + token_count = 0 + stack: Final = [text] + while stack: + item = stack.pop() + if item is None: + continue + if isinstance(item, list): + stack.extend(item) + continue + if isinstance(item, dict): + token_count += litellm.token_counter(model=model, text=json.dumps(item)) + continue + token_count += litellm.token_counter(model=model, text=str(item)) + return token_count diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index a033b89a6a8..61e597bf674 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -224,26 +224,121 @@ class _CacheTestResolver: @final class TokenCounter: - def __new__(cls, tokenizer_json: str) -> TokenCounter: ... @staticmethod - def from_cl100k_ranks(rank_file: str) -> TokenCounter: ... - @staticmethod - def from_o200k_ranks(rank_file: str) -> TokenCounter: ... - @staticmethod - def from_tiktoken(encoding: str) -> TokenCounter: ... + def from_tokenizer(tokenizer: Tokenizer, fast: bool = False) -> TokenCounter: ... def acount_request(self, body: bytes) -> Future[dict[str, object]]: ... +@final +class Tokenizer: + @staticmethod + def from_tiktoken(encoding: str) -> Tokenizer: ... + @staticmethod + def from_json(tokenizer_json: str) -> Tokenizer: ... + @staticmethod + def from_pretrained( + identifier: str, + revision: str = "main", + token: str | None = None, + ) -> Tokenizer: ... + def encode(self, text: str) -> list[int]: ... + def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str: ... + def count(self, text: str, fast: bool = False) -> int: ... + # tiktoken encodings + def encode_special(self, text: str, allowed: Sequence[str]) -> list[int]: ... + def encode_with_unstable(self, text: str, allowed: Sequence[str]) -> tuple[list[int], list[list[int]]]: ... + def encode_single_token(self, piece: bytes) -> int: ... + def special_tokens(self) -> dict[str, int]: ... + def max_token_value(self) -> int: ... + def is_special_token(self, token: int) -> bool: ... + def token_byte_values(self) -> list[bytes]: ... + def decode_bytes(self, ids: Sequence[int]) -> bytes: ... + # Hugging Face tokenizers + def to_json(self, pretty: bool = False) -> str: ... + def token_to_id(self, token: str) -> int | None: ... + def id_to_token(self, id: int) -> str | None: ... + def get_vocab(self, with_added_tokens: bool = True) -> dict[str, int]: ... + def get_vocab_size(self, with_added_tokens: bool = True) -> int: ... + def added_tokens_decoder(self) -> list[tuple[int, tuple[str, bool, bool, bool, bool, bool]]]: ... + def padding(self) -> dict[str, object] | None: ... + def truncation(self) -> dict[str, object] | None: ... + def num_special_tokens_to_add(self, is_pair: bool) -> int: ... + def encode_special_tokens(self) -> bool: ... + def encode_huggingface( + self, + sequence: str | Sequence[str], + pair: str | Sequence[str] | None = None, + is_pretokenized: bool = False, + add_special_tokens: bool = True, + fast: bool = False, + ) -> HuggingFaceEncoding: ... + def encode_batch_huggingface( + self, + inputs: Sequence[tuple[str | Sequence[str], str | Sequence[str] | None]], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + fast: bool = False, + ) -> list[HuggingFaceEncoding]: ... + @property + def name(self) -> str: ... + +@final +class HuggingFaceEncoding: + def __new__(cls, json: str | None = None) -> HuggingFaceEncoding: ... + @staticmethod + def merge(encodings: Sequence[HuggingFaceEncoding], growing_offsets: bool = True) -> HuggingFaceEncoding: ... + def __len__(self) -> int: ... + def __reduce__(self) -> tuple[type[HuggingFaceEncoding], tuple[str]]: ... + def word_to_tokens(self, word_index: int, sequence_index: int = 0) -> tuple[int, int] | None: ... + def word_to_chars(self, word_index: int, sequence_index: int = 0) -> tuple[int, int] | None: ... + def token_to_sequence(self, token_index: int) -> int | None: ... + def token_to_chars(self, token_index: int) -> tuple[int, int] | None: ... + def token_to_word(self, token_index: int) -> int | None: ... + def char_to_token(self, char_pos: int, sequence_index: int = 0) -> int | None: ... + def char_to_word(self, char_pos: int, sequence_index: int = 0) -> int | None: ... + def set_sequence_id(self, sequence_id: int) -> None: ... + def pad( + self, + length: int, + direction: str = "right", + pad_id: int = 0, + pad_type_id: int = 0, + pad_token: str = "[PAD]", + ) -> None: ... + def truncate(self, max_length: int, stride: int = 0, direction: str = "right") -> None: ... + @property + def ids(self) -> list[int]: ... + @property + def tokens(self) -> list[str]: ... + @property + def offsets(self) -> list[tuple[int, int]]: ... + @property + def type_ids(self) -> list[int]: ... + @property + def attention_mask(self) -> list[int]: ... + @property + def special_tokens_mask(self) -> list[int]: ... + @property + def word_ids(self) -> list[int | None]: ... + @property + def sequence_ids(self) -> list[int | None]: ... + @property + def overflowing(self) -> list[HuggingFaceEncoding]: ... + @property + def n_sequences(self) -> int: ... + def gil_stats() -> dict[str, int]: ... def process_state_started() -> bool: ... def reserve_process_for_forking() -> None: ... __all__ = [ "ForkedAfterNativeRuntimeStarted", + "HuggingFaceEncoding", "ProcessReservedForForking", "ResponsesWebSocketConnection", "RustBridgeDeclined", "RustUpstreamError", "TokenCounter", + "Tokenizer", "achat_completions", "amessages", "aocr", diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 74ceb1ba123..d7479631e04 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -22,6 +22,8 @@ class Route(str, Enum): RESPONSES = "responses" TRANSCRIPTION = "transcription" OCR = "ocr" + TOKEN_COUNTER = "token_counter" + TOKENIZER = "tokenizer" class Delivery(Enum): @@ -92,6 +94,8 @@ RULES: Final[Rules] = ( RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), RouteRule(Route.OCR, Rollout.RUST_OPT_OUT), RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN), + RouteRule(Route.TOKEN_COUNTER, Rollout.RUST_OPT_IN), + RouteRule(Route.TOKENIZER, Rollout.RUST_OPT_IN), RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.LOCAL})), CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.REDIS})), diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index cd468c80655..152ba632996 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -2,6 +2,7 @@ from __future__ import annotations import os from enum import Enum, auto +from functools import lru_cache from typing import Final from pydantic import TypeAdapter, ValidationError @@ -32,7 +33,9 @@ class _RustConfiguration: _CONFIGURATION: Final = _RustConfiguration() +@lru_cache(maxsize=16) def _parse_env_bool(value: str | None) -> bool | None: + """`LITELLM_RUST` as a bool; cached by raw value because `decision` runs per tokenizer call.""" if value is None: return None try: diff --git a/litellm/rust_bridge/token_counter.py b/litellm/rust_bridge/token_counter.py index d36234f56c1..250ad18d44c 100644 --- a/litellm/rust_bridge/token_counter.py +++ b/litellm/rust_bridge/token_counter.py @@ -5,18 +5,19 @@ from __future__ import annotations from collections.abc import Awaitable from dataclasses import dataclass from functools import lru_cache -from typing import Final, Literal, Protocol, cast # noqa: TID251 # native extension exposes untyped callables +from typing import TYPE_CHECKING, Final, Literal, Protocol, cast # noqa: TID251 # PyO3 binding validation from pydantic import TypeAdapter +from typing_extensions import assert_never import litellm -from litellm._logging import verbose_logger -from litellm.litellm_core_utils.default_encoding import cl100k_base_rank_file, o200k_base_rank_file -from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding, uses_legacy_message_accounting +from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding_name, uses_legacy_message_accounting +from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.configuration import rust_enabled -from litellm.rust_bridge.runtime import BridgeErrorContext, RustHandled, aattempt -from litellm.utils import claude_json_str, huggingface_tokenizer_kind +from litellm.utils import huggingface_tokenizer_kind + +if TYPE_CHECKING: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer RustTokenizer = Literal["anthropic", "cl100k_base", "o200k_base"] @@ -27,13 +28,7 @@ class RustTokenCounter(Protocol): class RustTokenCounterFactory(Protocol): - def __call__(self, tokenizer_json: str) -> RustTokenCounter: - raise NotImplementedError - - def from_cl100k_ranks(self, rank_file: str) -> RustTokenCounter: - raise NotImplementedError - - def from_o200k_ranks(self, rank_file: str) -> RustTokenCounter: + def from_tokenizer(self, tokenizer: NativeTokenizer, fast: bool = False) -> RustTokenCounter: raise NotImplementedError @@ -51,7 +46,7 @@ def _as_factory(value: object) -> RustTokenCounterFactory | None: cast( # cast-ok: native extension protocol is runtime-defined RustTokenCounterFactory, value ) - if callable(value) + if callable(getattr(value, "from_tokenizer", None)) else None ) @@ -73,7 +68,7 @@ def rust_tokenizer(model: str) -> RustTokenizer | None: return "anthropic" if kind is not None or uses_legacy_message_accounting(model): return None - match openai_tokenizer_encoding(model).name: + match openai_tokenizer_encoding_name(model): case "cl100k_base": return "cl100k_base" case "o200k_base": @@ -84,31 +79,26 @@ def rust_tokenizer(model: str) -> RustTokenizer | None: @lru_cache(maxsize=4) def _counter(factory: RustTokenCounterFactory, tokenizer: RustTokenizer) -> RustTokenCounter: + return factory.from_tokenizer(_native_tokenizer(tokenizer)) + + +def _native_tokenizer(tokenizer: RustTokenizer) -> NativeTokenizer: match tokenizer: case "anthropic": - return factory(claude_json_str) - case "cl100k_base": - return factory.from_cl100k_ranks(cl100k_base_rank_file()) - case "o200k_base": - return factory.from_o200k_ranks(o200k_base_rank_file()) + native = tokenizer_dispatch.native_anthropic() + case "cl100k_base" | "o200k_base": + native = tokenizer_dispatch.native_encoding(tokenizer) + case _: + assert_never(tokenizer) + if native is None: + raise RuntimeError(f"native {tokenizer} tokenizer is unavailable") + return native -async def count_input_tokens(body: bytes, tokenizer: RustTokenizer) -> InputTokenCount | None: - if not rust_enabled(): - return None - factory: Final = TOKEN_COUNTER.load() - if factory is None: - return None - try: - attempt: Final = await aattempt( - native_call=lambda: _counter(factory, tokenizer).acount_request(body), - adapt=_INPUT_TOKEN_COUNT.validate_python, - context=BridgeErrorContext(route="token_counter", provider=tokenizer, model=""), - ) - except (RuntimeError, ValueError) as error: - verbose_logger.debug("Rust token counter (%s) failed, counting in Python: %s", tokenizer, error) - return None - if not isinstance(attempt, RustHandled): - return None - verbose_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, attempt.value.input_tokens) - return attempt.value +async def native_count(factory: RustTokenCounterFactory, tokenizer: RustTokenizer, body: bytes) -> InputTokenCount: + """One native count, validated into the public shape. + + ``RustBridgeDeclined`` and upstream errors propagate so the caller's route + runner can map them onto its fallback policy; other failures (RuntimeError, + ValueError) propagate as-is.""" + return _INPUT_TOKEN_COUNT.validate_python(await _counter(factory, tokenizer).acount_request(body)) diff --git a/litellm/rust_bridge/tokenizer.py b/litellm/rust_bridge/tokenizer.py new file mode 100644 index 00000000000..5ed89813620 --- /dev/null +++ b/litellm/rust_bridge/tokenizer.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # native class is validated at the binding boundary + +import tiktoken +from tokenizers import Tokenizer as PythonHuggingFaceTokenizer + +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding +from litellm.rust_bridge import runtime +from litellm.rust_bridge.bindings import NativeBinding +from litellm.rust_bridge.catalog import Route, RouteContext + +if TYPE_CHECKING: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + +def _as_factory(value: object) -> type[NativeTokenizer] | None: + return ( + cast(type["NativeTokenizer"], value) # cast-ok: PyO3 class validated at the native boundary + if isinstance(value, type) + else None + ) + + +TOKENIZER: Final = NativeBinding("Tokenizer", validate=_as_factory) + +# The catalog contexts the tokenizer factories dispatch on. Callers that cache a tokenizer per +# backend key their cache on `decision(...)` of the same context, so key and dispatch agree. +TIKTOKEN_CONTEXT: Final = RouteContext(Route.TOKENIZER, provider="tiktoken") +HUGGINGFACE_CONTEXT: Final = RouteContext(Route.TOKENIZER, provider="huggingface") + + +@lru_cache(maxsize=8) +def _native_tiktoken(factory: type[NativeTokenizer], name: str) -> NativeTokenizer: + return factory.from_tiktoken(name) + + +@lru_cache(maxsize=1) +def _native_anthropic(factory: type[NativeTokenizer]) -> NativeTokenizer: + from litellm.utils import claude_json_str + + return factory.from_json(claude_json_str) + + +@lru_cache(maxsize=8) +def _native_encoding(factory: type[NativeTokenizer], name: str) -> OpenAIEncoding: + return OpenAIEncoding.wrap(_native_tiktoken(factory, name)) + + +def native_encoding(name: str) -> NativeTokenizer | None: + """The native tiktoken encoding behind `get_encoding(name)`, for a Rust route that counts + with the same loaded model; `None` without the extension.""" + factory: Final = TOKENIZER.load() + return None if factory is None else _native_tiktoken(factory, name) + + +def native_anthropic() -> NativeTokenizer | None: + """The native packaged Anthropic tokenizer behind `anthropic()`, parsed once per process.""" + factory: Final = TOKENIZER.load() + return None if factory is None else _native_anthropic(factory) + + +def _python_encoding(name: str) -> tiktoken.Encoding: + from litellm.litellm_core_utils.default_encoding import encoding + + return encoding if name == encoding.name else tiktoken.get_encoding(name) + + +def get_encoding(name: str) -> Encoding: + return runtime.run( + TIKTOKEN_CONTEXT, + binding=TOKENIZER, + native=lambda factory: _native_encoding(factory, name), + python=lambda: _python_encoding(name), + ) + + +def anthropic() -> HuggingFace: + """The packaged Anthropic tokenizer on the selected backend.""" + from litellm.utils import claude_json_str + + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer(_native_anthropic(factory)), + python=lambda: PythonHuggingFaceTokenizer.from_str(claude_json_str), + ) + + +def from_str(json: str) -> HuggingFace: + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer(factory.from_json(json)), + python=lambda: PythonHuggingFaceTokenizer.from_str(json), + ) + + +def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFace: + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer( + factory.from_pretrained(identifier, revision=revision, token=token) + ), + python=lambda: PythonHuggingFaceTokenizer.from_pretrained(identifier, revision=revision, token=token), + ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2e8c869b89d..65d66677b27 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -104,6 +104,8 @@ def _nested_selector( if TYPE_CHECKING: + from litellm.litellm_core_utils.tokenizer import Tokenizer + from .vector_stores import VectorStoreSearchResponse else: VectorStoreSearchResponse = Any @@ -4406,7 +4408,7 @@ class ProviderSpecificHeader(TypedDict): class SelectTokenizerResponse(TypedDict): type: Literal["openai_tokenizer", "huggingface_tokenizer"] - tokenizer: Any + tokenizer: ReadOnly["Tokenizer"] class LiteLLMFineTuningJob(FineTuningJob): diff --git a/litellm/utils.py b/litellm/utils.py index 812299560c7..e088a5988c8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -40,14 +40,11 @@ from types import MappingProxyType import dotenv import httpx import openai -import tiktoken from httpx import Proxy from httpx._utils import get_environment_proxies from openai.lib import _parsing, _pydantic from openai.types.chat.completion_create_params import ResponseFormat from pydantic import BaseModel -from tiktoken import Encoding -from tokenizers import Tokenizer import litellm import litellm.litellm_core_utils @@ -91,6 +88,10 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_fill_missing_generalizations, ) from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, strip_special_tokens +from litellm.rust_bridge import tokenizer as tokenizer_dispatch +from litellm.rust_bridge.catalog import decision +from litellm.rust_bridge.configuration import Decision _CachingHandlerResponse = None _LLMCachingHandler = None @@ -289,6 +290,8 @@ import importlib.metadata from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast, get_args +from typing_extensions import assert_never + from litellm import utils as litellm_utils # These are lazy loaded via __getattr__ @@ -2254,17 +2257,27 @@ def _select_tokenizer(model: str, custom_tokenizer: CustomHuggingfaceTokenizer | identifier=custom_tokenizer["identifier"], revision=custom_tokenizer["revision"], auth_token=custom_tokenizer["auth_token"], + backend=_huggingface_tokenizer_backend(), ) return _select_tokenizer_helper(model=model) +def _huggingface_tokenizer_backend() -> Decision: + """The backend `tokenizer_dispatch.from_str` / `from_pretrained` will select right now. + + Cached HuggingFace tokenizers are keyed on it, so flipping `LITELLM_RUST` or + `litellm.rust(...)` reaches a fresh object instead of the other backend's.""" + return decision(tokenizer_dispatch.HUGGINGFACE_CONTEXT) + + @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) -def _select_custom_tokenizer_helper(identifier: str, revision: str, auth_token: str | None) -> SelectTokenizerResponse: +def _select_custom_tokenizer_helper( + identifier: str, revision: str, auth_token: str | None, backend: Decision +) -> SelectTokenizerResponse: verbose_logger.debug("Loading custom HuggingFace tokenizer %s (revision %s)", identifier, revision) return create_pretrained_tokenizer(identifier=identifier, revision=revision, auth_token=auth_token) -@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: if litellm.disable_hf_tokenizer_download is True: return _return_openai_tokenizer(model) @@ -2274,6 +2287,10 @@ def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: if result is not None: return result except Exception as e: + from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking + + if isinstance(e, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): + raise verbose_logger.debug("Error selecting tokenizer: %s", e) # default - tiktoken @@ -2308,19 +2325,26 @@ def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | None: kind: Final = huggingface_tokenizer_kind(model) if kind is None: return None - return {"type": "huggingface_tokenizer", "tokenizer": _load_huggingface_tokenizer(kind)} + return { + "type": "huggingface_tokenizer", + "tokenizer": _load_huggingface_tokenizer(kind, _huggingface_tokenizer_backend()), + } -def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind) -> Tokenizer: +@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) +def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decision) -> HuggingFace: + """One tokenizer per kind and backend; `backend` is the cache key, the dispatch re-derives it.""" match kind: case "cohere": - return Tokenizer.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") + return tokenizer_dispatch.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") case "anthropic": - return Tokenizer.from_str(claude_json_str) + return tokenizer_dispatch.anthropic() case "llama2": - return Tokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") + return tokenizer_dispatch.from_pretrained("hf-internal-testing/llama-tokenizer") case "llama3": - return Tokenizer.from_pretrained("Xenova/llama-3-tokenizer") + return tokenizer_dispatch.from_pretrained("Xenova/llama-3-tokenizer") + case _: + assert_never(kind) def encode(model="", text="", custom_tokenizer: dict | None = None): @@ -2336,15 +2360,13 @@ def encode(model="", text="", custom_tokenizer: dict | None = None): enc: The encoded text. """ tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model=model) - if isinstance(tokenizer_json["tokenizer"], Encoding): - enc = tokenizer_json["tokenizer"].encode(text, disallowed_special=()) - else: - enc = tokenizer_json["tokenizer"].encode(text) - # Normalize: HuggingFace Tokenizer.encode() returns an Encoding object; - # extract .ids so the return type is always List[int]. - if hasattr(enc, "ids"): - return enc.ids - return enc + if tokenizer_json["type"] == "openai_tokenizer": + openai_tokenizer: Final = cast( # cast-ok: [LIT006] caller's explicit type tag selects this interface + Encoding, tokenizer_json["tokenizer"] + ) + return openai_tokenizer.encode(text, disallowed_special=()) + encoded: Final = tokenizer_json["tokenizer"].encode(text) + return encoded.ids if hasattr(encoded, "ids") else encoded def decode( @@ -2363,26 +2385,12 @@ def decode( """ tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model=model) if tokenizer_json["type"] == "huggingface_tokenizer": - if skip_special_tokens: - tokens = _strip_huggingface_special_token_ids(tokenizer_json["tokenizer"], tokens) - dec = tokenizer_json["tokenizer"].decode(tokens, skip_special_tokens=skip_special_tokens) - return dec - dec = tokenizer_json["tokenizer"].decode(tokens) - return dec - - -def _strip_huggingface_special_token_ids(tokenizer: Tokenizer, tokens: Sequence[int]) -> Sequence[int]: - try: - added_tokens_decoder: Final = tokenizer.get_added_tokens_decoder() - except Exception: - return tokens - - special_token_ids: Final = { - token_id for token_id, added_token in added_tokens_decoder.items() if getattr(added_token, "special", False) - } - if not special_token_ids: - return tokens - return [token for token in tokens if token not in special_token_ids] + ids: Final = strip_special_tokens(tokenizer_json["tokenizer"], tokens) if skip_special_tokens else tokens + hf_tokenizer: Final = cast( # cast-ok: [LIT006] caller's explicit type tag selects this interface + HuggingFace, tokenizer_json["tokenizer"] + ) + return hf_tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) + return tokenizer_json["tokenizer"].decode(tokens) def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: str | None = None): @@ -2398,7 +2406,7 @@ def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: st dict: A dictionary with the tokenizer and its type. """ - tokenizer: Final = Tokenizer.from_pretrained(identifier, revision=revision, token=auth_token) + tokenizer: Final = tokenizer_dispatch.from_pretrained(identifier, revision=revision, token=auth_token) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} @@ -2413,7 +2421,7 @@ def create_tokenizer(json: str): dict: A dictionary with the tokenizer and its type. """ - tokenizer: Final = Tokenizer.from_str(json) + tokenizer: Final = tokenizer_dispatch.from_str(json) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} diff --git a/migrations/Dockerfile b/migrations/Dockerfile index c6d1b0cc46e..f34940c0ce0 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -67,6 +67,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra extra_proxy \ --python python3.13 +RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ + COPY migrations/run.py /app/run.py # Pre-warm the Prisma binary cache so the Job pod doesn't reach the diff --git a/pyproject.toml b/pyproject.toml index a1b276e4e8c..f447343ff33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,11 +19,13 @@ dependencies = [ "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<3.0.0", "python-dotenv>=1.0.0,<2.0", + "pyyaml>=6.0.3,<7.0", + "packaging>=24.0", + "importlib-metadata>=8.0.0,<9.0", "tiktoken>=0.8.0,<1.0; python_version < '3.14'", "tiktoken>=0.12.0,<1.0; python_version >= '3.14'", - "importlib-metadata>=8.0.0,<9.0", - "packaging>=24.0", "tokenizers>=0.21.0,<1.0", + "huggingface-hub>=0.34.0,<2.0", "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", @@ -191,6 +193,7 @@ litellm-proxy = "litellm.proxy.client.cli:litellm_proxy_cli" [dependency-groups] dev = [ + "numpy>=1.26.0,<3.0", "diff-cover==9.7.2", "hypothesis==6.165.10", "reportlab==5.0.1", @@ -290,6 +293,12 @@ healthcheck = [ "httpx==0.28.1", "pyyaml==6.0.3", ] +benchmarks = [ + "pytest==9.0.3", + "pytest-codspeed==4.3.0", + "mcp>=2.2.0,<3", + "a2a-sdk==1.1.0", +] [build-system] requires = ["maturin==1.15.0"] diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index c9b31cfb7d7..309ecab9991 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -8,8 +8,11 @@ flipping results between runs. Running the executor inline keeps each benchmark's cost self-contained and deterministic. """ +import os +import sys from collections.abc import Callable, Iterator from concurrent.futures import Future +from pathlib import Path from typing import ParamSpec, TypeVar import pytest @@ -20,6 +23,21 @@ P = ParamSpec("P") R = TypeVar("R") +def pytest_configure(config: pytest.Config) -> None: + if os.environ.get("LITELLM_REQUIRE_INSTALLED_WHEEL") != "1": + return + + import litellm + import litellm.rust_bridge._native as native + + prefix = Path(sys.prefix).resolve() + for name, module_file in (("litellm", litellm.__file__), ("litellm.rust_bridge._native", native.__file__)): + path = Path(module_file).resolve() + if not path.is_relative_to(prefix): + raise pytest.UsageError(f"{name} resolved outside the benchmark environment: {path}") + print(f"{name}: {path}") # noqa: T201 # provenance evidence must be visible in CI logs + + def _submit_inline(fn: Callable[P, R], /, *args: P.args, **kwargs: P.kwargs) -> Future[R]: future: Future[R] = Future() try: diff --git a/tests/proxy_unit_tests/test_custom_tokenizer_bug.py b/tests/proxy_unit_tests/test_custom_tokenizer_bug.py index c4b1f4f3afd..23b6b302202 100644 --- a/tests/proxy_unit_tests/test_custom_tokenizer_bug.py +++ b/tests/proxy_unit_tests/test_custom_tokenizer_bug.py @@ -22,10 +22,8 @@ from litellm.proxy.proxy_server import token_counter def _fake_hf_tokenizer(num_tokens: int) -> MagicMock: - encoding = MagicMock() - encoding.__len__.return_value = num_tokens tokenizer = MagicMock() - tokenizer.encode_batch_fast.return_value = [encoding] + tokenizer.encode_batch_fast.return_value = [[0] * num_tokens] return tokenizer @@ -58,7 +56,7 @@ async def test_custom_tokenizer_from_model_info_is_used(monkeypatch): ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - with patch.object(litellm.utils, "Tokenizer") as mock_tokenizer_cls: + with patch.object(litellm.utils, "tokenizer_dispatch") as mock_tokenizer_cls: mock_tokenizer_cls.from_pretrained.return_value = _fake_hf_tokenizer(7) response = await token_counter( @@ -92,7 +90,7 @@ async def test_model_without_custom_tokenizer_uses_default(monkeypatch): ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - with patch.object(litellm.utils, "Tokenizer") as mock_tokenizer_cls: + with patch.object(litellm.utils, "tokenizer_dispatch") as mock_tokenizer_cls: response = await token_counter( request=TokenCountRequest( model="gpt-4", diff --git a/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py b/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py index d1a5f78a859..dd2daef5484 100644 --- a/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py +++ b/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py @@ -1,23 +1,11 @@ -from tokenizers import AddedToken, Tokenizer -from tokenizers.models import WordLevel -from tokenizers.pre_tokenizers import Whitespace -from tokenizers.processors import TemplateProcessing - from litellm import decode, encode +from tokenizers import Tokenizer + +TOKENIZER_JSON = """{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":3,"content":"[BOS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[BOS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],"pair":[{"Sequence":{"id":"A","type_id":0}},{"Sequence":{"id":"B","type_id":1}}],"special_tokens":{"[BOS]":{"id":"[BOS]","ids":[3],"tokens":["[BOS]"]}}},"decoder":null,"model":{"type":"WordLevel","vocab":{"[UNK]":0,"Hello":1,"World":2},"unk_token":"[UNK]"}}""" def _create_custom_tokenizer(): - tokenizer = Tokenizer( - WordLevel({"[UNK]": 0, "Hello": 1, "World": 2}, unk_token="[UNK]") - ) - tokenizer.pre_tokenizer = Whitespace() - tokenizer.add_special_tokens([AddedToken("[BOS]", special=True)]) - bos_token_id = tokenizer.token_to_id("[BOS]") - assert bos_token_id is not None - tokenizer.post_processor = TemplateProcessing( - single="[BOS] $A", - special_tokens=[("[BOS]", bos_token_id)], - ) + tokenizer = Tokenizer.from_str(TOKENIZER_JSON) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index f19a8891609..5ce6a4b1ce9 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -786,23 +786,23 @@ def test_token_counter(): import unittest -from litellm.utils import _select_tokenizer_helper, claude_json_str, encoding +from litellm.utils import _load_huggingface_tokenizer, _select_tokenizer_helper, claude_json_str, encoding # Clear the cache at module load to ensure clean state -_select_tokenizer_helper.cache_clear() +_load_huggingface_tokenizer.cache_clear() class TestTokenizerSelection(unittest.TestCase): def setUp(self): """Clear the LRU cache before each test method. - The _select_tokenizer_helper function is decorated with @lru_cache, - which can cause cache hits from previous tests when running with + The HuggingFace tokenizers behind _select_tokenizer_helper are cached with + @lru_cache, which can cause cache hits from previous tests when running with --dist=loadscope (tests from same file run on same worker). """ - _select_tokenizer_helper.cache_clear() + _load_huggingface_tokenizer.cache_clear() - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_llama3_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") @@ -817,7 +817,7 @@ class TestTokenizerSelection(unittest.TestCase): self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_cohere_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") @@ -837,10 +837,10 @@ class TestTokenizerSelection(unittest.TestCase): self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_str") - def test_claude_tokenizer_api_failure(self, mock_from_str): + @patch("litellm.utils.tokenizer_dispatch.anthropic") + def test_claude_tokenizer_api_failure(self, mock_anthropic): # Setup mock to raise an error - mock_from_str.side_effect = Exception("Failed to load tokenizer") + mock_anthropic.side_effect = Exception("Failed to load tokenizer") # Add Claude model to the list for testing litellm.anthropic_models = ["claude-2"] @@ -849,13 +849,13 @@ class TestTokenizerSelection(unittest.TestCase): result = _select_tokenizer_helper("claude-2") # Verify the attempt to load Claude tokenizer - mock_from_str.assert_called_once_with(claude_json_str) + mock_anthropic.assert_called_once_with() # Verify fallback to OpenAI tokenizer self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_llama2_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") diff --git a/tests/test_litellm/litellm_core_utils/test_tokenizer.py b/tests/test_litellm/litellm_core_utils/test_tokenizer.py new file mode 100644 index 00000000000..aa4a0fc6a1c --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_tokenizer.py @@ -0,0 +1,403 @@ +import copy +import os +import pickle +import subprocess +import sys +from pathlib import Path +from typing import Final, Literal + +import pytest +import tiktoken +from tokenizers import Tokenizer as ReferenceTokenizer + +import litellm +from litellm.caching._embedding_router import truncate_embedding_input +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding +from litellm.utils import claude_json_str +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + + +@pytest.mark.parametrize( + "name", ("cl100k_base", "o200k_base", "p50k_base", "p50k_edit", "r50k_base", "gpt2", "o200k_harmony") +) +@pytest.mark.parametrize( + "text", ("hello world", "café 漢字 🙂", "", "a\ud800b", "\ud83d\ude42", "🙂\ud83d\ude42\udfff", " " * 64) +) +def test_openai_encoding_matches_python_unicode_and_batches(name: str, text: str) -> None: + reference: Final = tiktoken.get_encoding(name) + encoding: Final = OpenAIEncoding.from_tiktoken(name) + expected: Final = reference.encode(text) + + assert encoding.encode(text) == expected + assert encoding.count(text) == len(expected) + assert encoding.encode_batch([text], num_threads=2) == reference.encode_batch([text], num_threads=2) + assert encoding.encode_ordinary_batch([text]) == reference.encode_ordinary_batch([text]) + assert encoding.decode_batch([expected]) == reference.decode_batch([expected]) + assert encoding.decode_bytes_batch([expected]) == reference.decode_bytes_batch([expected]) + + +@pytest.mark.parametrize("allowed", (frozenset(), frozenset({"<|endoftext|>"}), "all")) +@pytest.mark.parametrize("disallowed", (frozenset(), frozenset({"<|fim_prefix|>"}), "all")) +def test_openai_special_token_options_match_python( + allowed: frozenset[str] | Literal["all"], disallowed: frozenset[str] | Literal["all"] +) -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + encoding: Final = OpenAIEncoding.from_tiktoken(reference.name) + text: Final = "hello<|endoftext|><|fim_prefix|>world" + allowed_set: Final = reference.special_tokens_set if allowed == "all" else allowed + disallowed_set: Final = reference.special_tokens_set - allowed_set if disallowed == "all" else disallowed + if any(token in text for token in disallowed_set): + with pytest.raises(ValueError, match="disallowed special token"): + encoding.encode(text, allowed_special=allowed, disallowed_special=disallowed) + return + assert encoding.encode(text, allowed_special=allowed, disallowed_special=disallowed) == reference.encode( + text, allowed_special=allowed, disallowed_special=disallowed + ) + assert encoding.special_tokens_set == reference.special_tokens_set + assert encoding.eot_token == reference.eot_token + + +@pytest.mark.parametrize("errors", ("replace", "ignore", "backslashreplace", "strict")) +def test_openai_partial_token_decoding_preserves_error_policy(errors: str) -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + encoding: Final = OpenAIEncoding.from_tiktoken(reference.name) + tokens: Final = reference.encode("🙂")[:1] + assert encoding.decode_bytes(tokens) == reference.decode_bytes(tokens) + if errors == "strict": + with pytest.raises(UnicodeDecodeError): + encoding.decode(tokens, errors=errors) + return + assert encoding.decode(tokens, errors=errors) == reference.decode(tokens, errors=errors) + assert encoding.decode_tokens_bytes(tokens) == reference.decode_tokens_bytes(tokens) + + +def test_public_encoding_and_semantic_cache_preserve_truncated_unicode() -> None: + reference: Final = tiktoken.get_encoding(litellm.encoding.name) + text: Final = "🙂" + tokens: Final = reference.encode(text) + + assert litellm.encoding.encode(text, disallowed_special=()) == tokens + assert litellm.encoding.encode_batch([text]) == [tokens] + assert litellm.decode(tokens=tokens[:1]) == reference.decode(tokens[:1]) + assert truncate_embedding_input(text, "", 1) == reference.decode(tokens[:1]) + + +@pytest.mark.parametrize("add_special_tokens", (True, False)) +def test_huggingface_encoding_preserves_result_fields_and_serialization(add_special_tokens: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + tokenizer: Final = HuggingFaceTokenizer.from_str(TOKENIZER_JSON) + expected: Final = reference.encode("Hello World", add_special_tokens=add_special_tokens) + actual: Final = tokenizer.encode("Hello World", add_special_tokens=add_special_tokens) + + assert (actual.ids, actual.tokens, actual.type_ids, actual.offsets, actual.word_ids, actual.sequence_ids) == ( + expected.ids, + expected.tokens, + expected.type_ids, + expected.offsets, + expected.word_ids, + expected.sequence_ids, + ) + assert (actual.attention_mask, actual.special_tokens_mask, actual.n_sequences, len(actual)) == ( + expected.attention_mask, + expected.special_tokens_mask, + expected.n_sequences, + len(expected), + ) + assert copy.deepcopy(actual).ids == expected.ids + assert pickle.loads(pickle.dumps(actual)).offsets == expected.offsets + assert tokenizer.decode(actual.ids, skip_special_tokens=False) == reference.decode( + expected.ids, skip_special_tokens=False + ) + + +def test_huggingface_character_offsets_and_pretokenized_pairs_match_python() -> None: + reference: Final = ReferenceTokenizer.from_str(claude_json_str) + tokenizer: Final = HuggingFaceTokenizer.from_str(claude_json_str) + text: Final = "café 漢字 🙂" + actual: Final = tokenizer.encode(text) + expected: Final = reference.encode(text) + + assert actual.offsets == expected.offsets + assert actual.ids == expected.ids + assert ( + tokenizer.encode(["hello", "world"], ["again"], is_pretokenized=True).ids + == reference.encode(["hello", "world"], ["again"], is_pretokenized=True).ids + ) + + +def test_huggingface_batches_apply_padding_across_inputs() -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + reference.enable_padding(pad_id=0, pad_token="[UNK]") + tokenizer: Final = HuggingFaceTokenizer.from_str(reference.to_str()) + inputs: Final = ["Hello", ("Hello World", "World")] + expected: Final = reference.encode_batch(inputs) + actual: Final = tokenizer.encode_batch(inputs) + fast: Final = tokenizer.encode_batch_fast(inputs) + + assert [(item.ids, item.attention_mask, item.offsets) for item in actual] == [ + (item.ids, item.attention_mask, item.offsets) for item in expected + ] + assert [item.ids for item in fast] == [item.ids for item in expected] + assert tokenizer.decode_batch([item.ids for item in actual]) == reference.decode_batch( + [item.ids for item in expected] + ) + + +def test_caller_supplied_huggingface_tokenizer_preserves_public_encode_and_count() -> None: + tokenizer: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + custom: Final = {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + expected: Final = tokenizer.encode("Hello World").ids + + assert litellm.encode(text="Hello World", custom_tokenizer=custom) == expected + assert litellm.token_counter(text="Hello World", custom_tokenizer=custom) == len(expected) + assert litellm.decode(tokens=expected, custom_tokenizer=custom) == "Hello World" + + +def test_caller_supplied_tiktoken_treats_special_spellings_as_text() -> None: + tokenizer: Final = tiktoken.get_encoding("cl100k_base") + custom: Final = {"type": "openai_tokenizer", "tokenizer": tokenizer} + text: Final = "<|endoftext|>" + + assert litellm.encode(text=text, custom_tokenizer=custom) == tokenizer.encode(text, disallowed_special=()) + + +def test_public_tokenizer_objects_survive_pickle_and_deepcopy(tmp_path: Path) -> None: + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) + tokenizer: Final = custom["tokenizer"] + path: Final = tmp_path / "tokenizer.json" + tokenizer.save(str(path)) + + assert copy.deepcopy(custom)["tokenizer"].encode("Hello World").ids == tokenizer.encode("Hello World").ids + assert ( + pickle.loads(pickle.dumps(custom))["tokenizer"].encode("Hello World").ids == tokenizer.encode("Hello World").ids + ) + assert HuggingFaceTokenizer.from_file(str(path)).encode("Hello World").ids == tokenizer.encode("Hello World").ids + assert copy.deepcopy(litellm.encoding).encode("hello") == litellm.encoding.encode("hello") + assert pickle.loads(pickle.dumps(litellm.encoding)).encode("hello") == litellm.encoding.encode("hello") + + +@pytest.mark.parametrize("offline", ("0", "1")) +def test_hub_loader_preserves_environment_auth_cache_and_offline(tmp_path: Path, offline: str) -> None: + script: Final = """ +import json +import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import httpx +import huggingface_hub +from huggingface_hub.errors import LocalEntryNotFoundError +import litellm +payload = sys.argv[2].encode() +offline = sys.argv[3] == "1" +observed = [] +def handle(request): + assert not offline, "offline loading issued a request" + if request.url.path.endswith("/tokenizer.json"): + observed.append(request.headers.get("authorization")) + if request.headers.get("authorization") != "Bearer audit-fixture-token": + return httpx.Response(401) + return httpx.Response(200, headers={"content-length": str(len(payload)), "etag": '"fixture"', "x-repo-commit": "a" * 40}, content=payload if request.method == "GET" else b"") +if not offline: + huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) +try: + tokenizer = litellm.create_pretrained_tokenizer("test-fixture/tokenizer")["tokenizer"] +except LocalEntryNotFoundError: + assert offline + assert observed == [] +else: + assert not offline + assert "Bearer audit-fixture-token" in observed + assert tokenizer.decode(tokenizer.encode("Hello World").ids) == "Hello World" + assert tuple(Path(sys.argv[4]).rglob("tokenizer.json")) +print("compatible") +""" + result: Final = subprocess.run( + [ + sys.executable, + "-I", + "-c", + script, + str(Path(litellm.__file__).parent.parent), + TOKENIZER_JSON, + offline, + str(tmp_path / "cache"), + ], + capture_output=True, + text=True, + timeout=30, + env={ + **os.environ, + "HF_HOME": str(tmp_path / "home"), + "HF_HUB_CACHE": str(tmp_path / "cache"), + "HF_ENDPOINT": "http://127.0.0.1:9", + "HF_TOKEN": "audit-fixture-token", + "HF_HUB_OFFLINE": offline, + "HF_HUB_DISABLE_IMPLICIT_TOKEN": "0", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + + +@pytest.mark.parametrize("rust", (None, "0", "1")) +def test_tokenization_without_native_extension_stays_offline(tmp_path: Path, rust: str | None) -> None: + script: Final = """ +import importlib.abc +import sys +sys.path.insert(0, sys.argv[1]) +def reject_network(event, args): + if event == "socket.connect": + raise AssertionError("tokenizer attempted a network connection") +sys.addaudithook(reject_network) +class Block(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "litellm.rust_bridge._native": + raise ImportError("native extension is unavailable") +sys.meta_path.insert(0, Block()) +import litellm +from litellm.rust_bridge.tokenizer import get_encoding +import tiktoken +from tokenizers import Tokenizer +assert isinstance(litellm.encoding, tiktoken.Encoding) +for name in ("cl100k_base", "o200k_base", "o200k_harmony", "p50k_base", "p50k_edit"): + encoding = get_encoding(name) + text = "offline café 漢字 🙂" + " " * 64 + assert encoding.decode(encoding.encode(text)) == text +ids = litellm.encode(text="hello world") +assert litellm.decode(tokens=ids) == "hello world" +assert litellm.token_counter(model=None, text="hello world") == len(ids) +custom = litellm.create_tokenizer(sys.argv[2]) +assert isinstance(custom["tokenizer"], Tokenizer) +custom["tokenizer"].enable_padding(pad_id=0, pad_token="[UNK]") +assert litellm.decode(tokens=litellm.encode(text="Hello World", custom_tokenizer=custom), custom_tokenizer=custom) == "Hello World" +print("compatible") +""" + result: Final = subprocess.run( + [sys.executable, "-I", "-c", script, str(Path(litellm.__file__).parent.parent), TOKENIZER_JSON], + capture_output=True, + text=True, + timeout=30, + cwd=tmp_path, + env={ + **{key: value for key, value in os.environ.items() if key != "LITELLM_RUST"}, + **({"LITELLM_RUST": rust} if rust is not None else {}), + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "TIKTOKEN_CACHE_DIR": str(tmp_path / "unused-tokenizer-cache"), + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + assert not (tmp_path / "unused-tokenizer-cache").exists() + + +@pytest.mark.parametrize("is_pretokenized", (False, True)) +def test_huggingface_batch_sequence_containers_match_python(is_pretokenized: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + tokenizer: Final = HuggingFaceTokenizer.from_str(TOKENIZER_JSON) + inputs: Final = [["Hello", "World"], ("Hello", "World")] + actual: Final = tokenizer.encode_batch(inputs, is_pretokenized=is_pretokenized) + expected: Final = reference.encode_batch(inputs, is_pretokenized=is_pretokenized) + assert [(item.ids, item.type_ids, item.sequence_ids) for item in actual] == [ + (item.ids, item.type_ids, item.sequence_ids) for item in expected + ] + + +@pytest.mark.parametrize("name", ("cl100k_base", "o200k_base", "p50k_edit", "gpt2")) +def test_openai_encoding_exposes_the_tiktoken_vocabulary_surface(name: str) -> None: + reference: Final = tiktoken.get_encoding(name) + encoding: Final = OpenAIEncoding.from_tiktoken(name) + text: Final = "hello fanta" + + assert repr(encoding) == repr(reference) == f"" + assert (encoding.name, encoding.n_vocab, encoding.max_token_value) == ( + reference.name, + reference.n_vocab, + reference.max_token_value, + ) + assert encoding.token_byte_values() == reference.token_byte_values() + assert encoding.encode_single_token("hello") == reference.encode_single_token("hello") + assert encoding.encode_single_token(b"<|endoftext|>") == reference.eot_token + assert [encoding.is_special_token(token) for token in (0, reference.eot_token)] == [False, True] + assert encoding.decode_with_offsets(reference.encode(text)) == reference.decode_with_offsets(reference.encode(text)) + assert encoding.encode_to_numpy(text).tolist() == reference.encode_to_numpy(text).tolist() + stable, completions = encoding.encode_with_unstable(text) + expected_stable, expected_completions = reference.encode_with_unstable(text) + assert (stable, sorted(completions)) == (expected_stable, sorted(expected_completions)) + with pytest.raises(KeyError): + encoding.encode_single_token("<|not-a-token|>") + + +def test_huggingface_tokenizer_exposes_the_tokenizers_vocabulary_surface() -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + reference.enable_padding(pad_id=0, pad_token="[UNK]", length=4) + reference.enable_truncation(max_length=3, stride=1, strategy="only_first", direction="left") + tokenizer: Final = HuggingFaceTokenizer.from_str(reference.to_str()) + + assert tokenizer.token_to_id("Hello") == reference.token_to_id("Hello") == 1 + assert tokenizer.id_to_token(3) == reference.id_to_token(3) == "[BOS]" + assert tokenizer.id_to_token(99) is None + assert tokenizer.get_vocab() == reference.get_vocab() + assert tokenizer.get_vocab(with_added_tokens=False) == reference.get_vocab(with_added_tokens=False) + assert tokenizer.get_vocab_size() == reference.get_vocab_size() == 4 + assert tokenizer.get_vocab_size(with_added_tokens=False) == reference.get_vocab_size(with_added_tokens=False) + added: Final = tokenizer.get_added_tokens_decoder() + expected_added: Final = reference.get_added_tokens_decoder() + assert {token_id: str(token) for token_id, token in added.items()} == { + token_id: str(token) for token_id, token in expected_added.items() + } + assert added[3].special == expected_added[3].special + assert tokenizer.num_special_tokens_to_add(False) == reference.num_special_tokens_to_add(False) == 1 + assert tokenizer.num_special_tokens_to_add(True) == reference.num_special_tokens_to_add(True) == 0 + assert tokenizer.padding == reference.padding + assert tokenizer.truncation == reference.truncation + assert tokenizer.encode_special_tokens == reference.encode_special_tokens is False + assert HuggingFaceTokenizer.from_buffer(TOKENIZER_JSON.encode()).encode("Hello").ids == [3, 1] + assert HuggingFaceTokenizer.from_str(TOKENIZER_JSON).padding is None + assert HuggingFaceTokenizer.from_str(TOKENIZER_JSON).truncation is None + + +def test_huggingface_encoding_exposes_the_tokenizers_lookup_and_mutation_surface() -> None: + reference: Final = ReferenceTokenizer.from_str(claude_json_str) + tokenizer: Final = HuggingFaceTokenizer.from_str(claude_json_str) + text: Final = "hello wide world" + actual: Final = tokenizer.encode(text, "again") + expected: Final = reference.encode(text, "again") + + lookups: Final = ( + lambda encoding: [encoding.token_to_chars(index) for index in range(len(encoding))], + lambda encoding: [encoding.token_to_word(index) for index in range(len(encoding))], + lambda encoding: [encoding.token_to_sequence(index) for index in range(len(encoding))], + lambda encoding: [encoding.char_to_token(position) for position in range(len(text))], + lambda encoding: [encoding.char_to_word(position) for position in range(len(text))], + lambda encoding: [encoding.char_to_token(position, 1) for position in range(5)], + lambda encoding: [encoding.word_to_tokens(word) for word in range(3)], + lambda encoding: [encoding.word_to_chars(word) for word in range(3)], + lambda encoding: [encoding.word_to_tokens(0, 1), encoding.word_to_chars(0, 1)], + ) + for lookup in lookups: + assert lookup(actual) == lookup(expected) + assert repr(actual) == repr(expected) + + actual.truncate(4, stride=1, direction="left") + expected.truncate(4, stride=1, direction="left") + assert (actual.ids, [item.ids for item in actual.overflowing]) == ( + expected.ids, + [item.ids for item in expected.overflowing], + ) + actual.pad(6, direction="left", pad_id=7, pad_type_id=1, pad_token="") + expected.pad(6, direction="left", pad_id=7, pad_type_id=1, pad_token="") + assert (actual.ids, actual.attention_mask, actual.type_ids, actual.tokens) == ( + expected.ids, + expected.attention_mask, + expected.type_ids, + expected.tokens, + ) + actual.set_sequence_id(3) + expected.set_sequence_id(3) + assert actual.sequence_ids == expected.sequence_ids + merged: Final = type(actual).merge([actual, tokenizer.encode("more")]) + assert merged.ids == type(expected).merge([expected, reference.encode("more")]).ids + assert merged.offsets == type(expected).merge([expected, reference.encode("more")]).offsets + with pytest.raises(ValueError, match="direction"): + actual.pad(8, direction="sideways") diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py index 214b5cde7da..13488106df4 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -4,7 +4,7 @@ import json import math from datetime import datetime, timedelta, timezone from types import MappingProxyType -from typing import Final +from typing import Final, cast import pytest @@ -33,6 +33,7 @@ from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.rust_bridge import bindings, configuration from litellm.rust_bridge import token_counter as rust_token_counter +from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo TOKEN_COUNTING_ROUTES: Final = ( @@ -239,6 +240,22 @@ class _FakeUpstream(Exception): pass +class _FakeTokenizer: + """Stands in for one shared native `Tokenizer`; only its name identifies it.""" + + def __init__(self, name: str, json: str | None = None) -> None: + self.name = name + self.json = json + + +def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: + """Point the counter's tokenizer lookups at fakes; the codec path keeps falling back to Python.""" + fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} + anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) + monkeypatch.setattr(tokenizer_dispatch, "native_encoding", fakes.__getitem__) + monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic) + + class _FakeNative: RustBridgeDeclined = _FakeDeclined RustUpstreamError = _FakeUpstream @@ -257,19 +274,13 @@ class _RecordingCounter: class _RecordingFactory: - """Stands in for the native `TokenCounter` class: called with tokenizer JSON, or `from_*_ranks`.""" + """Stands in for the native `TokenCounter` class, built over a loaded `Tokenizer`.""" def __init__(self) -> None: self.calls: list[tuple[rust_token_counter.RustTokenizer, bytes]] = [] - def __call__(self, tokenizer_json: str) -> _RecordingCounter: - return _RecordingCounter(self, "anthropic") - - def from_cl100k_ranks(self, rank_file: str) -> _RecordingCounter: - return _RecordingCounter(self, "cl100k_base") - - def from_o200k_ranks(self, rank_file: str) -> _RecordingCounter: - return _RecordingCounter(self, "o200k_base") + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: + return _RecordingCounter(self, cast(rust_token_counter.RustTokenizer, tokenizer.name)) class _DecliningCounter: @@ -278,19 +289,14 @@ class _DecliningCounter: class _DecliningFactory: - def __call__(self, tokenizer_json: str) -> _DecliningCounter: - return _DecliningCounter() - - def from_cl100k_ranks(self, rank_file: str) -> _DecliningCounter: - return _DecliningCounter() - - def from_o200k_ranks(self, rank_file: str) -> _DecliningCounter: + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _DecliningCounter: return _DecliningCounter() @pytest.fixture def rust_counter(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) + _fake_native_tokenizers(monkeypatch) rust_token_counter._counter.cache_clear() configuration.reset_rust_configuration() yield diff --git a/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py b/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py new file mode 100644 index 00000000000..49bbe148386 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py @@ -0,0 +1,191 @@ +"""Tests for input-token counting shared across the reservation path's models.""" + +from __future__ import annotations + +import json +from types import MappingProxyType +from typing import Final, cast + +import pytest + +import litellm +from litellm.proxy.spend_tracking.input_tokens import ( + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, + count_input_tokens, + count_input_tokens_for_model, +) +from litellm.rust_bridge import bindings, configuration, token_counter +from litellm.rust_bridge import tokenizer as tokenizer_dispatch +from litellm.rust_bridge.token_counter import RustTokenizer + +ANTHROPIC_MODEL: Final = "claude-sonnet-4-5-20250929" +CL100K_MODEL: Final = "gpt-4" +O200K_MODEL: Final = "gpt-4o" +PYTHON_ONLY_MODEL: Final = "replicate/meta/llama-2-70b-chat" +MESSAGES: Final = [{"role": "user", "content": "hello"}] +RUST_TOKENS: Final = 777 + + +class _FakeDeclined(Exception): + pass + + +class _FakeUpstream(Exception): + pass + + +class _FakeTokenizer: + """Stands in for one shared native `Tokenizer`; only its name identifies it.""" + + def __init__(self, name: str, json: str | None = None) -> None: + self.name = name + self.json = json + + +def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: + """Point the counter's tokenizer lookups at fakes; the codec path keeps falling back to Python.""" + fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} + anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) + monkeypatch.setattr(tokenizer_dispatch, "native_encoding", fakes.__getitem__) + monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic) + + +class _FakeNative: + RustBridgeDeclined = _FakeDeclined + RustUpstreamError = _FakeUpstream + + +class _RecordingCounter: + def __init__(self, factory: _RecordingFactory, tokenizer: RustTokenizer) -> None: + self.factory = factory + self.tokenizer = tokenizer + + async def acount_request(self, body: bytes) -> object: + self.factory.calls.append((self.tokenizer, body)) + return {"model": "", "input_tokens": RUST_TOKENS} + + +class _RecordingFactory: + def __init__(self) -> None: + self.calls: list[tuple[RustTokenizer, bytes]] = [] + + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: + return _RecordingCounter(self, cast(RustTokenizer, tokenizer.name)) + + +class _DecliningCounter: + async def acount_request(self, body: bytes) -> object: + raise _FakeDeclined("unsupported request shape") + + +class _DecliningFactory: + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _DecliningCounter: + return _DecliningCounter() + + +@pytest.fixture(autouse=True) +def _reset_bridge(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) + _fake_native_tokenizers(monkeypatch) + token_counter.TOKEN_COUNTER.reset() + token_counter._counter.cache_clear() + configuration.reset_rust_configuration() + yield + token_counter.TOKEN_COUNTER.reset() + token_counter._counter.cache_clear() + configuration.reset_rust_configuration() + + +def _body(model: object) -> tuple[dict[str, object], bytes]: + body: Final = {"model": model, "messages": MESSAGES} + return body, json.dumps(body).encode() + + +@pytest.mark.asyncio +async def test_models_sharing_a_tokenizer_are_counted_once_and_merged() -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(factory) + request_body, raw_body = _body([ANTHROPIC_MODEL, CL100K_MODEL, O200K_MODEL, "gpt-5", PYTHON_ONLY_MODEL]) + + counts: Final = await count_input_tokens( + request_body=request_body, + raw_body=raw_body, + models=(ANTHROPIC_MODEL, CL100K_MODEL, O200K_MODEL, "gpt-5", PYTHON_ONLY_MODEL), + ) + + assert factory.calls == [("anthropic", raw_body), ("cl100k_base", raw_body), ("o200k_base", raw_body)] + assert dict(counts) == { + ANTHROPIC_MODEL: RUST_TOKENS, + CL100K_MODEL: RUST_TOKENS, + O200K_MODEL: RUST_TOKENS, + "gpt-5": RUST_TOKENS, + PYTHON_ONLY_MODEL: count_input_tokens_for_model(request_body=request_body, model=PYTHON_ONLY_MODEL), + } + + +@pytest.mark.asyncio +async def test_rust_disabled_counts_everything_in_python() -> None: + factory: Final = _RecordingFactory() + litellm.rust(False) + token_counter.TOKEN_COUNTER.override(factory) + request_body, raw_body = _body([ANTHROPIC_MODEL, CL100K_MODEL]) + + counts: Final = await count_input_tokens( + request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL, CL100K_MODEL) + ) + + assert factory.calls == [] + assert dict(counts) == { + model: count_input_tokens_for_model(request_body=request_body, model=model) + for model in (ANTHROPIC_MODEL, CL100K_MODEL) + } + + +@pytest.mark.asyncio +async def test_missing_raw_body_counts_in_python() -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(factory) + request_body, _ = _body(ANTHROPIC_MODEL) + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=None, models=(ANTHROPIC_MODEL,)) + + assert factory.calls == [] + assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) + + +@pytest.mark.asyncio +async def test_missing_binding_counts_in_python() -> None: + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(None) + request_body, raw_body = _body(ANTHROPIC_MODEL) + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL,)) + + assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) + + +@pytest.mark.asyncio +async def test_declined_request_counts_in_python() -> None: + litellm.rust(True) + token_counter.TOKEN_COUNTER.override(_DecliningFactory()) + request_body, raw_body = _body(ANTHROPIC_MODEL) + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw_body, models=(ANTHROPIC_MODEL,)) + + assert counts[ANTHROPIC_MODEL] == count_input_tokens_for_model(request_body=request_body, model=ANTHROPIC_MODEL) + assert counts[ANTHROPIC_MODEL] != RUST_TOKENS + + +@pytest.mark.asyncio +async def test_large_input_is_still_counted() -> None: + request_body: Final = { + "model": CL100K_MODEL, + "messages": [{"role": "user", "content": "x" * (TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS + 1)}], + } + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=None, models=(CL100K_MODEL,)) + + assert counts[CL100K_MODEL] == count_input_tokens_for_model(request_body=request_body, model=CL100K_MODEL) + assert isinstance(counts, MappingProxyType) diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 86bf896188f..b3913079bb2 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -39,8 +39,6 @@ from litellm.proxy.common_utils.user_api_key_cache import ( model_access_group_spend_counter_key, ) from litellm.proxy.spend_tracking.budget_reservation import ( - TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, - _approximate_input_size, _get_model_access_group_budget_counters, estimate_request_max_cost, get_budget_window_start, @@ -49,6 +47,10 @@ from litellm.proxy.spend_tracking.budget_reservation import ( release_budget_reservation_on_cancel, reserve_budget_for_request, ) +from litellm.proxy.spend_tracking.input_tokens import ( + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, + _approximate_input_size, +) from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 2792e176e0e..26bfd5c52bd 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -14877,7 +14877,7 @@ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_coun async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeypatch): - from tokenizers import Tokenizer + from litellm.rust_bridge._native import Tokenizer from litellm import Router from tests.test_litellm.litellm_core_utils.event_loop_lag import assert_loop_stayed_free, timed_with_loop_lags @@ -14890,7 +14890,7 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp time.sleep(0.3) return claude_tokenizer - monkeypatch.setattr(litellm.utils, "Tokenizer", SlowHubTokenizer) + monkeypatch.setattr("litellm.rust_bridge.tokenizer.from_pretrained", SlowHubTokenizer.from_pretrained) monkeypatch.setattr( "litellm.proxy.proxy_server.llm_router", Router( @@ -14914,7 +14914,7 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revision_and_token(monkeypatch): - from tokenizers import Tokenizer + from litellm.rust_bridge._native import Tokenizer from litellm import Router from litellm.types.router import DeploymentTypedDict @@ -14931,7 +14931,7 @@ async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revisi }, } - monkeypatch.setattr(litellm.utils, "Tokenizer", MagicMock(from_pretrained=from_pretrained)) + monkeypatch.setattr("litellm.rust_bridge.tokenizer.from_pretrained", from_pretrained) monkeypatch.setattr( "litellm.proxy.proxy_server.llm_router", Router( diff --git a/tests/test_litellm/rust_bridge/test_catalog.py b/tests/test_litellm/rust_bridge/test_catalog.py index 3fa5f3de7ab..82e3766e8a2 100644 --- a/tests/test_litellm/rust_bridge/test_catalog.py +++ b/tests/test_litellm/rust_bridge/test_catalog.py @@ -53,7 +53,7 @@ def test_shipped_decisions( enabled: Final = environment == "1" if environment is not None else process is not False assert catalog.rollout(context) is Rollout.RUST_OPT_OUT assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if enabled else Decision.PYTHON) - elif route is Route.MESSAGES: + elif route in (Route.MESSAGES, Route.TOKEN_COUNTER, Route.TOKENIZER): enabled: Final = environment == "1" if environment is not None else process is True assert catalog.rollout(context) is Rollout.RUST_OPT_IN assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if enabled else Decision.PYTHON) diff --git a/tests/test_litellm/rust_bridge/test_token_counter.py b/tests/test_litellm/rust_bridge/test_token_counter.py index 71aa79cc4bb..3da291c898d 100644 --- a/tests/test_litellm/rust_bridge/test_token_counter.py +++ b/tests/test_litellm/rust_bridge/test_token_counter.py @@ -12,25 +12,32 @@ from types import MappingProxyType from typing import Final import pytest -import tiktoken -from tokenizers import Tokenizer import litellm from litellm.constants import TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding -from litellm.proxy.spend_tracking.budget_reservation import _count_input_tokens +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model from litellm.rust_bridge import bindings, configuration from litellm.rust_bridge import token_counter as bridge +from litellm.rust_bridge import tokenizer as tokenizer_dispatch +from litellm.rust_bridge._native import Tokenizer from litellm.utils import claude_json_str MODEL: Final = "claude-sonnet-4-5-20250929" CL100K_MODEL: Final = "gpt-4" O200K_MODEL: Final = "gpt-4o" +MODEL_BY_TOKENIZER: Final[MappingProxyType[bridge.RustTokenizer, str]] = MappingProxyType( + {"anthropic": MODEL, "cl100k_base": CL100K_MODEL, "o200k_base": O200K_MODEL} +) TOKENIZERS: Final[tuple[bridge.RustTokenizer, ...]] = ("anthropic", "cl100k_base", "o200k_base") -RANK_FILE_LINES: Final = MappingProxyType({"cl100k_base": 100_256, "o200k_base": 199_998}) BODY: Final = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": "hello"}]}).encode() +def _counted(body: dict[str, object], model: str) -> tuple[bytes, dict[str, object]]: + raw: Final = json.dumps({**body, "model": model}).encode() + return raw, json.loads(raw) + + class _FakeDeclined(Exception): pass @@ -39,14 +46,41 @@ class _FakeUpstream(Exception): pass +class _FakeTokenizer: + """Stands in for one shared native `Tokenizer`; only its name identifies it.""" + + def __init__(self, name: str, json: str | None = None) -> None: + self.name = name + self.json = json + + +def _fake_native_tokenizers(monkeypatch: pytest.MonkeyPatch, anthropic_json: str | None = None) -> None: + """Point the counter's tokenizer lookups at fakes while the bridge is faked; the codec path + keeps falling back to Python. Parity tests that restore the real extension get the real + lookups back.""" + fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} + anthropic: Final = _FakeTokenizer("anthropic", anthropic_json) + real_encoding: Final = tokenizer_dispatch.native_encoding + real_anthropic: Final = tokenizer_dispatch.native_anthropic + + def faked() -> bool: + return isinstance(bindings.get_native_bridge(), _FakeNative) + + monkeypatch.setattr( + tokenizer_dispatch, "native_encoding", lambda name: fakes[name] if faked() else real_encoding(name) + ) + monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic if faked() else real_anthropic()) + + class _FakeNative: RustBridgeDeclined = _FakeDeclined RustUpstreamError = _FakeUpstream class _RecordingCounter: - def __init__(self, tokenizer_json: str) -> None: - self.tokenizer_json = tokenizer_json + def __init__(self, tokenizer: _FakeTokenizer, fast: bool) -> None: + self.tokenizer = tokenizer + self.fast = fast self.bodies: list[bytes] = [] async def acount_request(self, body: bytes) -> object: @@ -55,25 +89,16 @@ class _RecordingCounter: class _RecordingFactory: - """Stands in for the native `TokenCounter` class: callable for tokenizer JSON, `from_*_ranks` for rank files.""" + """Stands in for the native `TokenCounter` class, built over a loaded `Tokenizer`.""" def __init__(self) -> None: self.counters: list[_RecordingCounter] = [] - self.rank_files: list[str] = [] - def __call__(self, tokenizer_json: str) -> _RecordingCounter: - counter = _RecordingCounter(tokenizer_json) + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: + counter = _RecordingCounter(tokenizer, fast) self.counters.append(counter) return counter - def from_cl100k_ranks(self, rank_file: str) -> _RecordingCounter: - self.rank_files.append(rank_file) - return self("cl100k_base") - - def from_o200k_ranks(self, rank_file: str) -> _RecordingCounter: - self.rank_files.append(rank_file) - return self("o200k_base") - class _RaisingCounter: def __init__(self, error: Exception) -> None: @@ -89,13 +114,7 @@ class _RaisingFactory: def __init__(self, error: Exception) -> None: self.error = error - def __call__(self, tokenizer_json: str) -> _RaisingCounter: - return _RaisingCounter(self.error) - - def from_cl100k_ranks(self, rank_file: str) -> _RaisingCounter: - return _RaisingCounter(self.error) - - def from_o200k_ranks(self, rank_file: str) -> _RaisingCounter: + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RaisingCounter: return _RaisingCounter(self.error) @@ -105,6 +124,7 @@ def _reset_bridge(monkeypatch: pytest.MonkeyPatch): bridge._counter.cache_clear() configuration.reset_rust_configuration() monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) + _fake_native_tokenizers(monkeypatch, anthropic_json=claude_json_str) yield bridge.TOKEN_COUNTER.reset() bridge._counter.cache_clear() @@ -117,8 +137,12 @@ async def test_disabled_bridge_never_constructs_a_counter(tokenizer: bridge.Rust factory: Final = _RecordingFactory() litellm.rust(False) bridge.TOKEN_COUNTER.override(factory) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - assert await bridge.count_input_tokens(BODY, tokenizer) is None + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) + + assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) assert factory.counters == [] @@ -128,31 +152,33 @@ async def test_enabled_bridge_returns_typed_count_and_reuses_one_counter() -> No litellm.rust(True) bridge.TOKEN_COUNTER.override(factory) - first: Final = await bridge.count_input_tokens(BODY, "anthropic") - second: Final = await bridge.count_input_tokens(BODY, "anthropic") + first: Final = await bridge.native_count(factory, "anthropic", BODY) + second: Final = await bridge.native_count(factory, "anthropic", BODY) assert first == bridge.InputTokenCount(model=MODEL, input_tokens=42) assert second == first assert len(factory.counters) == 1 assert factory.counters[0].bodies == [BODY, BODY] - assert json.loads(factory.counters[0].tokenizer_json)["model"]["type"] == "BPE" + assert factory.counters[0].fast is False + assert factory.counters[0].tokenizer is tokenizer_dispatch.native_anthropic() + assert json.loads(factory.counters[0].tokenizer.json or "")["model"]["type"] == "BPE" @pytest.mark.asyncio @pytest.mark.parametrize("tokenizer", ("cl100k_base", "o200k_base")) -async def test_tiktoken_counter_is_built_from_the_vendored_rank_file_once(tokenizer: bridge.RustTokenizer) -> None: +async def test_tiktoken_counter_is_built_over_the_shared_encoding_once(tokenizer: bridge.RustTokenizer) -> None: factory: Final = _RecordingFactory() litellm.rust(True) bridge.TOKEN_COUNTER.override(factory) - first: Final = await bridge.count_input_tokens(BODY, tokenizer) - second: Final = await bridge.count_input_tokens(BODY, tokenizer) + first: Final = await bridge.native_count(factory, tokenizer, BODY) + second: Final = await bridge.native_count(factory, tokenizer, BODY) assert first == second == bridge.InputTokenCount(model=MODEL, input_tokens=42) - assert len(factory.rank_files) == 1 - assert factory.rank_files[0].startswith("IQ== 0\n") - assert factory.rank_files[0].count("\n") == RANK_FILE_LINES[tokenizer] - assert factory.counters[0].tokenizer_json == tokenizer + assert len(factory.counters) == 1 + assert factory.counters[0].tokenizer.name == tokenizer + assert factory.counters[0].tokenizer is tokenizer_dispatch.native_encoding(tokenizer) + assert factory.counters[0].fast is False assert factory.counters[0].bodies == [BODY, BODY] @@ -162,13 +188,13 @@ async def test_each_tokenizer_gets_its_own_cached_counter() -> None: litellm.rust(True) bridge.TOKEN_COUNTER.override(factory) - await bridge.count_input_tokens(BODY, "anthropic") - await bridge.count_input_tokens(BODY, "cl100k_base") - await bridge.count_input_tokens(BODY, "o200k_base") - await bridge.count_input_tokens(BODY, "anthropic") - await bridge.count_input_tokens(BODY, "o200k_base") + await bridge.native_count(factory, "anthropic", BODY) + await bridge.native_count(factory, "cl100k_base", BODY) + await bridge.native_count(factory, "o200k_base", BODY) + await bridge.native_count(factory, "anthropic", BODY) + await bridge.native_count(factory, "o200k_base", BODY) - assert [counter.tokenizer_json for counter in factory.counters][1:] == ["cl100k_base", "o200k_base"] + assert [counter.tokenizer.name for counter in factory.counters] == ["anthropic", "cl100k_base", "o200k_base"] assert [len(counter.bodies) for counter in factory.counters] == [2, 1, 2] @@ -176,8 +202,11 @@ async def test_each_tokenizer_gets_its_own_cached_counter() -> None: async def test_missing_native_module_falls_back(monkeypatch: pytest.MonkeyPatch) -> None: litellm.rust(True) monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, MODEL) - assert [await bridge.count_input_tokens(BODY, tokenizer) for tokenizer in TOKENIZERS] == [None, None, None] + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(MODEL,)) + + assert counts[MODEL] == count_input_tokens_for_model(request_body=request_body, model=MODEL) @pytest.mark.asyncio @@ -185,8 +214,12 @@ async def test_missing_native_module_falls_back(monkeypatch: pytest.MonkeyPatch) async def test_declined_request_falls_back(tokenizer: bridge.RustTokenizer) -> None: litellm.rust(True) bridge.TOKEN_COUNTER.override(_RaisingFactory(_FakeDeclined("request has no messages"))) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - assert await bridge.count_input_tokens(BODY, tokenizer) is None + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) + + assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) @pytest.mark.asyncio @@ -194,8 +227,12 @@ async def test_declined_request_falls_back(tokenizer: bridge.RustTokenizer) -> N async def test_runtime_failure_falls_back(tokenizer: bridge.RustTokenizer) -> None: litellm.rust(True) bridge.TOKEN_COUNTER.override(_RaisingFactory(RuntimeError("encode failed"))) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, request_body = _counted({"messages": [{"role": "user", "content": "hello"}]}, model) - assert await bridge.count_input_tokens(BODY, tokenizer) is None + counts: Final = await count_input_tokens(request_body=request_body, raw_body=raw, models=(model,)) + + assert counts[model] == count_input_tokens_for_model(request_body=request_body, model=model) @pytest.mark.parametrize( @@ -273,8 +310,8 @@ def test_rust_tokenizer_names_the_encoding_python_actually_counts_with(model: st "Hello, world! camelCase ABCdef \u00e9\u00e8 12345 \u3053\u3093\u306b\u3061\u306f <|endoftext|>\r\n" * 9 ) python_count: Final = litellm.token_counter(model=model, text=text) - cl100k_count: Final = len(tiktoken.get_encoding("cl100k_base").encode(text, disallowed_special=())) - o200k_count: Final = len(tiktoken.get_encoding("o200k_base").encode(text, disallowed_special=())) + cl100k_count: Final = Tokenizer.from_tiktoken("cl100k_base").count(text) + o200k_count: Final = Tokenizer.from_tiktoken("o200k_base").count(text) assert cl100k_count != o200k_count match bridge.rust_tokenizer(model): case "cl100k_base": @@ -282,7 +319,7 @@ def test_rust_tokenizer_names_the_encoding_python_actually_counts_with(model: st case "o200k_base": assert python_count == o200k_count case "anthropic": - assert python_count == len(Tokenizer.from_str(claude_json_str).encode(text).ids) + assert python_count == Tokenizer.from_json(claude_json_str).count(text) assert python_count not in {cl100k_count, o200k_count} case None: pytest.fail(f"{model} must have a Rust tokenizer") @@ -386,12 +423,11 @@ async def test_native_count_matches_python_budget_counter( litellm.rust(True) body: Final = json.dumps(request_body).replace(MODEL, model) - rust_count: Final = await bridge.count_input_tokens(body.encode(), tokenizer) - python_count: Final = _count_input_tokens(request_body=json.loads(body), model=model) + request_body_parsed: Final = json.loads(body) + counts: Final = await count_input_tokens(request_body=request_body_parsed, raw_body=body.encode(), models=(model,)) + python_count: Final = count_input_tokens_for_model(request_body=request_body_parsed, model=model) - assert rust_count is not None - assert rust_count.model == json.loads(body).get("model") - assert rust_count.input_tokens == python_count + assert counts[model] == python_count @pytest.mark.asyncio @@ -405,15 +441,14 @@ async def test_tiktoken_counts_long_text_exactly_where_python_chunks( litellm.rust(True) text: Final = "x " * 20_000 body: Final = {"model": model, "messages": [{"role": "user", "content": text}]} - encoding: Final = tiktoken.get_encoding(tokenizer) - exact: Final = 3 + len(encoding.encode("user")) + len(encoding.encode(text)) + 3 + encoding: Final = Tokenizer.from_tiktoken(tokenizer) + exact: Final = 3 + encoding.count("user") + encoding.count(text) + 3 chunks: Final = -(-len(text) // TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS) - rust_count: Final = await bridge.count_input_tokens(json.dumps(body).encode(), tokenizer) - python_count: Final = _count_input_tokens(request_body=body, model=model) + counts: Final = await count_input_tokens(request_body=body, raw_body=json.dumps(body).encode(), models=(model,)) + python_count: Final = count_input_tokens_for_model(request_body=body, model=model) - assert rust_count is not None - assert rust_count.input_tokens == exact + assert counts[model] == exact assert python_count is not None assert exact < python_count <= exact + chunks @@ -440,5 +475,9 @@ async def test_native_declines_shapes_python_prices_differently( native: Final = pytest.importorskip("litellm.rust_bridge._native") monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) litellm.rust(True) + model: Final = MODEL_BY_TOKENIZER[tokenizer] + raw, parsed = _counted(request_body, model) - assert await bridge.count_input_tokens(json.dumps(request_body).encode(), tokenizer) is None + counts: Final = await count_input_tokens(request_body=parsed, raw_body=raw, models=(model,)) + + assert counts.get(model) == count_input_tokens_for_model(request_body=parsed, model=model) diff --git a/tests/test_litellm/rust_bridge/test_tokenizer.py b/tests/test_litellm/rust_bridge/test_tokenizer.py new file mode 100644 index 00000000000..0de7ad50b1e --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_tokenizer.py @@ -0,0 +1,134 @@ +from collections.abc import Generator +from typing import Final + +import pytest +import tiktoken +from tokenizers import Tokenizer + +import litellm +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding +from litellm.rust_bridge import configuration, tokenizer +from litellm.utils import _select_tokenizer +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + + +@pytest.fixture(autouse=True) +def isolated_configuration(monkeypatch: pytest.MonkeyPatch) -> Generator[None]: + monkeypatch.delenv("LITELLM_RUST", raising=False) + configuration.reset_rust_configuration() + yield + tokenizer.TOKENIZER.reset() + configuration.reset_rust_configuration() + + +@pytest.mark.parametrize("environment", (None, "0", "1")) +@pytest.mark.parametrize("process", (None, False, True)) +def test_tokenizer_factories_follow_rollout( + monkeypatch: pytest.MonkeyPatch, environment: str | None, process: bool | None +) -> None: + configuration.rust(process) + if environment is not None: + monkeypatch.setenv("LITELLM_RUST", environment) + enabled: Final = environment == "1" if environment is not None else process is True + encoding: Final = tokenizer.get_encoding("cl100k_base") + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) + reference: Final = Tokenizer.from_str(TOKENIZER_JSON) + + assert isinstance(encoding, OpenAIEncoding if enabled else tiktoken.Encoding) + assert isinstance(custom["tokenizer"], HuggingFaceTokenizer if enabled else Tokenizer) + assert encoding.encode("café 漢字 🙂") == tiktoken.get_encoding(encoding.name).encode("café 漢字 🙂") + assert litellm.encode(text="Hello World", custom_tokenizer=custom) == reference.encode("Hello World").ids + assert litellm.token_counter(text="Hello World", custom_tokenizer=custom) == len(reference.encode("Hello World")) + + +def test_missing_native_binding_keeps_python_tokenizer_api() -> None: + configuration.rust(True) + tokenizer.TOKENIZER.override(None) + encoding: Final = tokenizer.get_encoding("cl100k_base") + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON)["tokenizer"] + + assert isinstance(encoding, tiktoken.Encoding) + assert isinstance(custom, Tokenizer) + custom.enable_padding(pad_id=0, pad_token="[UNK]") + assert [item.ids for item in custom.encode_batch(["Hello", "Hello World"])] == [[3, 1, 0], [3, 1, 2]] + + +def test_cached_selection_follows_backend_changes(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", True) + configuration.rust(True) + native: Final = _select_tokenizer("dispatch-fixture")["tokenizer"] + configuration.rust(False) + python: Final = _select_tokenizer("dispatch-fixture")["tokenizer"] + + assert isinstance(native, OpenAIEncoding) + assert isinstance(python, tiktoken.Encoding) + assert native.encode("hello") == python.encode("hello") + + +def test_declined_native_factory_falls_back_before_tokenizing() -> None: + from litellm.rust_bridge._native import RustBridgeDeclined + + class UnavailableTokenizer: + @staticmethod + def from_json(json: str) -> None: + raise RustBridgeDeclined("huggingface feature is disabled") + + configuration.rust(True) + binding: Final = tokenizer._as_factory(UnavailableTokenizer) + tokenizer.TOKENIZER.override(binding) + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) + + assert isinstance(custom["tokenizer"], Tokenizer) + assert ( + litellm.decode(tokens=litellm.encode(text="Hello World", custom_tokenizer=custom), custom_tokenizer=custom) + == "Hello World" + ) + + +@pytest.mark.parametrize( + ("model", "text"), + ( + ("gpt-4o", "hello <|endoftext|> world"), + ("gpt-3.5-turbo", "café 漢字 🙂"), + ("text-davinci-003", " def f():\n return 1\n"), + ("tokenizer-parity-fixture", "hello again"), + ), +) +def test_public_token_api_is_identical_across_backends(monkeypatch: pytest.MonkeyPatch, model: str, text: str) -> None: + """`litellm.token_counter`, `encode` and `decode` return the same values whichever backend + the catalog picks; only the object types differ.""" + monkeypatch.setattr(litellm, "anthropic_models", {*litellm.anthropic_models, "tokenizer-parity-fixture"}) + messages: Final = [{"role": "user", "content": text}, {"role": "assistant", "content": "ok"}] + + def observe() -> tuple[int, int, list[int], str]: + ids: Final = litellm.encode(model=model, text=text) + return ( + litellm.token_counter(model=model, text=text), + litellm.token_counter(model=model, messages=messages), + ids, + litellm.decode(model=model, tokens=ids), + ) + + configuration.rust(False) + python: Final = observe() + configuration.rust(True) + rust: Final = observe() + + assert rust == python + + +def test_cached_huggingface_tokenizers_follow_backend_changes(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer as RustHuggingFaceTokenizer + from litellm.utils import _load_huggingface_tokenizer + + monkeypatch.setattr(litellm, "anthropic_models", {*litellm.anthropic_models, "tokenizer-cache-fixture"}) + _load_huggingface_tokenizer.cache_clear() + configuration.rust(True) + native: Final = _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] + configuration.rust(False) + python: Final = _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] + configuration.rust(True) + + assert isinstance(native, RustHuggingFaceTokenizer) + assert isinstance(python, Tokenizer) + assert _select_tokenizer("tokenizer-cache-fixture")["tokenizer"] is native diff --git a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py index e449d4392d8..0eae041f535 100644 --- a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -56,10 +56,11 @@ def _write_wheel( metadata_tags: tuple[str, ...] | None = (_EXPECTED_TAG,), dist_info: str = _DIST_INFO, duplicate_wheel: bool = False, + native_bytes: bytes = b"synthetic native extension", ) -> Path: wheel: Final = tmp_path / f"litellm-1.100.0-{filename_tag}.whl" with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive: - archive.writestr(_NATIVE_MEMBER, b"synthetic native extension") + archive.writestr(_NATIVE_MEMBER, native_bytes) archive.writestr( f"{dist_info}/METADATA", "Metadata-Version: 2.1\nName: litellm\nVersion: 1.100.0\n", @@ -195,3 +196,17 @@ def test_rejects_production_module_exposing_panic_hook(tmp_path: Path) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) assert _run_verifier(wheel, exposes_panic=True) == 1 + + +@pytest.mark.parametrize("embedded", (False, True)) +def test_vocabulary_is_packaged_once(tmp_path: Path, embedded: bool) -> None: + ranks: Final = b"AA== 0\nAQ== 1\nAg== 2\n" + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + native_bytes=b"native engine" + (ranks if embedded else b""), + ) + with zipfile.ZipFile(wheel, "a") as archive: + archive.writestr("litellm/litellm_core_utils/tokenizers/" + "a" * 40, ranks) + + assert _run_verifier(wheel) == (1 if embedded else 0) diff --git a/tests/test_litellm_rust/test_fork_guard.py b/tests/test_litellm_rust/test_fork_guard.py index 086397bab5c..2a8fb6f9fca 100644 --- a/tests/test_litellm_rust/test_fork_guard.py +++ b/tests/test_litellm_rust/test_fork_guard.py @@ -1,5 +1,6 @@ import os import textwrap +from typing import Final import pytest @@ -144,3 +145,76 @@ def test_sdk_call_in_a_child_forked_after_native_use_raises_instead_of_hanging() result = run_child_interpreter(_SDK_CONTRACT, env=env, timeout=120) assert result.returncode == 0, result.stderr + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="fork only") +@pytest.mark.parametrize("warm_fast_counter", (False, True)) +def test_tokenizers_share_the_native_process_guard(warm_fast_counter: bool) -> None: + script: Final = """ +import asyncio +import os +import litellm +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens +from litellm.rust_bridge import _native +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer +from litellm.utils import claude_json_str + +litellm.anthropic_models = {*litellm.anthropic_models, "tokenizer-fork-fixture"} +_native.reserve_process_for_forking() +for create in ( + lambda: _native.Tokenizer.from_tiktoken("cl100k_base"), + lambda: _native.Tokenizer.from_json(claude_json_str), + lambda: litellm.token_counter(model="tokenizer-fork-fixture", text="hello"), +): + try: + create() + except _native.ProcessReservedForForking: + pass + else: + raise AssertionError("reserved parent ran a native tokenizer") +assert not _native.process_state_started() + +pid = os.fork() +if pid == 0: + tokenizer = HuggingFaceTokenizer.from_str(claude_json_str) + encoding = _native.Tokenizer.from_tiktoken("cl100k_base") + if os.environ["WARM_FAST_COUNTER"] == "True": + _native.TokenCounter.from_tokenizer(encoding, fast=True) + expected = [item.ids for item in tokenizer.encode_batch(["hello", "world"])] + assert _native.process_state_started() + grandchild = os.fork() + if grandchild == 0: + for call in ( + lambda: tokenizer.encode_batch(["hello", "world"]), + lambda: tokenizer.encode("hello"), + lambda: encoding.count("hello"), + lambda: encoding.count("hello", fast=True), + lambda: _native.TokenCounter.from_tokenizer(encoding), + lambda: _native.TokenCounter.from_tokenizer(encoding, fast=True), + lambda: _native.Tokenizer.from_tiktoken("cl100k_base"), + lambda: asyncio.run(count_input_tokens({"prompt": "hello"}, b'{"prompt": "hello"}', ("counter-fork-fixture",))), + ): + try: + call() + except _native.ForkedAfterNativeRuntimeStarted: + pass + else: + os._exit(1) + os._exit(0) + assert os.waitpid(grandchild, 0)[1] == 0 + assert [item.ids for item in tokenizer.encode_batch(["hello", "world"])] == expected + os._exit(0) +assert os.waitpid(pid, 0)[1] == 0 +""" + result: Final = run_child_interpreter( + script, + env={ + **os.environ, + "OBJC_DISABLE_INITIALIZE_FORK_SAFETY": "YES", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "WARM_FAST_COUNTER": str(warm_fast_counter), + }, + timeout=30, + ) + + assert result.returncode == 0, result.stderr diff --git a/tests/test_litellm_rust/test_tokenizer.py b/tests/test_litellm_rust/test_tokenizer.py new file mode 100644 index 00000000000..98d5259b652 --- /dev/null +++ b/tests/test_litellm_rust/test_tokenizer.py @@ -0,0 +1,130 @@ +import json +from typing import Final + +import pytest +import tiktoken +from tokenizers import Tokenizer as ReferenceTokenizer + +from litellm.rust_bridge import _native +from litellm.utils import claude_json_str +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + +pytestmark = pytest.mark.requires_rust_extension + + +def test_tiktoken_codec_round_trips_and_counts() -> None: + tokenizer: Final = _native.Tokenizer.from_tiktoken("cl100k_base") + encoded: Final = tokenizer.encode("hello world") + + assert tokenizer.name == "cl100k_base" + assert tokenizer.count("hello world") == len(encoded) + assert tokenizer.decode(encoded) == "hello world" + + +def test_huggingface_codec_skips_special_tokens() -> None: + tokenizer: Final = _native.Tokenizer.from_json(claude_json_str) + encoded: Final = tokenizer.encode("hello") + + assert "" in tokenizer.decode(encoded, skip_special_tokens=False) + assert tokenizer.decode(encoded, skip_special_tokens=True) == "hello" + + +def test_tiktoken_codec_keeps_the_requested_encoding_name() -> None: + assert _native.Tokenizer.from_tiktoken("gpt2").name == "gpt2" + assert _native.Tokenizer.from_tiktoken("r50k_base").name == "r50k_base" + assert _native.Tokenizer.from_tiktoken("gpt2").encode("hi") == _native.Tokenizer.from_tiktoken("r50k_base").encode( + "hi" + ) + + +def test_tiktoken_codec_exposes_its_vocabulary() -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + tokenizer: Final = _native.Tokenizer.from_tiktoken("cl100k_base") + + assert tokenizer.special_tokens() == reference._special_tokens + assert tokenizer.max_token_value() == reference.max_token_value + assert tokenizer.token_byte_values() == reference.token_byte_values() + assert tokenizer.encode_single_token(b"hello") == reference.encode_single_token("hello") + assert tokenizer.is_special_token(reference.eot_token) and not tokenizer.is_special_token(0) + with pytest.raises(KeyError): + tokenizer.encode_single_token(b"<|not-a-token|>") + + +def test_huggingface_codec_rejects_tiktoken_only_calls() -> None: + tokenizer: Final = _native.Tokenizer.from_json(claude_json_str) + with pytest.raises(ValueError, match="requires a tiktoken encoding"): + tokenizer.token_byte_values() + with pytest.raises(ValueError, match="requires a Hugging Face tokenizer"): + _native.Tokenizer.from_tiktoken("cl100k_base").get_vocab() + + +def test_unknown_tiktoken_encoding_raises_value_error() -> None: + with pytest.raises(ValueError, match="unsupported tokenizer"): + _native.Tokenizer.from_tiktoken("unknown-encoding") + + +def test_tiktoken_codec_decodes_truncated_unicode_like_python() -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + tokenizer: Final = _native.Tokenizer.from_tiktoken(reference.name) + encoded: Final = reference.encode("🙂漢字") + + assert tuple(tokenizer.decode(encoded[:end]) for end in range(1, len(encoded) + 1)) == tuple( + reference.decode(encoded[:end]) for end in range(1, len(encoded) + 1) + ) + + +FAST_TEXTS: Final = ( + "", + "hello world <|endoftext|>", + "café 漢字 ع 🙂 line\r\n indented 123456789", + "x a\u0301 fi", +) + + +def test_fast_counting_is_an_opt_in_over_the_same_loaded_tokenizer() -> None: + for tokenizer in ( + _native.Tokenizer.from_tiktoken("cl100k_base"), + _native.Tokenizer.from_tiktoken("o200k_base"), + _native.Tokenizer.from_json(claude_json_str), + ): + assert [tokenizer.count(text, fast=True) for text in FAST_TEXTS] == [ + tokenizer.count(text) for text in FAST_TEXTS + ] + + +@pytest.mark.parametrize( + "name", ("cl100k_base", "o200k_base", "o200k_harmony", "p50k_base", "p50k_edit", "r50k_base", "gpt2") +) +@pytest.mark.asyncio +async def test_token_counter_counts_over_a_shared_tokenizer(name: str) -> None: + messages: Final = [{"role": "user", "content": "hello wide world"}, {"role": "assistant", "content": "ok"}] + body: Final = json.dumps({"model": "gpt-4", "messages": messages}).encode() + tokenizer: Final = _native.Tokenizer.from_tiktoken(name) + reference: Final = tiktoken.get_encoding(name) + for text in FAST_TEXTS: + assert tokenizer.count(text, fast=True) == tokenizer.count(text) == len(reference.encode_ordinary(text)) + + exact: Final = await _native.TokenCounter.from_tokenizer(tokenizer).acount_request(body) + fast: Final = await _native.TokenCounter.from_tokenizer(tokenizer, fast=True).acount_request(body) + + assert exact == fast + assert exact["input_tokens"] == 3 + sum( + 3 + len(reference.encode_ordinary(message["role"])) + len(reference.encode_ordinary(message["content"])) + for message in messages + ) + + +@pytest.mark.parametrize("configured", (False, True)) +@pytest.mark.asyncio +async def test_fast_count_preserves_huggingface_configuration(configured: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + if configured: + reference.enable_truncation(max_length=3) + reference.enable_padding(pad_id=0, pad_token="[UNK]", length=5) + tokenizer: Final = _native.Tokenizer.from_json(reference.to_str()) + counter: Final = _native.TokenCounter.from_tokenizer(tokenizer, fast=True) + for text in ("", "Hello", "Hello World Hello World", "[BOS] Hello"): + expected: Final = len(reference.encode(text)) + assert tokenizer.count(text, fast=True) == tokenizer.count(text) == expected + result: Final = await counter.acount_request(json.dumps({"prompt": text}).encode()) + assert result["input_tokens"] == expected diff --git a/uv.lock b/uv.lock index 18d57aa991b..32d7580f61b 100644 --- a/uv.lock +++ b/uv.lock @@ -1162,14 +1162,11 @@ wheels = [ [[package]] name = "click" -version = "8.3.3" +version = "8.5.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, ] [[package]] @@ -1989,11 +1986,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.29.0" +version = "3.32.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/59/e19834834cb01a32febfbb0f8a23a9088088f5d45991824ff2bc3b5e8acb/filelock-3.32.7.tar.gz", hash = "sha256:37b8a3d9811b0f9aef7e5ec5c71bb320de52df51e6ca9bcd6f5ad81187660da7", size = 225154, upload-time = "2026-09-16T00:24:20.907Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/31098c5aeb4d966b553641472bd55fcf5fdfac953549894b8a765ba44e91/filelock-3.32.7-py3-none-any.whl", hash = "sha256:65ff0d0190ea42038b32bda4b77834fb05be2cad4c5b9b01aa4dfb3614536e52", size = 100157, upload-time = "2026-09-16T00:24:19.543Z" }, ] [[package]] @@ -2225,11 +2222,11 @@ wheels = [ [[package]] name = "fsspec" -version = "2026.4.0" +version = "2026.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d5/8d/1c51c094345df128ca4a990d633fe1a0ff28726c9e6b3c41ba65087bba1d/fsspec-2026.4.0.tar.gz", hash = "sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4", size = 312760, upload-time = "2026-04-29T20:42:38.635Z" } +sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, ] [[package]] @@ -3146,34 +3143,26 @@ wheels = [ [[package]] name = "hf-xet" -version = "1.5.0" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/d8/5c06fc76461418326a7decf8367480c35be11a41fd938633929c60a9ec6b/hf_xet-1.5.0.tar.gz", hash = "sha256:e0fb0a34d9f406eed88233e829a67ec016bec5af19e480eac65a233ea289a948", size = 837196, upload-time = "2026-05-06T06:18:15.583Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/68/9b/6912c99070915a4f28119e3c5b52a9abd1eec0ad5cb293b8c967a0c6f5a2/hf_xet-1.5.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:7d70fe2ce97b9db73b9c9b9c81fe3693640aec83416a966c446afea54acfae3c", size = 4023383, upload-time = "2026-05-06T06:17:53.947Z" }, - { url = "https://files.pythonhosted.org/packages/0f/6d/9563cfde59b5d8128a9c7ec972a087f4c782e4f7bac5a85234edfd5d5e49/hf_xet-1.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:73a0dae8c71de3b0633a45c73f4a4a5ed09e94b43441d82981a781d4f12baa42", size = 3792751, upload-time = "2026-05-06T06:17:51.791Z" }, - { url = "https://files.pythonhosted.org/packages/07/a5/ed5a0cf35b49a0571af5a8f53416dad1877a718c021c9937c3a53cb45781/hf_xet-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a60290ec57e9b71767fba7c3645ddafdd0759974b540441510c629c6db6db24a", size = 4456058, upload-time = "2026-05-06T06:17:40.735Z" }, - { url = "https://files.pythonhosted.org/packages/60/fb/3ae8bf2a7a37a4197d0195d7247fd25b3952e15cb8a599e285dfaa6f52b3/hf_xet-1.5.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e5de0f6deada0dada870bb376a11bcd1f08abf3a968a6d118f33e72d1b1eb480", size = 4250783, upload-time = "2026-05-06T06:17:38.412Z" }, - { url = "https://files.pythonhosted.org/packages/a2/9b/8bae40d4d91525085137196e84eb0ed49cf65b5e96e5c3ecdadd8bd0fac2/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c799d49f1a5544a0ef7591c0ee75e0d6b93d6f56dc7a4979f59f7518d2872216", size = 4445594, upload-time = "2026-05-06T06:18:04.219Z" }, - { url = "https://files.pythonhosted.org/packages/13/59/c74efbbd4e8728172b2cc72a2bc014d2947a4b7bdced932fbd3f5da1a4e5/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2baea1b0b989e5c152fe81425f7745ddc8901280ba3d97c98d8cdece7b706c60", size = 4663995, upload-time = "2026-05-06T06:18:06.1Z" }, - { url = "https://files.pythonhosted.org/packages/73/32/8e1e0410af64cda9b139d1dcebdc993a8ff9c8c7c0e2696ae356d75ccc0d/hf_xet-1.5.0-cp313-cp313t-win_amd64.whl", hash = "sha256:526345b3ed45f374f6317349df489167606736c876241ba984105afe7fd4839d", size = 3966608, upload-time = "2026-05-06T06:18:19.74Z" }, - { url = "https://files.pythonhosted.org/packages/fc/34/a8febc8f4edbea8b3e21b02ebc8b628679b84ba7e45cde624a7736b51500/hf_xet-1.5.0-cp313-cp313t-win_arm64.whl", hash = "sha256:786d28e2eb8315d5035544b9d137b4a842d600c434bb91bf7d0d953cce906ad4", size = 3796946, upload-time = "2026-05-06T06:18:17.568Z" }, - { url = "https://files.pythonhosted.org/packages/2a/20/8fc8996afe5815fa1a6be8e9e5c02f24500f409d599e905800d498a4e14d/hf_xet-1.5.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:872d5601e6deea30d15865ede55d29eac6daf5a534ab417b99b6ef6b076dd96c", size = 4023495, upload-time = "2026-05-06T06:18:01.94Z" }, - { url = "https://files.pythonhosted.org/packages/32/6a/93d84463c00cecb561a7508aa6303e35ee2894294eac14245526924415fe/hf_xet-1.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9929561f5abf4581c8ea79587881dfef6b8abb2a0d8a51915936fc2a614f4e73", size = 3792731, upload-time = "2026-05-06T06:18:00.021Z" }, - { url = "https://files.pythonhosted.org/packages/9d/5a/8ec8e0c863b382d00b3c2e2af6ded6b06371be617144a625903a6d562f4b/hf_xet-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7b7bbae318e583a86fb21e5a4a175d6721d628a2874f4bd022d0e660c32a682", size = 4456738, upload-time = "2026-05-06T06:17:49.574Z" }, - { url = "https://files.pythonhosted.org/packages/c5/ca/f7effa1a67717da2bcc6b6c28f71c6ca648c77acaec4e2c32f40cbe16d85/hf_xet-1.5.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:cf7b2dc6f31a4ea754bb50f74cde482dcf5d366d184076d8530b9872787f3761", size = 4251622, upload-time = "2026-05-06T06:17:47.096Z" }, - { url = "https://files.pythonhosted.org/packages/65/f2/19247dba3e231cf77dec59ddfb878f00057635ff773d099c9b59d37812c3/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8dbcbab554c9ef158ef2c991545c3e970ddd8cc7acdcd0a78c5a41095dab4ded", size = 4445667, upload-time = "2026-05-06T06:18:11.983Z" }, - { url = "https://files.pythonhosted.org/packages/7f/64/6f116801a3bcfb6f59f5c251f48cadc47ea54026441c4a385079286a94fa/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5906bf7718d3636dc13402914736abe723492cb730f744834f5f5b67d3a12702", size = 4664619, upload-time = "2026-05-06T06:18:13.771Z" }, - { url = "https://files.pythonhosted.org/packages/5c/e8/069542d37946ed08669b127e1496fa99e78196d71de8d41eda5e9f1b7a58/hf_xet-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f3dc2248fc01cc0a00cd392ab497f1ca373fcbc7e3f2da1f452480b384e839e", size = 3966802, upload-time = "2026-05-06T06:18:28.162Z" }, - { url = "https://files.pythonhosted.org/packages/f9/91/fc6fdec27b14d04e88c386ac0a0129732b53fa23f7c4a78f4b83a039c567/hf_xet-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b285cea1b5bab46b758772716ba8d6854a1a0310fed1c249d678a8b38601e5a0", size = 3797168, upload-time = "2026-05-06T06:18:26.287Z" }, - { url = "https://files.pythonhosted.org/packages/3d/fb/69ff198a82cae7eb1a69fb84d93b3a3e4816564d76817fe541ddc96874eb/hf_xet-1.5.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dad0dc84e941b8ba3c860659fe1fdc35c049d47cce293f003287757e971a8f56", size = 4030814, upload-time = "2026-05-06T06:17:57.933Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ff/edcc2b40162bef3ff78e14ab637e5f3b89243d6aee72f5949d3bb6a5af83/hf_xet-1.5.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a", size = 3798444, upload-time = "2026-05-06T06:17:55.79Z" }, - { url = "https://files.pythonhosted.org/packages/49/4d/103f76b04310e5e57656696cc184690d20c466af0bca3ca88f8c8ea5d4f3/hf_xet-1.5.0-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3531b1823a0e6d77d80f9ed15ca0e00f0d115094f8ac033d5cae88f4564cc949", size = 4465986, upload-time = "2026-05-06T06:17:44.886Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a2/546f47f464737b3edbab6f8ddb57f2599b93d2cbb66f06abb475ccb48651/hf_xet-1.5.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9a0ee58cd18d5ea799f7ed11290bbccbe56bdd8b1d97ca74b9cc49a3945d7a3b", size = 4259865, upload-time = "2026-05-06T06:17:42.639Z" }, - { url = "https://files.pythonhosted.org/packages/95/7f/1be593c1f28613be2e196473481cd81bfc5910795e30a34e8f744f6cac4f/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18", size = 4459835, upload-time = "2026-05-06T06:18:08.026Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b2/703569fc881f3284487e68cda7b42179978480da3c438042a6bbbb4a671c/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b35549ce62601b84da4ff9b24d970032ace3d4430f52d91bcbb26c901d6c690", size = 4672414, upload-time = "2026-05-06T06:18:09.864Z" }, - { url = "https://files.pythonhosted.org/packages/af/37/1b6def445c567286b50aa3b33828158e135b1be44938dde59f11382a500c/hf_xet-1.5.0-cp37-abi3-win_amd64.whl", hash = "sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4", size = 3977238, upload-time = "2026-05-06T06:18:23.621Z" }, - { url = "https://files.pythonhosted.org/packages/62/94/3b66b148778ee100dcfd69c2ca22b57b41b44d3063ceec934f209e9184ce/hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be", size = 3806916, upload-time = "2026-05-06T06:18:21.7Z" }, + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, ] [[package]] @@ -3381,22 +3370,23 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "1.14.0" +version = "1.32.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "click" }, { name = "filelock" }, { name = "fsspec" }, { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, { name = "httpx" }, { name = "packaging" }, { name = "pyyaml" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "tqdm" }, - { name = "typer" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/39/40/43109e943fd718b0ccd0cd61eb4f1c347df22bf81f5874c6f22adf44bcff/huggingface_hub-1.14.0.tar.gz", hash = "sha256:d6d2c9cd6be1d02ae9ec6672d5587d10a427f377db688e82528f426a041622c2", size = 782365, upload-time = "2026-05-06T14:14:34.278Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/0f/e83fdd856da8fca26bf78d71709ebd120432a0ce535e72b9597cab1eb5bf/huggingface_hub-1.32.0.tar.gz", hash = "sha256:ed70a45498abe86039df7c2f4e5f7575de524be908d3840e8f828d5525eafd6a", size = 1038662, upload-time = "2026-09-17T10:27:48.049Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/89/a5/33b49ba7bea7c41bb37f74ec0f8beea0831e052330196633fe2c77516ea6/huggingface_hub-1.14.0-py3-none-any.whl", hash = "sha256:efe075535c62e130b30e836b138e13785f6f043d1f0539e0a39aa411a99e90b8", size = 661479, upload-time = "2026-05-06T14:14:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/1b/cf/d98dd561d6d0d7b7d7a64d1563f8aaaa7c235daee41c1c9bcc3da62420ed/huggingface_hub-1.32.0-py3-none-any.whl", hash = "sha256:b0c7c80561969d9cdacdd55fce67ba9584cca0b9d4ea80957a3a5c1445fac5c8", size = 842906, upload-time = "2026-09-17T10:27:46.102Z" }, ] [[package]] @@ -4518,6 +4508,7 @@ dependencies = [ { name = "fastuuid" }, { name = "filelock" }, { name = "httpx", extra = ["http2"] }, + { name = "huggingface-hub" }, { name = "importlib-metadata" }, { name = "jinja2" }, { name = "jsonschema" }, @@ -4526,6 +4517,7 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, { name = "python-dotenv" }, + { name = "pyyaml" }, { name = "tiktoken" }, { name = "tokenizers" }, ] @@ -4650,6 +4642,12 @@ utils = [ ] [package.dev-dependencies] +benchmarks = [ + { name = "a2a-sdk" }, + { name = "mcp" }, + { name = "pytest" }, + { name = "pytest-codspeed" }, +] ci = [ { name = "aiodynamo" }, { name = "anthropic" }, @@ -4689,6 +4687,8 @@ dev = [ { name = "keyring" }, { name = "langfuse" }, { name = "mypy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "openapi-core" }, { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp" }, @@ -4787,6 +4787,7 @@ requires-dist = [ { name = "httpx", extras = ["http2"], specifier = ">=0.28.0,<1.0" }, { name = "httpx2", marker = "extra == 'mcp'", specifier = ">=2.5.0,<3" }, { name = "httpx2", marker = "extra == 'proxy'", specifier = ">=2.5.0,<3" }, + { name = "huggingface-hub", specifier = ">=0.34.0,<2.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" }, { name = "inquirerpy", marker = "extra == 'proxy'", specifier = ">=0.3.4,<1.0" }, @@ -4805,12 +4806,12 @@ requires-dist = [ { name = "numpydoc", marker = "extra == 'utils'", specifier = ">=1.8.0,<2.0" }, { name = "nvidia-riva-client", marker = "extra == 'stt-nvidia-riva'", specifier = ">=2.15.0" }, { name = "openai", specifier = ">=2.20.0,<3.0.0" }, - { name = "packaging", specifier = ">=24.0" }, { name = "opentelemetry-api", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "opentelemetry-exporter-otlp", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "opentelemetry-instrumentation-fastapi", marker = "extra == 'proxy-runtime'", specifier = "==0.49b0" }, { name = "opentelemetry-sdk", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "orjson", marker = "extra == 'proxy'", specifier = ">=3.11.6,<4.0" }, + { name = "packaging", specifier = ">=24.0" }, { name = "polars", marker = "extra == 'proxy'", specifier = ">=1.38.1,<2.0" }, { name = "prisma", marker = "extra == 'extra-proxy'", specifier = ">=0.11.0,<1.0" }, { name = "prometheus-client", marker = "extra == 'proxy-runtime'", specifier = ">=0.20.0,<1.0" }, @@ -4828,6 +4829,7 @@ requires-dist = [ { name = "python-dotenv", specifier = ">=1.0.0,<2.0" }, { name = "python-multipart", marker = "extra == 'proxy'", specifier = ">=0.0.27,<1.0" }, { name = "python3-saml", marker = "extra == 'saml'", specifier = ">=1.16.0,<2.0" }, + { name = "pyyaml", specifier = ">=6.0.3,<7.0" }, { name = "pyyaml", marker = "extra == 'cli'", specifier = ">=6.0.3,<7.0" }, { name = "pyyaml", marker = "extra == 'proxy'", specifier = ">=6.0.3,<7.0" }, { name = "redisvl", marker = "extra == 'extra-proxy'", specifier = ">=0.4.1,<1.0" }, @@ -4854,6 +4856,12 @@ requires-dist = [ provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] [package.metadata.requires-dev] +benchmarks = [ + { name = "a2a-sdk", specifier = "==1.1.0" }, + { name = "mcp", specifier = ">=2.2.0,<3" }, + { name = "pytest", specifier = "==9.0.3" }, + { name = "pytest-codspeed", specifier = "==4.3.0" }, +] ci = [ { name = "aiodynamo", specifier = "==24.7" }, { name = "anthropic", specifier = "==0.84.0" }, @@ -4893,6 +4901,7 @@ dev = [ { name = "keyring", specifier = "==25.7.0" }, { name = "langfuse", specifier = "==2.59.7" }, { name = "mypy", specifier = "==1.20.1" }, + { name = "numpy", specifier = ">=1.26.0,<3.0" }, { name = "openapi-core", specifier = "==0.22.0" }, { name = "opentelemetry-api", specifier = "==1.28.0" }, { name = "opentelemetry-exporter-otlp", specifier = "==1.28.0" }, @@ -9171,15 +9180,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, ] -[[package]] -name = "shellingham" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, -] - [[package]] name = "simple-websocket" version = "1.1.0" @@ -9920,21 +9920,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, ] -[[package]] -name = "typer" -version = "0.25.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "click" }, - { name = "rich" }, - { name = "shellingham" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e4/51/9aed62104cea109b820bbd6c14245af756112017d309da813ef107d42e7e/typer-0.25.1.tar.gz", hash = "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc", size = 122276, upload-time = "2026-04-30T19:32:16.964Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/f9/2b3ff4e56e5fa7debfaf9eb135d0da96f3e9a1d5b27222223c7296336e5f/typer-0.25.1-py3-none-any.whl", hash = "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89", size = 58409, upload-time = "2026-04-30T19:32:18.271Z" }, -] - [[package]] name = "types-awscrt" version = "0.34.1" From 55e95c0279928c30376b6ecb4421893ad6d02593 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:42:52 -0700 Subject: [PATCH 04/28] fix(presidio): mask PII in streaming /v1/messages output (#42351) * fix(presidio): mask PII in streaming /v1/messages output Raw Anthropic SSE frames were passed through the post_call output masking callback untouched, and ProxyLogging rerouted the callback to the unified apply_guardrail path on /v1/messages because mask_response_content was false. Buffer the raw frames, assemble them with the shared Anthropic SSE helpers, mask through Presidio, and re-emit the masked frames. Resolves LIT-8288 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(presidio): replay raw SSE frames when masking fails mid-stream Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(presidio): propagate upstream stream errors instead of returning an empty stream Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(presidio): extract buffered stream masking to satisfy complexity budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(presidio): let BLOCK on generated PII refuse the streaming /v1/messages response Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(presidio): fold the BLOCK re-raise into the existing except to stay within the complexity budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(presidio): move the blocked stream consumption into a helper so pytest.raises holds one statement Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(presidio): fail closed when output masking of a raw SSE stream errors A Presidio outage on streaming /v1/messages replayed the unscanned frames to the caller. Propagate the error instead, matching the non streaming path and the merge base Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(presidio): cover structured chat stream output masking and trailing bytes passthrough Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrails/guardrail_hooks/presidio.py | 85 +++-- .../guardrails/guardrail_initializers.py | 1 + tests/e2e/coverage_registry/guardrail.yaml | 1 + tests/e2e/guardrails/guardrails_client.py | 22 +- .../guardrails/test_presidio_masking_e2e.py | 195 ++++++++++- .../guardrail_hooks/test_presidio.py | 310 +++++++++++++++++- .../test_proxy_logging_hook_detection.py | 62 ++++ 7 files changed, 625 insertions(+), 51 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 70ea21320ee..fe91d6d7a28 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -11,7 +11,7 @@ import asyncio import json import threading -from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Sequence from contextlib import asynccontextmanager from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypedDict, cast @@ -39,6 +39,11 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.guardrails.anthropic_sse import ( + anthropic_sse_chunks_from_response, + assemble_anthropic_sse_stream, + model_response_text, +) from litellm.types.guardrails import ( GuardrailEventHooks, LitellmParams, @@ -1327,30 +1332,44 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) return response - async def _stream_apply_output_masking( - self, - response: AsyncIterable[object], - request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: - """Apply Presidio masking to streaming output (apply_to_output=True path).""" + async def _mask_buffered_model_response_stream( + self, all_chunks: Sequence[ModelResponseStream], request_data: dict + ) -> tuple[object, ...]: from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, ) from litellm.main import stream_chunk_builder from litellm.types.utils import ModelResponse + assembled: Final = stream_chunk_builder(chunks=list(all_chunks), messages=request_data.get("messages")) + if not isinstance(assembled, ModelResponse): + return tuple(all_chunks) + await self._process_response_for_pii(response=assembled, request_data=request_data, mode="mask") + return (convert_model_response_to_streaming(assembled),) + + async def _stream_apply_output_masking( + self, + response: AsyncIterable[object], + request_data: dict, + ) -> AsyncGenerator[object, None]: + """Apply Presidio masking to streaming output (apply_to_output=True path).""" all_chunks: list[ModelResponseStream] = [] passthrough_due_to_unknown_stream_shape = False try: - async for chunk in response: + stream: Final = response.__aiter__() + async for chunk in stream: if isinstance(chunk, ModelResponseStream): if passthrough_due_to_unknown_stream_shape: yield chunk else: all_chunks.append(chunk) elif isinstance(chunk, bytes): - yield chunk - continue + if passthrough_due_to_unknown_stream_shape or all_chunks: + yield chunk + continue + for masked_chunk in await self._mask_anthropic_sse_stream(chunk, stream, request_data): + yield masked_chunk + return else: if all_chunks: # Flush buffered chunks and switch to transparent passthrough for this stream shape. @@ -1375,33 +1394,39 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if not all_chunks: verbose_proxy_logger.warning( "Presidio apply_to_output: streaming response contained no " - "ModelResponseStream chunks (e.g. raw SSE bytes or an empty " - "upstream stream). Output PII masking was skipped for this " - "response." + "ModelResponseStream chunks (an empty upstream stream). " + "Output PII masking was skipped for this response." ) return - assembled_model_response = stream_chunk_builder(chunks=all_chunks, messages=request_data.get("messages")) - - if not isinstance(assembled_model_response, ModelResponse): - for chunk in all_chunks: - yield chunk - return - - await self._process_response_for_pii( - response=assembled_model_response, - request_data=request_data, - mode="mask", - ) - - mock_response_stream: Final = convert_model_response_to_streaming(assembled_model_response) - yield mock_response_stream + for masked_chunk in await self._mask_buffered_model_response_stream(all_chunks, request_data): + yield masked_chunk except Exception as e: + if not all_chunks or isinstance(e, BlockedPiiEntityError): + raise verbose_proxy_logger.error("Error masking streaming PII output: %s", e) for chunk in all_chunks: yield chunk + async def _mask_anthropic_sse_stream( + self, first_chunk: bytes, rest: AsyncIterator[object], request_data: dict + ) -> tuple[object, ...]: + rest_chunks: Final = [chunk async for chunk in rest] # mutable-ok: tuple() cannot consume an async iterator + chunks: Final = (first_chunk, *rest_chunks) + assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True) + if assembled is None: + verbose_proxy_logger.warning( + "Presidio apply_to_output: raw SSE stream could not be assembled into a response. " + "Output PII masking was skipped for this response." + ) + return chunks + original_text: Final = model_response_text(assembled) + await self._process_response_for_pii(response=assembled, request_data=request_data, mode="mask") + if model_response_text(assembled) == original_text: + return chunks + return anthropic_sse_chunks_from_response(assembled) + @staticmethod def _unmask_sse_bytes_chunk(chunk: bytes, pii_tokens: dict[str, str]) -> bytes: try: @@ -1460,7 +1485,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): self, response: AsyncIterable[object], request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: + ) -> AsyncGenerator[object, None]: """Apply PII unmasking to streaming output (output_parse_pii=True path).""" from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, @@ -1536,7 +1561,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): user_api_key_dict: UserAPIKeyAuth, response: AsyncIterable[object], request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: + ) -> AsyncGenerator[object, None]: """ Process streaming response chunks to unmask PII tokens when needed. diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 356eb7c96c6..7bafad26569 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -165,6 +165,7 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail) -> apply_to_output=True, event_hook=GuardrailEventHooks.post_call.value, output_parse_pii=False, + mask_response_content=True, ) if run_output else None diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml index 86eb44f6cb1..4fe9f949fae 100644 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ b/tests/e2e/coverage_registry/guardrail.yaml @@ -2,6 +2,7 @@ # Rolls up into the "Logging & Guardrails" dashboard module together with logging.* - {id: guardrail.presidio.pre_call.masks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "PII masking pre-call; data-leak blast radius"} - {id: guardrail.presidio.post_call.masks, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Mask PII in model output"} +- {id: guardrail.presidio.post_call.masks_generated_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, anthropic_messages_stream], source: "guardrail_hooks/presidio.py", rationale: "Mask model-generated credit-card output with the UI default scope"} - {id: guardrail.presidio.logging_only.masks, module: guardrail, tier: P0, hook_point: logging_only, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Redact in logs without blocking"} - {id: guardrail.presidio.pre_call.logs_masked_entities, module: guardrail, tier: P0, hook_point: pre_call, assertions: [logs_masked_entities], exercised_on: [chat_completions], source: "guardrail_hooks/presidio.py", rationale: "A masking run must record itself on the spend log: the dashboard's guardrail panel renders the masked-entity counts and per-entity scores straight off metadata.guardrail_information, so a run that masks but records nothing leaves an operator unable to audit it"} - {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"} diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 3ceac737399..97ecac0290f 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -46,7 +46,7 @@ class BlockedWordBody(BaseModel): class GuardrailParamsBase(BaseModel): - mode: GuardrailMode + mode: GuardrailMode | list[GuardrailMode] default_on: bool @@ -381,6 +381,26 @@ class GuardrailsClient: ), ) + def messages_stream_raw( + self, + key: str, + model: str, + text: str, + *, + guardrails: list[str] | None = None, + max_tokens: int = 64, + ) -> StreamingResponse: + return self.proxy.messages_stream( + key, + AnthropicMessagesBody( + model=model, + messages=[ChatMessage(role="user", content=text)], + max_tokens=max_tokens, + stream=True, + guardrails=guardrails, + ), + ) + def responses( self, key: str, diff --git a/tests/e2e/guardrails/test_presidio_masking_e2e.py b/tests/e2e/guardrails/test_presidio_masking_e2e.py index 49d698938ce..d47d64be9e3 100644 --- a/tests/e2e/guardrails/test_presidio_masking_e2e.py +++ b/tests/e2e/guardrails/test_presidio_masking_e2e.py @@ -30,6 +30,7 @@ this suite deliberately requires the detected-entity details to remain visible. from __future__ import annotations import os +import re import time from collections.abc import Callable from typing import Final, Literal @@ -65,9 +66,13 @@ GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS = 5.0 # angle brackets; the logged payload keeps the placeholder verbatim. MASKED_EMAIL_TOKEN = "EMAIL_ADDRESS" MASKED_PHONE_TOKEN = "PHONE_NUMBER" +MASKED_CREDIT_CARD_TOKEN = "CREDIT_CARD" # Fictional NANP 555 number; a standard format Presidio's phone recognizer detects. FAKE_PHONE = "+1 415-555-0134" +FAKE_VISA_TEST_CARD = "4111 1111 1111 1111" + +_CARD_DIGIT_RUN: Final = re.compile(r"(?:\d[ -]?){13,19}") def _presidio_bases() -> tuple[str, str]: @@ -86,8 +91,8 @@ def _register_presidio( resources: ResourceManager, *, name: str, - mode: GuardrailMode = "pre_call", - filter_scope: Literal["input", "output", "both"] = "input", + mode: GuardrailMode | list[GuardrailMode] = "pre_call", + filter_scope: Literal["input", "output", "both"] | None = "input", entities: dict[PiiEntity, PiiAction] | None = None, ) -> None: analyzer, anonymizer = _presidio_bases() @@ -123,6 +128,74 @@ def _first_content(response: ChatResponse) -> str: return (message.content if message else None) or "" +class _StreamDelta(BaseModel): + content: str | None = None + + +class _StreamChoice(BaseModel): + delta: _StreamDelta + + +class _StreamChunk(BaseModel): + choices: tuple[_StreamChoice, ...] = () + + +class _AnthropicStreamDelta(BaseModel): + type: str | None = None + text: str | None = None + + +class _AnthropicStreamEvent(BaseModel): + type: str + delta: _AnthropicStreamDelta | None = None + + +def _credit_card_prompt(marker: str) -> str: + return ( + f"{marker} Reply with only the well known Visa sandbox test card number that starts with 4111, " + "the 16 digits grouped in fours separated by spaces, and nothing else." + ) + + +def _passes_luhn(digits: str) -> bool: + checksum = sum( + digit if position % 2 == 0 else (digit * 2 - 9 if digit * 2 > 9 else digit * 2) + for position, digit in enumerate(int(char) for char in reversed(digits)) + ) + return checksum % 10 == 0 + + +def _contains_card_number(text: str) -> bool: + """Presidio's CREDIT_CARD recognizer only reports Luhn-valid digit runs, so a + Luhn-invalid number the model hallucinates is not something masking can catch.""" + return any( + 13 <= len(digits) <= 19 and _passes_luhn(digits) + for digits in (re.sub(r"[ -]", "", match.group()) for match in _CARD_DIGIT_RUN.finditer(text)) + ) + + +def _stream_content(result: StreamingResponse) -> str: + return "".join( + choice.delta.content + for event in result.stream_events + if event != "[DONE]" + for choice in _StreamChunk.model_validate_json(event).choices[:1] + if choice.delta.content + ) + + +def _anthropic_stream_content(result: StreamingResponse) -> str: + return "".join( + event.delta.text + for payload in result.stream_events + for event in [_AnthropicStreamEvent.model_validate_json(payload)] + if event.type == "content_block_delta" + and event.delta is not None + and event.delta.type == "text_delta" + and event.delta.text + ) + + def _messages_text(response: AnthropicMessagesResponse) -> str: """The text of a /v1/messages answer, whichever shape the proxy produced (Anthropic-native content blocks or OpenAI-normalized choices).""" @@ -290,6 +363,124 @@ class TestPresidioPostCallMasking: time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) +def _assert_eventually_masks_generated_card(fetch: Callable[[], str | None]) -> None: + deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS + last: str = "" + while True: + content = fetch() + if content is not None: + last = content + if _contains_card_number(content): + pytest.fail( + "the post_call output masking let a card number through: " + f"{content[:300]!r}" + ) + if MASKED_CREDIT_CARD_TOKEN in content: + return + if time.monotonic() >= deadline: + pytest.fail( + "presidio post_call output masking never masked the generated card within " + f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}" + ) + time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) + + +class TestPresidioCreditCardOutputMasking: + """Proves the UI-default Presidio scope masks model-generated card output.""" + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["chat_completions"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-chat-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=512) + match result: + case Success(data=data): + return _first_content(data) + case _: + return None + + _assert_eventually_masks_generated_card(fetch) + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["chat_completions_stream"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-stream-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.chat_stream_raw( + scoped_key, + MODEL, + prompt, + guardrails=[name], + max_tokens=512, + ) + if not result.ok or result.stream_error: + return None + return _stream_content(result) + + _assert_eventually_masks_generated_card(fetch) + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["anthropic_messages_stream"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_anthropic_messages( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-messages-stream-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.messages_stream_raw( + scoped_key, + MODEL, + prompt, + guardrails=[name], + max_tokens=512, + ) + if not result.ok or result.stream_error: + return None + return _anthropic_stream_content(result) + + _assert_eventually_masks_generated_card(fetch) + + _LOGGED_ENTITIES: dict[PiiEntity, PiiAction] = {"EMAIL_ADDRESS": "MASK", "PHONE_NUMBER": "MASK"} _ENTITY_LIST_ADAPTER: Final = TypeAdapter(list[GuardrailEntityMatch]) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 33614d2eeca..89e72debbf5 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -4,6 +4,7 @@ Tests PII detection and masking for different message formats """ import asyncio +import json from contextlib import asynccontextmanager from unittest.mock import MagicMock, patch @@ -18,7 +19,7 @@ from litellm.proxy.guardrails.guardrail_hooks.presidio import ( ) from litellm.exceptions import GuardrailRaisedException from litellm.types.guardrails import LitellmParams, PiiAction, PiiEntityType -from litellm.types.utils import Choices, Message, ModelResponse +from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices from litellm.exceptions import BlockedPiiEntityError @@ -2331,47 +2332,320 @@ async def test_apply_guardrail_masks_on_request(): assert "John Smith" not in result["texts"][0] +def _anthropic_sse(event_type: str, payload: dict) -> bytes: + return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode() + + +def _anthropic_text_deltas(chunks: list[bytes]) -> list[tuple[int, str]]: + deltas = [] + for line in b"".join(chunks).decode().split("\n"): + if not line.startswith("data: "): + continue + event = json.loads(line[6:]) + if event.get("type") == "content_block_delta" and event["delta"].get("type") == "text_delta": + deltas.append((event["index"], event["delta"]["text"])) + return deltas + + +def _chat_delta_chunk(text: str, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-out-mask", + choices=[StreamingChoices(index=0, delta=Delta(content=text, role="assistant"), finish_reason=finish_reason)], + created=1, + model="gpt-4", + object="chat.completion.chunk", + ) + + @pytest.mark.asyncio -async def test_apply_to_output_streaming_bytes_only_logs_warning(): +async def test_apply_to_output_streaming_chat_chunks_are_masked_as_one_response(): """ - Regression test: when apply_to_output=True and the stream contains only - bytes chunks (Anthropic native SSE), output masking is skipped. - A warning must be logged so operators are aware. + Structured chat completion chunks are buffered, assembled and masked as a + whole, so a card number split across deltas cannot reach the caller. """ guardrail = _OPTIONAL_PresidioPIIMasking( mock_testing=True, apply_to_output=True, + mock_redacted_text={"text": "my card is "}, + ) + + async def mock_stream(): + yield _chat_delta_chunk("my card is 4111") + yield _chat_delta_chunk(" 1111 1111 1111") + yield _chat_delta_chunk("", finish_reason="stop") + + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={"messages": [{"role": "user", "content": "what is my card"}]}, + ): + collected.append(chunk) + + assert all(isinstance(chunk, ModelResponseStream) for chunk in collected) + joined = "".join(chunk.choices[0].delta.content or "" for chunk in collected) + assert joined == "my card is " + assert collected[-1].choices[0].finish_reason == "stop" + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_bytes_after_chat_chunks_are_passed_through_in_order(): + """ + Once structured chunks have been buffered, a trailing bytes frame belongs to + the same stream and must be forwarded rather than treated as a new SSE stream. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": "hello"}, + ) + trailer = b"data: [DONE]\n\n" + + async def mock_stream(): + yield _chat_delta_chunk("hello", finish_reason="stop") + yield trailer + + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert collected[0] == trailer + assert len(collected) == 2 + assert isinstance(collected[1], ModelResponseStream) + assert collected[1].choices[0].delta.content == "hello" + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_masks_text_split_across_deltas(): + """ + Anthropic native /v1/messages streams reach the post_call hook as raw SSE + bytes. Output masking must run over the whole content block so a card + number split across text_delta events cannot reach the caller. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": ""}, ) byte_chunks = [ - b'data: {"type":"content_block_delta","delta":{"text":"Hello"}}\n\n', - b'data: {"type":"content_block_delta","delta":{"text":" world"}}\n\n', + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "4111"}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " 1111 1111 1111"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {}}), + _anthropic_sse("message_stop", {"type": "message_stop"}), ] async def mock_stream(): for b in byte_chunks: yield b - mock_user_api_key = UserAPIKeyAuth(api_key="test-key") + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert all(isinstance(chunk, bytes) for chunk in collected) + joined = b"".join(collected).decode() + assert "4111" not in joined + assert "".join(text for _, text in _anthropic_text_deltas(collected)) == "" + assert joined.count("event: message_start") == 1 + assert joined.count("event: message_stop") == 1 + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_without_pii_are_forwarded_unchanged(): + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": "Hello world"}, + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello"}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " world"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {}}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b collected = [] - with patch("litellm.proxy.guardrails.guardrail_hooks.presidio.verbose_proxy_logger") as mock_logger: + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert collected == byte_chunks + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_fail_closed_when_presidio_is_unreachable(): + """ + The raw SSE stream is fully drained before masking, so a Presidio outage + must surface as an error to the caller: replaying the unscanned frames + would hand over whatever PII the model generated. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + presidio_analyzer_api_base="http://127.0.0.1:9", + presidio_anonymizer_api_base="http://127.0.0.1:9", + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello world"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b + + collected = [] + + async def collect_masked_stream(): async for chunk in guardrail.async_post_call_streaming_iterator_hook( - user_api_key_dict=mock_user_api_key, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), response=mock_stream(), request_data={}, ): collected.append(chunk) - # All bytes should be yielded through - assert len(collected) == len(byte_chunks) - for original, received in zip(byte_chunks, collected): - assert original == received + with pytest.raises(Exception, match="Presidio PII analysis failed"): + await collect_masked_stream() - # Warning must be logged about skipped masking - mock_logger.warning.assert_called_once() - warning_msg = mock_logger.warning.call_args[0][0] - assert "Output PII masking was skipped" in warning_msg + assert collected == [] + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_block_action_raises_instead_of_replaying(): + """ + A BLOCK on generated PII must refuse the streaming /v1/messages response the + same way it refuses the non streaming one, not replay the raw frames. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + apply_to_output=True, + mock_testing=False, + presidio_analyzer_api_base="http://test-analyzer/", + presidio_anonymizer_api_base="http://test-anonymizer/", + pii_entities_config={PiiEntityType.CREDIT_CARD: PiiAction.BLOCK}, + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "4111 1111 1111 1111"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b + + analyzer_hit = [{"entity_type": "CREDIT_CARD", "score": 0.99, "start": 0, "end": 19}] + collected = [] + + async def collect_masked_stream(): + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + with patch.object(guardrail, "_get_session_iterator", _make_mock_session_iterator(analyzer_hit)): + with pytest.raises(BlockedPiiEntityError): + await collect_masked_stream() + + assert collected == [] + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_propagates_upstream_error_when_nothing_was_buffered(): + """ + An upstream guardrail that rejects the stream before the first chunk must + surface as an error to the caller, not as an empty 200 stream. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": ""}, + ) + + async def failing_stream(): + raise RuntimeError("upstream guardrail rejected the stream") + yield b"" + + with pytest.raises(RuntimeError, match="upstream guardrail rejected the stream"): + async for _ in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=failing_stream(), + request_data={}, + ): + pass @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index dd330d32ce6..f45715953f9 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -489,6 +489,68 @@ async def test_post_call_stream_masking_guardrail_keeps_own_iterator_on_anthropi assert delivered == chunks +@pytest.mark.asyncio +async def test_post_call_stream_presidio_output_masking_masks_anthropic_messages_stream(monkeypatch): + """Regression: the presidio output-masking callback built by initialize_presidio + was rerouted onto the unified scan-only path on /v1/messages, so a card number + the analyzer flagged still streamed to the caller unmasked.""" + import json + + from litellm.caching.caching import DualCache + from litellm.proxy.guardrails.guardrail_registry import InMemoryGuardrailHandler + from litellm.types.guardrails import SupportedGuardrailIntegrations + + handler = InMemoryGuardrailHandler() + result = handler.initialize_guardrail( + guardrail={ + "guardrail_name": "presidio-card-mask", + "litellm_params": { + "guardrail": SupportedGuardrailIntegrations.PRESIDIO.value, + "mode": ["pre_call", "post_call"], + "default_on": True, + "presidio_analyzer_api_base": "https://fakelink.com/v1/presidio/analyze", + "presidio_anonymizer_api_base": "https://fakelink.com/v1/presidio/anonymize", + "pii_entities_config": {"CREDIT_CARD": "MASK"}, + "mock_redacted_text": {"text": "", "items": []}, + }, + } + ) + guardrail_id = result["guardrail_id"] + callbacks = [ + handler.guardrail_id_to_custom_guardrail[guardrail_id], + *handler.guardrail_id_to_sibling_callbacks[guardrail_id], + ] + monkeypatch.setattr(litellm, "callbacks", callbacks) + + chunks = _anthropic_stream_chunks(["4111", " 1111 1111 1111"]) + + async def fake_stream(): + for chunk in chunks: + yield chunk + + delivered = [] + async for chunk in ProxyLogging(user_api_key_cache=DualCache()).async_post_call_streaming_iterator_hook( + response=fake_stream(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/messages"), + request_data={ + "model": "claude-sonnet-5", + "litellm_logging_obj": _streaming_logging_obj(), + "metadata": {}, + }, + ): + delivered.append(chunk) + + wire = b"".join(delivered).decode() + text_deltas = [ + json.loads(line[6:])["delta"]["text"] + for line in wire.split("\n") + if line.startswith("data: ") and json.loads(line[6:]).get("delta", {}).get("type") == "text_delta" + ] + assert "4111" not in wire, wire + assert "".join(text_deltas) == "", wire + assert wire.count("event: message_stop") == 1, wire + + class _AppliesGuardrail(CustomGuardrail): """Implements the unified interface only, so the proxy routes it to unified_guardrail.""" From 13b374873d7b75f400fc1702c759d68195620f62 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:57:44 -0700 Subject: [PATCH 05/28] fix(otel v2): map completions, images, speech, transcription and moderation output onto the Langfuse generation output (#42394) * fix(otel v2): map completions, images, speech, transcription and moderation output onto the generation output Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(redaction): redact text completion choices in the standard logging payload Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): compare decoded generation output text and follow the live moderation verdict Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel v2): compare logged byte counts with the received media and move e2e schemas into models.py Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(e2e): keep the otel_v2 Langfuse output e2e file out of the stage-mirror gate it cannot run in Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/e2e-stack/select_tests.py | 1 + litellm/integrations/otel/model/payloads.py | 105 ++++++- litellm/litellm_core_utils/litellm_logging.py | 2 + litellm/litellm_core_utils/redact_messages.py | 2 + litellm/types/llms/openai.py | 21 ++ .../test_e2e_changed_gate.py | 5 + tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/conftest.py | 6 + tests/e2e/e2e_config.py | 1 + tests/e2e/logging/logging_client.py | 35 +++ ..._otel_v2_langfuse_generation_output_e2e.py | 210 +++++++++++++ tests/e2e/models.py | 71 +++++ tests/e2e/pytest.ini | 1 + .../otel/test_otel_v2_sources_of_truth.py | 287 +++++++++++++++--- .../otel/test_otel_v2_vendor_mappers.py | 30 ++ .../test_redact_messages.py | 20 ++ 16 files changed, 738 insertions(+), 61 deletions(-) create mode 100644 tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index dbc4ae8f5c2..2386b184e54 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -9,6 +9,7 @@ UNSUPPORTED: Final = re.compile( r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$" r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" + r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$" ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index d3ad7234d93..7ecbeea255c 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -427,7 +427,7 @@ class LLMCallSpanData: # plain ``.get`` — no repeated ``isinstance`` guards. raw_response: Final = payload.get("response") response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {}) - choices_out: Final = _dicts(response.get("choices")) or _responses_choices(response) or _ocr_choices(response) + choices_out: Final = _output_choices(response) # ``finish_reasons`` is metadata, not content, so derive it from # ``choices_out`` before gating. The raw message/choice bodies are only # retained when content capture is enabled (see ``capture_span_content``); @@ -752,20 +752,99 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: return (choice,) -def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: - markdowns: Final = tuple( - text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None +def _output_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + """The response output as chat-shaped choices; images and binary bodies become size summaries, never bytes.""" + return ( + _completion_choices(response) + or _responses_choices(response) + or _ocr_choices(response) + or _transcription_choices(response) + or _moderation_choices(response) + or _image_choices(response) + or _binary_choices(response) ) - if not markdowns: + + +def _text_choice(content: str, finish_reason: str | None = None) -> _Choice: + message: Final[_AssistantMessage] = {"role": "assistant", "content": content, "refusal": None, "tool_calls": None} + return {"message": message, "finish_reason": finish_reason} + + +def _joined_choice(parts: tuple[str, ...]) -> tuple[_Choice, ...]: + return (_text_choice("\n\n".join(parts)),) if parts else () + + +def _completion_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return tuple( + _text_choice(text, as_str(choice.get("finish_reason"))) + if "message" not in choice and isinstance(text := choice.get("text"), str) + else choice + for choice in _dicts(response.get("choices")) + ) + + +def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple(text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None) + ) + + +def _transcription_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + text: Final = response.get("text") + return (_text_choice(text),) if isinstance(text, str) and text else () + + +def _moderation_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple( + _moderation_verdict(flagged, result.get("categories")) + for result in _dicts(response.get("results")) + if isinstance(flagged := result.get("flagged"), bool) + ) + ) + + +def _moderation_verdict(flagged: bool, categories: object) -> str: + if not flagged: + return "not flagged" + hits: Final = ( + tuple(name for name, hit in cast(Mapping[str, object], categories).items() if hit is True) + if isinstance(categories, dict) + else () + ) + return f"flagged: {', '.join(hits)}" if hits else "flagged" + + +def _image_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple(summary for item in _dicts(response.get("data")) if (summary := _image_summary(item)) is not None) + ) + + +def _image_summary(item: Mapping[str, object]) -> str | None: + location: Final = _image_location(item) + if location is None: + return None + revised: Final = as_str(item.get("revised_prompt")) + return f"{revised}\n{location}" if revised else location + + +def _image_location(item: Mapping[str, object]) -> str | None: + url: Final = as_str(item.get("url")) + if url is not None: + return url + encoded: Final = item.get("b64_json") + if not isinstance(encoded, str): + return None + return f"b64_json image ({len(encoded) * 3 // 4 - encoded[-2:].count('=')} bytes)" + + +def _binary_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + size: Final = as_int(response.get("num_bytes")) + if size is None: return () - message: Final[_AssistantMessage] = { - "role": "assistant", - "content": "\n\n".join(markdowns), - "refusal": None, - "tool_calls": None, - } - choice: Final[_Choice] = {"message": message, "finish_reason": None} - return (choice,) + content_type: Final = as_str(response.get("content_type")) + return (_text_choice(f"{content_type} ({size} bytes)" if content_type else f"{size} bytes"),) def _responses_parts_text(parts: tuple[Mapping[str, object], ...], part_type: str, field: str) -> str | None: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f5967185af0..0f14b46c8ba 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -6277,6 +6277,8 @@ def _extract_response_obj_and_hidden_params( hidden_params = getattr(init_response_obj, "_hidden_params", None) elif isinstance(init_response_obj, dict): response_obj = init_response_obj + elif isinstance(init_response_obj, HttpxBinaryResponseContent): + response_obj = dict(init_response_obj.logging_summary()) else: response_obj = {} diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index b409b181a79..15bccf0301e 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -245,6 +245,8 @@ def _redact_model_response_dict_choices(choices, redacted_str: str): if "audio" in choice["delta"]: choice["delta"]["audio"] = None _redact_tool_calls_dict(choice["delta"]) + elif choice.get("text") is not None: + choice["text"] = redacted_str else: _redact_choice_content(choice) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index bc44eb5b5b7..599b1a76249 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -110,6 +110,14 @@ FileTypes = ( EmbeddingInput = str | list[str] +class BinaryResponseSummary(TypedDict): + """What logging keeps of a binary response (speech audio, file content): size and media type, never the bytes.""" + + object: ReadOnly[Literal["binary"]] + content_type: ReadOnly[str | None] + num_bytes: ReadOnly[int] + + class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): _hidden_params: dict @@ -117,6 +125,19 @@ class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): super().__init__(response) self._hidden_params = {} # mutable-ok: mutable-dict contract shared with ModelResponse logging consumers + def logging_summary(self) -> BinaryResponseSummary: + return { + "object": "binary", + "content_type": self.response.headers.get("content-type"), + "num_bytes": self._num_bytes(), + } + + def _num_bytes(self) -> int: + try: + return len(self.response.content) + except httpx.ResponseNotRead: + return self.response.num_bytes_downloaded + def set_response_cost(self, response_cost: float | None) -> None: if response_cost is None: self._hidden_params.pop("response_cost", None) diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index 78e6562a4a8..2b37ab14e7f 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -212,6 +212,11 @@ def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]: (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()), (("tests/e2e/guardrails/test_presidio_masking_e2e.py",), ()), (("tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py",), ()), + (("tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py",), ()), + ( + ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), + ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), + ), ( ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 2b59e8770b5..15c2d6763d6 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -105,7 +105,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 8776d00d502..ca0fbd84c35 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -30,6 +30,7 @@ from e2e_config import ( FIXTURE_MODE_RAW, MANAGED_FILES_OPT_IN_ENV, MCP_OAUTH_LIVE_OPT_IN_ENV, + OTEL_V2_OPT_IN_ENV, PROMPT_CACHING_OPT_IN_ENV, PROVIDER_EDGE_HOST_OPT_IN_ENV, PROXY_BASE_URL, @@ -61,6 +62,7 @@ OPT_IN_MARKERS: Final = MappingProxyType( "cli_determinism": CLI_DETERMINISM_OPT_IN_ENV, "mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV, "provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV, + "otel_v2": OTEL_V2_OPT_IN_ENV, } ) @@ -150,6 +152,10 @@ def pytest_configure(config: pytest.Config) -> None: "provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the " "gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set", ) + config.addinivalue_line( + "markers", + "otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 14de4619664..311c944eeb4 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -147,6 +147,7 @@ REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE" PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE" +OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/logging/logging_client.py b/tests/e2e/logging/logging_client.py index 66dfa233ec4..c2f987ea33d 100644 --- a/tests/e2e/logging/logging_client.py +++ b/tests/e2e/logging/logging_client.py @@ -146,6 +146,29 @@ class LangfuseObservationList(BaseModel): data: list[LangfuseObservation] = [] +class LangfuseOtelMetadata(BaseModel): + """Langfuse stores every OTel span attribute under metadata.attributes.""" + + model_config = ConfigDict(extra="ignore") + + attributes: dict[str, str] = {} + + +def otel_attributes(obs: LangfuseObservation) -> dict[str, str]: + try: + return LangfuseOtelMetadata.model_validate(obs.metadata).attributes + except ValidationError: + return {} + + +def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool: + attributes = otel_attributes(obs) + return ( + attributes.get("langfuse.observation.type") == "generation" + and attributes.get("litellm.metadata.user_api_key_alias") == key_alias + ) + + class LangfuseListParams(BaseModel): model_config = ConfigDict(populate_by_name=True) @@ -630,6 +653,18 @@ class LoggingClient: time.sleep(POLL_INTERVAL) return last + def poll_langfuse_generation( + self, creds: LangfuseCreds, *, key_alias: str, from_start_time: str + ) -> LangfuseObservation | None: + """The OTel v2 generation the proxy exported for one key alias since from_start_time.""" + deadline = time.monotonic() + POLL_TIMEOUT + while time.monotonic() < deadline: + for obs in self.list_langfuse_observations(creds, from_start_time=from_start_time): + if is_otel_v2_generation(obs, key_alias=key_alias): + return obs + time.sleep(POLL_INTERVAL) + return None + def poll_langfuse_trace_observations( self, creds: LangfuseCreds, diff --git a/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py new file mode 100644 index 00000000000..71008d94557 --- /dev/null +++ b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py @@ -0,0 +1,210 @@ +"""Live e2e: the OTel v2 Langfuse generation carries output for every non-chat endpoint (LIT-8309). + +With LITELLM_OTEL_V2=true the proxy exports one generation per request to the +team's Langfuse destination. Chat, Responses, embeddings and OCR already fill +its output; this file pins the remaining five families. Each test registers a +real OpenAI deployment, drives the endpoint through the shared transport, then +reads the generation back from Langfuse and asserts its output reflects what +the caller received: the completion text, the transcript, the moderation +verdict, and for images and speech a bounded summary that never carries the +raw base64 or audio bytes. +""" + +from __future__ import annotations + +import base64 +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from logging_client import LangfuseCreds, LangfuseObservation, LoggingClient, load_langfuse_creds +from models import ( + CompletionBody, + CompletionResponse, + ImageGenerationBody, + ImageGenerationResponse, + LiteLLMParamsBody, + ModerationBody, + ModerationResponse, + SpeechBody, + TranscriptionForm, + TranscriptionResponse, +) +from pydantic import BaseModel, TypeAdapter, ValidationError + +pytestmark = [pytest.mark.e2e, pytest.mark.otel_v2] + +WEATHER_WAV: Final = ( + Path(__file__).resolve().parent.parent / "llm_translation" / "realtime" / "fixtures" / "weather_question_24k.wav" +) +BOUNDED_OUTPUT_CHARS: Final = 1024 + + +class _OutputMessage(BaseModel): + """One assistant message of the Langfuse generation output; only the text is read.""" + + content: str = "" + + +_OUTPUT_MESSAGES: Final = TypeAdapter(list[_OutputMessage]) + + +@pytest.fixture(scope="session") +def langfuse_creds() -> LangfuseCreds: + return load_langfuse_creds() + + +def _langfuse_key( + client: LoggingClient, creds: LangfuseCreds, resources: ResourceManager, params: LiteLLMParamsBody +) -> tuple[str, str, str]: + """A model registered for this run plus a key on a team whose Langfuse callback is `creds`.""" + model: Final = f"e2e-otel-out-{unique_marker()}" + model_id: Final = client.proxy.create_model(model, params) + resources.defer(lambda: client.proxy.delete_model(model_id)) + team_id: Final = client.create_team(f"otel-out-team-{unique_marker()}", models=[model]) + resources.defer(lambda: client.delete_team(team_id)) + client.add_team_langfuse_callback(team_id, creds) + alias: Final = f"otel-out-key-{unique_marker()}" + key: Final = client.key_with_alias(alias, models=[model], team_id=team_id) + resources.defer(lambda: client.delete_key(key)) + return model, key, alias + + +def _generation(client: LoggingClient, creds: LangfuseCreds, *, alias: str, started: datetime) -> LangfuseObservation: + since: Final = (started - timedelta(seconds=5)).isoformat() + observation: Final = client.poll_langfuse_generation(creds, key_alias=alias, from_start_time=since) + assert observation is not None, f"no OTel v2 generation reached Langfuse for key alias {alias!r}" + return observation + + +def _output_text(observation: LangfuseObservation) -> str: + assert observation.output not in (None, "", [], {}), f"generation output is empty: {observation!r}" + try: + messages: Final = _OUTPUT_MESSAGES.validate_python(observation.output) + except ValidationError: + pytest.fail(f"generation output is not a list of assistant messages: {observation!r}") + assert messages, f"generation output is empty: {observation!r}" + return "\n".join(message.content for message in messages) + + +def _openai(model: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody(model=model, api_key="os.environ/OPENAI_API_KEY") + + +class TestOtelV2LangfuseGenerationOutput: + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["completions"]) + def test_completions_output_is_the_completion_text( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-3.5-turbo-instruct")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/completions", + headers=client.proxy.transport.bearer(key), + json=CompletionBody(model=model, prompt=f"Repeat exactly: {unique_marker()}", n=2), + response_type=CompletionResponse, + ) + ) + texts: Final = tuple(choice.text.strip() for choice in response.choices) + assert len(texts) == 2 and all(texts), f"/v1/completions returned no text: {response!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert all(text in output for text in texts), ( + f"generation output lacks the completion texts {texts!r}: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["images_generations"]) + def test_images_output_is_a_bounded_summary_without_base64( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-image-1-mini")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/images/generations", + headers=client.proxy.transport.bearer(key), + json=ImageGenerationBody(model=model, prompt=f"a plain red square {unique_marker()}"), + response_type=ImageGenerationResponse, + timeout=180.0, + ) + ) + assert response.data, f"/v1/images/generations returned no data: {response!r}" + encoded: Final = response.data[0].b64_json or "" + assert encoded, f"expected a b64_json image from gpt-image-1-mini: {response.data[0].url!r}" + image_bytes: Final = len(base64.b64decode(encoded)) + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert len(output) <= BOUNDED_OUTPUT_CHARS, f"image generation output is not bounded ({len(output)} chars)" + assert encoded[:64] not in output, "image generation output leaks the raw base64 payload" + assert output == f"b64_json image ({image_bytes} bytes)", ( + f"image generation output does not report the {image_bytes} decoded bytes: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["audio_speech"]) + def test_speech_output_is_a_bounded_summary_without_audio_bytes( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-4o-mini-tts")) + started: Final = datetime.now(timezone.utc) + audio: Final = client.proxy.transport.stream_binary( + "/v1/audio/speech", + headers=client.proxy.transport.bearer(key), + json=SpeechBody(model=model, input=f"hello {unique_marker()}"), + ) + assert audio.ok and audio.total_bytes > 0, f"/v1/audio/speech returned no audio: {audio!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert len(output) <= BOUNDED_OUTPUT_CHARS, f"speech output is not bounded ({len(output)} chars)" + assert output.endswith(f" ({audio.total_bytes} bytes)"), ( + f"speech output does not report the {audio.total_bytes} audio bytes the caller received: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["audio_transcriptions"]) + def test_transcription_output_is_the_transcript( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-4o-mini-transcribe")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.upload( + "/v1/audio/transcriptions", + headers=client.proxy.transport.bearer(key), + form=TranscriptionForm(model=model), + filename=WEATHER_WAV.name, + content=WEATHER_WAV.read_bytes(), + file_content_type="audio/wav", + response_type=TranscriptionResponse, + ) + ) + transcript: Final = response.text.strip() + assert transcript, f"/v1/audio/transcriptions returned no text: {response!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert transcript in output, f"generation output lacks the transcript {transcript!r}: {output!r}" + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["moderations"]) + def test_moderations_output_is_the_verdict( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/omni-moderation-latest")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/moderations", + headers=client.proxy.transport.bearer(key), + json=ModerationBody(model=model, input=f"I will find you and hurt you badly {unique_marker()}"), + response_type=ModerationResponse, + ) + ) + assert response.results, f"/v1/moderations returned no results: {response!r}" + verdict: Final = "flagged: " if response.results[0].flagged else "not flagged" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert output.startswith(verdict), ( + f"generation output does not carry the moderation verdict {verdict!r}: {output!r}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 2608da2e4fc..d0b1e8824da 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -760,6 +760,77 @@ class OcrResponse(BaseModel): pages: list[OcrPage] = [] +# ---------- completions ---------- + + +class CompletionBody(BaseModel): + model: str + prompt: str + max_tokens: int = 8 + n: int = 1 + + +class CompletionChoice(BaseModel): + text: str = "" + + +class CompletionResponse(BaseModel): + choices: list[CompletionChoice] = [] + + +# ---------- images ---------- + + +class ImageGenerationBody(BaseModel): + model: str + prompt: str + n: int = 1 + size: str = "1024x1024" + quality: str = "low" + + +class ImageDatum(BaseModel): + url: str | None = None + b64_json: str | None = None + + +class ImageGenerationResponse(BaseModel): + data: list[ImageDatum] = [] + + +# ---------- audio ---------- + + +class SpeechBody(BaseModel): + model: str + input: str + voice: str = "alloy" + + +class TranscriptionForm(BaseModel): + model: str + + +class TranscriptionResponse(BaseModel): + text: str = "" + + +# ---------- moderations ---------- + + +class ModerationBody(BaseModel): + model: str + input: str + + +class ModerationResult(BaseModel): + flagged: bool + + +class ModerationResponse(BaseModel): + results: list[ModerationResult] = [] + + # ---------- spend logs ---------- diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index c6acd449884..6f9f57d333e 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -14,3 +14,4 @@ markers = redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set + otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 01f2a13d252..70ab4fe07de 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -4,6 +4,7 @@ and the typed StandardLoggingPayload adapter. These need no OTel SDK.""" import json import logging import re +from collections.abc import Mapping from pathlib import Path from typing import Final @@ -151,9 +152,7 @@ def test_llm_call_span_name(): def _all_constants(cls): return { - getattr(cls, name) - for name in vars(cls) - if not name.startswith("__") and isinstance(getattr(cls, name), str) + getattr(cls, name) for name in vars(cls) if not name.startswith("__") and isinstance(getattr(cls, name), str) } @@ -465,9 +464,7 @@ def test_mcp_tool_call_content_gated_off_by_default(): off = MCPToolCallSpanData.from_standard_logging_payload(_mcp_payload()) assert off.arguments_json is None and off.result_json is None - on = MCPToolCallSpanData.from_standard_logging_payload( - _mcp_payload(), capture_content=True - ) + on = MCPToolCallSpanData.from_standard_logging_payload(_mcp_payload(), capture_content=True) assert on.arguments_json is not None and '"Paris"' in on.arguments_json assert on.result_json is not None and "21" in on.result_json @@ -689,9 +686,7 @@ def test_content_capture_gated_off_by_default(): payload = _sample_payload( messages=[{"role": "user", "content": "secret prompt"}], ) - payload["response"]["choices"] = [ - {"finish_reason": "stop", "message": {"role": "assistant", "content": "secret"}} - ] + payload["response"]["choices"] = [{"finish_reason": "stop", "message": {"role": "assistant", "content": "secret"}}] data = LLMCallSpanData.from_standard_logging_payload(payload) assert data.messages_in == () assert data.choices_out == () @@ -911,6 +906,221 @@ def test_ocr_pages_without_markdown_stay_empty(): assert data.choices_out == () +def _assistant_choice(content: str, finish_reason: str | None = None) -> dict[str, object]: + return { + "message": {"role": "assistant", "content": content, "refusal": None, "tool_calls": None}, + "finish_reason": finish_reason, + } + + +def _route_payload(call_type: str, model: str, response: Mapping[str, object]) -> dict[str, object]: + return _sample_payload(call_type=call_type, model=model, messages=None, response=response) + + +def test_text_completion_choices_become_assistant_messages_in_choice_order() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "atext_completion", + "gpt-3.5-turbo-instruct", + { + "id": "cmpl-1", + "object": "text_completion", + "choices": [ + {"index": 0, "text": " first", "finish_reason": "length", "logprobs": None}, + {"index": 1, "text": " second", "finish_reason": "stop", "logprobs": None}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice(" first", "length"), _assistant_choice(" second", "stop")) + assert data.finish_reasons == ("length", "stop") + assert data.response_id == "cmpl-1" + + +def test_text_completion_choices_follow_the_content_capture_gate_but_finish_reasons_do_not() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "atext_completion", "gpt-3.5-turbo-instruct", {"choices": [{"text": "x", "finish_reason": "stop"}]} + ) + ) + + assert data.choices_out == () + assert data.finish_reasons == ("stop",) + + +def test_chat_choices_with_a_message_are_passed_through_untouched_even_beside_a_stray_text_key() -> None: + choice: Final = { + "index": 0, + "finish_reason": "stop", + "text": "no", + "message": {"role": "assistant", "content": "chat"}, + } + data: Final = LLMCallSpanData.from_standard_logging_payload( + _sample_payload(response={"choices": [choice]}), capture_content=True + ) + + assert data.choices_out == (choice,) + + +def test_transcription_text_becomes_one_assistant_choice() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("atranscription", "gpt-4o-mini-transcribe", {"text": "What is the weather like?", "task": "x"}), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("What is the weather like?"),) + assert data.finish_reasons == () + + +def test_empty_transcription_text_stays_empty() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("atranscription", "gpt-4o-mini-transcribe", {"text": ""}), capture_content=True + ) + + assert data.choices_out == () + + +def test_moderation_results_become_one_verdict_per_input_naming_the_hit_categories() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "amoderation", + "omni-moderation-latest", + { + "id": "modr-1", + "results": [ + { + "flagged": True, + "categories": {"harassment": False, "violence": True, "self-harm": True}, + "category_scores": {"harassment": 0.01, "violence": 0.98, "self-harm": 0.7}, + }, + {"flagged": False, "categories": {"violence": False}}, + {"flagged": True}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("flagged: violence, self-harm\n\nnot flagged\n\nflagged"),) + assert data.response_id == "modr-1" + + +def test_moderation_output_follows_the_content_capture_gate() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("amoderation", "omni-moderation-latest", {"results": [{"flagged": True}]}) + ) + + assert data.choices_out == () + + +def test_moderation_results_without_a_verdict_produce_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("amoderation", "omni-moderation-latest", {"results": [{"categories": {"violence": True}}]}), + capture_content=True, + ) + + assert data.choices_out == () + + +def test_image_data_becomes_a_size_summary_and_never_carries_the_base64_payload() -> None: + encoded: Final = "QUJDRA==" + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "aimage_generation", + "gpt-image-1-mini", + { + "created": 1, + "data": [ + {"b64_json": encoded, "revised_prompt": "a red bicycle"}, + {"url": "https://images.example/cat.png"}, + {"b64_json": "QUJDREVGR0g="}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == ( + _assistant_choice( + "a red bicycle\nb64_json image (4 bytes)\n\nhttps://images.example/cat.png\n\nb64_json image (8 bytes)" + ), + ) + assert encoded not in json.dumps(data.choices_out) + + +def test_image_data_without_a_url_or_payload_stays_empty_and_embeddings_are_not_images() -> None: + images: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aimage_generation", "gpt-image-1-mini", {"data": [{"revised_prompt": "x"}]}), + capture_content=True, + ) + embeddings: Final = LLMCallSpanData.from_standard_logging_payload( + _embedding_payload([[0.1, 0.2]]), capture_content=True + ) + + assert images.choices_out == () + assert embeddings.choices_out == () + + +def test_speech_summary_becomes_a_media_type_and_byte_count_choice() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "aspeech", "gpt-4o-mini-tts", {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 48210} + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("audio/mpeg (48210 bytes)"),) + + +def test_speech_summary_without_a_media_type_is_the_byte_count_and_follows_the_capture_gate() -> None: + response: Final = {"object": "binary", "content_type": None, "num_bytes": 7} + shown: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aspeech", "gpt-4o-mini-tts", response), capture_content=True + ) + gated: Final = LLMCallSpanData.from_standard_logging_payload(_route_payload("aspeech", "gpt-4o-mini-tts", response)) + + assert shown.choices_out == (_assistant_choice("7 bytes"),) + assert gated.choices_out == () + + +def test_speech_response_without_a_byte_count_produces_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aspeech", "gpt-4o-mini-tts", {"object": "binary", "content_type": "audio/mpeg"}), + capture_content=True, + ) + + assert data.choices_out == () + + +def test_speech_binary_response_is_logged_as_its_summary_not_dropped() -> None: + import httpx + + from litellm.litellm_core_utils.litellm_logging import _extract_response_obj_and_hidden_params + from litellm.types.llms.openai import HttpxBinaryResponseContent + + raw: Final = httpx.Response(200, headers={"content-type": "audio/mpeg"}, content=b"\x00" * 1234) + response_obj, hidden_params = _extract_response_obj_and_hidden_params(HttpxBinaryResponseContent(raw), None) + + assert response_obj == {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 1234} + assert hidden_params is None + + +def test_speech_binary_response_still_streaming_reports_the_bytes_downloaded_so_far() -> None: + import httpx + + from litellm.types.llms.openai import HttpxBinaryResponseContent + + unread: Final = httpx.Response(200, stream=httpx.ByteStream(b"\x00" * 10)) + + assert HttpxBinaryResponseContent(unread).logging_summary() == { + "object": "binary", + "content_type": None, + "num_bytes": 0, + } + + def test_request_identity_prefers_canonical_team_keys(): from litellm.integrations.otel.model.payloads import RequestIdentity @@ -931,9 +1141,7 @@ def test_request_identity_prefers_canonical_team_keys(): def test_request_identity_falls_back_to_legacy_team_keys(): from litellm.integrations.otel.model.payloads import RequestIdentity - payload = _sample_payload( - metadata={"team_id": "legacy-team", "team_alias": "legacy"} - ) + payload = _sample_payload(metadata={"team_id": "legacy-team", "team_alias": "legacy"}) ident = RequestIdentity.from_payload(payload) assert ident.team_id == "legacy-team" assert ident.team_alias == "legacy" @@ -952,7 +1160,10 @@ def test_request_identity_falls_back_to_legacy_team_keys(): }, "from-header", ), - ({"proxy_server_request": {"headers": {"langfuse_trace_name": ""}}, "metadata": {"trace_name": "body"}}, "body"), + ( + {"proxy_server_request": {"headers": {"langfuse_trace_name": ""}}, "metadata": {"trace_name": "body"}}, + "body", + ), ({"proxy_server_request": {"headers": {}}, "metadata": {"user_api_key_team_id": "t1"}}, None), ({}, None), ], @@ -1002,7 +1213,15 @@ def test_caller_trace_name_prefers_the_langfuse_header_over_body_metadata(reques ), ({}, TraceControls()), ], - ids=["body", "headers-beat-body", "anthropic-body", "non-string-tags-dropped", "scalar-coercion", "mutation-controls-ignored", "empty"], + ids=[ + "body", + "headers-beat-body", + "anthropic-body", + "non-string-tags-dropped", + "scalar-coercion", + "mutation-controls-ignored", + "empty", + ], ) def test_caller_trace_controls_carry_user_session_and_tags(request_data, expected): assert caller_trace_controls({"litellm_params": request_data}) == expected @@ -1168,9 +1387,7 @@ def test_content_capture_opt_in_retains_bodies(): payload = _sample_payload( messages=[{"role": "user", "content": "secret prompt"}], ) - payload["response"]["choices"] = [ - {"finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}} - ] + payload["response"]["choices"] = [{"finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}] data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True) assert data.messages_in and data.messages_in[0]["content"] == "secret prompt" assert data.choices_out and data.choices_out[0]["message"]["content"] == "hi" @@ -1187,41 +1404,17 @@ def test_capture_span_content_resolves_modes(): # default (no_content) → off assert OpenTelemetryV2Config().capture_span_content is False + assert OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.SPAN_ONLY).capture_span_content is True assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.SPAN_ONLY - ).capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.SPAN_AND_EVENT - ).capture_span_content - is True + OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.SPAN_AND_EVENT).capture_span_content is True ) # event-only does not authorize span-attribute content - assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.EVENT_ONLY - ).capture_span_content - is False - ) + assert OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.EVENT_ONLY).capture_span_content is False # V1 accepted UPPER_SNAKE_CASE; the env value is case-insensitive so an # operator carrying ``SPAN_AND_EVENT`` forward still enables capture. - assert ( - OpenTelemetryV2Config( - capture_message_content="SPAN_AND_EVENT" - ).capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config(capture_message_content="SPAN_ONLY").capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config(capture_message_content="NO_CONTENT").capture_span_content - is False - ) + assert OpenTelemetryV2Config(capture_message_content="SPAN_AND_EVENT").capture_span_content is True + assert OpenTelemetryV2Config(capture_message_content="SPAN_ONLY").capture_span_content is True + assert OpenTelemetryV2Config(capture_message_content="NO_CONTENT").capture_span_content is False def test_capture_message_content_normalizer_only_touches_strings(): diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py index 9fa198c4ec5..1e2ae24a329 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py @@ -6,6 +6,8 @@ backends, so one trace lights up every configured destination. """ import json +from collections.abc import Mapping +from typing import Final import pytest @@ -249,6 +251,34 @@ def test_langfuse_mapper_renders_an_ocr_call_with_the_page_markdown_as_output(): assert attrs["langfuse.observation.type"] == "generation" +@pytest.mark.parametrize( + ("call_type", "response", "expected_content"), + [ + ("atext_completion", {"choices": [{"text": "Paris.", "finish_reason": "stop"}]}, "Paris."), + ("atranscription", {"text": "What is the weather like?"}, "What is the weather like?"), + ("amoderation", {"results": [{"flagged": True, "categories": {"violence": True}}]}, "flagged: violence"), + ("aimage_generation", {"data": [{"b64_json": "QUJDRA=="}]}, "b64_json image (4 bytes)"), + ("aspeech", {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 9}, "audio/mpeg (9 bytes)"), + ], +) +def test_langfuse_mapper_renders_every_non_chat_route_output_as_an_assistant_message( + call_type: str, response: Mapping[str, object], expected_content: str +) -> None: + payload: Final[dict[str, object]] = { + "call_type": call_type, + "custom_llm_provider": "openai", + "model": "m", + "messages": None, + "response": response, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True)) + + assert json.loads(attrs["langfuse.observation.output"]) == [ + {"role": "assistant", "content": expected_content, "refusal": None, "tool_calls": None} + ] + assert attrs["langfuse.observation.type"] == "generation" + + # --------------------------------------------------------------------------- # # Weave # --------------------------------------------------------------------------- # diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index 276a67e0bd4..22298c00219 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -304,6 +304,26 @@ class TestPerformRedaction: assert delta["thinking_blocks"] is None assert delta["audio"] is None + def test_redacts_text_completion_choices_in_standard_logging_object(self): + details = { + "standard_logging_object": { + "response": { + "object": "text_completion", + "choices": [ + {"text": " Paris.", "finish_reason": "stop", "index": 0}, + {"text": "\n\nBlue", "finish_reason": "length", "index": 1}, + ], + } + } + } + + perform_redaction(details, None) + + assert details["standard_logging_object"]["response"]["choices"] == [ + {"text": "redacted-by-litellm", "finish_reason": "stop", "index": 0}, + {"text": "redacted-by-litellm", "finish_reason": "length", "index": 1}, + ] + def test_redacts_object_choices_inside_model_response_dict(self): result = { "choices": [ From 8603d6e259fdcba31970b7b52e920282a57a194a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:15:35 -0700 Subject: [PATCH 06/28] fix(cost): honor per-second custom pricing on chat completions for every provider (#42403) * fix(cost): honor per-second custom pricing on chat completions for every provider * test(cost): assert a per-second priced deployment bills instead of staying at $0 The zero-cost diagnostic tests from #42345 used a per-second-only entry as their misconfigured fixture, which this branch now bills. Switch that fixture to a per-query-only entry, which is still selected as the deployment's own pricing and still prices chat usage at $0, and add a per-second test asserting the call duration is billed with no diagnostic Also let a caller's explicit total_time outrank the logging window in completion_cost, so the SDK precedence stays stamped response, caller, logging * test(response_metadata): move the per-second pricing regression into the mapped tests/unit file * fix(cost_calculator): keep media-mode per-second rates off the wall-clock path A video, transcription, speech, or realtime entry's per-second rates price media seconds, which their dedicated cost paths bill from the media itself. The generic per-second branch now skips those modes, so a video status poll on a per-second video model bills nothing instead of the seconds the poll took to answer. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 119 +++++++++----- litellm/litellm_core_utils/litellm_logging.py | 18 ++- .../llm_response_utils/response_metadata.py | 2 +- .../test_litellm_logging.py | 51 ++++-- tests/test_litellm/test_cost_calculator.py | 151 +++++++++++++++++- .../test_response_metadata.py | 58 ++++++- 6 files changed, 336 insertions(+), 63 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 37743a9ce33..7b02a5ede00 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -115,6 +115,7 @@ from litellm.types.utils import ( LlmProviders, LlmProvidersSet, ModelInfo, + ModelInfoBase, PromptTokensDetailsWrapper, ServiceTier, StandardBuiltInToolsParams, @@ -322,6 +323,48 @@ class OCRPricing(TypedDict, total=False): annotation_cost_per_page: ReadOnly[float | None] +_WALL_CLOCK_PRICED_MODES: Final = frozenset({"chat", "completion", "embedding", "responses"}) + + +def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool: + return ( + (model_info.get("input_cost_per_token") or 0.0) > 0 + or (model_info.get("output_cost_per_token") or 0.0) > 0 + or model_info.get("tiered_pricing") is not None + ) + + +def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool: + mode: Final = model_info.get("mode") + return mode is None or mode in _WALL_CLOCK_PRICED_MODES + + +def _per_second_pricing_cost( + model: str, + custom_llm_provider: str | None, + response_time_ms: float | None, +) -> tuple[float, float] | None: + try: + model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model + return None + if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): + return None + input_cost_per_second: Final = model_info.get("input_cost_per_second") + output_cost_per_second: Final = model_info.get("output_cost_per_second") + if input_cost_per_second is None and output_cost_per_second is None: + return None + seconds: Final = (response_time_ms or 0.0) / 1000 + verbose_logger.debug( + "For model=%s - input_cost_per_second: %s; output_cost_per_second: %s; response time: %s", + model, + input_cost_per_second, + output_cost_per_second, + response_time_ms, + ) + return (input_cost_per_second or 0.0) * seconds, (output_cost_per_second or 0.0) * seconds + + def cost_per_token( model: str = "", prompt_tokens: int = 0, @@ -448,9 +491,6 @@ def cost_per_token( if response_cost is not None: return response_cost[0], response_cost[1] - # given - prompt_tokens_cost_usd_dollar: float = 0 - completion_tokens_cost_usd_dollar: float = 0 model_cost_ref: Final = litellm.model_cost # Only callers that explicitly pass `custom_llm_provider` get the # dedup/prefix-join treatment. When provider is omitted, preserve legacy @@ -611,6 +651,14 @@ def cost_per_token( number_of_queries=number_of_queries or 1, optional_params=(getattr(response, "_hidden_params", None) if response else None), ) + elif ( + per_second_cost := _per_second_pricing_cost( + model=model, + custom_llm_provider=custom_llm_provider, + response_time_ms=response_time_ms, + ) + ) is not None: + return per_second_cost elif custom_llm_provider == "vertex_ai": cost_router: Final = google_cost_router( model=model_without_prefix, @@ -685,12 +733,7 @@ def cost_per_token( ) else: model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) - - if ( - (model_info.get("input_cost_per_token") or 0.0) > 0 - or (model_info.get("output_cost_per_token") or 0.0) > 0 - or model_info.get("tiered_pricing") is not None - ): + if _has_token_or_tiered_pricing(model_info): return generic_cost_per_token( model=model, usage=usage_block, @@ -698,36 +741,8 @@ def cost_per_token( service_tier=service_tier, data_residency=data_residency, ) - - input_cost_per_second: Final = model_info.get("input_cost_per_second") - if input_cost_per_second is not None and response_time_ms is not None: - verbose_logger.debug( - "For model=%s - input_cost_per_second: %s; response time: %s", - model, - input_cost_per_second, - response_time_ms, - ) - ## COST PER SECOND ## - prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000 - - output_cost_per_second: Final = model_info.get("output_cost_per_second") - if output_cost_per_second is not None and response_time_ms is not None: - verbose_logger.debug( - "For model=%s - output_cost_per_second: %s; response time: %s", - model, - output_cost_per_second, - response_time_ms, - ) - ## COST PER SECOND ## - completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000 - - verbose_logger.debug( - "Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s", - model, - prompt_tokens_cost_usd_dollar, - completion_tokens_cost_usd_dollar, - ) - return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar + verbose_logger.debug("No per-token, tiered, or per-second pricing for model=%s; cost is 0", model) + return 0.0, 0.0 def get_replicate_completion_pricing(completion_response: dict, total_time=0.0): @@ -1222,6 +1237,21 @@ def _split_responses_ws_logging_object_by_service_tier( ) +def _response_time_ms_for_cost( + completion_response: object, + litellm_logging_obj: LitellmLoggingObject | None, + total_time: float | None, +) -> float: + stamped: Final = getattr(completion_response, "_response_ms", None) + if isinstance(stamped, (int, float)): + return float(stamped) + if total_time: + return total_time + if litellm_logging_obj is not None: + return litellm_logging_obj.get_response_ms() + return 0.0 + + def completion_cost( completion_response: object | None = None, model: str | None = None, @@ -1443,8 +1473,6 @@ def completion_cost( prompt_tokens_details = _usage.get("prompt_tokens_details") or {} cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0) - total_time = getattr(completion_response, "_response_ms", 0) - hidden_params = getattr(completion_response, "_hidden_params", None) if hidden_params is not None: custom_llm_provider = hidden_params.get("custom_llm_provider", custom_llm_provider or None) @@ -1676,6 +1704,11 @@ def completion_cost( ) return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj) + response_time_ms = _response_time_ms_for_cost( + completion_response=completion_response, + litellm_logging_obj=litellm_logging_obj, + total_time=total_time, + ) # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" @@ -1686,7 +1719,7 @@ def completion_cost( # see https://replicate.com/pricing elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost: # for unmapped replicate model, default to replicate's time tracking logic - return get_replicate_completion_pricing(completion_response, total_time) + return get_replicate_completion_pricing(completion_response, response_time_ms) if model is None: raise ValueError( @@ -1718,7 +1751,7 @@ def completion_cost( prompt_tokens=prompt_tokens or 0, completion_tokens=completion_tokens or 0, custom_llm_provider=custom_llm_provider, - response_time_ms=total_time, + response_time_ms=response_time_ms, region_name=None if explicit_pricing else region_name, custom_cost_per_second=custom_cost_per_second, custom_cost_per_token=custom_cost_per_token, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0f14b46c8ba..d06780dda53 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -496,6 +496,14 @@ def mask_api_base_credentials(api_base: str) -> str: return api_base[:key_end] + "*" * 5 + api_base[-4:] +def _timestamp_seconds(moment: object) -> float | None: + if isinstance(moment, datetime.datetime): + return moment.timestamp() + if isinstance(moment, (int, float)): + return float(moment) + return None + + class Logging(LiteLLMLoggingBaseClass): global \ supabaseClient, \ @@ -1634,10 +1642,12 @@ class Logging(LiteLLMLoggingBaseClass): return response.mcp_tool_call_response def get_response_ms(self) -> float: - return ( - self.model_call_details.get("end_time", datetime.datetime.now()) - - self.model_call_details.get("start_time", datetime.datetime.now()) - ).total_seconds() * 1000 + now: Final = datetime.datetime.now() + start_seconds: Final = _timestamp_seconds(self.model_call_details.get("start_time", now)) + end_seconds: Final = _timestamp_seconds(self.model_call_details.get("end_time", now)) + if start_seconds is None or end_seconds is None: + return 0.0 + return (end_seconds - start_seconds) * 1000 def set_cost_breakdown( self, diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 4a3c8de78c5..503814cc143 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -251,6 +251,6 @@ def update_response_metadata( return metadata: Final = ResponseMetadata(result) - metadata.set_hidden_params(logging_obj, model, kwargs) metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead) + metadata.set_hidden_params(logging_obj, model, kwargs) metadata.apply() diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 1bfc55d1486..021e012f29d 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -305,14 +305,15 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): class TestZeroCostDiagnostic: - DEPLOYMENT_ID: Final = "lit7898-per-second-priced-deployment" - MODEL_GROUP: Final = "per-second-priced-chat" + DEPLOYMENT_ID: Final = "lit7898-query-only-priced-deployment" + MODEL_GROUP: Final = "query-only-priced-chat" + QUERY_ONLY_PRICING: Final = {"input_cost_per_query": 0.00042} PER_SECOND_PRICING: Final = {"input_cost_per_second": 0.00042, "output_cost_per_second": 0.00042} FREE_PRICING: Final = {"input_cost_per_token": 0, "output_cost_per_token": 0} - @pytest.fixture(params=["per_second", "free"]) + @pytest.fixture(params=["query_only", "free"]) def deployment_pricing(self, request: pytest.FixtureRequest) -> Iterator[Mapping[str, float]]: - pricing: Final = self.PER_SECOND_PRICING if request.param == "per_second" else self.FREE_PRICING + pricing: Final = self.QUERY_ONLY_PRICING if request.param == "query_only" else self.FREE_PRICING litellm.register_model(model_cost={self.DEPLOYMENT_ID: pricing}, persist_across_reloads=False) try: yield pricing @@ -557,11 +558,11 @@ class TestZeroCostDiagnostic: priced_pricing: Final = {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06} usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) litellm.register_model( - model_cost={self.DEPLOYMENT_ID: self.PER_SECOND_PRICING, priced_id: priced_pricing}, + model_cost={self.DEPLOYMENT_ID: self.QUERY_ONLY_PRICING, priced_id: priced_pricing}, persist_across_reloads=False, ) try: - logging_obj: Final = self._logging_obj(self.PER_SECOND_PRICING) + logging_obj: Final = self._logging_obj(self.QUERY_ONLY_PRICING) with caplog.at_level(logging.WARNING, logger="LiteLLM"): assert logging_obj._response_cost_calculator(result=self._response(usage)) == 0.0 self._assert_flagged(logging_obj, caplog) @@ -570,7 +571,7 @@ class TestZeroCostDiagnostic: assert logging_obj._response_cost_calculator(result=self._response(usage)) == pytest.approx(5e-05) assert logging_obj.model_call_details["zero_cost_diagnostic"] is None - self._route_to_deployment(logging_obj, self.PER_SECOND_PRICING) + self._route_to_deployment(logging_obj, self.QUERY_ONLY_PRICING) assert logging_obj._response_cost_calculator(result=self._response(usage)) == 0.0 assert logging_obj.model_call_details["zero_cost_diagnostic"]["reason"] == "missing_pricing_key" @@ -585,7 +586,7 @@ class TestZeroCostDiagnostic: dated_model: Final = "lit7898-nano-2026-03-17" requested_model: Final = "lit7898-nano" usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) - cost_map_entry: Final = {"litellm_provider": "openai", "mode": "chat", **self.PER_SECOND_PRICING} + cost_map_entry: Final = {"litellm_provider": "openai", "mode": "chat", **self.QUERY_ONLY_PRICING} litellm.register_model( model_cost={dated_model: cost_map_entry, requested_model: cost_map_entry}, persist_across_reloads=False ) @@ -646,6 +647,24 @@ class TestZeroCostDiagnostic: assert logging_obj.model_call_details["zero_cost_diagnostic"] is None assert self._zero_cost_warnings(caplog) == [] + def test_per_second_priced_deployment_bills_the_call_duration_and_stays_silent( + self, caplog: pytest.LogCaptureFixture + ) -> None: + per_second_id: Final = "lit8315-per-second-priced-deployment" + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + litellm.register_model(model_cost={per_second_id: self.PER_SECOND_PRICING}, persist_across_reloads=False) + try: + logging_obj: Final = self._logging_obj(self.PER_SECOND_PRICING, deployment_id=per_second_id) + response: Final = self._response(usage) + response._response_ms = 1000.0 + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=response) == pytest.approx(0.00084) + + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + finally: + litellm.model_cost.pop(per_second_id, None) + @pytest.mark.parametrize("spilled_over", [True, False]) def test_ptu_deployment_is_judged_by_the_entry_the_calculator_priced_with( self, spilled_over: bool, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture @@ -663,7 +682,7 @@ class TestZeroCostDiagnostic: litellm.register_model( model_cost={ router_model_id: {**self.FREE_PRICING, "litellm_provider": "azure", "mode": "chat"}, - served_model: {**self.PER_SECOND_PRICING, "litellm_provider": "azure", "mode": "chat"}, + served_model: {**self.QUERY_ONLY_PRICING, "litellm_provider": "azure", "mode": "chat"}, }, persist_across_reloads=False, ) @@ -3504,6 +3523,20 @@ def _make_logging_obj(stream: bool) -> LitellmLogging: ) +def test_get_response_ms_measures_a_float_start_time_against_a_datetime_end_time(): + """The files paths construct the logging object with ``time.time()`` while the success + handler stamps a datetime end, and the per-second cost path reads this window.""" + logging_obj = _make_logging_obj(stream=False) + logging_obj.update_environment_variables( + model="openai/codex-mini-latest", user="", optional_params={}, litellm_params={} + ) + start_seconds = logging_obj.model_call_details["start_time"] + assert isinstance(start_seconds, float) + logging_obj.model_call_details["end_time"] = datetime.datetime.fromtimestamp(start_seconds + 1.5) + + assert logging_obj.get_response_ms() == pytest.approx(1500) + + def test_get_assembled_streaming_response_returns_none_for_non_streaming(): """Non-streaming requests should return None so the streaming block is skipped.""" import datetime diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 84c60e029cb..bdeedb5f38c 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,3 +1,4 @@ +import datetime import time from typing import Final @@ -29,6 +30,7 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, Usage, ) +from litellm.types.videos.main import VideoObject @pytest.fixture @@ -3038,21 +3040,23 @@ def test_completion_cost_logs_cache_and_reasoning_breakdown_for_custom_pricing() assert total == pytest.approx(100 * 1e-6 + 800 * 1e-7 + 100 * 1.25e-6 + 500 * 2e-6) -def test_cost_per_token_per_second_pricing(monkeypatch): +@pytest.mark.parametrize("custom_llm_provider", ["together_ai", "openai", "anthropic", "bedrock", "azure"]) +def test_cost_per_token_per_second_pricing(monkeypatch, custom_llm_provider: str): """ Models priced by duration (input/output_cost_per_second) with no per-token rates - must be billed as cost_per_second * response_time_ms / 1000 in cost_per_token. + must be billed as cost_per_second * response_time_ms / 1000 in cost_per_token, + whether or not the provider has its own cost calculator. """ monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - model = "test-per-second-pricing-model" + model = f"test-per-second-pricing-{custom_llm_provider}" litellm.register_model( model_cost={ model: { "input_cost_per_second": 0.02, "output_cost_per_second": 0.04, - "litellm_provider": "together_ai", + "litellm_provider": custom_llm_provider, "mode": "chat", } } @@ -3060,7 +3064,7 @@ def test_cost_per_token_per_second_pricing(monkeypatch): prompt_cost, completion_cost_value = cost_per_token( model=model, - custom_llm_provider="together_ai", + custom_llm_provider=custom_llm_provider, prompt_tokens=10, completion_tokens=20, response_time_ms=1500.0, @@ -3070,6 +3074,143 @@ def test_cost_per_token_per_second_pricing(monkeypatch): assert completion_cost_value == pytest.approx(0.04 * 1.5) +def test_cost_per_token_keeps_token_pricing_when_per_second_rates_are_also_set(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + model = "test-token-and-per-second-pricing-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + + prompt_cost, completion_cost_value = cost_per_token( + model=model, + custom_llm_provider="openai", + prompt_tokens=10, + completion_tokens=20, + response_time_ms=1500.0, + ) + + assert prompt_cost == pytest.approx(10 * 1e-6) + assert completion_cost_value == pytest.approx(20 * 2e-6) + + +def _logging_obj_with_call_window(duration_ms: float) -> Logging: + start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0) + logging_obj: Final = Logging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="per-second-call-window", + function_id="f", + ) + logging_obj.model_call_details["start_time"] = start_time + logging_obj.model_call_details["end_time"] = start_time + datetime.timedelta(milliseconds=duration_ms) + return logging_obj + + +@pytest.mark.parametrize( + ("stamped_response_ms", "total_time", "logged_duration_ms", "expected_seconds"), + [(None, 0.0, 1500.0, 1.5), (3000.0, 0.0, 1500.0, 3.0), (None, 2500.0, 1500.0, 2.5), (3000.0, 2500.0, 1500.0, 3.0)], +) +def test_completion_cost_per_second_deployment_bills_the_call_duration( + monkeypatch, + stamped_response_ms: float | None, + total_time: float, + logged_duration_ms: float, + expected_seconds: float, +): + """ + A deployment priced only per second bills the stamped ``_response_ms`` when there is one, + then the caller's explicit ``total_time``, and the logging object's start/end window otherwise + (a streamed response is never stamped). + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + deployment_id = "per-second-openai-deployment" + litellm.register_model( + model_cost={ + deployment_id: { + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + response = ModelResponse( + model="gpt-5.4-nano", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + response._response_ms = stamped_response_ms + + cost = completion_cost( + completion_response=response, + model="openai/gpt-5.4-nano", + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + total_time=total_time, + litellm_logging_obj=_logging_obj_with_call_window(logged_duration_ms), + ) + + assert cost == pytest.approx((0.02 + 0.04) * expected_seconds) + + +@pytest.mark.parametrize("mode", ["audio_transcription", "audio_speech", "video_generation", "realtime"]) +def test_cost_per_token_leaves_media_second_rates_to_their_dedicated_paths(monkeypatch, mode: str): + """ + A media-mode entry's per-second rates price audio or video seconds, which the dedicated + transcription, speech, video, and realtime paths bill from the media itself, so a call that + reaches the generic path with only a wall-clock duration must not bill them. + """ + model = f"test-media-per-second-{mode}" + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_second": 0.02, "output_cost_per_second": 0.4, "litellm_provider": "openai", "mode": mode}, + ) + + assert cost_per_token(model=model, custom_llm_provider="openai", response_time_ms=2000.0) == (0.0, 0.0) + + +def test_completion_cost_video_status_poll_bills_nothing_on_a_per_second_video_model(monkeypatch): + """ + Polling a video job returns a ``VideoObject`` with no stamped duration, so the cost path falls + back to the logging object's call window; on a video model priced per output second that + window must not be billed, or every status poll would charge for the seconds it took to answer. + """ + model = "test-veo-per-second-poll" + monkeypatch.setitem( + litellm.model_cost, + model, + {"output_cost_per_second": 0.4, "litellm_provider": "vertex_ai", "mode": "video_generation"}, + ) + video = VideoObject(id="video_1", object="video", status="completed", model=model, progress=100) + + cost = completion_cost( + completion_response=video, + model=model, + custom_llm_provider="vertex_ai", + call_type=CallTypes.video_retrieve.value, + litellm_logging_obj=_logging_obj_with_call_window(2000.0), + ) + + assert cost == 0.0 + + def _batch_cache_usage() -> Usage: return Usage( prompt_tokens=11000, diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py index 50409b2ea2c..37201e8155b 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -7,10 +7,12 @@ through _hidden_params to the x-litellm-callback-duration-ms response header. import asyncio import datetime +from typing import Final from unittest.mock import MagicMock import pytest +import litellm import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod import litellm.proxy.common_request_processing as common_request_processing_mod from litellm.litellm_core_utils.litellm_logging import Logging @@ -22,7 +24,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.utils import ModelResponse +from litellm.types.utils import ModelResponse, Usage class TestCallbackDurationMs: @@ -583,3 +585,57 @@ class TestLoggingInitCallbackDuration: # Should still be set (deep copy of None is essentially a no-op) assert hasattr(obj, "callback_duration_ms") assert obj.callback_duration_ms >= 0 + + +def test_update_response_metadata_prices_per_second_deployment_from_its_stamped_duration(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + deployment_id: Final = "per-second-deployment-response-metadata" + litellm.register_model( + model_cost={ + deployment_id: { + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0) + logging_obj: Final = Logging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="per-second-response-metadata", + function_id="f", + ) + logging_obj.update_environment_variables( + model="gpt-5.4-nano", + litellm_params={ + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "metadata": {"model_info": {"id": deployment_id}}, + }, + optional_params={}, + custom_llm_provider="openai", + ) + logging_obj.model_call_details["end_time"] = start_time + datetime.timedelta(seconds=10) + result: Final = ModelResponse( + model="gpt-5.4-nano", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + + update_response_metadata( + result=result, + logging_obj=logging_obj, + model="gpt-5.4-nano", + kwargs={"model_info": {"id": deployment_id}}, + start_time=start_time, + end_time=start_time + datetime.timedelta(seconds=2), + ) + + assert result._response_ms == pytest.approx(2000) + assert result._hidden_params["response_cost"] == pytest.approx((0.02 + 0.04) * 2) From 798d45970bf879053d1932ec16bd17868d4e8f83 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:29:26 -0700 Subject: [PATCH 07/28] chore(prices): sync OpenRouter prices: 2 models (#42418) openrouter/nex-agi/nex-n2.5-mini: supports_response_schema openrouter/nex-agi/nex-n2.5-pro: supports_tool_choice, supports_response_schema, supports_function_calling Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 8 ++++---- model_prices_and_context_window.json | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 77cada25d25..0f02ec8da5f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -77544,7 +77544,7 @@ "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_response_schema": false, + "supports_response_schema": true, "supports_tool_choice": false, "supports_vision": true, "supports_web_search": false @@ -77560,12 +77560,12 @@ "output_cost_per_token": 2.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, - "supports_function_calling": false, + "supports_function_calling": true, "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": false, + "supports_response_schema": true, + "supports_tool_choice": true, "supports_vision": true, "supports_web_search": false } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 77cada25d25..0f02ec8da5f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -77544,7 +77544,7 @@ "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_response_schema": false, + "supports_response_schema": true, "supports_tool_choice": false, "supports_vision": true, "supports_web_search": false @@ -77560,12 +77560,12 @@ "output_cost_per_token": 2.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, - "supports_function_calling": false, + "supports_function_calling": true, "supports_pdf_input": false, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": false, + "supports_response_schema": true, + "supports_tool_choice": true, "supports_vision": true, "supports_web_search": false } From 4936427094689f3006c208c08292f561b3dd3d91 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:30:41 -0700 Subject: [PATCH 08/28] feat(errors): prefilled GitHub issue link on unmapped internal errors (#42065) * feat(errors): prefilled GitHub issue link on unmapped internal errors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(errors): scope bug report link to unmapped exceptions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(errors): cover bug report link on unmapped SDK exceptions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(errors): exercise unmapped branch without changing openai mapping Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style: format bug report changes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): guard bug report generation for invalid exceptions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep bug report notice out of client error bodies Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(errors): strip only the notice separator from client messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(errors): only link bug reports for exceptions without a provider status Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(errors): allowlist-only bug report link, drop exception message and model from URL Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): read request.url.path for the unhandled-exception bug report route Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bug_report): drop non-string provider values before the allowlist lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(bug_report): drop the OS line from the prefilled issue Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: ryan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/bug_report.py | 155 ++++++++++++++++ .../exception_mapping_utils.py | 21 ++- litellm/proxy/common_request_processing.py | 35 +++- litellm/proxy/proxy_server.py | 17 ++ litellm/proxy/utils.py | 10 +- .../litellm_core_utils/test_bug_report.py | 170 ++++++++++++++++++ .../test_exception_mapping_utils.py | 27 ++- .../proxy/test_common_request_processing.py | 108 +++++++++++ tests/test_litellm/proxy/test_proxy_utils.py | 17 +- 9 files changed, 553 insertions(+), 7 deletions(-) create mode 100644 litellm/litellm_core_utils/bug_report.py create mode 100644 tests/test_litellm/litellm_core_utils/test_bug_report.py diff --git a/litellm/litellm_core_utils/bug_report.py b/litellm/litellm_core_utils/bug_report.py new file mode 100644 index 00000000000..6b4cf992dad --- /dev/null +++ b/litellm/litellm_core_utils/bug_report.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import os +import platform +import traceback +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal +from urllib.parse import urlencode + +import litellm +from litellm._version import version as litellm_version +from litellm.types.utils import LlmProviders + +ISSUE_URL_BASE: Final = "https://github.com/BerriAI/litellm/issues/new" +MAX_URL_LENGTH: Final = 6000 +MAX_FRAMES: Final = 12 +DISABLE_ENV_VAR: Final = "LITELLM_DISABLE_BUG_REPORT_LINK" +NOTICE_PREFIX: Final = "This looks like a bug in LiteLLM rather than in your request." +KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders) + +Surface = Literal["sdk", "proxy"] + + +@dataclass(frozen=True, slots=True) +class BugReport: + surface: Surface + exception_type: str + litellm_frames: tuple[str, ...] + litellm_version: str + python_version: str + call_type: str | None + custom_llm_provider: str | None + + +def bug_report_enabled() -> bool: + return os.getenv(DISABLE_ENV_VAR, "").lower() != "true" + + +def should_report_bug(exc: object) -> bool: + return bug_report_enabled() and isinstance(exc, BaseException) and getattr(exc, "status_code", None) is None + + +def _format_frame(frame: traceback.FrameSummary, package_dir: Path, package_parent: Path) -> str | None: + frame_path: Final = Path(frame.filename).resolve() + try: + frame_path.relative_to(package_dir) + relative_path: Final = frame_path.relative_to(package_parent) + except ValueError: + return None + return f"{relative_path.as_posix()}:{frame.lineno} in {frame.name}" + + +def _get_litellm_frames(exc: BaseException) -> tuple[str, ...]: + if exc.__traceback__ is None: + return () + package_dir: Final = Path(litellm.__file__).resolve().parent + package_parent: Final = package_dir.parent + return tuple( + frame_text + for frame in traceback.extract_tb(exc.__traceback__) + if (frame_text := _format_frame(frame, package_dir, package_parent)) is not None + )[-MAX_FRAMES:] + + +def allowlisted(value: object, allowed: frozenset[str]) -> str | None: + return value if isinstance(value, str) and value in allowed else None + + +def build_bug_report( + exc: BaseException, + *, + surface: Surface, + call_type: str | None = None, + custom_llm_provider: object = None, +) -> BugReport: + return BugReport( + surface=surface, + exception_type=type(exc).__name__, + litellm_frames=_get_litellm_frames(exc), + litellm_version=litellm_version, + python_version=platform.python_version(), + call_type=call_type, + custom_llm_provider=allowlisted(custom_llm_provider, KNOWN_PROVIDERS), + ) + + +def _domain(report: BugReport) -> str: + if report.surface == "sdk": + return "Python SDK: the litellm package itself" + if report.custom_llm_provider is not None: + return "LLM translation: a specific provider's request or response" + return "Proxy core: startup, config, health checks, endpoints" + + +def _title(report: BugReport, frames: tuple[str, ...]) -> str: + location: Final = frames[-1].split(":", 1)[0] if frames else "litellm" + return f"[Bug]: {report.exception_type} in {location}" + + +def _description(report: BugReport, frames: tuple[str, ...]) -> str: + frame_block: Final = "LiteLLM frames:\n```\n" + "\n".join(frames) + "\n```\n\n" if frames else "" + return ( + "Auto-generated by LiteLLM's bug report link. It carries no request data or error text. " + "Please describe what you were doing, and paste the error message from your log below " + "if it contains nothing sensitive.\n\n" + "```\n\n```\n\n" + f"Exception: `{report.exception_type}`\n\n" + f"{frame_block}" + f"Surface: {report.surface}\n" + f"Endpoint / call: {report.call_type or 'unknown'}\n" + f"Provider: {report.custom_llm_provider or 'unknown'}\n" + f"LiteLLM: {report.litellm_version}\n" + f"Python: {report.python_version}\n" + ) + + +def _issue_url(report: BugReport, frames: tuple[str, ...]) -> str: + deployment: Final[tuple[tuple[str, str], ...]] = ( + (("deployment", "pip / Python SDK"),) + if report.surface == "sdk" + else (("deployment", "Docker"),) + if os.path.exists("/.dockerenv") + else () + ) + fields: Final = ( + ("template", "bug_report.yml"), + ("labels", "bug"), + ("title", _title(report, frames)), + ("version", report.litellm_version), + ("domain", _domain(report)), + ("description", _description(report, frames)), + ) + deployment + return f"{ISSUE_URL_BASE}?{urlencode(fields)}" + + +def bug_report_issue_url(report: BugReport) -> str: + frames: Final = report.litellm_frames + candidates: Final = tuple(_issue_url(report, frames[index:]) for index in range(len(frames) + 1)) + return next((candidate for candidate in candidates if len(candidate) <= MAX_URL_LENGTH), candidates[-1]) + + +def strip_bug_report_notice(message: str) -> str: + index: Final = message.find(NOTICE_PREFIX) + if index == -1: + return message + head: Final = message[:index] + return head.removesuffix("\n") + + +def bug_report_notice(report: BugReport) -> str: + return ( + f"{NOTICE_PREFIX} File it with one click " + f"(prefilled, no request data or error text, review before submitting): {bug_report_issue_url(report)}" + ) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 425714d730f..da2f11f2593 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -10,6 +10,11 @@ import httpx import litellm from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger +from litellm.litellm_core_utils.bug_report import ( + bug_report_notice, + build_bug_report, + should_report_bug, +) from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.types.utils import LlmProviders @@ -2673,7 +2678,21 @@ def exception_type( ) else: raise APIConnectionError( - message=f"{original_exception}\n{_redact_string(traceback.format_exc())}", + message=( + f"{original_exception}\n{_redact_string(traceback.format_exc())}" + + ( + "\n" + + bug_report_notice( + build_bug_report( + original_exception, + surface="sdk", + custom_llm_provider=custom_llm_provider, + ) + ) + if should_report_bug(original_exception) + else "" + ) + ), llm_provider=custom_llm_provider, model=model, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f6e0d56127f..c95cc3e0529 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -19,6 +19,7 @@ from typing import ( overload, runtime_checkable, ) +from urllib.parse import urlparse import anyio import httpx @@ -45,6 +46,13 @@ from litellm.constants import ( UNSAFE_PROXY_RESPONSE_HEADERS, ) from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.bug_report import ( + allowlisted, + bug_report_notice, + build_bug_report, + should_report_bug, + strip_bug_report_notice, +) from litellm.litellm_core_utils.core_helpers import ( get_or_create_metadata_bucket, independent_snapshot, @@ -64,7 +72,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.streaming_handler import ( backfill_missing_cache_usage_fields, ) -from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, request_skips_budget_checks, @@ -107,6 +115,10 @@ from litellm.types.router_weights import validate_router_weights _LateResponseT = TypeVar("_LateResponseT", bound=Response) _LlmCallT = TypeVar("_LlmCallT") +KNOWN_PROXY_ROUTES: Final = frozenset( + route for member in LiteLLMRoutes for route in member.value if route.startswith("/") +) + ProxyRouteType: TypeAlias = Literal[ "acompletion", "aembedding", @@ -3669,8 +3681,27 @@ class ProxyBaseLLMRequestProcessing: _code = _exc_status_code else: _code = status.HTTP_500_INTERNAL_SERVER_ERROR + if should_report_bug(e): + proxy_server_request: Final = self.data.get("proxy_server_request") + request_url: Final = ( + proxy_server_request.get("url") if isinstance(proxy_server_request, Mapping) else None + ) + request_path: Final = urlparse(str(request_url)).path if request_url is not None else None + verbose_proxy_logger.error( + bug_report_notice( + build_bug_report( + e, + surface="proxy", + call_type=allowlisted(request_path, KNOWN_PROXY_ROUTES), + custom_llm_provider=self.data.get("custom_llm_provider"), + ) + ) + ) + client_message: Final = getattr(e, "message", error_msg) raise ProxyException( - message=redact_internal_details_from_client_message(getattr(e, "message", error_msg)), + message=redact_internal_details_from_client_message( + strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg + ), type=openai_error_type(e, _code), param=openai_error_param(e), openai_code=getattr(e, "code", None), diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6fa7a7d761f..77246804dcd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -75,6 +75,12 @@ from litellm.constants import ( RUNTIME_UPDATABLE_ROUTER_SETTINGS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.bug_report import ( + allowlisted, + bug_report_notice, + build_bug_report, + should_report_bug, +) from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, ) @@ -371,6 +377,7 @@ from litellm.proxy.batches_endpoints.endpoints import router as batches_router ## Import All Misc routes here ## from litellm.proxy.caching_routes import router as caching_router from litellm.proxy.common_request_processing import ( + KNOWN_PROXY_ROUTES, ProxyBaseLLMRequestProcessing, _is_azure_model_router_request, _should_return_raw_model_name, @@ -1965,6 +1972,16 @@ async def otel_unhandled_exception_handler(request: Request, exc: Exception): if isinstance(exc, (ProxyException, HTTPException, RequestValidationError)): raise exc verbose_proxy_logger.exception("Unhandled exception in request: %s", type(exc).__name__) + if should_report_bug(exc): + verbose_proxy_logger.error( + bug_report_notice( + build_bug_report( + exc, + surface="proxy", + call_type=allowlisted(request.url.path, KNOWN_PROXY_ROUTES), + ) + ) + ) _close_dangling_otel_server_span(request, 500, exc=exc) return JSONResponse( status_code=500, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bfd412db1a7..b4846d529ab 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -57,6 +57,12 @@ from litellm.constants import ( SPEND_LOG_WRITE_BATCH_MAX_BYTES, SPEND_LOG_WRITE_BATCH_MAX_ROWS, ) +from litellm.litellm_core_utils.bug_report import ( + bug_report_notice, + build_bug_report, + should_report_bug, + strip_bug_report_notice, +) from litellm.proxy._types import ( CommonProxyErrors, ProxyErrorTypes, @@ -7989,8 +7995,10 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) elif isinstance(e, ProxyException): return with_litellm_call_id(e, litellm_call_id) _status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) + if should_report_bug(e): + verbose_proxy_logger.error(bug_report_notice(build_bug_report(e, surface="proxy"))) return ProxyException( - message=str(e), + message=strip_bug_report_notice(str(e)), type=ProxyErrorTypes.internal_server_error, param=openai_error_param(e), headers=headers, diff --git a/tests/test_litellm/litellm_core_utils/test_bug_report.py b/tests/test_litellm/litellm_core_utils/test_bug_report.py new file mode 100644 index 00000000000..c9189ebbbdd --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_bug_report.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import cast +from urllib.parse import parse_qs, urlparse + +import httpx +import pytest + +import litellm +from litellm._version import version +from litellm.exceptions import APIConnectionError, BadRequestError, InternalServerError +from litellm.litellm_core_utils.bug_report import ( + DISABLE_ENV_VAR, + ISSUE_URL_BASE, + MAX_FRAMES, + MAX_URL_LENGTH, + allowlisted, + bug_report_enabled, + bug_report_issue_url, + bug_report_notice, + build_bug_report, + should_report_bug, + strip_bug_report_notice, +) +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + +def test_build_bug_report_keeps_only_litellm_frames(): + with pytest.raises(BadRequestError) as raised: + get_llm_provider(cast(str, None)) + report = build_bug_report(raised.value, surface="sdk") + + assert report.litellm_frames + assert all(frame.startswith("litellm/") for frame in report.litellm_frames) + assert all("test_bug_report.py" not in frame for frame in report.litellm_frames) + + +def test_issue_url_never_contains_the_exception_message(): + secret = "sk-abcdefghijklmnopqrstuvwxyz1234567890" + prompt = "my social security number is 123-45-6789" + report = build_bug_report(RuntimeError(f"{secret} {prompt}"), surface="sdk") + url = bug_report_issue_url(report) + query = parse_qs(urlparse(url).query) + + assert url.startswith(ISSUE_URL_BASE) + assert secret not in url and "123-45-6789" not in url and "social" not in url + assert query["title"] == ["[Bug]: RuntimeError in litellm"] + assert query["version"] == [version] + assert query["template"] == ["bug_report.yml"] + assert query["domain"] == ["Python SDK: the litellm package itself"] + assert query["deployment"] == ["pip / Python SDK"] + assert "Exception: `RuntimeError`" in query["description"][0] + assert "Python:" in query["description"][0] + + +def test_issue_url_drops_unknown_provider_and_call_type(): + report = build_bug_report( + ValueError("boom"), + surface="proxy", + custom_llm_provider="acme-internal-gateway", + ) + query = parse_qs(urlparse(bug_report_issue_url(report)).query) + + assert report.custom_llm_provider is None + assert "acme" not in bug_report_issue_url(report) + assert "Provider: unknown" in query["description"][0] + assert "Endpoint / call: unknown" in query["description"][0] + + +def test_allowlisted_only_passes_exact_members(): + allowed = frozenset({"/v1/chat/completions"}) + + assert allowlisted("/v1/chat/completions", allowed) == "/v1/chat/completions" + assert allowlisted("/v1/chat/completions/../../admin", allowed) is None + assert allowlisted(None, allowed) is None + assert allowlisted(["/v1/chat/completions"], allowed) is None + assert allowlisted({"provider": "openai"}, allowed) is None + + +def test_build_bug_report_survives_unhashable_provider_from_request_data(): + report = build_bug_report( + KeyError("missing"), + surface="proxy", + custom_llm_provider={"name": "openai"}, + ) + + assert report.custom_llm_provider is None + + +def test_frames_are_capped_and_url_is_bounded(): + namespace: dict[str, object] = {} + exec( + compile( + "def recurse(depth):\n if depth == 0:\n raise RuntimeError('deep')\n recurse(depth - 1)\n", + str(Path(litellm.__file__).with_name("fake_deep_module.py")), + "exec", + ), + namespace, + ) + recurse = cast(Callable[[int], None], namespace["recurse"]) + + with pytest.raises(RuntimeError) as raised: + recurse(200) + report = build_bug_report(raised.value, surface="proxy") + + assert len(report.litellm_frames) == MAX_FRAMES + assert all(frame.startswith("litellm/fake_deep_module.py:") for frame in report.litellm_frames) + assert len(bug_report_issue_url(report)) <= MAX_URL_LENGTH + + +def test_issue_url_builds_without_a_traceback(): + exc = RuntimeError("no traceback") + assert exc.__traceback__ is None + report = build_bug_report(exc, surface="proxy") + + assert report.litellm_frames == () + assert bug_report_issue_url(report).startswith(ISSUE_URL_BASE) + + +def test_bug_report_can_be_disabled(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(DISABLE_ENV_VAR, "true") + + assert bug_report_enabled() is False + assert should_report_bug(RuntimeError("boom")) is False + + +@pytest.mark.parametrize( + "exc", + [ + InternalServerError(message="upstream 500", llm_provider="openai", model="gpt-4"), + APIConnectionError( + message="connection reset", + llm_provider="openai", + model="gpt-4", + request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), + ), + BadRequestError(message="bad input", llm_provider="openai", model="gpt-4"), + "not an exception", + ], +) +def test_should_report_bug_skips_provider_and_network_errors(exc: object): + assert should_report_bug(exc) is False + + +def test_should_report_bug_accepts_plain_python_errors(): + assert should_report_bug(KeyError("missing")) is True + + +def test_proxy_known_provider_uses_translation_domain(): + report = build_bug_report( + RuntimeError("proxy failure"), + surface="proxy", + call_type="/v1/chat/completions", + custom_llm_provider="openai", + ) + query = parse_qs(urlparse(bug_report_issue_url(report)).query) + + assert report.custom_llm_provider == "openai" + assert query["domain"] == ["LLM translation: a specific provider's request or response"] + assert "Endpoint / call: /v1/chat/completions" in query["description"][0] + + +def test_strip_bug_report_notice(): + report = build_bug_report(RuntimeError("boom"), surface="sdk") + notice = bug_report_notice(report) + + assert strip_bug_report_notice(f"boom\n\n{notice}") == "boom\n" + assert strip_bug_report_notice("boom") == "boom" diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index cfe7470fa76..5c5c2c9536b 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -3,8 +3,6 @@ import openai import pytest import litellm - - from litellm.litellm_core_utils.exception_mapping_utils import ( ExceptionCheckers, _get_body_error_code, @@ -974,6 +972,31 @@ def test_an_unmapped_exception_with_no_model_or_provider_is_a_connection_error(q assert "boom" in raised.value.message +def test_unmapped_sdk_exception_includes_bug_report_link(quiet_exception_mapping): + with pytest.raises(litellm.APIConnectionError) as raised: + exception_type( + model="my-model", + custom_llm_provider="minimax", + original_exception=ValueError("boom"), + ) + + assert "https://github.com/BerriAI/litellm/issues/new?" in str(raised.value) + assert "ValueError" in str(raised.value) + + +def test_unmapped_sdk_exception_bug_report_link_can_be_disabled(quiet_exception_mapping, monkeypatch): + monkeypatch.setenv("LITELLM_DISABLE_BUG_REPORT_LINK", "true") + + with pytest.raises(litellm.APIConnectionError) as raised: + exception_type( + model="my-model", + custom_llm_provider="minimax", + original_exception=ValueError("boom"), + ) + + assert "https://github.com/BerriAI/litellm/issues/new?" not in str(raised.value) + + def _raise_and_map(model: str | None, original_exception: Exception, custom_llm_provider: str | None) -> None: """Calls exception_type() from inside the except block, as litellm/main.py does, so traceback.format_exc() has a real stack.""" diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 218d8246715..af1604219d1 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -4,6 +4,7 @@ import datetime import json from types import MappingProxyType, SimpleNamespace from typing import AsyncGenerator, Callable, Final, Iterator, Optional, Sequence +from urllib.parse import unquote_plus from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -13,6 +14,12 @@ from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid +from litellm.litellm_core_utils.bug_report import ( + DISABLE_ENV_VAR, + ISSUE_URL_BASE, + bug_report_notice, + build_bug_report, +) from litellm.constants import ( CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_LITELLM_CALL_ID_LENGTH, @@ -4290,6 +4297,21 @@ class TestHandleLLMApiExceptionRetryAfter: proxy_exc = await self._invoke(ValueError("some other failure")) assert "retry-after" not in proxy_exc.headers + async def test_handle_llm_api_exception_strips_bug_report_notice_from_client_message(self, caplog): + report = build_bug_report(RuntimeError("boom"), surface="sdk") + notice = bug_report_notice(report) + exc = litellm.APIConnectionError( + message=f"boom\n{notice}", + model="gpt-4o", + llm_provider="openai", + ) + + with caplog.at_level("ERROR"): + proxy_exc = await self._invoke(exc) + + assert ISSUE_URL_BASE not in proxy_exc.message + assert ISSUE_URL_BASE in caplog.text + async def test_handle_llm_api_exception_retry_after_survives_callback_headers(self): from litellm.types.router import RouterRateLimitError @@ -9374,6 +9396,92 @@ async def test_handle_llm_api_exception_forwards_litellm_response_headers_when_r assert exc_info.value.headers["llm_provider-x-request-id"] == "req_openai_400" +@pytest.mark.asyncio +async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + monkeypatch.delenv(DISABLE_ENV_VAR, raising=False) + processor = ProxyBaseLLMRequestProcessing( + data={ + "proxy_server_request": {"url": "https://example.test/v1/chat/completions?debug=true"}, + "model": "acme-prod-gpt4", + "custom_llm_provider": "openai", + } + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + with pytest.raises(ProxyException): + await processor._handle_llm_api_exception( + e=RuntimeError("unmapped for user@example.com"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + issue_url = next(word for word in caplog.text.split() if word.startswith(ISSUE_URL_BASE)) + assert "Endpoint / call: /v1/chat/completions" in unquote_plus(issue_url) + assert "Provider: openai" in unquote_plus(issue_url) + assert "acme-prod-gpt4" not in unquote_plus(issue_url) + assert "user@example.com" not in unquote_plus(issue_url) + + +@pytest.mark.asyncio +async def test_handle_llm_api_exception_bug_report_drops_unknown_route( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + monkeypatch.delenv(DISABLE_ENV_VAR, raising=False) + processor = ProxyBaseLLMRequestProcessing( + data={"proxy_server_request": {"url": "https://example.test/v1/files/file-customer-123/content"}} + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + with pytest.raises(ProxyException): + await processor._handle_llm_api_exception( + e=RuntimeError("unmapped"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + issue_url = next(word for word in caplog.text.split() if word.startswith(ISSUE_URL_BASE)) + assert "Endpoint / call: unknown" in unquote_plus(issue_url) + assert "file-customer-123" not in unquote_plus(issue_url) + + +@pytest.mark.asyncio +async def test_handle_llm_api_exception_skips_bug_report_for_provider_status( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + monkeypatch.delenv(DISABLE_ENV_VAR, raising=False) + + class ProviderRateLimitError(Exception): + def __init__(self, message: str): + super().__init__(message) + self.status_code = 429 + + processor = ProxyBaseLLMRequestProcessing(data={}) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + with pytest.raises(ProxyException): + await processor._handle_llm_api_exception( + e=ProviderRateLimitError("rate limited"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + assert ISSUE_URL_BASE not in caplog.text + + class TestBackgroundResponseRetrievalGovernance: """LIT-7175: retrieving a background Response attaches the model's post_call policy pipelines.""" diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 734408d9b61..0fc7295a717 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -6,10 +6,12 @@ import pytest from fastapi import HTTPException from litellm.caching.caching import DualCache +from litellm.exceptions import InternalServerError from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.bug_report import ISSUE_URL_BASE from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth -from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy from litellm.types.guardrails import GuardrailEventHooks @@ -2418,3 +2420,16 @@ def test_mcp_auth_policy_uses_original_request_model(monkeypatch, model, expecte synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs) assert ("model-rule" in synthetic["metadata"]["guardrails"]) is expected assert "request-rule" in synthetic["metadata"]["guardrails"] + + +def test_handle_exception_on_proxy_logs_bug_report_only_for_unmapped_500(caplog): + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + provider_result = handle_exception_on_proxy( + InternalServerError(message="upstream 500", llm_provider="openai", model="gpt-4") + ) + assert ISSUE_URL_BASE not in caplog.text + internal_result = handle_exception_on_proxy(KeyError("missing")) + + assert provider_result.code == internal_result.code == "500" + assert ISSUE_URL_BASE in caplog.text + assert ISSUE_URL_BASE not in internal_result.message From 30d8b1251276a051130d0d69acbcf0a83247c529 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:32:56 -0700 Subject: [PATCH 09/28] test(e2e): add conversational matrix across chat, messages and responses (#42359) * test(e2e): add conversational matrix across chat, messages and responses Parameterizes one behavioral contract (reply, stream, cost log, tool call, tool round trip) across /v1/chat/completions, /v1/messages and /v1/responses, OpenAI and Anthropic models, and env-ref vs stored-credential auth, with record/replay fixtures. Adds general_settings.disable_model_info_refresh so the proxy fronting a replay fixture does not poll every OpenAI-compatible deployment's /v1/models in the background, which otherwise leaves unconsumed interactions in the recorded bundle. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): force the weather tool on the first turn and rename Provider to Deployment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 30 +- .../coverage_registry/llm_conversational.yaml | 12 + tests/e2e/gateway/record_replay_ci_config.yml | 1 + .../llm_translation/conversational_matrix.py | 544 ++++++++++++++++++ .../test_conversational_matrix_e2e.py | 163 ++++++ .../proxy/proxy_server/test_lifecycle.py | 29 + 6 files changed, 767 insertions(+), 12 deletions(-) create mode 100644 tests/e2e/llm_translation/conversational_matrix.py create mode 100644 tests/e2e/llm_translation/test_conversational_matrix_e2e.py diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 77246804dcd..f8945694051 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1494,23 +1494,29 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: ## Initialize shared aiohttp session for connection reuse shared_aiohttp_session = await _initialize_shared_aiohttp_session() - model_info_scheduler: Final = scheduler if scheduler is not None else AsyncIOScheduler() - model_info_scheduler.add_job( - ProxyStartupEvent.refresh_model_info, - "interval", - seconds=MODEL_INFO_REFRESH_SECONDS, - id="refresh_model_info", - next_run_time=datetime.now(timezone.utc), - max_instances=1, - replace_existing=True, + model_info_refresh_disabled: Final = ( + "disable_model_info_refresh" in general_settings and general_settings["disable_model_info_refresh"] is True ) - if not model_info_scheduler.running: - model_info_scheduler.start() + model_info_scheduler: Final = ( + None if model_info_refresh_disabled else scheduler if scheduler is not None else AsyncIOScheduler() + ) + if model_info_scheduler is not None: + model_info_scheduler.add_job( + ProxyStartupEvent.refresh_model_info, + "interval", + seconds=MODEL_INFO_REFRESH_SECONDS, + id="refresh_model_info", + next_run_time=datetime.now(timezone.utc), + max_instances=1, + replace_existing=True, + ) + if not model_info_scheduler.running: + model_info_scheduler.start() # End of startup event yield - if model_info_scheduler.running: + if model_info_scheduler is not None and model_info_scheduler.running: model_info_scheduler.remove_job("refresh_model_info") if model_info_scheduler is not scheduler: model_info_scheduler.shutdown(wait=False) diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 49d4d92ff0b..e4e1ac2c7b6 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -94,3 +94,15 @@ - {id: llm.messages.together_ai.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together over /v1/messages streaming"} - {id: llm.messages.together_ai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over /v1/messages"} - {id: llm.messages.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip over /v1/messages"} +- {id: llm.chat_completions.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /chat/completions: cost header and spend row agree"} +- {id: llm.chat_completions.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /chat/completions"} +- {id: llm.messages.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /v1/messages"} +- {id: llm.messages.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI models served on the Anthropic Messages contract"} +- {id: llm.messages.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI over /v1/messages streams the Anthropic event grammar"} +- {id: llm.messages.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI over /v1/messages: cost header and spend row agree"} +- {id: llm.messages.openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI tool calls translated to Anthropic tool_use blocks"} +- {id: llm.messages.openai.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI tool result round trip over /v1/messages"} +- {id: llm.responses.openai.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI function_call_output round trip over /v1/responses"} +- {id: llm.responses.anthropic.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses streams Responses events"} +- {id: llm.responses.anthropic.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses: cost header and spend row agree"} +- {id: llm.responses.anthropic.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic function_call_output round trip over /v1/responses"} diff --git a/tests/e2e/gateway/record_replay_ci_config.yml b/tests/e2e/gateway/record_replay_ci_config.yml index 08972969cf0..5b3db0ff530 100644 --- a/tests/e2e/gateway/record_replay_ci_config.yml +++ b/tests/e2e/gateway/record_replay_ci_config.yml @@ -1,3 +1,4 @@ general_settings: master_key: os.environ/LITELLM_MASTER_KEY store_model_in_db: true + disable_model_info_refresh: true diff --git a/tests/e2e/llm_translation/conversational_matrix.py b/tests/e2e/llm_translation/conversational_matrix.py new file mode 100644 index 00000000000..0d6f6ed3d4e --- /dev/null +++ b/tests/e2e/llm_translation/conversational_matrix.py @@ -0,0 +1,544 @@ +"""The endpoint x deployment x auth matrix behind test_conversational_matrix_e2e.py. + +One conversation, three wire formats. Each `Surface` speaks its own API through +the customer SDK (chat completions and Responses through the OpenAI SDK, Messages +through the Anthropic SDK) and folds what came back into the surface-neutral +`Reply` / `StreamedReply`, so a single behavior test asserts the same contract on +every cell. A new model, from an existing or a new provider, is one `Deployment` row +in DEPLOYMENTS; a new way of handing the proxy a provider credential is one +`AuthMethod`. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal, Protocol + +import anthropic +import openai +import pytest +from _pytest.mark.structures import ParameterSet +from anthropic.types import ( + MessageParam, + RawMessageStreamEvent, + TextBlock, + ToolChoiceToolParam, + ToolParam, + ToolResultBlockParam, + ToolUseBlock, + ToolUseBlockParam, +) +from e2e_config import provider_edge_base, unique_marker +from lifecycle import ResourceManager +from llm_translation.sdk_clients import NO_PROXY_CACHE, SdkClients, response_header +from models import CredentialCreateBody, LiteLLMParamsBody +from openai.types.chat import ( + ChatCompletionAssistantMessageParam, + ChatCompletionChunk, + ChatCompletionMessageFunctionToolCallParam, + ChatCompletionMessageParam, + ChatCompletionNamedToolChoiceParam, + ChatCompletionToolMessageParam, + ChatCompletionToolParam, +) +from openai.types.chat.chat_completion_message_function_tool_call import ChatCompletionMessageFunctionToolCall +from openai.types.responses import ( + FunctionToolParam, + ResponseFunctionToolCall, + ResponseFunctionToolCallParam, + ResponseInputParam, + ResponseStreamEvent, + ToolChoiceFunctionParam, +) +from openai.types.responses.response_input_param import FunctionCallOutput +from proxy_client import ProxyClient +from pydantic import BaseModel + +SurfaceName = Literal["chat_completions", "messages", "responses"] +AuthMethod = Literal["env_ref", "stored_credential"] +Capability = Literal["basic", "tool_use", "multi_turn"] +Streaming = Literal["stream", "nonstream"] +Assertion = Literal["works", "cost_logged"] +ToolMode = Literal["none", "forced", "offered"] + +SURFACES: Final[tuple[SurfaceName, ...]] = ("chat_completions", "messages", "responses") +AUTH_METHODS: Final[tuple[AuthMethod, ...]] = ("env_ref", "stored_credential") + +MAX_OUTPUT_TOKENS: Final = 512 +INSTRUCTIONS: Final = "You are a terse assistant. Answer in one short sentence." +GREETING_PROMPT: Final = "Say hello." +WEATHER_PROMPT: Final = "What is the weather in Paris right now? Use the get_weather tool." +WEATHER_REPORT: Final = "Paris: 22 degrees Celsius, clear skies" +WEATHER_TOOL_NAME: Final = "get_weather" +WEATHER_TOOL_DESCRIPTION: Final = "Current weather for a city" +WEATHER_TOOL_SCHEMA: Final[Mapping[str, object]] = MappingProxyType( + { + "type": "object", + "properties": {"location": {"type": "string", "description": "City name"}}, + "required": ["location"], + } +) + + +@dataclass(frozen=True, slots=True) +class Deployment: + """One deployment target: the litellm backend string plus how to wire it.""" + + route: Literal["openai", "anthropic"] + label: str + backend: str + api_key_env: str + edge_mount: str + edge_suffix: str + + def api_base(self) -> str | None: + base: Final = provider_edge_base(self.edge_mount) + return None if base is None else f"{base}{self.edge_suffix}" + + def api_key(self) -> str: + key: Final = os.environ.get(self.api_key_env, "") + assert key, f"{self.api_key_env} is not set in the test process environment" + return key + + +DEPLOYMENTS: Final[tuple[Deployment, ...]] = ( + Deployment( + route="openai", + label="gpt-4o-mini", + backend="openai/gpt-4o-mini", + api_key_env="OPENAI_API_KEY", + edge_mount="openai", + edge_suffix="/v1", + ), + Deployment( + route="openai", + label="gpt-5.4-mini", + backend="openai/gpt-5.4-mini", + api_key_env="OPENAI_API_KEY", + edge_mount="openai", + edge_suffix="/v1", + ), + Deployment( + route="anthropic", + label="claude-haiku-4-5", + backend="anthropic/claude-haiku-4-5", + api_key_env="ANTHROPIC_API_KEY", + edge_mount="anthropic", + edge_suffix="", + ), +) + + +@dataclass(frozen=True, slots=True) +class Cell: + surface: SurfaceName + deployment: Deployment + auth: AuthMethod + + @property + def id(self) -> str: + return f"{self.surface}-{self.deployment.label}-{self.auth}" + + def registry_id(self, capability: Capability, streaming: Streaming, assertion: Assertion) -> str: + return f"llm.{self.surface}.{self.deployment.route}.{capability}.{streaming}.{assertion}" + + +CELLS: Final[tuple[Cell, ...]] = tuple( + Cell(surface=surface, deployment=deployment, auth=auth) + for surface in SURFACES + for deployment in DEPLOYMENTS + for auth in AUTH_METHODS +) + + +def cells_covering(capability: Capability, streaming: Streaming, assertion: Assertion) -> tuple[ParameterSet, ...]: + """Every cell as a pytest param carrying the registry id its test proves.""" + return tuple( + pytest.param(cell, id=cell.id, marks=pytest.mark.covers(cell.registry_id(capability, streaming, assertion))) + for cell in CELLS + ) + + +DeploymentKey = tuple[str, AuthMethod] + + +@dataclass(frozen=True, slots=True) +class Deployments: + """Model aliases registered on the proxy, one per (deployment, auth).""" + + aliases: Mapping[DeploymentKey, str] + + def alias(self, cell: Cell) -> str: + return self.aliases[(cell.deployment.label, cell.auth)] + + +def _litellm_params(deployment: Deployment, auth: AuthMethod, credential_name: str) -> LiteLLMParamsBody: + match auth: + case "env_ref": + return LiteLLMParamsBody( + model=deployment.backend, api_key=f"os.environ/{deployment.api_key_env}", api_base=deployment.api_base() + ) + case "stored_credential": + return LiteLLMParamsBody( + model=deployment.backend, litellm_credential_name=credential_name, api_base=deployment.api_base() + ) + + +def _register(proxy: ProxyClient, resources: ResourceManager, deployment: Deployment, auth: AuthMethod) -> str: + marker: Final = unique_marker() + credential_name: Final = f"e2e-matrix-{deployment.label}-{marker}" + if auth == "stored_credential": + proxy.create_credential( + CredentialCreateBody(credential_name=credential_name, credential_values={"api_key": deployment.api_key()}) + ) + resources.defer(lambda: proxy.delete_credential(credential_name)) + alias: Final = f"e2e-matrix-{deployment.label}-{auth}-{marker}" + model_id: Final = proxy.create_model(alias, _litellm_params(deployment, auth, credential_name)) + resources.defer(lambda: proxy.delete_model(model_id)) + return alias + + +def register_deployments(proxy: ProxyClient) -> Iterator[Deployments]: + resources: Final = ResourceManager(client=proxy) + try: + yield Deployments( + aliases=MappingProxyType( + { + (deployment.label, auth): _register(proxy, resources, deployment, auth) + for deployment in DEPLOYMENTS + for auth in AUTH_METHODS + } + ) + ) + finally: + resources.teardown() + + +class WeatherArgs(BaseModel): + location: str + + +@dataclass(frozen=True, slots=True) +class ToolCall: + call_id: str + name: str + arguments: str + + def parsed(self) -> WeatherArgs: + return WeatherArgs.model_validate_json(self.arguments) + + +@dataclass(frozen=True, slots=True) +class Usage: + input_tokens: int + output_tokens: int + + +@dataclass(frozen=True, slots=True) +class Reply: + """What every surface owes the caller for one non-streamed turn.""" + + response_id: str + text: str + tool_calls: tuple[ToolCall, ...] + usage: Usage | None + call_id_header: str | None + cost_header: str | None + + +@dataclass(frozen=True, slots=True) +class StreamedReply: + """The reassembled stream: its text, whether the surface's own terminal event + arrived, and whether usage was reported anywhere in the stream.""" + + text: str + finished: bool + usage_reported: bool + event_count: int + + +class Surface(Protocol): + @property + def name(self) -> SurfaceName: ... + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: ... + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: ... + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: ... + + +def _chat_tool() -> ChatCompletionToolParam: + return { + "type": "function", + "function": { + "name": WEATHER_TOOL_NAME, + "description": WEATHER_TOOL_DESCRIPTION, + "parameters": dict(WEATHER_TOOL_SCHEMA), + }, + } + + +def _messages_tool() -> ToolParam: + return { + "name": WEATHER_TOOL_NAME, + "description": WEATHER_TOOL_DESCRIPTION, + "input_schema": dict(WEATHER_TOOL_SCHEMA), + } + + +def _responses_tool() -> FunctionToolParam: + return { + "type": "function", + "name": WEATHER_TOOL_NAME, + "description": WEATHER_TOOL_DESCRIPTION, + "parameters": dict(WEATHER_TOOL_SCHEMA), + "strict": False, + } + + +def _chat_tool_choice() -> ChatCompletionNamedToolChoiceParam: + return {"type": "function", "function": {"name": WEATHER_TOOL_NAME}} + + +def _messages_tool_choice() -> ToolChoiceToolParam: + return {"type": "tool", "name": WEATHER_TOOL_NAME, "disable_parallel_tool_use": True} + + +def _responses_tool_choice() -> ToolChoiceFunctionParam: + return {"type": "function", "name": WEATHER_TOOL_NAME} + + +def _usage(input_tokens: int | None, output_tokens: int | None) -> Usage | None: + if input_tokens is None or output_tokens is None: + return None + return Usage(input_tokens=input_tokens, output_tokens=output_tokens) + + +@dataclass(frozen=True, slots=True) +class ChatCompletionsSurface: + sdk: SdkClients + name: SurfaceName = "chat_completions" + + def _turn(self, key: str, model: str, messages: Sequence[ChatCompletionMessageParam], tool: ToolMode) -> Reply: + raw: Final = self.sdk.openai(key).chat.completions.with_raw_response.create( + model=model, + messages=list(messages), + max_completion_tokens=MAX_OUTPUT_TOKENS, + tools=openai.omit if tool == "none" else [_chat_tool()], + tool_choice=_chat_tool_choice() if tool == "forced" else openai.omit, + parallel_tool_calls=False if tool == "forced" else openai.omit, + extra_body=NO_PROXY_CACHE, + ) + completion: Final = raw.parse() + message: Final = completion.choices[0].message + calls: Final = tuple( + ToolCall(call_id=call.id, name=call.function.name, arguments=call.function.arguments) + for call in message.tool_calls or () + if isinstance(call, ChatCompletionMessageFunctionToolCall) + ) + return Reply( + response_id=completion.id, + text=message.content or "", + tool_calls=calls, + usage=None + if completion.usage is None + else _usage(completion.usage.prompt_tokens, completion.usage.completion_tokens), + call_id_header=response_header(raw.headers, "x-litellm-call-id"), + cost_header=response_header(raw.headers, "x-litellm-response-cost"), + ) + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: + return self._turn(key, model, _chat_history(prompt), "forced" if with_tool else "none") + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: + chunks: Final[tuple[ChatCompletionChunk, ...]] = tuple( + self.sdk.openai(key).chat.completions.create( + model=model, + messages=_chat_history(prompt), + max_completion_tokens=MAX_OUTPUT_TOKENS, + stream=True, + stream_options={"include_usage": True}, + extra_body=NO_PROXY_CACHE, + ) + ) + return StreamedReply( + text="".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices), + finished=any(chunk.choices[0].finish_reason is not None for chunk in chunks if chunk.choices), + usage_reported=any(chunk.usage is not None for chunk in chunks), + event_count=len(chunks), + ) + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: + tool_call: Final[ChatCompletionMessageFunctionToolCallParam] = { + "id": call.call_id, + "type": "function", + "function": {"name": call.name, "arguments": call.arguments}, + } + assistant: Final[ChatCompletionAssistantMessageParam] = {"role": "assistant", "tool_calls": [tool_call]} + tool_result: Final[ChatCompletionToolMessageParam] = { + "role": "tool", + "tool_call_id": call.call_id, + "content": result, + } + return self._turn(key, model, (*_chat_history(prompt), assistant, tool_result), "offered") + + +def _chat_history(prompt: str) -> tuple[ChatCompletionMessageParam, ...]: + return ({"role": "system", "content": INSTRUCTIONS}, {"role": "user", "content": prompt}) + + +@dataclass(frozen=True, slots=True) +class MessagesSurface: + sdk: SdkClients + name: SurfaceName = "messages" + + def _turn(self, key: str, model: str, messages: Sequence[MessageParam], tool: ToolMode) -> Reply: + raw: Final = self.sdk.anthropic(key).messages.with_raw_response.create( + model=model, + max_tokens=MAX_OUTPUT_TOKENS, + system=INSTRUCTIONS, + messages=list(messages), + tools=anthropic.omit if tool == "none" else [_messages_tool()], + tool_choice=_messages_tool_choice() if tool == "forced" else anthropic.omit, + extra_body=NO_PROXY_CACHE, + ) + message: Final = raw.parse() + return Reply( + response_id=message.id, + text="".join(block.text for block in message.content if isinstance(block, TextBlock)), + tool_calls=tuple( + ToolCall(call_id=block.id, name=block.name, arguments=json.dumps(block.input)) + for block in message.content + if isinstance(block, ToolUseBlock) + ), + usage=_usage(message.usage.input_tokens, message.usage.output_tokens), + call_id_header=response_header(raw.headers, "x-litellm-call-id"), + cost_header=response_header(raw.headers, "x-litellm-response-cost"), + ) + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: + return self._turn(key, model, ({"role": "user", "content": prompt},), "forced" if with_tool else "none") + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: + events: Final[tuple[RawMessageStreamEvent, ...]] = tuple( + self.sdk.anthropic(key).messages.create( + model=model, + max_tokens=MAX_OUTPUT_TOKENS, + system=INSTRUCTIONS, + messages=[{"role": "user", "content": prompt}], + stream=True, + extra_body=NO_PROXY_CACHE, + ) + ) + return StreamedReply( + text="".join( + event.delta.text + for event in events + if event.type == "content_block_delta" and event.delta.type == "text_delta" + ), + finished=any(event.type == "message_stop" for event in events), + usage_reported=any(event.type == "message_delta" and event.usage.output_tokens > 0 for event in events), + event_count=len(events), + ) + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: + tool_use: Final[ToolUseBlockParam] = { + "type": "tool_use", + "id": call.call_id, + "name": call.name, + "input": call.parsed().model_dump(), + } + tool_result: Final[ToolResultBlockParam] = { + "type": "tool_result", + "tool_use_id": call.call_id, + "content": result, + } + history: Final[tuple[MessageParam, ...]] = ( + {"role": "user", "content": prompt}, + {"role": "assistant", "content": [tool_use]}, + {"role": "user", "content": [tool_result]}, + ) + return self._turn(key, model, history, "offered") + + +@dataclass(frozen=True, slots=True) +class ResponsesSurface: + sdk: SdkClients + name: SurfaceName = "responses" + + def _turn(self, key: str, model: str, history: ResponseInputParam, tool: ToolMode) -> Reply: + raw: Final = self.sdk.openai(key).responses.with_raw_response.create( + model=model, + input=history, + instructions=INSTRUCTIONS, + max_output_tokens=MAX_OUTPUT_TOKENS, + tools=openai.omit if tool == "none" else [_responses_tool()], + tool_choice=_responses_tool_choice() if tool == "forced" else openai.omit, + parallel_tool_calls=False if tool == "forced" else openai.omit, + extra_body=NO_PROXY_CACHE, + ) + response: Final = raw.parse() + return Reply( + response_id=response.id, + text=response.output_text, + tool_calls=tuple( + ToolCall(call_id=item.call_id, name=item.name, arguments=item.arguments) + for item in response.output + if isinstance(item, ResponseFunctionToolCall) + ), + usage=None if response.usage is None else _usage(response.usage.input_tokens, response.usage.output_tokens), + call_id_header=response_header(raw.headers, "x-litellm-call-id"), + cost_header=response_header(raw.headers, "x-litellm-response-cost"), + ) + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: + return self._turn(key, model, [{"role": "user", "content": prompt}], "forced" if with_tool else "none") + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: + events: Final[tuple[ResponseStreamEvent, ...]] = tuple( + self.sdk.openai(key).responses.create( + model=model, + input=prompt, + instructions=INSTRUCTIONS, + max_output_tokens=MAX_OUTPUT_TOKENS, + stream=True, + extra_body=NO_PROXY_CACHE, + ) + ) + return StreamedReply( + text="".join(event.delta for event in events if event.type == "response.output_text.delta"), + finished=bool(events) and events[-1].type == "response.completed", + usage_reported=any( + event.type == "response.completed" and event.response.usage is not None for event in events + ), + event_count=len(events), + ) + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: + function_call: Final[ResponseFunctionToolCallParam] = { + "type": "function_call", + "call_id": call.call_id, + "name": call.name, + "arguments": call.arguments, + } + output: Final[FunctionCallOutput] = { + "type": "function_call_output", + "call_id": call.call_id, + "output": result, + } + return self._turn(key, model, [{"role": "user", "content": prompt}, function_call, output], "offered") + + +def build_surfaces(sdk: SdkClients) -> Mapping[SurfaceName, Surface]: + return MappingProxyType[SurfaceName, Surface]( + { + "chat_completions": ChatCompletionsSurface(sdk), + "messages": MessagesSurface(sdk), + "responses": ResponsesSurface(sdk), + } + ) diff --git a/tests/e2e/llm_translation/test_conversational_matrix_e2e.py b/tests/e2e/llm_translation/test_conversational_matrix_e2e.py new file mode 100644 index 00000000000..a5cfc9ce70b --- /dev/null +++ b/tests/e2e/llm_translation/test_conversational_matrix_e2e.py @@ -0,0 +1,163 @@ +"""The same conversation contract on every (endpoint, deployment, auth) cell. +A deployment is one provider model (openai/gpt-4o-mini, anthropic/claude-haiku-4-5, ...). + +/chat/completions, /v1/messages and /v1/responses each have their own +translation code in the proxy, so a bug fixed on one surface tends to survive +on the others. Every test here runs once per cell in `CELLS` +(conversational_matrix.py), so a change to a shared helper is proven against all +surfaces and providers at once, and a new model or provider is one row in `DEPLOYMENTS`. + +Edge-wired: OpenAI and Anthropic traffic goes through the provider edge in +record and replay, so the whole matrix replays with zero provider calls. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from typing import Final + +import pytest +from e2e_config import unique_marker +from lifecycle import ResourceManager +from llm_translation.conversational_matrix import ( + GREETING_PROMPT, + WEATHER_PROMPT, + WEATHER_REPORT, + WEATHER_TOOL_NAME, + Cell, + Deployments, + Surface, + SurfaceName, + ToolCall, + build_surfaces, + cells_covering, + register_deployments, +) +from llm_translation.sdk_clients import SdkClients +from models import SpendLogRow +from proxy_client import ProxyClient + +pytestmark = [pytest.mark.e2e, pytest.mark.replayable] + + +def _approx_equal(actual: float, expected: float) -> bool: + return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) + + +@pytest.fixture(scope="module") +def deployments(proxy: ProxyClient) -> Iterator[Deployments]: + yield from register_deployments(proxy) + + +@pytest.fixture(scope="module") +def surfaces(sdk: SdkClients) -> Mapping[SurfaceName, Surface]: + return build_surfaces(sdk) + + +def _weather_call(surface: Surface, key: str, model: str) -> ToolCall: + first: Final = surface.reply(key, model, WEATHER_PROMPT, with_tool=True) + assert len(first.tool_calls) == 1, ( + f"{surface.name} forced tool_choice={WEATHER_TOOL_NAME} with parallel calls off, " + f"got {len(first.tool_calls)} tool call(s): {first.tool_calls} text={first.text!r}" + ) + call: Final = first.tool_calls[0] + assert call.name == WEATHER_TOOL_NAME, f"{surface.name} called {call.name!r}, not the forced {WEATHER_TOOL_NAME!r}" + assert call.call_id, f"{surface.name} tool call has no id, so the caller cannot answer it: {call}" + assert "paris" in call.parsed().location.lower(), f"{surface.name} tool arguments lost the location: {call}" + return call + + +class TestConversationalMatrix: + @pytest.mark.parametrize("cell", cells_covering("basic", "nonstream", "works")) + def test_reply_carries_assistant_text_and_usage( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + surface: Final = surfaces[cell.surface] + reply: Final = surface.reply(resources.key(), deployments.alias(cell), GREETING_PROMPT) + + assert reply.response_id, f"{cell.id}: response has no id" + assert reply.text.strip(), f"{cell.id}: response carried no assistant text" + assert reply.usage is not None and reply.usage.input_tokens > 0 and reply.usage.output_tokens > 0, ( + f"{cell.id}: usage missing or zero, so the caller cannot account for this call: {reply.usage}" + ) + assert reply.call_id_header, f"{cell.id}: x-litellm-call-id header missing" + + @pytest.mark.parametrize("cell", cells_covering("basic", "stream", "works")) + def test_stream_delivers_text_usage_and_a_terminal_event( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + surface: Final = surfaces[cell.surface] + streamed: Final = surface.stream(resources.key(), deployments.alias(cell), GREETING_PROMPT) + + assert streamed.event_count > 1, f"{cell.id}: stream arrived as {streamed.event_count} event(s), not a stream" + assert streamed.text.strip(), f"{cell.id}: stream carried no text deltas" + assert streamed.finished, f"{cell.id}: stream never sent its terminal event" + assert streamed.usage_reported, f"{cell.id}: stream never reported usage" + + @pytest.mark.parametrize("cell", cells_covering("basic", "nonstream", "cost_logged")) + def test_cost_header_matches_the_spend_log( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + proxy: ProxyClient, + ) -> None: + key: Final = resources.key() + surface: Final = surfaces[cell.surface] + reply: Final = surface.reply(key, deployments.alias(cell), f"{GREETING_PROMPT} {unique_marker()}") + + assert reply.cost_header is not None, f"{cell.id}: x-litellm-response-cost header missing" + header_cost: Final = float(reply.cost_header) + assert header_cost > 0, f"{cell.id}: x-litellm-response-cost is not positive: {header_cost}" + + def _priced(rows: list[SpendLogRow]) -> bool: + return any(row.spend is not None and row.spend > 0 for row in rows) + + rows: Final = proxy.poll_logs_for_key(key, predicate=_priced) + priced: Final = tuple(row for row in rows if row.spend is not None and row.spend > 0) + assert len(priced) == 1, f"{cell.id}: expected exactly one priced spend row for a fresh key, got {rows}" + row: Final = priced[0] + assert (row.prompt_tokens or 0) > 0 and (row.completion_tokens or 0) > 0, ( + f"{cell.id}: spend row has no token counts, so the cost is not real usage: {row}" + ) + assert row.spend is not None and _approx_equal(row.spend, header_cost), ( + f"{cell.id}: logged spend {row.spend} disagrees with x-litellm-response-cost {header_cost}" + ) + assert row.model and cell.deployment.backend.endswith(row.model), ( + f"{cell.id}: spend row logged model {row.model!r}, not the deployment's {cell.deployment.backend!r}" + ) + + @pytest.mark.parametrize("cell", cells_covering("tool_use", "nonstream", "works")) + def test_tool_call_is_returned_named_and_addressable( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + _weather_call(surfaces[cell.surface], resources.key(), deployments.alias(cell)) + + @pytest.mark.parametrize("cell", cells_covering("multi_turn", "nonstream", "works")) + def test_tool_result_round_trip_reaches_the_model( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + key: Final = resources.key() + model: Final = deployments.alias(cell) + surface: Final = surfaces[cell.surface] + call: Final = _weather_call(surface, key, model) + + answer: Final = surface.reply_to_tool_result(key, model, WEATHER_PROMPT, call, WEATHER_REPORT) + assert "22" in answer.text, f"{cell.id}: the model never saw the tool result: {answer.text!r}" diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index e9695685ca5..9954351fa2e 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -818,6 +818,35 @@ async def test_proxy_startup_event_prunes_dead_workers_live_gauges(tmp_path): assert counter.exists() +@pytest.mark.asyncio +@pytest.mark.parametrize(("disable_model_info_refresh", "job_scheduled"), [(True, False), (False, True)]) +async def test_proxy_startup_event_honors_disable_model_info_refresh( + disable_model_info_refresh: bool, job_scheduled: bool +) -> None: + """``general_settings.disable_model_info_refresh: true`` keeps the proxy from polling every + OpenAI-compatible deployment's ``/v1/models`` in the background, so a proxy fronting a replay + fixture (or a metered upstream) makes only the calls its clients asked for.""" + scheduler = AsyncIOScheduler() + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} | { + "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true" + } + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.object(ps, "scheduler", scheduler), + patch.dict(ps.general_settings, {"disable_model_info_refresh": disable_model_info_refresh}), + ): + try: + async with proxy_startup_event(app=None): + job = scheduler.get_job("refresh_model_info") + finally: + if scheduler.running: + scheduler.shutdown(wait=False) + + assert (job is not None) is job_scheduled, ( + f"disable_model_info_refresh={disable_model_info_refresh} but refresh_model_info job is {job}" + ) + + def test_otel_global_provider_published_after_callback_init(): """The OTel V2 global-provider publish must run after callback initialization in ``proxy_startup_event``. From fa8483b6aed3cba954c6a8aa135869e9b0cd4168 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:37:33 -0700 Subject: [PATCH 10/28] fix(proxy): answer 503 no_db_connection on management routes when the caller's user read hits a database outage (#42410) * fix(proxy): answer 503 no_db_connection on management routes when the caller's user read hits a database outage Under allow_requests_on_db_unavailable, once the caller's key row lapses the request runs as the restricted fallback identity, and its own user read fails on the outage. /v2/team/list, /user/list, and /user/filter/ui answered a bare 500 for that; every route now answers the same 503 body auth gives, through one shared builder consulted by the generic exception handler and by ui_view_users' own catch-all * fix(proxy): log the database outage before answering 503 on user search * chore(proxy): drop the docstrings on the db outage 503 helper and its tests --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/auth/auth_exception_handler.py | 7 +- litellm/proxy/db/exception_handler.py | 10 ++ .../internal_user_endpoints.py | 4 + litellm/proxy/proxy_server.py | 5 + .../test_internal_user_endpoints.py | 126 ++++++++++++++++++ .../test_team_endpoints.py | 81 +++++++++++ .../proxy_server/test_exception_handlers.py | 35 +++++ 7 files changed, 262 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index bbe4b0f5c35..9bf7f6cab96 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -71,12 +71,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException: if isinstance(e, ProxyException): return e if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): - return ProxyException( - message=PrismaDBExceptionHandler.database_unavailable_message(e), - type=ProxyErrorTypes.no_db_connection, - param="None", - code=status.HTTP_503_SERVICE_UNAVAILABLE, - ) + return PrismaDBExceptionHandler.service_unavailable_proxy_exception(e) return ProxyException( message="Authentication Error, " + str(e), type=ProxyErrorTypes.auth_error, diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 99167c1275c..38d6fb9b99d 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -1,5 +1,6 @@ import re from collections.abc import Awaitable, Callable, Iterator +from http import HTTPStatus from typing import Final, Protocol, TypeVar from pydantic import TypeAdapter, ValidationError @@ -378,6 +379,15 @@ class PrismaDBExceptionHandler: "The proxy deployment needs attention." ) + @staticmethod + def service_unavailable_proxy_exception(e: Exception) -> ProxyException: + return ProxyException( + message=PrismaDBExceptionHandler.database_unavailable_message(e), + type=ProxyErrorTypes.no_db_connection, + param="None", + code=HTTPStatus.SERVICE_UNAVAILABLE.value, + ) + @staticmethod def find_database_service_unavailable_error_in_chain(e: BaseException) -> Exception | None: """The exception in the ``__cause__`` / ``__context__`` chain that diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 60ac7e55eaf..6509977f7ff 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -47,6 +47,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( object_permission_cache_key, user_object_permission_id_cache_key, ) +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.hooks.model_max_budget_limiter import build_model_max_budget_usage from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks from litellm.proxy.management_endpoints.common_daily_activity import ( @@ -2810,6 +2811,9 @@ async def ui_view_users( except HTTPException: raise except Exception as e: + if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e): + verbose_proxy_logger.warning("Database unavailable during user search: %s", type(e).__name__) + raise PrismaDBExceptionHandler.service_unavailable_proxy_exception(e) from e verbose_proxy_logger.exception("Error searching users: %s", e) raise HTTPException(status_code=500, detail=f"Error searching users: {e}") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f8945694051..815fab0dba0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1977,6 +1977,11 @@ async def otel_request_validation_exception_handler(request: Request, exc: Reque async def otel_unhandled_exception_handler(request: Request, exc: Exception): if isinstance(exc, (ProxyException, HTTPException, RequestValidationError)): raise exc + if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(exc): + verbose_proxy_logger.warning("Database unavailable during request: %s", type(exc).__name__) + return await openai_exception_handler( + request=request, exc=PrismaDBExceptionHandler.service_unavailable_proxy_exception(exc) + ) verbose_proxy_logger.exception("Unhandled exception in request: %s", type(exc).__name__) if should_report_bug(exc): verbose_proxy_logger.error( diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 75545a574e3..f684e2040bd 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -1,8 +1,10 @@ import hashlib import json +import logging from datetime import datetime, timezone from types import SimpleNamespace from typing import Final +from unittest.mock import AsyncMock, MagicMock import httpx import pytest @@ -17,12 +19,15 @@ from litellm.proxy._types import ( LiteLLM_UserTableFiltered, LitellmUserRoles, NewUserRequest, + ProxyErrorTypes, ProxyException, UpdateUserRequest, UserAPIKeyAuth, ) from litellm.proxy.management_endpoints.internal_user_endpoints import ( LiteLLM_UserTableWithKeyCount, + _authorize_user_list_request, + _resolve_org_filter_for_user_search, _resolve_user_email_metadata, _update_internal_user_params, get_user_key_counts, @@ -4653,3 +4658,124 @@ async def test_delete_user_evicts_cached_user_rows(mocker: MockerFixture) -> Non assert await cache.async_get_cache(key=deleted.user_id, model_type=LiteLLM_UserTable) is None assert await cache.async_get_cache(key=survivor.user_id, model_type=LiteLLM_UserTable) == survivor broadcast.assert_awaited_once_with(cache_key=deleted.user_id) + + +_DB_OUTAGE_503_BODY: Final = { + "error": { + "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + "type": "no_db_connection", + "param": "None", + "code": "503", + } +} + + +def _user_read_raising(mocker: MockerFixture, error: Exception) -> tuple[MagicMock, MagicMock]: + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock(side_effect=error) + cache = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + cache.async_set_cache = AsyncMock() + mocker.patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True) + return prisma_client, cache + + +def _db_unavailable_fallback_identity(route: str) -> UserAPIKeyAuth: + from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID + + return UserAPIKeyAuth( + key_name="failed-to-connect-to-db", + token="failed-to-connect-to-db", + user_id=DB_UNAVAILABLE_FALLBACK_USER_ID, + user_role=LitellmUserRoles.INTERNAL_USER, + request_route=route, + ) + + +@pytest.mark.asyncio +async def test_authorize_user_list_request_propagates_a_db_outage_instead_of_answering_403(mocker): + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + + with pytest.raises(httpx.ConnectError): + await _authorize_user_list_request( + user_api_key_dict=_db_unavailable_fallback_identity("/user/list"), + organization_ids=None, + prisma_client=prisma_client, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +@pytest.mark.asyncio +async def test_resolve_org_filter_for_user_search_propagates_a_db_outage_instead_of_answering_403(mocker): + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + mocker.patch( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_ui_settings_cached", + return_value={"scope_user_search_to_org": True}, + ) + + with pytest.raises(httpx.ConnectError): + await _resolve_org_filter_for_user_search( + user_api_key_dict=_db_unavailable_fallback_identity("/user/filter/ui"), + team_id=None, + prisma_client=prisma_client, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +@pytest.mark.asyncio +async def test_ui_view_users_answers_a_db_outage_as_503_no_db_connection_not_as_its_own_500(mocker, caplog): + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + mocker.patch( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_ui_settings_cached", + return_value={"scope_user_search_to_org": True}, + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.service_logging_obj.async_service_failure_hook = AsyncMock() + mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma_client) + mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) + mocker.patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"), pytest.raises(ProxyException) as raised: + await ui_view_users( + user_api_key_dict=_db_unavailable_fallback_identity("/user/filter/ui"), + user_id=None, + user_email="lit", + team_id=None, + page=1, + page_size=50, + ) + + assert raised.value.code == "503" + assert raised.value.type == ProxyErrorTypes.no_db_connection + assert isinstance(raised.value.__cause__, httpx.ConnectError) + outage_logs: Final = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and "ConnectError" in r.getMessage()] + assert outage_logs == ["Database unavailable during user search: ConnectError"] + + +@pytest.mark.parametrize( + ("route", "params"), + [("/user/list", {}), ("/user/filter/ui", {"user_email": "lit"})], + ids=["user_list", "user_filter_ui"], +) +def test_user_routes_answer_503_no_db_connection_when_the_callers_user_read_hits_a_db_outage( + mocker, route: str, params: dict[str, str] +): + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + mocker.patch( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_ui_settings_cached", + return_value={"scope_user_search_to_org": True}, + ) + mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma_client) + mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) + app.dependency_overrides[user_api_key_auth] = lambda: _db_unavailable_fallback_identity(route) + try: + response = TestClient(app, raise_server_exceptions=False).get(route, params=params) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 503, response.text + assert response.json() == _DB_OUTAGE_503_BODY diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index e95359ace8a..8bf9c598b1a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -7,6 +7,7 @@ from collections.abc import Sequence from typing import Final, Optional, cast from unittest.mock import AsyncMock, MagicMock, PropertyMock, call, patch +import httpx import pytest from fastapi import HTTPException from fastapi.testclient import TestClient @@ -42,6 +43,8 @@ from litellm.proxy.management_endpoints.team_endpoints import ( _STRIP_DELETED_TEAM_FROM_USERS_SQL, GetTeamMemberPermissionsResponse, UpdateTeamMemberPermissionsRequest, + _build_team_list_where_conditions, + _get_org_admin_org_ids, _persist_deleted_team_records, _save_deleted_team_records, _transform_teams_to_deleted_records, @@ -16569,3 +16572,81 @@ def test_team_member_update_request_rejects_unusable_temp_budget_increase(increa TeamMemberUpdateRequest( team_id="team-1", user_id="user-1", temp_budget_increase=increase, temp_budget_expiry="2030-01-01T00:00:00Z" ) + + +_DB_OUTAGE_503_BODY: Final = { + "error": { + "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + "type": "no_db_connection", + "param": "None", + "code": "503", + } +} + + +def _user_read_raising(error: Exception) -> tuple[MagicMock, MagicMock]: + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock(side_effect=error) + cache = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + cache.async_set_cache = AsyncMock() + return prisma_client, cache + + +def _db_unavailable_fallback_identity(route: str) -> UserAPIKeyAuth: + from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID + + return UserAPIKeyAuth( + key_name="failed-to-connect-to-db", + token="failed-to-connect-to-db", + user_id=DB_UNAVAILABLE_FALLBACK_USER_ID, + user_role=LitellmUserRoles.INTERNAL_USER, + request_route=route, + ) + + +@pytest.mark.asyncio +async def test_get_org_admin_org_ids_propagates_a_db_outage_instead_of_answering_not_an_org_admin(): + prisma_client, cache = _user_read_raising(httpx.ConnectError("All connection attempts failed")) + + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): + with pytest.raises(httpx.ConnectError): + await _get_org_admin_org_ids( + user_id="outage-probe-user", + prisma_client=prisma_client, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +@pytest.mark.asyncio +async def test_build_team_list_where_conditions_propagates_a_db_outage_instead_of_answering_user_not_found(): + prisma_client, cache = _user_read_raising(httpx.ConnectError("All connection attempts failed")) + + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): + with pytest.raises(httpx.ConnectError): + await _build_team_list_where_conditions( + prisma_client=prisma_client, + team_id=None, + team_alias=None, + organization_id=None, + user_id="outage-probe-user", + use_deleted_table=False, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +def test_list_team_v2_answers_503_no_db_connection_when_the_callers_user_read_hits_a_db_outage(monkeypatch): + prisma_client, cache = _user_read_raising(httpx.ConnectError("All connection attempts failed")) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + app.dependency_overrides[user_api_key_auth] = lambda: _db_unavailable_fallback_identity("/v2/team/list") + try: + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): + response = TestClient(app, raise_server_exceptions=False).get("/v2/team/list") + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 503, response.text + assert response.json() == _DB_OUTAGE_503_BODY diff --git a/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py b/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py index b3028ae71dd..089c2d57594 100644 --- a/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py +++ b/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py @@ -11,8 +11,10 @@ from __future__ import annotations import json from types import SimpleNamespace +from typing import Final from unittest.mock import MagicMock +import httpx import pytest from fastapi import HTTPException from fastapi.exceptions import RequestValidationError @@ -364,6 +366,39 @@ async def test_otel_unhandled_exception_handler_returns_500_generic_payload(): } +_DB_OUTAGE_503_BODY: Final = { + "error": { + "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + "type": "no_db_connection", + "param": "None", + "code": "503", + } +} + + +def _raised_from(outer: Exception, cause: Exception) -> Exception: + try: + raise outer from cause + except Exception as chained: + return chained + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "exc", + [ + httpx.ConnectError("All connection attempts failed"), + _raised_from(RuntimeError("user read failed"), httpx.ConnectError("All connection attempts failed")), + ], + ids=["raw_connect_error", "connect_error_as_cause"], +) +async def test_otel_unhandled_exception_handler_answers_a_db_outage_with_503_no_db_connection(exc): + response = await otel_unhandled_exception_handler(request=_make_request(path="/v2/team/list"), exc=exc) + + assert response.status_code == 503 + assert json.loads(response.body) == _DB_OUTAGE_503_BODY + + @pytest.mark.asyncio async def test_otel_unhandled_exception_handler_reraises_proxy_exception_error(): """ProxyException / HTTPException / RequestValidationError are re-raised From 77d656a8ba65472ecba5835bab29b11ba3a20e80 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:42:13 -0700 Subject: [PATCH 11/28] fix(e2e-stack): print add-mask lines only under GitHub Actions (#42423) * fix(e2e-stack): print add-mask lines only under GitHub Actions * refactor(e2e-stack): inline the add-mask lines into main --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .github/e2e-stack/secrets_to_env.py | 14 +++++++++---- .../test_e2e_changed_gate.py | 21 ++++++++++++++++++- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py index 7913b25918b..691c10203bf 100644 --- a/.github/e2e-stack/secrets_to_env.py +++ b/.github/e2e-stack/secrets_to_env.py @@ -9,6 +9,7 @@ from pydantic import TypeAdapter, ValidationError secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str]) ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") MIN_MASKED_LENGTH: Final = 8 +ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS" def main() -> int: @@ -30,10 +31,15 @@ def main() -> int: f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n" ) return 1 - for value in secrets.values(): - if len(value) >= MIN_MASKED_LENGTH: - _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n") - sys.stdout.flush() + if os.environ.get(ACTIONS_RUNNER_FLAG) == "true": + _ = sys.stdout.write( + "".join( + f"::add-mask::{value.replace('%', '%25')}\n" + for value in secrets.values() + if len(value) >= MIN_MASKED_LENGTH + ) + ) + sys.stdout.flush() lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value) try: with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle: diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index 2b37ab14e7f..758d579f67d 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -106,10 +106,11 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat env_path: Final = tmp_path / ".env" result: Final = subprocess.run( - [sys.executable, str(SECRETS_TO_ENV), str(env_path)], + [sys.executable, "-I", str(SECRETS_TO_ENV), str(env_path)], input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}', capture_output=True, text=True, + env={**os.environ, "GITHUB_ACTIONS": "true"}, ) assert result.returncode == 0, result.stderr @@ -117,6 +118,24 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" +def test_outside_actions_no_value_is_printed(tmp_path: Path) -> None: + env_path: Final = tmp_path / ".env" + local_env: Final = {key: value for key, value in os.environ.items() if key != "GITHUB_ACTIONS"} + + result: Final = subprocess.run( + [sys.executable, "-I", str(SECRETS_TO_ENV), str(env_path)], + input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}', + capture_output=True, + text=True, + env=local_env, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout == "" + assert "sk-0123456789abcdef" not in result.stderr + assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" + + def redact_output(tmp_path: Path, values: tuple[str, ...], text: str) -> tuple[subprocess.CompletedProcess[str], Path]: env_path: Final = tmp_path / ".env" _ = env_path.write_text("".join(f"{name}='{value}'\n" for name, value in zip(("A", "B", "C"), values))) From a9cea9d644502ca3a7325138d731bedbff54d59a Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 21 Sep 2026 22:51:24 -0700 Subject: [PATCH 12/28] feat(errors): add stream and safe config flags to the bug report link (#42428) Proxy bug reports now carry the request's stream flag and a config block built from dotted paths like router_settings.routing_strategy. A line is emitted only when its key is defined by a LiteLLM schema and its value is a bool or a LiteLLM-defined value (providers, callbacks, routing strategies, cache types, guardrail integrations and modes, key management systems). Secrets, URLs, numbers and custom values leave no line --- litellm/litellm_core_utils/bug_report.py | 35 +++- litellm/proxy/bug_report_config.py | 185 ++++++++++++++++++ litellm/proxy/common_request_processing.py | 6 +- litellm/proxy/proxy_server.py | 5 +- litellm/proxy/utils.py | 4 +- litellm/router_utils/routing_groups.py | 7 +- .../litellm_core_utils/test_bug_report.py | 36 +++- .../proxy/test_bug_report_config.py | 154 +++++++++++++++ .../proxy/test_common_request_processing.py | 2 + 9 files changed, 417 insertions(+), 17 deletions(-) create mode 100644 litellm/proxy/bug_report_config.py create mode 100644 tests/test_litellm/proxy/test_bug_report_config.py diff --git a/litellm/litellm_core_utils/bug_report.py b/litellm/litellm_core_utils/bug_report.py index 6b4cf992dad..4a8f4bc65ad 100644 --- a/litellm/litellm_core_utils/bug_report.py +++ b/litellm/litellm_core_utils/bug_report.py @@ -31,6 +31,8 @@ class BugReport: python_version: str call_type: str | None custom_llm_provider: str | None + stream: bool | None + config_lines: tuple[str, ...] def bug_report_enabled() -> bool: @@ -73,6 +75,8 @@ def build_bug_report( surface: Surface, call_type: str | None = None, custom_llm_provider: object = None, + stream: object = None, + config_lines: tuple[str, ...] = (), ) -> BugReport: return BugReport( surface=surface, @@ -82,6 +86,8 @@ def build_bug_report( python_version=platform.python_version(), call_type=call_type, custom_llm_provider=allowlisted(custom_llm_provider, KNOWN_PROVIDERS), + stream=stream if isinstance(stream, bool) else None, + config_lines=config_lines, ) @@ -98,8 +104,14 @@ def _title(report: BugReport, frames: tuple[str, ...]) -> str: return f"[Bug]: {report.exception_type} in {location}" -def _description(report: BugReport, frames: tuple[str, ...]) -> str: +def _description(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str: frame_block: Final = "LiteLLM frames:\n```\n" + "\n".join(frames) + "\n```\n\n" if frames else "" + stream_line: Final = "" if report.stream is None else f"Stream: {str(report.stream).lower()}\n" + config_block: Final = ( + "\nConfig (true/false flags and LiteLLM-defined values only):\n```\n" + "\n".join(config_lines) + "\n```\n" + if config_lines + else "" + ) return ( "Auto-generated by LiteLLM's bug report link. It carries no request data or error text. " "Please describe what you were doing, and paste the error message from your log below " @@ -112,10 +124,12 @@ def _description(report: BugReport, frames: tuple[str, ...]) -> str: f"Provider: {report.custom_llm_provider or 'unknown'}\n" f"LiteLLM: {report.litellm_version}\n" f"Python: {report.python_version}\n" + f"{stream_line}" + f"{config_block}" ) -def _issue_url(report: BugReport, frames: tuple[str, ...]) -> str: +def _issue_url(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str: deployment: Final[tuple[tuple[str, str], ...]] = ( (("deployment", "pip / Python SDK"),) if report.surface == "sdk" @@ -129,15 +143,26 @@ def _issue_url(report: BugReport, frames: tuple[str, ...]) -> str: ("title", _title(report, frames)), ("version", report.litellm_version), ("domain", _domain(report)), - ("description", _description(report, frames)), + ("description", _description(report, frames, config_lines)), ) + deployment return f"{ISSUE_URL_BASE}?{urlencode(fields)}" def bug_report_issue_url(report: BugReport) -> str: frames: Final = report.litellm_frames - candidates: Final = tuple(_issue_url(report, frames[index:]) for index in range(len(frames) + 1)) - return next((candidate for candidate in candidates if len(candidate) <= MAX_URL_LENGTH), candidates[-1]) + config_lines: Final = report.config_lines + candidates: Final = ( + *((frames, config_lines[:count]) for count in range(len(config_lines), -1, -1)), + *((frames[index:], ()) for index in range(1, len(frames) + 1)), + ) + return next( + ( + url + for candidate_frames, candidate_config in candidates + if len(url := _issue_url(report, candidate_frames, candidate_config)) <= MAX_URL_LENGTH + ), + _issue_url(report, (), ()), + ) def strip_bug_report_notice(message: str) -> str: diff --git a/litellm/proxy/bug_report_config.py b/litellm/proxy/bug_report_config.py new file mode 100644 index 00000000000..fa4a2604209 --- /dev/null +++ b/litellm/proxy/bug_report_config.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import ast +import functools +import inspect +from collections.abc import Mapping, Sequence +from pathlib import Path +from types import MappingProxyType +from typing import Final + +from pydantic import JsonValue, TypeAdapter, ValidationError + +import litellm +from litellm.litellm_core_utils.bug_report import KNOWN_PROVIDERS, BugReport, allowlisted, build_bug_report +from litellm.proxy._types import ConfigGeneralSettings +from litellm.router_utils.routing_groups import VALID_ROUTING_STRATEGIES +from litellm.types.caching import LiteLLMCacheType +from litellm.types.guardrails import GuardrailEventHooks, LitellmParams, SupportedGuardrailIntegrations +from litellm.types.secret_managers.main import KeyManagementSystem + +_OBJECT_MAP: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_OBJECT_LIST: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +def _object_map(value: object) -> Mapping[str, object]: + try: + return _OBJECT_MAP.validate_python(value) + except ValidationError: + return MappingProxyType({}) + + +def _object_list(value: object) -> Sequence[object]: + try: + return _OBJECT_LIST.validate_python(value) + except ValidationError: + return () + + +@functools.cache +def _known_values() -> frozenset[str]: + from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry + + return frozenset( + ( + *VALID_ROUTING_STRATEGIES, + *KNOWN_PROVIDERS, + *litellm._known_custom_logger_compatible_callbacks, # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownArgumentType] # untyped List of the callback Literal's args, no public alias + *CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE, + *(member.value for member in LiteLLMCacheType), + *(member.value for member in KeyManagementSystem), + *(member.value for member in SupportedGuardrailIntegrations), + *(member.value for member in GuardrailEventHooks), + ) + ) + + +def _module_level_names(node: ast.stmt) -> tuple[str, ...]: + match node: + case ast.Assign(targets=targets): + return tuple(target.id for target in targets if isinstance(target, ast.Name)) + case ast.AnnAssign(target=ast.Name(id=name)): + return (name,) + case ast.ImportFrom(names=aliases): + return tuple(alias.asname or alias.name for alias in aliases) + case _: + return () + + +@functools.cache +def _litellm_settings_keys() -> frozenset[str]: + tree: Final = ast.parse(Path(litellm.__file__).read_text()) + return frozenset(name for node in tree.body for name in _module_level_names(node)) + + +@functools.cache +def _router_settings_keys() -> frozenset[str]: + from litellm.router import Router + + return frozenset(name for name in inspect.signature(Router.__init__).parameters if name != "self") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped params, only names are read + + +@functools.cache +def _cache_params_keys() -> frozenset[str]: + from litellm.caching.caching import Cache + + return frozenset(name for name in inspect.signature(Cache.__init__).parameters if name != "self") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped params, only names are read + + +def _render_json(value: JsonValue) -> str | None: + match value: + case bool(): + return str(value).lower() + case str(): + return value if value in _known_values() else None + case list(): + known_items: Final = tuple(rendered for item in value if (rendered := _render_json(item)) is not None) + return f"[{', '.join(known_items)}]" if known_items else None + case _: + return None + + +def _render(value: object) -> str | None: + try: + return _render_json(_JSON.validate_python(value)) + except ValidationError: + return None + + +def _section_lines(section: str, values: Mapping[str, object], known_keys: frozenset[str]) -> tuple[str, ...]: + return tuple( + f"{section}.{key} = {rendered}" + for key, value in values.items() + if key in known_keys and (rendered := _render(value)) is not None + ) + + +def _guardrail_lines(guardrails: object) -> tuple[str, ...]: + known_keys: Final = frozenset(LitellmParams.model_fields) + return tuple( + line + for index, guardrail in enumerate(_object_list(guardrails)) + for line in _section_lines( + f"guardrails[{index}].litellm_params", _object_map(_object_map(guardrail).get("litellm_params")), known_keys + ) + ) + + +def _deployment_provider(model: object) -> str | None: + prefix: Final = model.split("/", 1)[0] if isinstance(model, str) and "/" in model else None + return allowlisted(prefix, KNOWN_PROVIDERS) + + +def _model_list_lines(model_list: object) -> tuple[str, ...]: + providers: Final = tuple( + sorted( + frozenset( + provider + for deployment in _object_list(model_list) + if ( + provider := _deployment_provider( + _object_map(_object_map(deployment).get("litellm_params")).get("model") + ) + ) + is not None + ) + ) + ) + return (f"model_list[*].provider = [{', '.join(providers)}]",) if providers else () + + +def safe_config_lines(config: Mapping[str, object], general_settings: Mapping[str, object]) -> tuple[str, ...]: + litellm_settings: Final = _object_map(config.get("litellm_settings")) + return ( + *_section_lines("general_settings", general_settings, frozenset(ConfigGeneralSettings.model_fields)), + *_section_lines("litellm_settings", litellm_settings, _litellm_settings_keys()), + *_section_lines( + "litellm_settings.cache_params", _object_map(litellm_settings.get("cache_params")), _cache_params_keys() + ), + *_section_lines("router_settings", _object_map(config.get("router_settings")), _router_settings_keys()), + *_guardrail_lines(config.get("guardrails")), + *_model_list_lines(config.get("model_list")), + ) + + +def build_proxy_bug_report( + exc: BaseException, + *, + call_type: str | None = None, + custom_llm_provider: object = None, + stream: object = None, +) -> BugReport: + from litellm.proxy import proxy_server + + return build_bug_report( + exc, + surface="proxy", + call_type=call_type, + custom_llm_provider=custom_llm_provider, + stream=stream, + config_lines=safe_config_lines( + proxy_server.proxy_config.config, + _object_map(proxy_server.general_settings), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # bare dict global, validated by _object_map + ), + ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index c95cc3e0529..e7c7a2607c4 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -49,7 +49,6 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.bug_report import ( allowlisted, bug_report_notice, - build_bug_report, should_report_bug, strip_bug_report_notice, ) @@ -79,6 +78,7 @@ from litellm.proxy.auth.auth_checks import ( tag_max_budget_check_for_tags, ) from litellm.proxy.auth.auth_utils import check_response_size_is_safe, get_request_route +from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.callback_utils import ( get_logging_caching_headers, get_remaining_tokens_and_requests_from_request_data, @@ -3689,11 +3689,11 @@ class ProxyBaseLLMRequestProcessing: request_path: Final = urlparse(str(request_url)).path if request_url is not None else None verbose_proxy_logger.error( bug_report_notice( - build_bug_report( + build_proxy_bug_report( e, - surface="proxy", call_type=allowlisted(request_path, KNOWN_PROXY_ROUTES), custom_llm_provider=self.data.get("custom_llm_provider"), + stream=self.data.get("stream"), ) ) ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 815fab0dba0..18bc02467d1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -78,7 +78,6 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.bug_report import ( allowlisted, bug_report_notice, - build_bug_report, should_report_bug, ) from litellm.litellm_core_utils.litellm_logging import ( @@ -373,6 +372,7 @@ from litellm.proxy.auth.user_api_key_auth import ( user_api_key_auth_websocket, ) from litellm.proxy.batches_endpoints.endpoints import router as batches_router +from litellm.proxy.bug_report_config import build_proxy_bug_report ## Import All Misc routes here ## from litellm.proxy.caching_routes import router as caching_router @@ -1986,9 +1986,8 @@ async def otel_unhandled_exception_handler(request: Request, exc: Exception): if should_report_bug(exc): verbose_proxy_logger.error( bug_report_notice( - build_bug_report( + build_proxy_bug_report( exc, - surface="proxy", call_type=allowlisted(request.url.path, KNOWN_PROXY_ROUTES), ) ) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b4846d529ab..e1a92b4380e 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -59,7 +59,6 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.bug_report import ( bug_report_notice, - build_bug_report, should_report_bug, strip_bug_report_notice, ) @@ -70,6 +69,7 @@ from litellm.proxy._types import ( SpendLogsMetadata, SpendLogsPayload, ) +from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.openai_error_payload import ( litellm_call_id_headers, openai_error_param, @@ -7996,7 +7996,7 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) return with_litellm_call_id(e, litellm_call_id) _status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) if should_report_bug(e): - verbose_proxy_logger.error(bug_report_notice(build_bug_report(e, surface="proxy"))) + verbose_proxy_logger.error(bug_report_notice(build_proxy_bug_report(e))) return ProxyException( message=strip_bug_report_notice(str(e)), type=ProxyErrorTypes.internal_server_error, diff --git a/litellm/router_utils/routing_groups.py b/litellm/router_utils/routing_groups.py index ba65ddf8643..772c8bd805c 100644 --- a/litellm/router_utils/routing_groups.py +++ b/litellm/router_utils/routing_groups.py @@ -4,18 +4,19 @@ from typing import Final from litellm._logging import verbose_router_logger from litellm.types.router import RoutingGroup, RoutingStrategy +VALID_ROUTING_STRATEGIES: Final = ("simple-shuffle", "lar1", *(s.value for s in RoutingStrategy)) + def validate_routing_strategy(routing_strategy: RoutingStrategy | str | None) -> None: if routing_strategy is None: return - valid_strategy_strings: Final = ("simple-shuffle", "lar1", *(s.value for s in RoutingStrategy)) - is_valid_string: Final = isinstance(routing_strategy, str) and routing_strategy in valid_strategy_strings + is_valid_string: Final = isinstance(routing_strategy, str) and routing_strategy in VALID_ROUTING_STRATEGIES is_valid_enum: Final = isinstance(routing_strategy, RoutingStrategy) if not is_valid_string and not is_valid_enum: raise ValueError( f"Invalid routing_strategy: '{routing_strategy}'. " - f"Valid options: {list(valid_strategy_strings)}. " + f"Valid options: {list(VALID_ROUTING_STRATEGIES)}. " f"Check 'router_settings.routing_strategy' in your config.yaml " f"or the 'routing_strategy' parameter if using the Router SDK directly." ) diff --git a/tests/test_litellm/litellm_core_utils/test_bug_report.py b/tests/test_litellm/litellm_core_utils/test_bug_report.py index c9189ebbbdd..f47361dda2e 100644 --- a/tests/test_litellm/litellm_core_utils/test_bug_report.py +++ b/tests/test_litellm/litellm_core_utils/test_bug_report.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Callable from pathlib import Path from typing import cast -from urllib.parse import parse_qs, urlparse +from urllib.parse import parse_qs, unquote_plus, urlparse import httpx import pytest @@ -168,3 +168,37 @@ def test_strip_bug_report_notice(): assert strip_bug_report_notice(f"boom\n\n{notice}") == "boom\n" assert strip_bug_report_notice("boom") == "boom" + + +def test_issue_description_renders_stream_and_config_block(): + report = build_bug_report( + RuntimeError("boom"), + surface="proxy", + stream=True, + config_lines=("router_settings.routing_strategy = least-busy", "litellm_settings.drop_params = true"), + ) + description = parse_qs(urlparse(bug_report_issue_url(report)).query)["description"][0] + + assert "Stream: true\n" in description + assert "```\nrouter_settings.routing_strategy = least-busy\nlitellm_settings.drop_params = true\n```" in description + + +@pytest.mark.parametrize("stream", [None, "true", 1]) +def test_issue_description_omits_stream_unless_it_is_a_bool(stream: object): + report = build_bug_report(RuntimeError("boom"), surface="proxy", stream=stream) + + assert report.stream is None + assert "Stream:" not in unquote_plus(bug_report_issue_url(report)) + + +def test_oversized_config_is_trimmed_from_the_end_before_any_frame(): + with pytest.raises(BadRequestError) as raised: + get_llm_provider(cast(str, None)) + config_lines = tuple(f"general_settings.flag_{index:04d} = true" for index in range(400)) + report = build_bug_report(raised.value, surface="proxy", config_lines=config_lines) + description = parse_qs(urlparse(url := bug_report_issue_url(report)).query)["description"][0] + + assert len(url) <= MAX_URL_LENGTH + assert all(frame in description for frame in report.litellm_frames) + assert "general_settings.flag_0000 = true" in description + assert "general_settings.flag_0399 = true" not in description diff --git a/tests/test_litellm/proxy/test_bug_report_config.py b/tests/test_litellm/proxy/test_bug_report_config.py new file mode 100644 index 00000000000..ce4baa946f9 --- /dev/null +++ b/tests/test_litellm/proxy/test_bug_report_config.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from collections.abc import Iterator, Mapping + +import pytest + +from litellm.proxy import proxy_server +from litellm.proxy.bug_report_config import build_proxy_bug_report, safe_config_lines + +CUSTOMER_STRINGS = ( + "acme", + "sk-live-secret", + "hunter2", + "postgres://", + "10.0.0.7", +) + +CUSTOMER_CONFIG: Mapping[str, object] = { + "model_list": [ + { + "model_name": "acme-prod-gpt4", + "litellm_params": { + "model": "azure/acme-gpt4o-deployment", + "api_base": "https://acme-eastus.openai.azure.com", + "api_key": "sk-live-secret-1", + "rpm": 600, + "acme_extra_param": "acme", + }, + }, + { + "model_name": "acme-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-live-secret-2"}, + }, + { + "model_name": "acme-backup", + "litellm_params": {"model": "azure/acme-backup-deployment", "api_key": "sk-live-secret-3"}, + }, + {"model_name": "acme-bare", "litellm_params": {"model": "acme-custom-model"}}, + ], + "litellm_settings": { + "callbacks": ["langfuse", "acme_hooks.audit_logger"], + "drop_params": True, + "num_retries": 3, + "acme_internal_flag": True, + "cache": True, + "cache_params": { + "type": "redis", + "host": "10.0.0.7", + "port": 6379, + "password": "hunter2", + "acme_cache_option": "acme", + }, + }, + "router_settings": { + "routing_strategy": "latency-based-routing", + "redis_host": "10.0.0.7", + "acme_router_option": True, + }, + "guardrails": [ + {"guardrail_name": "acme-pii-mask", "litellm_params": {"guardrail": "presidio", "mode": "pre_call"}}, + { + "guardrail_name": "acme-policy", + "litellm_params": {"guardrail": "acme_guardrails.PolicyCheck", "api_key": "sk-live-secret-4"}, + }, + ], + "environment_variables": {"ACME_PROD_OPENAI_KEY": "sk-live-secret-5", "ACME_TENANT": "acme"}, +} + +CUSTOMER_GENERAL_SETTINGS: Mapping[str, object] = { + "master_key": "sk-live-secret-master", + "database_url": "postgres://user:hunter2@10.0.0.7/litellm", + "key_management_system": "aws_secret_manager", + "store_model_in_db": True, + "health_check_interval": 300, + "acme_sso_tenant": "acme-prod", +} + + +def test_safe_config_lines_keep_only_flags_and_litellm_defined_values(): + lines = safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS) + + assert lines == ( + "general_settings.key_management_system = aws_secret_manager", + "general_settings.store_model_in_db = true", + "litellm_settings.callbacks = [langfuse]", + "litellm_settings.drop_params = true", + "litellm_settings.cache = true", + "litellm_settings.cache_params.type = redis", + "router_settings.routing_strategy = latency-based-routing", + "guardrails[0].litellm_params.guardrail = presidio", + "guardrails[0].litellm_params.mode = pre_call", + "model_list[*].provider = [azure, openai]", + ) + assert not any(customer_string in "\n".join(lines) for customer_string in CUSTOMER_STRINGS) + + +@pytest.mark.parametrize( + ("config", "expected_lines"), + [ + ({"router_settings": {"routing_strategy": "acme-strategy"}}, ()), + ({"litellm_settings": {"cache_params": {"type": "acme-cache"}}}, ()), + ( + {"litellm_settings": {"success_callback": ["acme_logger", "langsmith"]}}, + ("litellm_settings.success_callback = [langsmith]",), + ), + ({"litellm_settings": {"callbacks": ["acme_hooks.audit_logger"]}}, ()), + ], +) +def test_string_values_show_only_when_litellm_defines_them( + config: Mapping[str, object], expected_lines: tuple[str, ...] +): + assert safe_config_lines(config, {}) == expected_lines + + +def test_secrets_numbers_and_unknown_values_leave_no_line(): + general_settings: Mapping[str, object] = { + "master_key": "sk-live-secret-master", + "health_check_interval": 300, + "store_model_in_db": object(), + "alerting": {"acme": "webhook"}, + "background_health_checks": False, + } + + assert safe_config_lines({}, general_settings) == ("general_settings.background_health_checks = false",) + + +def test_malformed_sections_produce_no_lines(): + config: Mapping[str, object] = { + "litellm_settings": "acme", + "router_settings": ["acme"], + "guardrails": {"acme": {"litellm_params": {"guardrail": "presidio"}}}, + "model_list": "acme", + "environment_variables": None, + } + + assert safe_config_lines(config, {}) == () + + +@pytest.fixture +def loaded_proxy_config(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + previous_config = proxy_server.proxy_config.get_config_state() + proxy_server.proxy_config.update_config_state(config=CUSTOMER_CONFIG) + monkeypatch.setattr(proxy_server, "general_settings", dict(CUSTOMER_GENERAL_SETTINGS)) + yield + proxy_server.proxy_config.update_config_state(config=previous_config) + + +@pytest.mark.usefixtures("loaded_proxy_config") +def test_build_proxy_bug_report_reads_the_loaded_proxy_config(): + report = build_proxy_bug_report(RuntimeError("boom"), stream=False) + + assert report.surface == "proxy" + assert report.stream is False + assert report.config_lines == safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index af1604219d1..f96142def06 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -9407,6 +9407,7 @@ async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error( "proxy_server_request": {"url": "https://example.test/v1/chat/completions?debug=true"}, "model": "acme-prod-gpt4", "custom_llm_provider": "openai", + "stream": True, } ) proxy_logging_obj = MagicMock() @@ -9424,6 +9425,7 @@ async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error( issue_url = next(word for word in caplog.text.split() if word.startswith(ISSUE_URL_BASE)) assert "Endpoint / call: /v1/chat/completions" in unquote_plus(issue_url) assert "Provider: openai" in unquote_plus(issue_url) + assert "Stream: true" in unquote_plus(issue_url) assert "acme-prod-gpt4" not in unquote_plus(issue_url) assert "user@example.com" not in unquote_plus(issue_url) From 5a8c4f48e41619b670dad5eec7e2dc3b48811e99 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 21 Sep 2026 22:52:29 -0700 Subject: [PATCH 13/28] feat(router): native compact-to-fit across conversation APIs (#42074) * feat(router): native compact-to-fit across conversation APIs * fix(router): preserve compaction admission and shared client boundaries * fix(router): honor compaction fit fallbacks and router-scoped access * fix(router): charge compaction usage to caller token limits * test(http): keep FastAPI inside proxy tests * fix(router): check compactor capacity before skipping escalation --- litellm/anthropic_beta_headers_config.json | 1 + .../initialize_dynamic_callback_params.py | 24 +- litellm/llms/anthropic/chat/transformation.py | 34 +- litellm/llms/anthropic/common_utils.py | 21 + litellm/llms/anthropic/compaction.py | 71 +++ .../adapters/transformation.py | 60 +- .../messages/handler.py | 4 +- .../messages/transformation.py | 5 + litellm/llms/compaction.py | 48 ++ litellm/llms/custom_httpx/asgi_handler.py | 46 ++ litellm/llms/custom_httpx/http_handler.py | 16 +- ...odel_prices_and_context_window_backup.json | 13 + litellm/proxy/auth/auth_checks.py | 5 +- litellm/proxy/common_request_processing.py | 3 +- .../hooks/parallel_request_limiter_v3.py | 7 +- litellm/proxy/native_compaction.py | 90 +++ litellm/router.py | 59 +- .../complexity_router/complexity_router.py | 13 +- .../complexity_router/config.py | 19 + .../complexity_router/context_compaction.py | 515 ++++++++++++++++++ .../router_utils/auto_router_model_naming.py | 5 +- litellm/types/llms/anthropic.py | 23 +- .../anthropic_messages/anthropic_response.py | 9 +- litellm/types/llms/custom_http.py | 1 + litellm/types/utils.py | 3 + litellm/utils.py | 1 + model_prices_and_context_window.json | 13 + model_prices_and_context_window.schema.json | 3 + ...test_initialize_dynamic_callback_params.py | 20 +- .../test_anthropic_chat_transformation.py | 72 ++- ...al_pass_through_adapters_transformation.py | 14 + .../llms/custom_httpx/test_asgi_handler.py | 53 ++ .../hooks/test_parallel_request_limiter_v3.py | 22 +- .../proxy/test_native_compaction.py | 163 ++++++ .../test_context_compaction.py | 503 +++++++++++++++++ .../router_strategy/test_complexity_router.py | 2 + .../test_anthropic_beta_headers_filtering.py | 1 + tests/test_litellm/test_utils.py | 9 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 25 + 39 files changed, 1936 insertions(+), 60 deletions(-) create mode 100644 litellm/llms/anthropic/compaction.py create mode 100644 litellm/llms/compaction.py create mode 100644 litellm/llms/custom_httpx/asgi_handler.py create mode 100644 litellm/proxy/native_compaction.py create mode 100644 litellm/router_strategy/complexity_router/context_compaction.py create mode 100644 tests/test_litellm/llms/custom_httpx/test_asgi_handler.py create mode 100644 tests/test_litellm/proxy/test_native_compaction.py create mode 100644 tests/test_litellm/router_strategy/complexity_router/test_context_compaction.py diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 917bfbd5ae9..3a28d65e47c 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -7,6 +7,7 @@ "bash_20250124": null, "code-execution-2025-08-25": "code-execution-2025-08-25", "compact-2026-01-12": "compact-2026-01-12", + "compact-2026-09-04": "compact-2026-09-04", "computer-use-2025-01-24": "computer-use-2025-01-24", "computer-use-2025-11-24": "computer-use-2025-11-24", "context-1m-2025-08-07": "context-1m-2025-08-07", diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index e4744079622..34d0ded618c 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -1,10 +1,24 @@ import re -from collections.abc import Iterator, Mapping +from collections.abc import Generator, Iterator, Mapping +from contextlib import contextmanager +from contextvars import ContextVar from typing import Any, Final from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams _CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata") +_inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar( + "inherited_message_logging_disabled", default=False +) + + +@contextmanager +def inherit_message_logging_privacy(disabled: bool) -> Generator[None]: + token: Final = _inherited_message_logging_disabled.set(_inherited_message_logging_disabled.get() or disabled) + try: + yield + finally: + _inherited_message_logging_disabled.reset(token) def iter_client_callback_metadata_dicts( @@ -143,7 +157,7 @@ def get_trusted_callback_params(kwargs: Mapping[str, Any] | None) -> tuple[tuple def initialize_standard_callback_dynamic_params( - kwargs: dict | None = None, + kwargs: dict[str, object] | None = None, ) -> StandardCallbackDynamicParams: """ Initialize the standard callback dynamic params from the kwargs @@ -179,4 +193,10 @@ def initialize_standard_callback_dynamic_params( if param in _trusted_overlay_callback_params: standard_callback_dynamic_params[param] = trusted_value + if _inherited_message_logging_disabled.get(): + private_params: Final[StandardCallbackDynamicParams] = { + **standard_callback_dynamic_params, + "turn_off_message_logging": True, + } + return private_params return standard_callback_dynamic_params diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index c221e9f1505..b7c2ce3c568 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -96,6 +96,7 @@ from ..common_utils import ( AnthropicModelInfo, eager_input_streaming_flag, process_anthropic_headers, + requires_native_compaction_beta, strip_advisor_blocks_from_messages, ) @@ -1770,7 +1771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return tools - def _ensure_beta_header(self, headers: dict, beta_value: str) -> None: + def _ensure_beta_header(self, headers: dict[str, str], beta_value: str) -> None: """ Ensure a beta header value is present in the anthropic-beta header. Merges with existing values instead of overriding them. @@ -1779,13 +1780,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): headers: Dictionary of headers to update beta_value: The beta header value to add """ - existing_beta: Final = headers.get("anthropic-beta") - if existing_beta is None: - headers["anthropic-beta"] = beta_value - return - existing_values: Final = [beta.strip() for beta in existing_beta.split(",")] - if beta_value not in existing_values: - headers["anthropic-beta"] = f"{existing_beta}, {beta_value}" + existing_values: Final = tuple( + beta.strip() + for key, value in headers.items() + if key.lower() == "anthropic-beta" + for beta in value.split(",") + if beta.strip() + ) + for key in tuple(headers): + if key.lower() == "anthropic-beta": + headers.pop(key) + headers["anthropic-beta"] = ", ".join(dict.fromkeys((*existing_values, beta_value))) def _ensure_context_management_beta_header(self, headers: dict, context_management: object) -> None: """ @@ -1823,7 +1828,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, ) - def update_headers_with_optional_anthropic_beta(self, headers: dict, optional_params: dict) -> dict: + def update_headers_with_optional_anthropic_beta( + self, headers: dict, optional_params: dict, messages: Sequence[object] = () + ) -> dict: """Update headers with optional anthropic beta.""" # Skip adding beta headers for Vertex requests @@ -1832,6 +1839,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if is_vertex_request: return headers + if requires_native_compaction_beta(self._resolved_provider, optional_params, messages): + self._ensure_beta_header(headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) + _tools: Final = optional_params.get("tools", []) for tool in _tools: if tool.get("type", None) and tool.get("type").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value): @@ -1928,8 +1938,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): custom_llm_provider=self._resolved_provider, ) - headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params) - # === Tool-name sanitization (single chokepoint) === # Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We # sanitize *here* -- not in map_openai_params -- because: @@ -1976,6 +1984,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): message=f"{e}\nReceived Messages={messages}", ) # don't use verbose_logger.exception, if exception is raised + self.update_headers_with_optional_anthropic_beta( + headers=headers, optional_params=optional_params, messages=anthropic_messages + ) + ## Auto-strip advisor blocks from history if advisor tool is absent. ## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool. _all_tools: Final = optional_params.get("tools") or [] diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index c6015e7884e..0c3c8996789 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -79,6 +79,27 @@ _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") +def requires_native_compaction_beta( + custom_llm_provider: str, + optional_params: Mapping[str, object], + messages: Sequence[object], +) -> bool: + return custom_llm_provider == "anthropic" and ( + optional_params.get("compaction") is not None + or any( + isinstance(block, Mapping) + and block.get("type") == "compaction" + and isinstance(block.get("signature"), str) + and bool(block.get("signature")) + for message in messages + if isinstance(message, Mapping) + for content in (message.get("content"),) + if isinstance(content, (list, tuple)) + for block in content + ) + ) + + def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool: from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.utils import supports_prompt_caching diff --git a/litellm/llms/anthropic/compaction.py b/litellm/llms/anthropic/compaction.py new file mode 100644 index 00000000000..cd09f936248 --- /dev/null +++ b/litellm/llms/anthropic/compaction.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter + +from litellm.llms.compaction import CompactionProtocol +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicCompaction + +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_HEADERS: Final = TypeAdapter(dict[str, str]) +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_CONFLICTS: Final = ("context_management", "response_format", "stop", "stop_sequences", "tool_choice") + + +def supports_native_compaction(params: Mapping[str, object]) -> bool: + from litellm.utils import get_model_info + + if params.get("custom_llm_provider") not in (None, "anthropic", "openai"): + return False + model: Final = str(params.get("model", "")).removeprefix("openai/").removeprefix("anthropic/") + try: + return get_model_info(model=model, custom_llm_provider="anthropic").get("supports_anthropic_compaction") is True + except Exception: + return False + + +def compatible_defaults(payload: Mapping[str, object]) -> bool: + return all(payload.get(key) is None for key in _CONFLICTS) + + +def request_kwargs() -> Mapping[str, object]: + operation: Final[AnthropicCompaction] = {"type": "summarize"} + return MappingProxyType( + { + "compaction": operation, + "extra_headers": _HEADERS.validate_python( + MappingProxyType({"anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}) + ), + } + ) + + +def _native_blocks(protocol: CompactionProtocol, response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + if protocol == "messages": + return ( + _OBJECTS.validate_python(response.get("content", ())) if response.get("stop_reason") == "compaction" else () + ) + choices: Final = _OBJECTS.validate_python(response.get("choices", ())) + choice: Final = choices[0] if len(choices) == 1 else _EMPTY + message: Final = _MAPPING.validate_python(choice.get("message", _EMPTY)) + fields: Final = _MAPPING.validate_python(message.get("provider_specific_fields") or _EMPTY) + return _OBJECTS.validate_python(fields.get("compaction_blocks", ())) + + +def extract_summary(protocol: CompactionProtocol, response: Mapping[str, object]) -> str | None: + blocks: Final = _native_blocks(protocol, response) + block: Final = blocks[0] if len(blocks) == 1 else _EMPTY + content: Final = block.get("content") + return ( + content + if block.get("type") == "compaction" + and isinstance(block.get("signature"), str) + and block.get("signature") + and isinstance(content, str) + and content.strip() + else None + ) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 1a85cf80bff..85431a5a637 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -2,8 +2,11 @@ import copy import hashlib import json from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast +from pydantic import JsonValue, TypeAdapter + import litellm from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, @@ -16,6 +19,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64 TOOL_NAME_HASH_LENGTH: Final = 8 TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55 PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"}) +_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue]) def _optional_attr(source: object, name: str) -> object: @@ -36,6 +40,20 @@ def _thought_signature(provider_specific_fields: object) -> str | None: return signature if isinstance(signature, str) else None +def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]: + fields: Final = _as_string_mapping(provider_specific_fields) + raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None + return ( + tuple( + block + for raw_block in raw_blocks + if (block := _as_string_mapping(raw_block)) is not None and block.get("type") == "compaction" + ) + if isinstance(raw_blocks, (list, tuple)) + else () + ) + + _ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset( {"name", "type", "input_schema", "description", "cache_control", "strict"} ) @@ -1330,7 +1348,11 @@ class LiteLLMAnthropicMessagesAdapter: tool_name_mapping: dict[str, str] | None = None, ) -> list[dict[str, Any]]: new_content: Final[list[dict[str, Any]]] = [] - for choice in choices: + for choice, compaction_blocks in ( + (choice, _compaction_blocks(_optional_attr(choice.message, "provider_specific_fields"))) + for choice in choices + ): + new_content.extend(_COMPACTION_BLOCK.validate_python(block) for block in compaction_blocks) # Handle thinking blocks first if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks: for thinking_block in choice.message.thinking_blocks: @@ -1365,7 +1387,7 @@ class LiteLLMAnthropicMessagesAdapter: ) # Handle text content - if choice.message.content is not None: + if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks): new_content.append( AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump() ) @@ -1545,21 +1567,35 @@ class LiteLLMAnthropicMessagesAdapter: openai_finish_reason=openai_finish_reason ) anthropic_finish_reason: Final = ( - "refusal" + "compaction" + if len(anthropic_content) == 1 and anthropic_content[0].get("type") == "compaction" + else "refusal" if refusal_text is not None and translated_finish_reason != "max_tokens" else translated_finish_reason ) # extract usage usage: Final[Usage] = getattr(response, "usage") - anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage) - - if polyfill_result is not None and polyfill_result.iterations_usage is not None: - message_iteration: Final[UsageIteration] = { - "type": "message", - "input_tokens": anthropic_usage["input_tokens"], - "output_tokens": usage.completion_tokens or 0, - } - anthropic_usage["iterations"] = list(polyfill_result.iterations_usage) + [message_iteration] + message_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage) + polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None + anthropic_usage: Final[AnthropicUsage] = ( + TypeAdapter(AnthropicUsage).validate_python( + MappingProxyType( + { + **message_usage, + "iterations": ( + *polyfill_iterations, + UsageIteration( + type="message", + input_tokens=message_usage.get("input_tokens", 0), + output_tokens=usage.completion_tokens or 0, + ), + ), + } + ) + ) + if polyfill_iterations is not None + else message_usage + ) translated_obj: Final = AnthropicMessagesResponse( id=response.id, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index d87cb0a64f5..ac4240690c1 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -580,7 +580,9 @@ def anthropic_messages_handler( ) if anthropic_messages_provider_config is None: # Route to Responses API for OpenAI / Azure, chat/completions for everything else. - if _should_route_to_responses_api(custom_llm_provider, original_model, model): + if kwargs.get("compaction") is None and _should_route_to_responses_api( + custom_llm_provider, original_model, model + ): return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( max_tokens=max_tokens, messages=messages, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index eed30c2698c..a83e23d83d5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -24,6 +24,7 @@ from ...common_utils import ( AnthropicError, AnthropicModelInfo, optionally_handle_anthropic_oauth, + requires_native_compaction_beta, strip_advisor_blocks_from_messages, strip_encrypted_reasoning_blocks_from_anthropic_messages, ) @@ -74,6 +75,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "tool_choice", "thinking", "context_management", + *(("compaction",) if self._resolved_provider == "anthropic" else ()), "output_format", "inference_geo", "speed", @@ -637,6 +639,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ) beta_values.update(existing_beta) + if requires_native_compaction_beta(custom_llm_provider, optional_params, messages): + beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) + # Check for context management context_management_param: Final = optional_params.get("context_management") if context_management_param is not None: diff --git a/litellm/llms/compaction.py b/litellm/llms/compaction.py new file mode 100644 index 00000000000..16afa2e2c25 --- /dev/null +++ b/litellm/llms/compaction.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias + +from pydantic import TypeAdapter + +from litellm.types.llms.openai import AllMessageValues + +if TYPE_CHECKING: + from litellm.router import Router + +CompactionProtocol: TypeAlias = Literal["chat", "messages"] +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_MESSAGES: Final = TypeAdapter(list[AllMessageValues]) + + +class NativeCompactionProvider(Protocol): + def supports_native_compaction(self, params: Mapping[str, object]) -> bool: ... + + def compatible_defaults(self, payload: Mapping[str, object]) -> bool: ... + + def request_kwargs(self) -> Mapping[str, object]: ... + + def extract_summary(self, protocol: CompactionProtocol, response: Mapping[str, object]) -> str | None: ... + + +def get_native_compaction_provider(params: Mapping[str, object]) -> NativeCompactionProvider | None: + from litellm.llms.anthropic import compaction + + return compaction if compaction.supports_native_compaction(params) else None + + +async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mapping[str, object]) -> Mapping[str, object]: + if protocol == "messages": + return _MAPPING.validate_python( + await router.aanthropic_messages(custom_llm_provider=None, client=None, **payload) + ) + response: Final = await router.acompletion( + model=str(payload["model"]), + messages=_MESSAGES.validate_python(payload["messages"]), + stream=False, + **MappingProxyType( + {key: value for key, value in payload.items() if key not in ("model", "messages", "stream")} + ), + ) + return _MAPPING.validate_python(response.model_dump()) diff --git a/litellm/llms/custom_httpx/asgi_handler.py b/litellm/llms/custom_httpx/asgi_handler.py new file mode 100644 index 00000000000..ab704ab12a6 --- /dev/null +++ b/litellm/llms/custom_httpx/asgi_handler.py @@ -0,0 +1,46 @@ +from collections.abc import Generator +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Final + +import httpx +from starlette.types import ASGIApp, Receive, Scope, Send + +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared cache retains its legacy parameter mapping +) +from litellm.types.llms.custom_http import httpxSpecialProvider + + +@dataclass(frozen=True, slots=True) +class _ASGITarget: + app: ASGIApp + root_path: str + client: tuple[str, int] | None + + +_target: Final[ContextVar[_ASGITarget]] = ContextVar("httpx_asgi_target") + + +async def _dispatch(scope: Scope, receive: Receive, send: Send) -> None: + target: Final = _target.get() + await target.app({**scope, "root_path": target.root_path, "client": target.client}, receive, send) + + +_TRANSPORT: Final = httpx.ASGITransport(app=_dispatch, raise_app_exceptions=False) + + +@contextmanager +def get_async_asgi_client( + app: ASGIApp, root_path: str = "", client: tuple[str, int] | None = None +) -> Generator[httpx.AsyncClient]: + handler: Final = get_async_httpx_client( + llm_provider=httpxSpecialProvider.ASGI, + params={"transport": _TRANSPORT, "timeout": httpx.Timeout(None), "follow_redirects": False}, + ) + token: Final = _target.set(_ASGITarget(app, root_path, client)) + try: + yield handler.client + finally: + _target.reset(token) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index b7b2477e85c..fcc05e54bc5 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -614,11 +614,15 @@ class AsyncHTTPHandler: client_alias: str | None = None, # name for client in logs ssl_verify: VerifyTypes | None = None, shared_session: Optional["ClientSession"] = None, + transport: httpx.AsyncBaseTransport | None = None, + follow_redirects: bool = True, ): self.timeout = timeout self.event_hooks = event_hooks self.ssl_verify = ssl_verify self.shared_session = shared_session + self.transport = transport + self.follow_redirects = follow_redirects self._owns_client = True self._client = self.create_client( timeout=timeout, @@ -651,6 +655,16 @@ class AsyncHTTPHandler: ssl_verify: VerifyTypes | None = None, shared_session: Optional["ClientSession"] = None, ) -> httpx.AsyncClient: + if self.transport is not None: + return httpx.AsyncClient( + transport=self.transport, + event_hooks=event_hooks, + timeout=timeout if timeout is not None else _DEFAULT_TIMEOUT, + headers=get_default_headers(), + cookies=blocked_cookie_jar(), + follow_redirects=self.follow_redirects, + trust_env=False, + ) # Get unified SSL configuration ssl_config: Final = get_ssl_configuration(ssl_verify) @@ -680,7 +694,7 @@ class AsyncHTTPHandler: cert=cert, headers=default_headers, cookies=blocked_cookie_jar(), - follow_redirects=True, + follow_redirects=self.follow_redirects, http2=http2_enabled(), ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0f02ec8da5f..b5106670a2b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14516,6 +14516,7 @@ "source": "https://docs.anthropic.com/en/docs/about-claude/pricing" }, "claude-sonnet-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -14555,6 +14556,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-sonnet-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -14770,6 +14772,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14808,6 +14811,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6-20260205": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14845,6 +14849,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14884,6 +14889,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-7-20260416": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14922,6 +14928,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -14961,6 +14968,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-fable-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -15001,6 +15009,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15043,6 +15052,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-8": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -62148,6 +62158,7 @@ "supports_audio_output": true }, "claude-mythos-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -62187,6 +62198,7 @@ } }, "claude-mythos-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -62227,6 +62239,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-mythos-preview": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f0588b3fa79..d2fed1eb421 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -4310,7 +4310,10 @@ def _can_object_call_model( ) return True - potential_models: Final = [model] + from litellm.router_strategy.complexity_router.context_compaction import native_compaction_parent + + compaction_parent: Final = native_compaction_parent(model) + potential_models: Final = [model, compaction_parent] if compaction_parent is not None else [model] if model in litellm.model_alias_map: potential_models.append(litellm.model_alias_map[model]) elif llm_router and model in llm_router.model_group_alias: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index e7c7a2607c4..0c0f2492e56 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -103,6 +103,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( ) from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression +from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import route_request from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails from litellm.router import Router @@ -2555,7 +2556,7 @@ class ProxyBaseLLMRequestProcessing: user_model=user_model, user_api_key_dict=user_api_key_dict, ) - llm_call_task: Final = asyncio.create_task(llm_call) + llm_call_task: Final = asyncio.create_task(with_proxy_compaction_executor(llm_call, request)) tasks.append(llm_call_task) llm_responses: Final = asyncio.gather(*tasks) # run the moderation check in parallel to the actual llm api call diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a6b00be1091..9d35178891c 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -4475,9 +4475,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # 'metadata' and 'litellm_metadata' fields from litellm_params standard_logging_object: Final = kwargs.get("standard_logging_object") or {} request_metadata: Final = get_litellm_metadata_from_kwargs(kwargs) - if request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY): - # Internal sub-calls bill spend to the caller but are not the caller's - # traffic; charging them here would let background evals eat TPM headroom. + origin: Final = request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) + if origin and origin != "autorouter_compaction": + # Background evaluations keep their exemption; foreground compaction + # is necessary caller traffic and consumes the caller's token limits. return [] standard_logging_metadata: Final = standard_logging_object.get("metadata") or {} diff --git a/litellm/proxy/native_compaction.py b/litellm/proxy/native_compaction.py new file mode 100644 index 00000000000..fd27e7fbbdc --- /dev/null +++ b/litellm/proxy/native_compaction.py @@ -0,0 +1,90 @@ +import asyncio +from collections.abc import Awaitable, Mapping +from contextvars import Context +from types import MappingProxyType +from typing import Final, Literal, TypeVar + +from fastapi import Request +from pydantic import TypeAdapter, ValidationError +from starlette.types import ASGIApp + +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + inherit_message_logging_privacy, + initialize_standard_callback_dynamic_params, +) +from litellm.llms.custom_httpx.asgi_handler import get_async_asgi_client +from litellm.proxy.litellm_pre_call_utils import UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS +from litellm.router_strategy.complexity_router.context_compaction import ( + compaction_executor, + native_compaction_call, +) + +_ResultT: Final = TypeVar("_ResultT") +_ASGI_APP: Final = TypeAdapter[ASGIApp](ASGIApp) +_ROOT_PATH: Final = TypeAdapter(str) +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +_REMOVED_HEADERS: Final = frozenset( + ( + b"content-length", + b"x-litellm-call-id", + b"x-litellm-num-retries", + b"x-litellm-timeout", + b"x-litellm-stream-timeout", + ) +) + + +async def with_proxy_compaction_executor(call: Awaitable[_ResultT], request: Request) -> _ResultT: + async def execute( + protocol: Literal["chat", "messages"], payload: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: + logging_disabled: Final = initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is True + + async def dispatch() -> Mapping[str, object]: + scope: Final = _JSON_OBJECT.validate_python(request.scope) + root_path: Final = _ROOT_PATH.validate_python(scope.get("root_path", "")) + path: Final = "/v1/chat/completions" if protocol == "chat" else "/v1/messages" + url: Final = str(request.url.replace(path=root_path.rstrip("/") + path, query="", fragment="")) + headers: Final = tuple( + (name, value) + for name, value in request.headers.raw + if name.lower() not in _REMOVED_HEADERS + and not (logging_disabled and name.decode("latin-1").lower() in UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS) + ) + with ( + native_compaction_call(parent_model, str(payload["model"])), + inherit_message_logging_privacy(logging_disabled), + ): + with get_async_asgi_client( + app=_ASGI_APP.validate_python(scope["app"]), + root_path=root_path, + client=request.client, + ) as client: + async with client.stream( + "POST", url, headers=headers, json=_JSON_OBJECT.validate_python(payload) + ) as response: + if not response.is_success: + raise BadRequestError( + message=f"Native compaction child request failed (HTTP {response.status_code})", + model="context_compaction", + llm_provider="", + ) + body: Final = await response.aread() + try: + return MappingProxyType(_JSON_OBJECT.validate_json(body)) + except ValidationError: + raise BadRequestError( + message="Native compaction child returned an invalid JSON object", + model="context_compaction", + llm_provider="", + ) from None + + task: Final = Context().run(asyncio.create_task, dispatch()) + return await task + + token: Final = compaction_executor.set(execute) + try: + return await call + finally: + compaction_executor.reset(token) diff --git a/litellm/router.py b/litellm/router.py index 9a5c770e78a..06a65b85cc3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -118,6 +118,15 @@ from litellm.llms.openai_like.model_info import ( get_openai_compatible_model_info, ) from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.router_strategy.complexity_router.context_compaction import ( + arm_compaction, + compact_to_fit, + compaction_pending, + initialize_compaction_state, + is_native_compaction_call, + reject_recursive_compactor, + surface_for_call, +) from litellm.router_strategy.least_busy import LeastBusyLoggingHandler from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler @@ -3635,6 +3644,7 @@ class Router: kwargs=kwargs, client_type="max_parallel_requests", ) + compacted_input: Final = await compact_to_fit(self, deployment, input_kwargs, "chat") async with contextlib.AsyncExitStack() as deployment_slot: if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit): deployment_slot.enter_context(max_parallel_requests_limit) @@ -3643,7 +3653,7 @@ class Router: logging_obj=logging_obj, parent_otel_span=parent_otel_span, ) - response = await litellm.acompletion(**input_kwargs) + response = await litellm.acompletion(**compacted_input) ## CHECK CONTENT FILTER ERROR ## if isinstance(response, ModelResponse): @@ -5247,8 +5257,14 @@ class Router: if custom_llm_provider is not None: response_kwargs["custom_llm_provider"] = custom_llm_provider + compacted_input: Final = await compact_to_fit( + self, + deployment, + response_kwargs, + surface_for_call(getattr(original_generic_function, "__name__", "")), + ) async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await original_generic_function(**response_kwargs) + response = await original_generic_function(**compacted_input) if self._should_raise_anthropic_refusal_error( model=model, @@ -7382,6 +7398,11 @@ class Router: If it fails after num_retries, fall back to another model group """ model_group: Final[str | None] = kwargs.get("model") + compaction_surface: Final = surface_for_call( + getattr(kwargs.get("original_generic_function") or kwargs.get("original_function"), "__name__", "") + ) + if compaction_surface is not None: + kwargs["_context_compaction_state"] = initialize_compaction_state(kwargs, compaction_surface) clear_pre_routing_selection(kwargs) # pyright: ignore[reportUnknownArgumentType] # **kwargs is untyped at this boundary if not isinstance(kwargs.get("attempted_targets"), AttemptedFallbackTargets): _fallback_metadata_key: Final = _get_router_metadata_variable_name( @@ -12090,8 +12111,8 @@ class Router: def _count_pre_call_check_tokens( self, - messages: list[dict[str, str]] | None, - input: str | list | None, + messages: Sequence[Mapping[str, object]] | None, + input: str | list[object] | None, request_kwargs: Mapping[str, object] | None = None, ) -> int: """ @@ -12238,7 +12259,9 @@ class Router: _rate_limit_error = False parent_otel_span: Final = _get_parent_otel_span_from_kwargs(request_kwargs) - has_countable_input: Final = messages is not None or input is not None + has_countable_input: Final = (messages is not None or input is not None) and not compaction_pending( + request_kwargs + ) ## get model group RPM ## dt: Final = get_utc_datetime() @@ -13437,6 +13460,8 @@ class Router: registered_model_name: str, request_kwargs: Mapping[str, object], ) -> str: + if is_native_compaction_call(): + return registered_model_name if not any((self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers)): return registered_model_name cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs) @@ -13515,6 +13540,7 @@ class Router: model=registered_model_name, request_kwargs=request_kwargs ) if selected_strategy is None: + await arm_compaction(request_kwargs, None) self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) self._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None @@ -13525,6 +13551,29 @@ class Router: return None from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference + from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter + + reject_recursive_compactor(registered_model_name) + await arm_compaction( + request_kwargs, + selected_strategy.strategy.config.context_compaction + if isinstance(selected_strategy.strategy, ComplexityRouter) + else None, + tuple( + dict.fromkeys( + member + for pool in selected_strategy.strategy.config.tiers.values() + for member in ((pool,) if isinstance(pool, str) else pool) + ) + ) + if isinstance(selected_strategy.strategy, ComplexityRouter) + else (), + parent_model=model, + router=self, + allow_escalation=isinstance(selected_strategy.strategy, ComplexityRouter) + and selected_strategy.strategy.config.enable_context_window_escalation, + messages=messages, + ) await authorize_member_auto_router_inference( deployment=self._selected_strategy_marker_deployment( diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index f0fef3974f4..1f4285a7960 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -56,6 +56,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.router_strategy.adaptive_router.classifier import classify_prompt +from litellm.router_strategy.complexity_router.context_compaction import compaction_pending from litellm.router_strategy.complexity_router.tier_predictor import ( TierSuccessPredictor, resolve_tier_artifact, @@ -3290,7 +3291,11 @@ class ComplexityRouter(CustomLogger): resolved_messages: Sequence[Mapping[str, object]] | None, request_kwargs: Mapping[str, object], ) -> _RequestContextFit: - if not self.config.enable_context_window_escalation or not resolved_messages: + if ( + compaction_pending(request_kwargs) + or not self.config.enable_context_window_escalation + or not resolved_messages + ): return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer) names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset( (self.config.default_model,) if self.config.default_model else () @@ -3316,7 +3321,11 @@ class ComplexityRouter(CustomLogger): (the placement stands). Only a real tokenizer count ever moves a request, escalation lands only on groups whose every deployment declares a fitting window, and a group with no resolvable window is never moved on faith in either direction.""" - if not self.config.enable_context_window_escalation or not resolved_messages: + if ( + compaction_pending(request_kwargs) + or not self.config.enable_context_window_escalation + or not resolved_messages + ): return None pools: Final = self._tier_pools() pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ())) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index c4dbe68ebe2..dbc70631298 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -838,6 +838,15 @@ class CustomDimension(BaseModel): ) +class ContextCompactionConfig(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + model: str | None = Field(default=None, min_length=1) + trigger_ratio: float = Field(default=0.9, gt=0, lt=1) + max_tokens: int = Field(default=4096, ge=512) + timeout_seconds: float = Field(default=120, gt=0) + + class ComplexityRouterConfig(BaseModel): """Configuration for the ComplexityRouter.""" @@ -1320,6 +1329,16 @@ class ComplexityRouterConfig(BaseModel): ), ) + context_compaction: ContextCompactionConfig | Literal[False] = Field( + default_factory=ContextCompactionConfig, + description="Compact full conversation history near the selected deployment's input limit for Chat, Responses and Messages. Uses a capable configured tier model unless model is specified. Set false or null to disable. Stored and client-managed native history keep their existing behavior.", + ) + + @field_validator("context_compaction", mode="before") + @classmethod + def _normalize_context_compaction(cls, value: object) -> object: + return False if value is None else value + enable_context_window_escalation: bool = Field( default=False, description=( diff --git a/litellm/router_strategy/complexity_router/context_compaction.py b/litellm/router_strategy/complexity_router/context_compaction.py new file mode 100644 index 00000000000..d82090f3b99 --- /dev/null +++ b/litellm/router_strategy/complexity_router/context_compaction.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +from collections.abc import Generator, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from itertools import takewhile +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, NoReturn, Protocol, TypeAlias + +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + inherit_message_logging_privacy, + initialize_standard_callback_dynamic_params, +) +from litellm.litellm_core_utils.internal_call_metadata import parent_session_kwargs, sanitized_forwardable_call_metadata +from litellm.litellm_core_utils.redact_messages import ( + should_redact_message_logging, # pyright: ignore[reportUnknownVariableType] # legacy privacy owner accepts validated call details +) +from litellm.llms.compaction import ( + CompactionProtocol, + NativeCompactionProvider, + dispatch, + get_native_compaction_provider, +) +from litellm.router_strategy.complexity_router.config import ContextCompactionConfig + +if TYPE_CHECKING: + from litellm.router import Router + +Surface: TypeAlias = Literal["chat", "messages", "responses"] +_SURFACES: Final[Mapping[str, Surface]] = MappingProxyType( + {"_acompletion": "chat", "anthropic_messages": "messages", "aresponses": "responses"} +) +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_DICT: Final = TypeAdapter(dict[str, object]) +_ITEMS: Final = TypeAdapter(list[dict[str, object]]) +_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_INPUT: Final = TypeAdapter[str | list[object] | None](str | list[object] | None) +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_STATE_KEY: Final = "_context_compaction_state" +_native_child: Final[ContextVar[bool]] = ContextVar("native_compaction_child", default=False) +_native_parent: Final[ContextVar[tuple[str, str] | None]] = ContextVar("native_compaction_parent", default=None) + + +class CompactionExecutor(Protocol): + async def __call__( + self, protocol: CompactionProtocol, payload: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: ... + + +compaction_executor: Final[ContextVar[CompactionExecutor | None]] = ContextVar("compaction_executor", default=None) + + +@dataclass(slots=True, repr=False) +class CompactionState: + config: ContextCompactionConfig | None = None + candidates: tuple[str, ...] = () + summary: tuple[str, asyncio.Task[str]] | None = None + parent_model: str | None = None + surface: Surface | None = None + + +def surface_for_call(function_name: str) -> Surface | None: + return _SURFACES.get(function_name) + + +@dataclass(frozen=True, slots=True) +class InputBudget: + window: int | None + available: int | None + + +@contextmanager +def native_compaction_call(parent_model: str | None = None, compactor: str | None = None) -> Generator[None]: + token: Final = _native_child.set(True) + parent: Final = _native_parent.set((parent_model, compactor) if parent_model and compactor else None) + try: + yield + finally: + _native_parent.reset(parent) + _native_child.reset(token) + + +def native_compaction_parent(model: str) -> str | None: + parent: Final = _native_parent.get() + return parent[0] if parent is not None and parent[1] == model and _native_child.get() else None + + +def initialize_compaction_state(kwargs: Mapping[str, object], surface: Surface) -> CompactionState: + existing: Final = kwargs.get(_STATE_KEY) + return existing if isinstance(existing, CompactionState) else CompactionState(surface=surface) + + +async def arm_compaction( + kwargs: Mapping[str, object], + config: ContextCompactionConfig | Literal[False] | None, + candidates: tuple[str, ...] = (), + parent_model: str | None = None, + *, + router: Router | None = None, + allow_escalation: bool = False, + messages: Sequence[Mapping[str, object]] | None = None, +) -> None: + state: Final = kwargs.get(_STATE_KEY) + if isinstance(state, CompactionState): + state.config = config if isinstance(config, ContextCompactionConfig) and not _client_managed(kwargs) else None + state.candidates = candidates + state.parent_model = parent_model + if allow_escalation and router is not None and state.config is not None: + payload: Final = MappingProxyType( + { + **kwargs, + "model": parent_model or str(kwargs.get("model", "")), + **({"messages": messages} if messages is not None and state.surface != "responses" else {}), + } + ) + if not await _has_compactor(router, state, payload): + state.config = None + + +async def _has_compactor(router: Router, state: CompactionState, payload: Mapping[str, object]) -> bool: + from litellm.exceptions import ContextWindowExceededError + + if state.surface is None or state.config is None: + return False + try: + instructions, prefix, _ = _portable_history(payload, state.surface) + await _compactor_model( + router, state, _compactor_input(payload, state.surface, instructions, prefix, state.config.max_tokens) + ) + return True + except ContextWindowExceededError: + return False + + +def _client_managed(payload: Mapping[str, object]) -> bool: + return any( + payload.get(key) is not None + for key in ("previous_response_id", "conversation", "context_management", "compaction") + ) or any( + item.get("type") in ("reasoning", "compaction", "item_reference") + or item.get("encrypted_content") is not None + or any(block.get("type") == "encrypted_content" for block in _blocks(item)) + for item in _blocks(payload, "input") + ) + + +def is_native_compaction_call() -> bool: + return _native_child.get() + + +def reject_recursive_compactor(model: str) -> None: + if _native_child.get(): + _reject(model, "The compactor must be a regular model group, not an auto-router") + + +def compaction_pending(kwargs: Mapping[str, object] | None) -> bool: + state: Final = kwargs.get(_STATE_KEY) if kwargs is not None else None + return isinstance(state, CompactionState) and state.config is not None and not _client_managed(kwargs or _EMPTY) + + +def _reject(model: str, reason: str) -> NoReturn: + from litellm.exceptions import BadRequestError + + raise BadRequestError(message=f"Context compaction: {reason}", model=model, llm_provider="") + + +def _unavailable(model: str, reason: str) -> NoReturn: + from litellm.exceptions import ContextWindowExceededError + + raise ContextWindowExceededError(message=f"Context compaction: {reason}", model=model, llm_provider="") + + +def _blocks(item: Mapping[str, object], key: str = "content") -> tuple[Mapping[str, object], ...]: + value: Final = item.get(key) + return _OBJECTS.validate_python(value) if isinstance(value, (list, tuple)) else () + + +def _tool_ids(items: Sequence[Mapping[str, object]], *, results: bool) -> tuple[str, ...]: + return tuple( + identifier if isinstance(identifier, str) else "" + for item in items + for identifier in ( + *((item.get("tool_call_id"),) if results and item.get("role") == "tool" else ()), + *( + (item.get("call_id"),) + if item.get("type") == ("function_call_output" if results else "function_call") + else () + ), + *( + block.get("tool_use_id" if results else "id") + for block in _blocks(item) + if block.get("type") == ("tool_result" if results else "tool_use") + ), + *(call.get("id") for call in _blocks(item, "tool_calls") if not results), + ) + ) + + +def _history( + items: Sequence[Mapping[str, object]], model: str +) -> tuple[tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...]]: + instructions: Final = tuple(takewhile(lambda item: item.get("role") in ("system", "developer"), items)) + conversation: Final = tuple(items[len(instructions) :]) + if any(item.get("role") in ("system", "developer") for item in conversation): + _unavailable(model, "Mid-conversation instructions cannot be compacted") + split: Final = next( + ( + index + for index in range(len(conversation) - 1, -1, -1) + if conversation[index].get("role") == "user" + and not any(block.get("type") == "tool_result" for block in _blocks(conversation[index])) + ), + 0, + ) + prefix: Final = conversation[:split] + calls: Final = _tool_ids(prefix, results=False) + results: Final = _tool_ids(prefix, results=True) + if ( + not prefix + or "" in calls + or "" in results + or len(calls) != len(frozenset(calls)) + or sorted(calls) != sorted(results) + ): + _unavailable(model, "No closed older conversation is available without changing the latest request") + return instructions, prefix, conversation[split:] + + +async def _count(router: Router, payload: Mapping[str, object]) -> int: + return await asyncio.to_thread( + router._count_pre_call_check_tokens, # pyright: ignore[reportPrivateUsage] # shared Router admission counter + messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None, + input=_INPUT.validate_python(payload.get("input")), + request_kwargs=payload, + ) + + +def _budget( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], ratio: float +) -> InputBudget: + model: Final = str(payload.get("model", "")) + info: Final = _MAPPING.validate_python( + router.get_router_model_info(deployment=_DICT.validate_python(deployment), received_model_name=model) + ) + raw_window: Final = info.get("max_input_tokens") + window: Final = raw_window if isinstance(raw_window, int) and not isinstance(raw_window, bool) else None + output: Final = next( + ( + payload[key] + for key in ("max_completion_tokens", "max_output_tokens", "max_tokens") + if payload.get(key) is not None + ), + info.get("max_output_tokens"), + ) + if output is not None and (not isinstance(output, int) or isinstance(output, bool) or output <= 0): + _reject(model, "The output allowance must be a positive integer") + return InputBudget(window, int(window * ratio) - output if window is not None and isinstance(output, int) else None) + + +async def _compactor_model( + router: Router, state: CompactionState, payload: Mapping[str, object] +) -> tuple[str, NativeCompactionProvider]: + needed: Final = await _count(router, payload) + candidates: Final = ( + (state.config.model,) if state.config is not None and state.config.model is not None else state.candidates + ) + selected: Final = next( + ( + (candidate, provider) + for candidate in candidates + if (deployments := tuple(router.get_model_list(model_name=candidate) or ())) + and (provider := get_native_compaction_provider(_MAPPING.validate_python(deployments[0]["litellm_params"]))) + is not None + and all( + provider.supports_native_compaction(params := _MAPPING.validate_python(deployment["litellm_params"])) + and provider.compatible_defaults(params) + and (budget := _budget(router, deployment, payload, 0.9)).available is not None + and needed <= budget.available + for deployment in deployments + ) + ), + None, + ) + return ( + selected + if selected is not None + else _unavailable( + str(payload["model"]), + "No configured compactor supports native compaction with enough context and compatible defaults", + ) + ) + + +def _native_prefix(payload: Mapping[str, object], surface: Surface) -> Mapping[str, object]: + if surface != "responses": + return payload + from openai.types.responses.response_create_params import ResponseInputParam + + from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig + + messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=TypeAdapter(ResponseInputParam).validate_python(payload["input"]), + responses_api_request=_DICT.validate_python(payload), + ) + tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + _OBJECTS.validate_python(payload.get("tools") or ()) + ) + return MappingProxyType({**payload, "messages": _ITEMS.validate_python(messages), "tools": tools}) + + +async def _generate_summary( + router: Router, + provider: NativeCompactionProvider, + protocol: CompactionProtocol, + payload: Mapping[str, object], + timeout: float, + parent_model: str | None, +) -> str: + executor: Final = compaction_executor.get() + with native_compaction_call(): + response: Final = await asyncio.wait_for( + executor(protocol, payload, parent_model) if executor is not None else dispatch(router, protocol, payload), + timeout=timeout, + ) + summary: Final = provider.extract_summary(protocol, response) + return ( + summary + if summary is not None + else _reject(str(payload["model"]), "The provider did not return one complete native compaction block") + ) + + +def _compactor_input( + payload: Mapping[str, object], + surface: Surface, + instructions: Sequence[Mapping[str, object]], + prefix: Sequence[Mapping[str, object]], + output: int, +) -> Mapping[str, object]: + key: Final = "input" if surface == "responses" else "messages" + older: Final = _native_prefix( + MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *prefix))}), surface + ) + return MappingProxyType( + { + "model": str(payload["model"]), + "messages": older["messages"], + "max_tokens": output, + **{key: older[key] for key in ("system", "tools", "user") if key in older}, + } + ) + + +async def compact_to_fit( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None +) -> Mapping[str, object]: + from litellm.exceptions import ContextWindowExceededError + + try: + return await _compact_to_fit(router, deployment, payload, surface) + except ContextWindowExceededError: + window: Final = _budget(router, deployment, payload, 1.0).window + if not _native_child.get() and window is not None and await _count(router, payload) <= window: + return payload + raise + + +async def _check_client_managed_admission( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object] +) -> None: + if not router.enable_pre_call_checks: + return + router._pre_call_checks( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # restore legacy admission after deployment defaults + model=str(payload["model"]), + healthy_deployments=_ITEMS.validate_python((deployment,)), + messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None, # pyright: ignore[reportArgumentType] # legacy annotation omits structured content + input=_INPUT.validate_python(payload.get("input")), + request_kwargs=_DICT.validate_python(payload), + input_token_count=await _count(router, payload), + skip_inline_token_count=True, + ) + + +def _portable_history( + payload: Mapping[str, object], surface: Surface +) -> tuple[tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...]]: + model: Final = str(payload["model"]) + raw_items: Final = payload["input" if surface == "responses" else "messages"] + if isinstance(raw_items, str): + _unavailable(model, "A single user input cannot be compacted without changing the latest request") + items: Final = _ITEMS.validate_python(raw_items) + if surface == "responses" and any( + item.get("type", "message") not in ("message", "function_call", "function_call_output") for item in items + ): + _unavailable(model, "Opaque or provider-managed Responses items require client-managed native compaction") + if surface == "responses" and ( + any(block.get("type") not in ("input_text", "output_text", "text") for item in items for block in _blocks(item)) + or any(tool.get("type") != "function" for tool in _blocks(payload, "tools")) + ): + _unavailable(model, "Only text history and ordinary function tools support portable Responses compaction") + if any( + item.get("thinking_blocks") + or any(block.get("type") in ("thinking", "redacted_thinking", "compaction") for block in _blocks(item)) + for item in items + ): + _unavailable(model, "Native reasoning or compaction blocks require client-managed native compaction") + return _history(items, model) + + +async def _compact_to_fit( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None +) -> Mapping[str, object]: + state: Final = payload.get(_STATE_KEY) + config: Final = state.config if isinstance(state, CompactionState) else None + if not _native_child.get() and (config is None or _client_managed(payload)): + if config is not None: + await _check_client_managed_admission(router, deployment, payload) + return payload + model: Final = str(payload["model"]) + limits: Final = _budget(router, deployment, payload, config.trigger_ratio if config is not None else 0.9) + budget: Final = limits.available + if budget is None or budget <= 0: + if ( + config is not None + and config.model is None + and (limits.window is None or await _count(router, payload) <= limits.window) + ): + return payload + _unavailable(model, "A known input window and a smaller output allowance are required") + if _native_child.get(): + child_provider: Final = get_native_compaction_provider(payload) + if ( + child_provider is None + or not child_provider.compatible_defaults(payload) + or await _count(router, payload) > budget + ): + _reject(model, "The selected compactor's effective request is incompatible or exceeds its input budget") + return payload + if await _count(router, payload) <= budget: + return payload + if surface is None or config is None or not isinstance(state, CompactionState): + _unavailable(model, "This request surface cannot be compacted") + key: Final = "input" if surface == "responses" else "messages" + instructions, prefix, tail = _portable_history(payload, surface) + retained: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *tail))}) + if await _count(router, retained) >= budget: + _unavailable(model, "Retained instructions, tools and the latest turn leave no room for a summary") + older: Final = _compactor_input(payload, surface, instructions, prefix, config.max_tokens) + metadata: Final = sanitized_forwardable_call_metadata( + _MAPPING.validate_python(payload.get("litellm_metadata") or payload.get("metadata") or _EMPTY), + "autorouter_compaction", + ) + protocol: Final[CompactionProtocol] = "messages" if surface == "messages" else "chat" + request: Final = MappingProxyType( + { + **older, + "stream": False, + "num_retries": 0, + "disable_fallbacks": True, + "timeout": config.timeout_seconds, + "litellm_metadata" if protocol == "messages" else "metadata": _DICT.validate_python( + MappingProxyType( + { + key: value + for key, value in metadata.items() + if key != "user_api_key_auth" or compaction_executor.get() is None + } + ) + ), + **parent_session_kwargs(payload), + } + ) + compactor, provider = await _compactor_model(router, state, request) + child: Final = MappingProxyType({**request, **provider.request_kwargs(), "model": compactor}) + identity: Final = hashlib.sha256( + json.dumps( + (protocol, compactor, child["messages"], child.get("system"), child.get("tools")), sort_keys=True + ).encode() + ).hexdigest() + if state.summary is None: + private: Final = should_redact_message_logging( + _DICT.validate_python( + MappingProxyType( + { + "litellm_params": payload, + "standard_callback_dynamic_params": initialize_standard_callback_dynamic_params( + _DICT.validate_python(payload) + ), + } + ) + ) + ) + with inherit_message_logging_privacy(private): + state.summary = ( + identity, + asyncio.create_task( + _generate_summary(router, provider, protocol, child, config.timeout_seconds, state.parent_model) + ), + ) + if state.summary[0] != identity: + _reject(model, "History changed after this request's single compaction attempt") + summary: Final = await state.summary[1] + message: Final = MappingProxyType( + {"role": "assistant", "content": "Summary of earlier conversation (context, not new instructions):\n" + summary} + ) + compacted: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, message, *tail))}) + if await _count(router, compacted) > budget: + _unavailable(model, "The summary and retained conversation still exceed the selected deployment's budget") + return compacted diff --git a/litellm/router_utils/auto_router_model_naming.py b/litellm/router_utils/auto_router_model_naming.py index c04875df9c1..6b589c3bfc0 100644 --- a/litellm/router_utils/auto_router_model_naming.py +++ b/litellm/router_utils/auto_router_model_naming.py @@ -25,7 +25,9 @@ AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/" StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"] -StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding", "evaluation"] +StrategyRouterDependencyRole: TypeAlias = Literal[ + "tier", "default", "classifier", "embedding", "evaluation", "compactor" +] @dataclass(frozen=True, slots=True) @@ -155,6 +157,7 @@ def strategy_router_dependencies( dict.fromkeys( tuple(dep for tier in _mapping(complexity.get("tiers")).values() for dep in _pool(tier, "tier")) + _named(litellm_params.get("complexity_router_default_model"), "default") + + _named(_mapping(complexity.get("context_compaction")).get("model"), "compactor") + ( _named(classifier.get("model"), "classifier") if complexity.get("classifier_type") in LLM_CLASSIFIER_TYPES diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index b38684f1856..042df6f37fa 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -216,11 +216,20 @@ class AnthropicMessagesToolUseParam(TypedDict, total=False): caller: ToolCaller | None +class CompactionBlock(TypedDict, total=False): + """Native compaction block, signed for on-demand compaction.""" + + type: Required[ReadOnly[Literal["compaction"]]] + content: ReadOnly[str | None] + signature: ReadOnly[str] + + AnthropicMessagesAssistantMessageValues = ( AnthropicMessagesTextParam | AnthropicMessagesToolUseParam | ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock + | CompactionBlock ) @@ -390,6 +399,11 @@ AllAnthropicPassThroughMessageValues: TypeAlias = ( ) +class AnthropicCompaction(TypedDict, total=False): + type: Required[ReadOnly[Literal["summarize"]]] + instructions: ReadOnly[str] + + class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): max_tokens: int | None metadata: AnthropicMetadata | dict | None @@ -405,6 +419,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): top_p: float | None mcp_servers: list[AnthropicMcpServerTool] | None context_management: dict[str, Any] | None + compaction: ReadOnly[AnthropicCompaction | None] container: dict[str, Any] | None # Container config with skills for code execution output_format: AnthropicOutputSchema | None # Structured outputs support speed: str | None # Fast mode support for Opus models @@ -566,13 +581,6 @@ class ContextManagementResponse(TypedDict, total=False): applied_edits: list[AppliedEdit] -class CompactionBlock(TypedDict, total=False): - """Synthesized ``compaction`` content block (compact_20260112).""" - - type: Required[Literal["compaction"]] - content: str | None - - class UsageIteration(TypedDict, total=False): """One sampling iteration's token usage (compact_20260112).""" @@ -746,6 +754,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): WEB_SEARCH_2025_03_05 = "web-search-2025-03-05" CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27" COMPACT_2026_01_12 = "compact-2026-01-12" + COMPACT_2026_09_04 = "compact-2026-09-04" STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13" ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20" FAST_MODE_2026_02_01 = "fast-mode-2026-02-01" diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 1d4c3cdc864..5cf988bd19e 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from typing import Any, Literal, TypeAlias from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -6,8 +7,10 @@ from litellm.types.llms.anthropic import ( AnthropicResponseContentBlockText, AnthropicResponseContentBlockToolUse, AnthropicStopDetails, + CompactionBlock, ContextManagementResponse, ServerToolUsage, + UsageIteration, ) @@ -56,6 +59,7 @@ AnthropicResponseContentBlock: TypeAlias = ( | AnthropicResponseToolUseBlock | AnthropicResponseThinkingBlock | AnthropicResponseRedactedThinkingBlock + | CompactionBlock ) @@ -66,6 +70,7 @@ class AnthropicUsage(TypedDict, total=False): input_tokens: int output_tokens: int + iterations: ReadOnly[Sequence[UsageIteration]] """ Cache Tokens Used @@ -91,7 +96,9 @@ class AnthropicMessagesResponse(TypedDict, total=False): id: str model: str | None # This represents the Model type from Anthropic role: Literal["assistant"] | None - stop_reason: Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"] | None + stop_reason: ReadOnly[ + Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] | None + ] stop_details: NotRequired[ReadOnly[AnthropicStopDetails | None]] stop_sequence: str | None type: Literal["message"] | None diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py index 793893451df..06982a16755 100644 --- a/litellm/types/llms/custom_http.py +++ b/litellm/types/llms/custom_http.py @@ -32,6 +32,7 @@ class httpxSpecialProvider(str, Enum): Sandbox = "sandbox" ModelCostMap = "model_cost_map" PasswordBreachCheck = "password_breach_check" + ASGI = "asgi" VerifyTypes = str | bool | ssl.SSLContext diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 65d66677b27..346181d9d9e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -198,6 +198,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_output_config: bool | None supports_image_size: bool | None supports_anthropic_thinking_payload: ReadOnly[bool | None] + supports_anthropic_compaction: ReadOnly[bool | None] supported_audio_formats: ReadOnly[Sequence[Literal["mp3", "wav"]] | None] vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"] | None] bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None @@ -3025,6 +3026,7 @@ RoutingDecisionCause = Literal[ InternalCallOrigin = Literal[ "autorouter_classifier", + "autorouter_compaction", "shadow_eval_router", "shadow_eval_judge", "llm_as_a_judge_guardrail", @@ -3938,6 +3940,7 @@ all_litellm_params = ( agentic_loop_internal_litellm_params + [TRUSTED_CALLBACK_VARS_FIELD, ADDRESSED_RESPONSE_ID_FIELD, *bedrock_batch_litellm_params] + [ + "_context_compaction_state", "metadata", "litellm_metadata", "keepalive_seconds", diff --git a/litellm/utils.py b/litellm/utils.py index e088a5988c8..df2a6adbe1f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6158,6 +6158,7 @@ def _get_model_info_helper( supports_tool_search=_model_info.get("supports_tool_search", None), supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None), + supports_anthropic_compaction=_model_info.get("supports_anthropic_compaction", None), supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None), supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None), supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0f02ec8da5f..b5106670a2b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14516,6 +14516,7 @@ "source": "https://docs.anthropic.com/en/docs/about-claude/pricing" }, "claude-sonnet-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -14555,6 +14556,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-sonnet-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -14770,6 +14772,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14808,6 +14811,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6-20260205": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14845,6 +14849,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14884,6 +14889,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-7-20260416": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14922,6 +14928,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -14961,6 +14968,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-fable-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -15001,6 +15009,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15043,6 +15052,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-8": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -62148,6 +62158,7 @@ "supports_audio_output": true }, "claude-mythos-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -62187,6 +62198,7 @@ } }, "claude-mythos-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -62227,6 +62239,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-mythos-preview": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index f3a4e614f59..16b762f7803 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -839,6 +839,9 @@ "supports_adaptive_thinking": { "type": "boolean" }, + "supports_anthropic_compaction": { + "type": "boolean" + }, "supports_anthropic_thinking_payload": { "type": "boolean" }, diff --git a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py index fe965f75f8f..521244c6ede 100644 --- a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py +++ b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py @@ -1,8 +1,12 @@ +from types import MappingProxyType +from typing import Final + import pytest - +from pydantic import TypeAdapter from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + inherit_message_logging_privacy, initialize_standard_callback_dynamic_params, iter_client_callback_metadata_dicts, ) @@ -189,6 +193,20 @@ def test_empty_kwargs_returns_empty_params(): assert dict(params) == {} +@pytest.mark.parametrize("child_privacy", (False, True)) +def test_inherited_privacy_only_strengthens_child_and_resets(child_privacy: bool) -> None: + kwargs: Final = TypeAdapter(dict[str, object]).validate_python( + MappingProxyType({"turn_off_message_logging": child_privacy}) + ) + with inherit_message_logging_privacy(False): + assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy + with inherit_message_logging_privacy(True), inherit_message_logging_privacy(False): + params: Final = initialize_standard_callback_dynamic_params(kwargs) + assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy + assert params["turn_off_message_logging"] is True + assert initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is None + + def test_newrelic_callback_params_are_not_extracted_from_request_kwargs(): kwargs = { "newrelic_api_key": "caller-key", diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 633dd1d9460..7167a67d80d 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,8 +1,12 @@ -import pytest - +import json +from typing import Final from unittest.mock import MagicMock, patch +import httpx +import pytest +import respx + import litellm from litellm.constants import ( ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, @@ -3771,6 +3775,70 @@ def test_multiple_compaction_blocks(): assert compaction_blocks[1]["content"] == "Second summary..." +@pytest.mark.parametrize("messages_api,gateway,native_endpoint", [ + (False, False, False), (True, False, False), (False, True, False), (True, True, False), (True, True, True), +]) +async def test_native_compaction_wire_roundtrip( + messages_api: bool, gateway: bool, native_endpoint: bool, + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True") + monkeypatch.setattr(litellm.anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None) + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False) + block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"} + operation: Final = {"type": "summarize", "instructions": "Keep identifiers"} + usage: Final = {"input_tokens": 0, "output_tokens": 0, + "iterations": [{"type": "compaction", "input_tokens": 103, "output_tokens": 165}]} + chat_wire: Final = gateway and not native_endpoint + base: Final = "https://gateway.test/v1" if gateway else "https://api.anthropic.com/v1" + route: Final = respx_mock.post(f"{base}/{'chat/completions' if chat_wire else 'messages'}") + + def respond(request: httpx.Request) -> httpx.Response: + payload: Final = json.loads(request.content) + assert len(request.headers.get_list("anthropic-beta")) == 1 + assert {value.strip() for value in request.headers["anthropic-beta"].split(",")} == { + "compact-2026-09-04", "interleaved-thinking-2025-05-14", + } + if "compaction" in payload: + assert payload["compaction"] == operation + else: + assert payload["messages"][0] == {"role": "assistant", "content": [block]} + body: Final = ( + {"id": "chatcmpl_compact", "object": "chat.completion", "created": 1, "model": "claude-sonnet-5", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "", + "provider_specific_fields": {"compaction_blocks": [block]}}}], + "usage": {"prompt_tokens": 103, "completion_tokens": 165, "total_tokens": 268}} + if chat_wire else + {"id": "msg_compact", "type": "message", "role": "assistant", "model": "claude-sonnet-5", + "content": [block], "stop_reason": "compaction", "usage": usage} + ) + return httpx.Response(200, json=body) + + route.mock(side_effect=respond) + call: Final = litellm.anthropic.messages.acreate if messages_api else litellm.acompletion + params: Final = dict( + model=f"{'openai/' if gateway else ''}anthropic/claude-sonnet-5", api_key="test", max_tokens=512, + api_base=base if gateway else "https://api.anthropic.com", + extra_headers={"Anthropic-Beta": f"interleaved-thinking-2025-05-14{',compact-2026-09-04' if gateway else ''}"}, + model_info={"supported_endpoints": ["/v1/messages"]} if native_endpoint else {}, + ) + response: Final = await call( + messages=[{"role": "user", "content": "Remember identifiers"}], compaction=operation, **params + ) + message: Final = response if messages_api else response.choices[0].message.model_dump() + blocks: Final = message["content"] if messages_api else message["provider_specific_fields"]["compaction_blocks"] + assert blocks == [block] + if messages_api: + assert response["stop_reason"] == "compaction" + if not chat_wire: + assert response["usage"] == usage + if not gateway: + replay: Final = {"role": "assistant", "content": blocks} if messages_api else message + await call(messages=[replay, {"role": "user", "content": "Continue"}], **params) + assert route.call_count == (1 if gateway else 2) + + def test_compaction_block_request_transformation(): """ Test that compaction blocks from provider_specific_fields are correctly diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index b9a82e3fc68..d03174bc2c6 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -60,6 +60,20 @@ def test_translate_openai_response_to_anthropic_empty_choices() -> None: assert result["usage"]["input_tokens"] == 10 +@pytest.mark.parametrize("text,count,expected_stop", [ + ("", 1, "compaction"), (None, 1, "compaction"), ("Answer", 1, "max_tokens"), + (" ", 1, "max_tokens"), ("", 2, "max_tokens"), ("", 0, "max_tokens"), +]) +def test_native_compaction_response_roundtrip(text: str | None, count: int, expected_stop: str) -> None: + block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"} + message: Final = Message(content=text, provider_specific_fields={"compaction_blocks": [block] * count}) + response: Final = ModelResponse(choices=[Choices(message=message, finish_reason="length")], usage=Usage()) + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + expected_text: Final = [{"type": "text", "text": text}] if text is not None and (text != "" or not count) else [] + assert result["content"] == [*([block] * count), *expected_text] + assert result["stop_reason"] == expected_stop + + def test_translate_chat_refusal_to_anthropic_response(): response = ModelResponse( id="chatcmpl-refusal", diff --git a/tests/test_litellm/llms/custom_httpx/test_asgi_handler.py b/tests/test_litellm/llms/custom_httpx/test_asgi_handler.py new file mode 100644 index 00000000000..dbfa066a02d --- /dev/null +++ b/tests/test_litellm/llms/custom_httpx/test_asgi_handler.py @@ -0,0 +1,53 @@ +import asyncio +from collections.abc import Mapping +from typing import Final + +import pytest +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse, RedirectResponse +from starlette.routing import Route + +from litellm.llms.custom_httpx.asgi_handler import get_async_asgi_client + + +@pytest.mark.asyncio +async def test_cached_client_isolates_concurrent_apps_and_request_credentials() -> None: + ready: Final = (asyncio.Event(), asyncio.Event()) + + async def call(index: int) -> Mapping[str, object]: + async def endpoint(request: Request) -> JSONResponse: + ready[index].set() + await ready[1 - index].wait() + assert request.scope["root_path"] == f"/gateway-{index}" + assert request.client == (f"192.0.2.{index + 1}", 4321) + assert request.headers["authorization"] == f"Bearer key-{index}" + return JSONResponse({"app": index}, headers={"set-cookie": f"session=app-{index}; Path=/"}) + + app: Final = Starlette(routes=[Route("/child", endpoint, methods=["POST"])]) + with get_async_asgi_client(app, f"/gateway-{index}", (f"192.0.2.{index + 1}", 4321)) as client: + response: Final = await client.post( + f"https://proxy.test/gateway-{index}/child", headers={"authorization": f"Bearer key-{index}"}, + ) + assert response.status_code == 200 + assert not client.cookies + with get_async_asgi_client(app) as reused: + assert reused is client + return response.json() + + results: Final = await asyncio.wait_for(asyncio.gather(call(0), call(1)), timeout=5) + assert results == [{"app": 0}, {"app": 1}] + + +@pytest.mark.asyncio +async def test_internal_client_does_not_follow_redirects_or_environment_proxies(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HTTPS_PROXY", "http://unreachable.invalid:8080") + + async def endpoint(request: Request) -> RedirectResponse: + return RedirectResponse("https://external.invalid/credentials") + + app: Final = Starlette(routes=[Route("/redirect", endpoint, methods=["POST"])]) + with get_async_asgi_client(app) as client: + response: Final = await client.post("https://proxy.test/redirect", headers={"authorization": "Bearer fixture"}) + assert response.status_code == 307 + assert not response.history diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 4907b4ea054..0f19675edb9 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -6183,10 +6183,10 @@ async def test_configured_estimate_blocks_the_overrun_the_static_floor_admits(mo assert await admitted({"default_estimated_output_tokens": 3000}) == 2 -def test_internal_call_origin_success_ops_are_skipped(): - """Internal sub-calls (auto-router classifier, shadow eval shadow/judge) bill spend - to the caller's key but must not consume its TPM counters: the same kwargs charge - ops without the origin stamp and none with it.""" +@pytest.mark.parametrize("origin", ["shadow_eval_judge", "autorouter_compaction"]) +@pytest.mark.parametrize("rate_limit_type", ["input", "output", "total"]) +def test_internal_call_origin_success_ops_are_skipped(origin, rate_limit_type): + """Foreground compaction charges the same scopes as ordinary caller traffic.""" handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(DualCache()) ) @@ -6202,23 +6202,27 @@ def test_internal_call_origin_success_ops_are_skipped(): def _kwargs(metadata: Dict[str, Any]) -> Dict[str, Any]: return { "standard_logging_object": { - "metadata": {"user_api_key_hash": hash_token("sk-internal-origin")} + "metadata": { + "user_api_key_hash": hash_token("sk-internal-origin"), + "user_api_key_team_id": "compaction-team", + "user_api_key_project_id": "compaction-project", + } }, "litellm_params": {"metadata": metadata}, "model": "gpt-4o-mini", } charged = handler._build_success_event_pipeline_operations( - kwargs=_kwargs({}), response_obj=response, rate_limit_type="output" + kwargs=_kwargs({}), response_obj=response, rate_limit_type=rate_limit_type ) skipped = handler._build_success_event_pipeline_operations( - kwargs=_kwargs({INTERNAL_CALL_ORIGIN_METADATA_KEY: "shadow_eval_judge"}), + kwargs=_kwargs({INTERNAL_CALL_ORIGIN_METADATA_KEY: origin}), response_obj=response, - rate_limit_type="output", + rate_limit_type=rate_limit_type, ) assert charged - assert skipped == [] + assert skipped == (charged if origin == "autorouter_compaction" else []) def _conflicting_budget_bodies() -> Dict[str, Dict[str, object]]: diff --git a/tests/test_litellm/proxy/test_native_compaction.py b/tests/test_litellm/proxy/test_native_compaction.py new file mode 100644 index 00000000000..d24624aacc5 --- /dev/null +++ b/tests/test_litellm/proxy/test_native_compaction.py @@ -0,0 +1,163 @@ +import asyncio +from collections.abc import Awaitable, Mapping +from types import MappingProxyType +from typing import Final, Literal + +import pytest +from fastapi import FastAPI, Request +from pydantic import TypeAdapter + +from litellm.caching.caching import DualCache +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.redact_messages import should_redact_message_logging +from litellm.proxy import common_request_processing, proxy_server +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import can_key_call_model +from litellm.proxy._types import ProxyException +from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash, get_request_stash +from litellm.proxy.native_compaction import with_proxy_compaction_executor +from litellm.router import Router +from litellm.router_strategy.complexity_router.context_compaction import compaction_executor, reject_recursive_compactor +from litellm.types.utils import ModelResponse + +_HEADERS: Final = ( + (b"authorization", b"Bearer sk-compaction-fixture"), (b"cookie", b"session=fixture"), + (b"content-length", b"99999"), (b"x-litellm-call-id", b"parent"), + (b"litellm-disable-message-redaction", b"true"), (b"x-litellm-num-retries", b"8"), + (b"X-LiteLLM-Timeout", b"600"), (b"x-litellm-stream-timeout", b"500"), +) + + +async def _child( + protocol: Literal["chat", "messages"] = "chat", forged: bool = False, parent_model: str | None = None +) -> Mapping[str, object]: + executor: Final = compaction_executor.get() + assert executor is not None + payload: Final = TypeAdapter(Mapping[str, object]).validate_json( + b'{"model":"compactor","messages":[{"role":"user","content":"history"}],' + b'"num_retries":0,"timeout":7,"stream_timeout":7,"disable_fallbacks":true,"stream":false,' + b'"metadata":{"turn_off_message_logging":true}}' + ) + return await executor(protocol, MappingProxyType({ + "litellm_metadata" if protocol == "messages" and key == "metadata" else key: value + for key, value in payload.items() if forged or key != "metadata" + }), parent_model) + + +def _request(app: FastAPI) -> Request: + return Request(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({ + "type": "http", "app": app, "scheme": "https", "server": ("proxy.test", 443), + "path": "/gateway/parent", "root_path": "/gateway", "query_string": b"parent=1", + "client": ("192.0.2.1", 4321), "headers": _HEADERS, + }))) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("protocol", ("chat", "messages")) +async def test_child_preserves_credentials_and_isolates_context(protocol: Literal["chat", "messages"]) -> None: + app: Final = FastAPI() + stash: Final = get_or_create_request_stash() + + @app.post("/v1/chat/completions" if protocol == "chat" else "/v1/messages") + async def endpoint(request: Request) -> Mapping[str, object]: + assert get_request_stash() is None and compaction_executor.get() is None + assert request.client == ("192.0.2.1", 4321) and request.url.scheme == "https" + assert request.scope["root_path"] == "/gateway" and request.cookies["session"] == "fixture" + assert request.headers["authorization"] == "Bearer sk-compaction-fixture" and not request.query_params + assert "x-litellm-call-id" not in request.headers + assert "litellm-disable-message-redaction" not in request.headers + assert int(request.headers["content-length"]) == len(await request.body()) + with pytest.raises(BadRequestError, match="regular model group"): + reject_recursive_compactor("auto-router") + return MappingProxyType({"summary": "compacted"}) + + with inherit_message_logging_privacy(True): + assert (await with_proxy_compaction_executor(_child(protocol), _request(app)))["summary"] == "compacted" + assert get_request_stash() is stash and compaction_executor.get() is None + reject_recursive_compactor("auto-router") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("protocol", ("chat", "messages")) +@pytest.mark.parametrize("policy", ("allowed", "denied", "forged", "router_alias", "unrelated_alias")) +async def test_real_proxy_child_auth_privacy_and_body_policy( + monkeypatch: pytest.MonkeyPatch, protocol: Literal["chat", "messages"], policy: str, +) -> None: + cache: Final = DualCache() + token: Final = proxy_server.hash_token("sk-compaction-fixture") + models: Final = {"denied": ("answer",), "router_alias": ("auto",), "unrelated_alias": ("other-auto",)}.get(policy, ("compactor",)) + auth: Final = UserAPIKeyAuth.model_validate(MappingProxyType({"token": token, "models": models})) + await cache.async_set_cache(key=token, value=auth) + dispatched: Final = asyncio.Event() + allowed: Final = policy in ("allowed", "router_alias") + + async def route( + data: Mapping[str, object], llm_router: Router | None, user_model: str | None, + route_type: str, user_api_key_dict: UserAPIKeyAuth | None, + ) -> Awaitable[ModelResponse]: + dispatched.set() + assert allowed + if policy == "router_alias": + with pytest.raises(ProxyException): + await can_key_call_model("unrelated-compactor", None, auth, None) + assert (data["num_retries"], data["timeout"], data["stream_timeout"]) == (0, 7, 7) + assert data["disable_fallbacks"] is True and data["stream"] is False + logging: Final = data["litellm_logging_obj"] + assert isinstance(logging, Logging) + assert logging.standard_callback_dynamic_params.get("turn_off_message_logging") is True + assert should_redact_message_logging(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({ + "litellm_params": data, "standard_callback_dynamic_params": logging.standard_callback_dynamic_params, + }))) + return asyncio.sleep(0, result=ModelResponse(id="private-summary", model="compactor")) + + monkeypatch.setattr(proxy_server.app, "dependency_overrides", {}) + monkeypatch.setattr(proxy_server, "master_key", "sk-master-fixture") + monkeypatch.setattr(proxy_server, "prisma_client", object()) + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(common_request_processing, "route_request", route) + with inherit_message_logging_privacy(True): + call: Final = with_proxy_compaction_executor( + _child(protocol, policy == "forged", "auto" if policy.endswith("alias") else None), _request(proxy_server.app) + ) + if allowed: + assert (await call)["id"] == "private-summary" + else: + status: Final = 401 if policy == "forged" else 403 + with pytest.raises(BadRequestError, match=rf"child request failed \(HTTP {status}\)"): + await call + assert dispatched.is_set() is allowed + assert compaction_executor.get() is None + if policy.endswith("alias"): + with pytest.raises(ProxyException): + await can_key_call_model("compactor", None, auth, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("timeout", [False, True]) +async def test_cancelling_parent_cancels_and_drains_child(timeout: bool) -> None: + app: Final = FastAPI() + started: Final = asyncio.Event() + stopped: Final = asyncio.Event() + + @app.post("/v1/chat/completions") + async def endpoint() -> None: + started.set() + try: + await asyncio.Event().wait() + finally: + stopped.set() + + parent: Final = asyncio.create_task(with_proxy_compaction_executor(_child(), _request(app))) + await asyncio.wait_for(started.wait(), timeout=5) + if timeout: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(parent, timeout=0) + else: + parent.cancel() + with pytest.raises(asyncio.CancelledError): + await parent + assert stopped.is_set() and compaction_executor.get() is None diff --git a/tests/test_litellm/router_strategy/complexity_router/test_context_compaction.py b/tests/test_litellm/router_strategy/complexity_router/test_context_compaction.py new file mode 100644 index 00000000000..c6242f775c8 --- /dev/null +++ b/tests/test_litellm/router_strategy/complexity_router/test_context_compaction.py @@ -0,0 +1,503 @@ +import asyncio +import json +from collections.abc import Iterator, Mapping +from copy import deepcopy +from functools import partial +from typing import Final, Literal + +import httpx +import pytest +import respx + +import litellm +from litellm.llms import compaction as native +from litellm.router_strategy.complexity_router.config import ContextCompactionConfig +from litellm.router_strategy.complexity_router.context_compaction import ( + CompactionState, + Surface, + arm_compaction, + compact_to_fit, + compaction_executor, +) +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES +from litellm.types.router import Deployment + +pytestmark: Final = [pytest.mark.asyncio, pytest.mark.usefixtures("local_model_cost_map")] +SCHEMA: Final = {"type": "object", "properties": {"code": {"type": "string"}}} + + +@pytest.fixture(autouse=True) +def native_catalog(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setenv("LITELLM_LICENSE", "") + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False) + monkeypatch.setitem(litellm.model_cost, "summary-fixture", { + "litellm_provider": "anthropic", "mode": "chat", "max_input_tokens": 32000, + "max_output_tokens": 4096, "supports_anthropic_compaction": True, + }) + + +def make_router( + window: int | None = 512, settings: Mapping[str, object] | None = None, *, compactor_window: int = 32000, + conflict: bool = False, output: int | None = 64, + answer_defaults: Mapping[str, object] | None = None, + context_fallback: bool = False, +) -> litellm.Router: + config: Final = { + "tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "large", "REASONING": "large"}, + "keyword_tier_rules": [{"keywords": ["answer", "tail result"], "tier": "SIMPLE"}], + "enable_context_window_escalation": False, "max_tokens_from_tier_model": False, + **(settings or {}), + } + return litellm.Router(model_list=[ + {"model_name": "auto", "litellm_params": { + "model": "auto_router/complexity_router", "complexity_router_config": config, + }}, + {"model_name": "small", "litellm_params": { + "model": "openai/arbitrary-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test", "max_retries": 0, + **(answer_defaults or {}), + }, "model_info": {"id": "pinned-answer", "max_input_tokens": window, "max_output_tokens": output}}, + {"model_name": "large", "litellm_params": { + "model": "anthropic/summary-fixture", "api_base": "https://compact.test", "api_key": "compact-test", + **({"stop": ["deployment policy"]} if conflict else {}), + }, "model_info": {"id": "native-compactor", "max_input_tokens": compactor_window, "max_output_tokens": 4096}}, + {"model_name": "backup", "litellm_params": { + "model": "anthropic/summary-fixture", "api_base": "https://compact.test", "api_key": "backup-test", + }, "model_info": {"id": "backup-compactor", "max_input_tokens": 32000, "max_output_tokens": 4096}}, + ], enable_pre_call_checks=True, num_retries=0, disable_cooldowns=True, + retry_policy={"InternalServerErrorRetries": 1}, + context_window_fallbacks=[{"auto": ["large"]}] if context_fallback else []) + + +def exchange(surface: Surface, phase: str) -> list[dict[str, object]]: + identifier: Final = f"{phase}-call" + result: Final = f"{phase} result" + if surface == "responses": + return [ + {"type": "function_call", "call_id": identifier, "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "call_id": identifier, "output": result}, + ] + if surface == "messages": + return [ + {"role": "assistant", "content": [{"type": "tool_use", "id": identifier, "name": "lookup", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": identifier, "content": result}]}, + ] + return [ + {"role": "assistant", "tool_calls": [ + {"id": identifier, "type": "function", "function": {"name": "lookup", "arguments": "{}"}}, + ]}, + {"role": "tool", "tool_call_id": identifier, "content": result}, + ] + + +def history(surface: Surface) -> dict[str, object]: + conversation: Final = [ + {"role": "user", "content": "Project code MAPLE-47. Background detail. " * 150}, + {"role": "assistant", "content": "Recorded"}, + *exchange(surface, "prefix"), + {"role": "user", "content": "Answer with the project code"}, + *exchange(surface, "tail"), + ] + function: Final = {"name": "lookup", "parameters": SCHEMA} + if surface == "responses": + return {"instructions": "Keep the code exact", "tools": [{"type": "function", **function}], "input": [ + {"role": "developer", "content": "Retain the original spelling"}, *conversation, + ]} + if surface == "messages": + return {"system": "Keep the code exact", "tools": [{"name": "lookup", "input_schema": SCHEMA}], "messages": [ + *conversation, + ]} + return {"tools": [{"type": "function", "function": function}], "messages": [ + {"role": "system", "content": "Keep the code exact"}, *conversation, + ]} + + +def native_reply(summary: str = "Project code MAPLE-47", signed: bool = True, truncated: bool = False) -> httpx.Response: + return httpx.Response(200, json={ + "id": "msg_compact", "type": "message", "role": "assistant", "model": "summary-fixture", + "content": [{"type": "compaction", "content": summary, **({"signature": "native-signature"} if signed else {})}], + "stop_reason": "max_tokens" if truncated else "compaction", "usage": {"input_tokens": 0, "output_tokens": 0, "iterations": [ + {"type": "compaction", "input_tokens": 1200, "output_tokens": 20}, + ]}, + }) + + +def answer_reply(request: httpx.Request, expected_model: str = "arbitrary-answer") -> httpx.Response: + payload: Final = json.loads(request.content) + assert payload["model"] == expected_model + assert request.headers["authorization"] == "Bearer answer-test" + if request.url.path.endswith("responses"): + return httpx.Response(200, json={ + "id": "resp_answer", "object": "response", "created_at": 0, "status": "completed", + "model": payload["model"], "output": [{"id": "msg_answer", "type": "message", "role": "assistant", + "status": "completed", "content": [{"type": "output_text", "text": "MAPLE-47", "annotations": []}]}], + "usage": {"input_tokens": 60, "output_tokens": 8, "total_tokens": 68}, + }) + return httpx.Response(200, json={ + "id": "answer", "object": "chat.completion", "created": 0, "model": payload["model"], + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "MAPLE-47"}}], + "usage": {"prompt_tokens": 60, "completion_tokens": 8, "total_tokens": 68}, + }) + + +@pytest.fixture +def wire() -> Iterator[tuple[respx.Route, respx.Route]]: + with respx.mock(assert_all_called=False) as transport: + compactor: Final = transport.post("https://compact.test/v1/messages").mock(return_value=native_reply()) + answer: Final = transport.route(method="POST", host="answer.test").mock(side_effect=answer_reply) + yield compactor, answer + + +async def invoke(router: litellm.Router, surface: Surface, payload: Mapping[str, object], retries: int = 0) -> object: + if surface == "responses": + return await router.aresponses(model="auto", max_output_tokens=64, num_retries=retries, **payload) + if surface == "messages": + return await router.aanthropic_messages(model="auto", max_tokens=64, num_retries=retries, **payload) + return await router.acompletion(model="auto", max_tokens=64, num_retries=retries, **payload) + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("near", [False, True]) +@pytest.mark.parametrize("configured", [False, True]) +async def test_all_surfaces_compact_and_keep_selected_answerer( + wire: tuple[respx.Route, respx.Route], surface: Surface, near: bool, configured: bool, +) -> None: + payload: Final = history(surface) + original: Final = deepcopy(payload) + counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload) + window: Final = int((counted + 32) / ContextCompactionConfig().trigger_ratio) + 1 if near else 512 + assert (counted < window) is near + settings: Final = {"enable_context_window_escalation": True, + **({"context_compaction": {"model": "large", "max_tokens": 512}} if configured else {})} + router: Final = make_router(window, settings) + captured: Final = asyncio.Queue[Mapping[str, object]]() + compactor, answer = wire + retry: Final = near and configured + + def answer_after_retry(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, json={"error": {"message": "retry answer"}}) if answer.call_count == 0 else answer_reply(request) + + if retry: + answer.mock(side_effect=answer_after_retry) + + async def execute( + protocol: native.CompactionProtocol, request: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: + assert parent_model == "auto" + result: Final = await native.dispatch(router, protocol, request) + captured.put_nowait(result) + return result + + token: Final = compaction_executor.set(execute) + try: + response: Final = await invoke(router, surface, payload, retries=int(retry)) + finally: + compaction_executor.reset(token) + assert compactor.call_count == captured.qsize() == 1 + assert answer.call_count == router.total_calls["openai/arbitrary-answer"] == 1 + int(retry) + compact_request: Final = compactor.calls[0].request + compact_body: Final = json.loads(compact_request.content) + answer_body: Final = json.loads(answer.calls[0].request.content) + assert answer_body == json.loads(answer.calls[-1].request.content) + assert compact_body["model"] == "summary-fixture" and compact_body["compaction"] == {"type": "summarize"} + assert ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value in compact_request.headers["anthropic-beta"].split(",") + assert compact_body["max_tokens"] == (512 if configured else ContextCompactionConfig().max_tokens) + assert "Background detail" in str(compact_body) and "tail-call" not in str(compact_body) + assert "prefix-call" in str(compact_body) and "prefix result" in str(compact_body) + assert "Keep the code exact" in str(compact_body["system"]) + assert compact_body["tools"][0]["input_schema"] == SCHEMA + assert "MAPLE-47" in str(answer_body) and "Background detail" not in str(answer_body) + assert "prefix-call" not in str(answer_body) + assert "native-signature" not in str(answer_body) and "compaction" not in answer_body + assert "tail-call" in str(answer_body) and "tail result" in str(answer_body) + assert "MAPLE-47" in str(response) + usage: Final = captured.get_nowait()["usage"] + if surface == "messages": + assert usage["iterations"][0]["input_tokens"] == 1200 and usage["iterations"][0]["output_tokens"] == 20 + else: + assert usage["prompt_tokens"] == 1200 and usage["completion_tokens"] == 20 + if surface == "responses": + assert answer_body["input"][-3:] == original["input"][-3:] + assert answer_body["input"][0] == original["input"][0] + assert answer_body["instructions"] == original["instructions"] and answer_body["tools"] == original["tools"] + assert payload == original + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("mode", ["fitting", "false", "null"]) +async def test_fitting_and_disabled_requests_do_not_compact( + wire: tuple[respx.Route, respx.Route], surface: Surface, mode: str, +) -> None: + settings: Final = {} if mode == "fitting" else {"context_compaction": False if mode == "false" else None} + payload: Final = history(surface) + original: Final = deepcopy(payload) + counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload) + await invoke(make_router(20000 if mode == "fitting" else counted + 64, settings), surface, payload) + compactor, answer = wire + assert compactor.call_count == 0 and answer.call_count == 1 + assert "Background detail" in answer.calls[0].request.content.decode() + assert payload == original + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("reason", ["single", "unclosed", "no_compactor"]) +async def test_fitting_request_survives_unavailable_compaction( + wire: tuple[respx.Route, respx.Route], surface: Surface, reason: str, +) -> None: + key: Final = "input" if surface == "responses" else "messages" + items: Final = [{"role": "user", "content": "Answer with MAPLE-47. Detail. " * 80}] + payload: Final = history(surface) if reason == "no_compactor" else {key: ( + items if reason == "single" else [*items, *exchange(surface, "open")[:1], {"role": "user", "content": "Answer"}] + )} + counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload) + settings: Final = {"tiers": {"SIMPLE": "small", "MEDIUM": "small", "COMPLEX": "small", "REASONING": "small"}} if reason == "no_compactor" else {} + await invoke(make_router(counted + 1, settings), surface, payload) + compactor, answer = wire + assert compactor.call_count == 0 and answer.call_count == 1 + assert "Detail" in str(answer.calls[0].request.content) or "Background detail" in str(answer.calls[0].request.content) + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +async def test_uncompactable_overflow_uses_explicit_context_fallback( + wire: tuple[respx.Route, respx.Route], surface: Surface, +) -> None: + compactor, answer = wire + compactor.mock(return_value=httpx.Response(200, json={**native_reply().json(), + "content": [{"type": "text", "text": "MAPLE-47"}], "stop_reason": "end_turn"})) + payload: Final = {"input" if surface == "responses" else "messages": [ + {"role": "user", "content": "Answer with MAPLE-47. Detail. " * 150}, + ]} + await invoke(make_router(context_fallback=True), surface, payload) + assert answer.call_count == 0 and compactor.call_count == 1 + assert "compaction" not in json.loads(compactor.calls[0].request.content) + + +@pytest.mark.parametrize("escalate", [False, True]) +async def test_no_native_compactor_respects_explicit_escalation( + wire: tuple[respx.Route, respx.Route], monkeypatch: pytest.MonkeyPatch, escalate: bool, +) -> None: + monkeypatch.setitem(litellm.model_cost["summary-fixture"], "supports_anthropic_compaction", False) + router: Final = make_router(settings={"enable_context_window_escalation": escalate}) + compactor, answer = wire + compactor.mock(return_value=httpx.Response(200, json={**native_reply().json(), + "content": [{"type": "text", "text": "MAPLE-47"}], "stop_reason": "end_turn"})) + if not escalate: + with pytest.raises(litellm.ContextWindowExceededError, match="No configured compactor"): + await invoke(router, "chat", history("chat")) + assert compactor.call_count == 0 + else: + await invoke(router, "chat", history("chat")) + assert compactor.call_count == 1 + assert "compaction" not in json.loads(compactor.calls[0].request.content) + assert answer.call_count == 0 + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +async def test_undersized_native_compactor_does_not_block_explicit_escalation( + wire: tuple[respx.Route, respx.Route], surface: Surface, +) -> None: + router: Final = make_router(compactor_window=512, settings={ + "enable_context_window_escalation": True, + "tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "wide", "REASONING": "wide"}, + }) + router.add_deployment(Deployment( + model_name="wide", litellm_params={ + "model": "openai/wide-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test", + }, model_info={"id": "wide-answer", "max_input_tokens": 32000, "max_output_tokens": 64}, + )) + compactor, answer = wire + answer.mock(side_effect=partial(answer_reply, expected_model="wide-answer")) + await invoke(router, surface, history(surface)) + assert compactor.call_count == 0 and answer.call_count == 1 + assert json.loads(answer.calls[0].request.content)["model"] == "wide-answer" + + +@pytest.mark.parametrize( + ("surface", "failure"), + [(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")] + + [("messages", "truncated")], +) +async def test_bad_native_result_never_reaches_answerer( + wire: tuple[respx.Route, respx.Route], surface: Surface, failure: str, +) -> None: + compactor, answer = wire + reply: Final = httpx.Response(500, json={"error": {"type": "api_error", "message": "failed"}}) if failure == "provider" else native_reply( + "too large " * 2000 if failure == "oversized" else "MAPLE-47", signed=failure != "unsigned", + truncated=failure == "truncated", + ) + compactor.mock(return_value=reply) + with pytest.raises((litellm.BadRequestError, litellm.InternalServerError)): + await invoke(make_router(), surface, history(surface)) + assert compactor.call_count == 1 and answer.call_count == 0 + + +@pytest.mark.parametrize("failure", ["item", "content", "tools", "unclosed", "missing", "duplicate", "instructions", "retained"]) +@pytest.mark.parametrize("needed", [False, True]) +async def test_unsafe_responses_reject_only_when_compaction_needed( + wire: tuple[respx.Route, respx.Route], failure: str, needed: bool, +) -> None: + payload: Final = history("responses") + extra: Final = { + "item": [{"type": "computer_call", "call_id": "opaque-tool"}], + "content": [{"role": "assistant", "content": [{"type": "refusal", "refusal": "cannot"}]}], + "unclosed": [{"type": "function_call", "call_id": "unclosed", "name": "lookup", "arguments": "{}"}], + "missing": [{"type": "function_call", "name": "lookup", "arguments": "{}"}], + "duplicate": exchange("responses", "prefix"), + "instructions": [{"role": "developer", "content": "Changed instructions"}], + } + request: Final = { + **payload, "input": [*payload["input"][:3], *extra.get(failure, []), *payload["input"][3:]], + **({"tools": [{"type": "computer_use_preview", "display_width": 800, "display_height": 600}]} if failure == "tools" else {}), + **({"instructions": "Keep every instruction " * 600} if failure == "retained" else {}), + } + if needed: + with pytest.raises(litellm.BadRequestError, match="Context compaction"): + await invoke(make_router(), "responses", request) + assert all(route.call_count == 0 for route in wire) + else: + await invoke(make_router(20000), "responses", request) + compactor, answer = wire + assert compactor.call_count == 0 and answer.call_count == 1 + + +@pytest.mark.parametrize("owned", [ + {"previous_response_id": "resp_parent"}, {"conversation": "conv_parent"}, + {"context_management": [{"type": "compaction", "compact_threshold": 1000}]}, {"compaction": {"type": "summarize"}}, + {"input": [{"type": "reasoning", "encrypted_content": "opaque"}]}, + {"input": [{"type": "reasoning", "summary": [{"type": "summary_text", "text": "prior reasoning"}]}]}, + {"input": [{"type": "compaction", "encrypted_content": "opaque"}]}, + {"input": [{"type": "item_reference", "id": "item_parent"}]}, + {"input": [{"role": "assistant", "content": "visible", "encrypted_content": "opaque"}]}, + {"input": [{"role": "user", "content": [{"type": "encrypted_content", "encrypted_content": "opaque"}]}]}, +]) +@pytest.mark.parametrize("arm_first", [False, True]) +async def test_client_owned_history_bypasses_compaction( + wire: tuple[respx.Route, respx.Route], owned: Mapping[str, object], arm_first: bool, +) -> None: + router: Final = make_router() + deployment: Final = router.get_deployment(model_id="pinned-answer") + assert deployment is not None + state: Final = CompactionState() + request: Final = {**history("responses"), **owned, "model": "small", "max_tokens": 64, "_context_compaction_state": state} + original: Final = deepcopy({key: value for key, value in request.items() if key != "_context_compaction_state"}) + before_defaults: Final = {"_context_compaction_state": state} if arm_first else request + await arm_compaction(before_defaults, ContextCompactionConfig(), ("large",)) + counted: Final = router._count_pre_call_check_tokens(None, request["input"], request) + if arm_first and counted > 512: + with pytest.raises(litellm.ContextWindowExceededError): + await compact_to_fit(router, deployment.model_dump(), request, "responses") + else: + result: Final = await compact_to_fit(router, deployment.model_dump(), request, "responses") + assert result is request + assert {key: value for key, value in request.items() if key != "_context_compaction_state"} == original + assert all(route.call_count == 0 for route in wire) + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("source", ["request", "deployment"]) +async def test_client_managed_overflow_keeps_context_window_admission( + wire: tuple[respx.Route, respx.Route], surface: Surface, source: str, +) -> None: + managed: Final = {"context_management": {"edits": []}} + router: Final = make_router(answer_defaults=managed if source == "deployment" else None) + payload: Final = {**history(surface), **(managed if source == "request" else {})} + compactor, answer = wire + if source == "deployment": + with pytest.raises(litellm.ContextWindowExceededError): + await invoke(router, surface, payload) + assert compactor.call_count == 0 + else: + await invoke(router, surface, payload) + assert compactor.call_count == 1 + assert "compaction" not in json.loads(compactor.calls[0].request.content) + assert answer.call_count == 0 + + +@pytest.mark.parametrize("case", ["conflicting_defaults", "small_window", "capability_false", "capability_missing"]) +async def test_automatic_compactor_skips_conflicts_and_requires_capacity_and_capability( + wire: tuple[respx.Route, respx.Route], monkeypatch: pytest.MonkeyPatch, case: str, +) -> None: + if case.startswith("capability"): + metadata: Final = {key: value for key, value in litellm.model_cost["summary-fixture"].items() + if key != "supports_anthropic_compaction"} + monkeypatch.setitem(litellm.model_cost, "summary-fixture", { + **metadata, **({"supports_anthropic_compaction": False} if case == "capability_false" else {}), + }) + conflict: Final = case == "conflicting_defaults" + settings: Final = {"tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "backup", "REASONING": "backup"}} if conflict else {} + router: Final = make_router(settings=settings, conflict=conflict, compactor_window=512 if case == "small_window" else 32000) + if conflict: + await invoke(router, "chat", history("chat")) + compactor, answer = wire + assert compactor.call_count == answer.call_count == 1 + assert compactor.calls[0].request.headers["x-api-key"] == "backup-test" + assert "stop_sequences" not in json.loads(compactor.calls[0].request.content) + else: + with pytest.raises(litellm.BadRequestError, match="No configured compactor"): + await invoke(router, "chat", history("chat")) + assert all(route.call_count == 0 for route in wire) + + +@pytest.mark.parametrize("unknown_output", [False, True]) +@pytest.mark.parametrize("overflow", [False, True]) +async def test_unusable_output_budget_still_enforces_known_input_window( + wire: tuple[respx.Route, respx.Route], unknown_output: bool, overflow: bool, +) -> None: + payload: Final = history("chat") + counted: Final = make_router(output=None)._count_pre_call_check_tokens(payload["messages"], None, payload) + window: Final = 512 if overflow else counted + 64 + output: Final = None if unknown_output else window + router: Final = make_router(window, output=output) + deployment: Final = router.get_deployment(model_id="pinned-answer") + assert deployment is not None + state: Final = CompactionState(config=ContextCompactionConfig(), candidates=("large",)) + request: Final = {**payload, "model": "small", "max_tokens": output, "_context_compaction_state": state} + if overflow: + with pytest.raises(litellm.BadRequestError, match="known input window"): + await compact_to_fit(router, deployment.model_dump(), request, "chat") + else: + assert await compact_to_fit(router, deployment.model_dump(), request, "chat") is request + assert all(route.call_count == 0 for route in wire) + + +@pytest.mark.parametrize("outcome", ["success", "timeout", "cancel"]) +async def test_retry_reuses_summary_or_terminal_cancellation(outcome: Literal["success", "timeout", "cancel"]) -> None: + router: Final = make_router() + deployment: Final = router.get_deployment(model_id="pinned-answer") + assert deployment is not None + state: Final = CompactionState(config=ContextCompactionConfig(model="large", max_tokens=512, timeout_seconds=0.02)) + request: Final = {**history("messages"), "model": "small", "max_tokens": 64, "_context_compaction_state": state} + calls: Final = asyncio.Queue[None]() + started: Final = asyncio.Event() + stopped: Final = asyncio.Event() + + async def execute( + protocol: native.CompactionProtocol, payload: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: + calls.put_nowait(None) + started.set() + try: + return native_reply().json() if outcome == "success" else await asyncio.Future[Mapping[str, object]]() + finally: + stopped.set() + + token: Final = compaction_executor.set(execute) + try: + first: Final = asyncio.create_task(compact_to_fit(router, deployment.model_dump(), request, "messages")) + await asyncio.wait_for(started.wait(), timeout=2) + if outcome == "cancel": + first.cancel() + if outcome == "success": + assert await first == await compact_to_fit(router, deployment.model_dump(), request, "messages") + changed: Final = {**request, "messages": [{"role": "user", "content": "new history"}, *request["messages"]]} + with pytest.raises(litellm.BadRequestError, match="History changed"): + await compact_to_fit(router, deployment.model_dump(), changed, "messages") + else: + error: Final = asyncio.CancelledError if outcome == "cancel" else asyncio.TimeoutError + with pytest.raises(error): + await first + with pytest.raises(error): + await compact_to_fit(router, deployment.model_dump(), request, "messages") + assert calls.qsize() == 1 and stopped.is_set() + finally: + compaction_executor.reset(token) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index baa15ac1568..4e146b59b61 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -15317,6 +15317,7 @@ class TestHealthFallbackDispatch: router: Final = self._router( config={ + "context_compaction": False, "tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}, "enable_context_window_escalation": True, } @@ -15399,6 +15400,7 @@ class TestHealthFallbackDispatch: async def test_modality_default_must_also_fit_context(self, default_fits: bool) -> None: router: Final = self._router( config={ + "context_compaction": False, "modality_routing": True, "tiers": {"SIMPLE": "primary"}, "enable_context_window_escalation": True, diff --git a/tests/test_litellm/test_anthropic_beta_headers_filtering.py b/tests/test_litellm/test_anthropic_beta_headers_filtering.py index 19b26120672..8656a7564d2 100644 --- a/tests/test_litellm/test_anthropic_beta_headers_filtering.py +++ b/tests/test_litellm/test_anthropic_beta_headers_filtering.py @@ -83,6 +83,7 @@ class TestAnthropicBetaHeadersFiltering: filtered = filter_and_transform_beta_headers( beta_headers=all_headers, provider=provider ) + assert ("compact-2026-09-04" in filtered) is (provider == "anthropic") for header in unsupported_headers: assert ( diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index cf61a6d9f65..38aad3ace1c 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -178,6 +178,14 @@ def test_potential_model_names_keeps_provider_prefixed_candidate(): assert bare["provider_prefixed_model_name"] == bare["combined_model_name"] == "perplexity/glm-5.2" +@pytest.mark.parametrize("capability", [True, False, None]) +def test_get_model_info_anthropic_compaction( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, capability: bool | None +) -> None: + monkeypatch.setitem(litellm.model_cost["claude-sonnet-5"], "supports_anthropic_compaction", capability) + assert litellm.get_model_info("claude-sonnet-5")["supports_anthropic_compaction"] is capability + + def test_get_model_info_strips_openai_finetune_ids_without_a_custom_suffix(local_model_cost_map): info = litellm.get_model_info(model="ft:gpt-4o-2024-08-06:my-org::abc123", custom_llm_provider="openai") assert info["key"] == "ft:gpt-4o-2024-08-06" @@ -861,6 +869,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "source": {"type": "string"}, "comment": {"type": "string"}, "supports_assistant_prefill": {"type": "boolean"}, + "supports_anthropic_compaction": {"type": "boolean"}, "supports_audio_input": {"type": "boolean"}, "supports_audio_output": {"type": "boolean"}, "gemini_native_audio": {"type": "boolean"}, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6a184d0765d..78b929de36d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27678,6 +27678,26 @@ export interface components { */ pattern_type: "prebuilt" | "regex"; }; + /** ContextCompactionConfig */ + ContextCompactionConfig: { + /** + * Max Tokens + * @default 4096 + */ + max_tokens: number; + /** Model */ + model?: string | null; + /** + * Timeout Seconds + * @default 120 + */ + timeout_seconds: number; + /** + * Trigger Ratio + * @default 0.9 + */ + trigger_ratio: number; + }; /** * CoordinationRedisNode * @description A single startup node of a cluster-mode Redis used for proxy coordination. @@ -36962,6 +36982,11 @@ export interface components { * @description Keywords indicating code-related content */ code_keywords?: string[] | null; + /** + * Context Compaction + * @description Compact full conversation history near the selected deployment's input limit for Chat, Responses and Messages. Uses a capable configured tier model unless model is specified. Set false or null to disable. Stored and client-managed native history keep their existing behavior. + */ + context_compaction?: components["schemas"]["ContextCompactionConfig"] | false; /** * Context Window Escalation Buffer * @description Fraction of a model's declared context window the estimated prompt must fit within. The token count is an estimate, so fitting against the full window would dispatch prompts that the provider's own tokenizer then rejects; 0.95 leaves room for that drift plus the response tokens. From 25af172b854219ebe39c67db1d479ddb7b744548 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 22 Sep 2026 05:56:21 +0000 Subject: [PATCH 14/28] fix(mcp): keep config-defined servers read-only (#42299) * fix(mcp): persist config server edits in the database * test(mcp): cover config loading and failed promotion responses * test(mcp): preserve auth policy during config server promotion * fix(mcp): preserve existing YAML metadata during config loading * fix(mcp): reuse bounded traversal for config secret checks * fix(mcp): honor database access groups after config promotion * fix(mcp): keep config-defined servers read-only * fix(ui): clear frontend warnings and require warning-free green checks * fix(ui): preserve legacy MCP access group labels * ci: remove remaining frontend action runtime warnings * fix(mcp): limit read-only fix to ticket scope * fix(mcp): preserve API stability and remove unrelated guidance --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/models/mcp_server.py | 1 + .../mcp_server/mcp_server_manager.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 12 ++++++ .../mcp_server/test_mcp_server_manager.py | 15 +++++++ .../test_mcp_management_endpoints.py | 40 +++++++++++++++++++ .../_components/mcp_server_view.test.tsx | 34 ++++++++++++++++ .../_components/mcp_server_view.tsx | 14 +++++-- .../src/components/mcp_tools/types.tsx | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 +++ 9 files changed, 120 insertions(+), 4 deletions(-) diff --git a/litellm/models/mcp_server.py b/litellm/models/mcp_server.py index 7ccff9434a7..9125d708e79 100644 --- a/litellm/models/mcp_server.py +++ b/litellm/models/mcp_server.py @@ -50,6 +50,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase): """Represents a LiteLLM_MCPServerTable record""" server_id: str + is_config: bool = Field(default=False, description="Whether this server is defined in config and is read-only.") server_name: str | None = None alias: str | None = None description: str | None = None diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 4a2713cb19c..20c114a2f3e 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -7081,6 +7081,7 @@ class MCPServerManager: def _build_mcp_server_table(self, server: MCPServer) -> LiteLLM_MCPServerTable: return LiteLLM_MCPServerTable( server_id=server.server_id, + is_config=self.is_config_declared_server(server.server_id) and server.server_id not in self.registry, server_name=server.server_name, alias=server.alias, description=(server.mcp_info.get("description") if server.mcp_info else None), diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 03122f25870..4dbca14b917 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -24767,6 +24767,12 @@ "title": "Is Byok", "type": "boolean" }, + "is_config": { + "default": false, + "description": "Whether this server is defined in config and is read-only.", + "title": "Is Config", + "type": "boolean" + }, "issuer": { "anyOf": [ { @@ -27817,6 +27823,12 @@ "title": "Is Byok", "type": "boolean" }, + "is_config": { + "default": false, + "description": "Whether this server is defined in config and is read-only.", + "title": "Is Config", + "type": "boolean" + }, "issuer": { "anyOf": [ { diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 9f42a523350..7725aca1948 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -14470,6 +14470,21 @@ async def test_request_selected_during_guardrail_runs_concurrently_with_tool(mon assert result.content[0].text == "executed" +@pytest.mark.asyncio +@pytest.mark.parametrize("in_config,in_db,expected", [(True, False, True), (False, True, False), (True, True, False)]) +async def test_server_response_identifies_read_only_config(in_config, in_db, expected): + manager = MCPServerManager() + server = MCPServer(server_id="source-server", name="source_server", transport=MCPTransport.http) + manager.config_mcp_servers = {server.server_id: server} if in_config else {} + manager.registry = {server.server_id: server} if in_db else {} + + listed = await manager.get_all_mcp_servers_unfiltered() + + assert len(listed) == 1 + assert listed[0].model_dump().get("is_config") is expected + assert manager._build_mcp_server_table(server).model_dump().get("is_config") is expected + + @pytest.mark.asyncio @pytest.mark.parametrize("with_caller,legacy_factory", [(True, False), (False, False), (True, True)]) async def test_client_sampling_does_not_fill_explicit_context_from_another_ambient_caller(with_caller, legacy_factory): diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 47540eb5d6d..53645e62034 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -7900,3 +7900,43 @@ class TestGetMcpToolsWireShape: assert tool["outputSchema"] == {"type": "integer"} assert "_meta" in tool assert not {"input_schema", "output_schema", "meta"} & tool.keys() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role,expected_status", [ + (LitellmUserRoles.PROXY_ADMIN, 404), + (LitellmUserRoles.INTERNAL_USER, 403), +]) +async def test_config_server_edit_preserves_api_contract_without_creating_rows(role, expected_status): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + + manager = MCPServerManager() + server = generate_mock_mcp_server_config_record(server_id="read-only-config") + manager.config_mcp_servers = {server.server_id: server} + original = server.model_dump() + prisma = MagicMock() + prisma.db.litellm_mcpservertable.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_mcpservertable.update = AsyncMock(return_value=None) + with ( + patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), + patch.object(mgmt_endpoints, "get_prisma_client_or_throw", return_value=prisma), + ): + with pytest.raises(HTTPException) as exc: + await mgmt_endpoints.edit_mcp_server( + payload=UpdateMCPServerRequest(server_id=server.server_id, description="UI edit"), + user_api_key_dict=UserAPIKeyAuth(user_id="actor", user_role=role), + ) + + assert exc.value.status_code == expected_status + if role == LitellmUserRoles.PROXY_ADMIN: + assert exc.value.detail == { + "error": f"MCP Server not found, passed server_id={server.server_id}" + } + prisma.db.litellm_mcpservertable.update.assert_awaited_once() + else: + prisma.db.litellm_mcpservertable.update.assert_not_awaited() + prisma.db.litellm_mcpservertable.create.assert_not_called() + prisma.db.litellm_mcpservertable.create_many.assert_not_called() + prisma.tx.assert_not_called() + assert server.model_dump() == original + assert manager.registry == {} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx index da564f23de5..58e1e9bcce9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx @@ -4,6 +4,8 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { MCPServerView } from "./mcp_server_view"; import * as networking from "@/components/networking"; +import { setSecureItem } from "@/utils/secureStorage"; +import { EDIT_OAUTH_UI_STATE_KEY } from "./mcp_server_edit"; import type { MCPServer } from "@/components/mcp_tools/types"; vi.mock(".", () => ({ @@ -68,6 +70,7 @@ const openUserCredentials = async (props: Record) => { describe("MCPServerView", () => { beforeEach(() => { vi.clearAllMocks(); + sessionStorage.clear(); }); // Name, alias and description each label the header and a Settings row, so @@ -146,6 +149,37 @@ describe("MCPServerView", () => { expect(screen.queryByRole("button", { name: "Edit Settings" })).not.toBeInTheDocument(); }); + it.each([false, true])("keeps config settings read-only with isEditing=%s", async (isEditing) => { + renderView({ is_config: true }, { isEditing }); + + await userEvent.click(screen.getByRole("tab", { name: "Settings" })); + + expect(screen.getByRole("button", { name: "Edit Settings" })).toBeDisabled(); + expect(screen.getByText("Defined in config. Edit your YAML configuration to make changes")).toBeVisible(); + expect(screen.queryByText("edit form")).not.toBeInTheDocument(); + }); + + it.each([true, false])("honors config read-only state on OAuth return: %s", async (isConfig) => { + setSecureItem(EDIT_OAUTH_UI_STATE_KEY, JSON.stringify({ serverId: "srv-1" })); + renderView({ is_config: isConfig }); + + await userEvent.click(screen.getByRole("tab", { name: "Settings" })); + + if (isConfig) { + expect(screen.queryByText("edit form")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Edit Settings" })).toBeDisabled(); + } else { + expect(screen.getByText("edit form")).toBeVisible(); + } + }); + + it("does not open the editor for a view-only admin", async () => { + renderView({}, { isViewOnly: true, isEditing: true }); + await userEvent.click(screen.getByRole("tab", { name: "Settings" })); + expect(screen.getByRole("button", { name: "Edit Settings" })).toBeDisabled(); + expect(screen.queryByText("edit form")).not.toBeInTheDocument(); + }); + it("opens on the tab named by initialTabIndex", async () => { renderView({}, { initialTabIndex: 1 }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx index a346c7d986b..6045be4607d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx @@ -62,7 +62,8 @@ export const MCPServerView: React.FC = ({ }) => { // Open the editing Settings tab on first render when returning from the edit OAuth // redirect, so the "token fetched" feedback shows where the user left off (Settings=2). - const returningFromEditOAuth = isReturningFromEditOAuth(isProxyAdmin, mcpServer.server_id); + const canEdit = isProxyAdmin && !isViewOnly && !mcpServer.is_config; + const returningFromEditOAuth = isReturningFromEditOAuth(canEdit, mcpServer.server_id); const [editing, setEditing] = useState(isEditing || returningFromEditOAuth); const [showFullUrl, setShowFullUrl] = useState(false); const [copiedStates, setCopiedStates] = useState>({}); @@ -224,13 +225,18 @@ export const MCPServerView: React.FC = ({

MCP Server Settings

- {editing ? null : ( - )}
- {editing ? ( + {mcpServer.is_config && ( +

+ Defined in config. Edit your YAML configuration to make changes +

+ )} + {editing && canEdit ? ( Date: Tue, 22 Sep 2026 01:05:58 -0700 Subject: [PATCH 15/28] ci(code-quality): allowlist _render_json in the recursive detector (#42442) Co-authored-by: yuneng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/code_coverage_tests/recursive_detector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 071191183df..b77348b5d73 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -67,6 +67,7 @@ IGNORE_FUNCTIONS = [ "_redact_agent_params_tree", # max depth set (default 10), same shape as _redact_sensitive_litellm_params. "_restore_redacted_nested_value", # max depth set (default 10), mirrors _redact_agent_params_tree on the write side. "_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible). + "_render_json", # bounded by the nesting depth of a pydantic-validated JsonValue from the operator's config (a finite JSON tree, no cycles possible). "completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None. ] From 1a714548a46694bf82ed4d285941bcb2d5d86c0e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 01:19:22 -0700 Subject: [PATCH 16/28] fix(guardrails): store the masked output in spend logs when Presidio masks the response (#42441) * fix(guardrails): store the masked output in spend logs when a post_call guardrail rewrites the response Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): record served output without re-narrowing the logging object Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): overlay the served output before message redaction so turn_off_message_logging still wins Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(logging): type the monkeypatch fixture in the redaction ordering regression Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): poll through raw card output until the guardrail reaches the serving worker Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): keep blanked, multi-choice and disconnected served output out of raw spend logs Served text keeps empty strings and tracks unavailable choices as None so a guardrail that blanks the output still overrides the raw provider text. Stream choices are sized from the highest choice index, served chunks are recorded before a client disconnect or stream failure propagates, and message-logging redaction drops the served text from callback kwargs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 13 +- litellm/litellm_core_utils/redact_messages.py | 2 + .../litellm_core_utils/served_output_texts.py | 171 ++++++++++++++++++ litellm/proxy/common_request_processing.py | 5 + litellm/proxy/utils.py | 16 ++ tests/e2e/coverage_registry/guardrail.yaml | 1 + tests/e2e/guardrails/guardrails_client.py | 20 ++ .../guardrails/test_presidio_masking_e2e.py | 144 +++++++++++++++ tests/e2e/models.py | 2 + .../test_litellm_logging.py | 49 +++++ .../test_redact_messages.py | 8 + .../test_served_output_texts.py | 114 ++++++++++++ .../proxy/test_common_request_processing.py | 54 ++++++ .../test_proxy_logging_hook_detection.py | 54 ++++++ 14 files changed, 647 insertions(+), 6 deletions(-) create mode 100644 litellm/litellm_core_utils/served_output_texts.py create mode 100644 tests/test_litellm/litellm_core_utils/test_served_output_texts.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d06780dda53..306812cca0a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -107,6 +107,10 @@ from litellm.litellm_core_utils.redact_messages import ( redact_streaming_responses_for_custom_logger, should_redact_message_logging, ) +from litellm.litellm_core_utils.served_output_texts import ( + SERVED_OUTPUT_TEXTS_KEY, + overlay_served_output_texts, +) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils @@ -5843,15 +5847,12 @@ class StandardLoggingPayloadSetup: modified_final_response_obj: Final = redact_message_input_output_from_logging( model_call_details=kwargs, - result=final_response_obj, + result=overlay_served_output_texts(final_response_obj, kwargs.get(SERVED_OUTPUT_TEXTS_KEY)), ) if modified_final_response_obj is not None and isinstance(modified_final_response_obj, BaseModel): - final_response_obj = modified_final_response_obj.model_dump() - else: - final_response_obj = modified_final_response_obj - - return final_response_obj + return modified_final_response_obj.model_dump() + return modified_final_response_obj @staticmethod def get_additional_headers( diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 15bccf0301e..8c77ef32cfb 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.classifier_logging import without_classifier_aud from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) +from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY from litellm.llms.vertex_ai.common_utils import ( redact_vertex_ai_metadata_from_litellm_params, redact_vertex_ai_metadata_from_logged_object, @@ -267,6 +268,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] model_call_details["prompt"] = "" model_call_details["input"] = "" + model_call_details.pop(SERVED_OUTPUT_TEXTS_KEY, None) standard_logging_object: Final = model_call_details.get("standard_logging_object") if isinstance(standard_logging_object, Mapping): model_call_details["standard_logging_object"] = _redact_standard_logging_object(standard_logging_object) diff --git a/litellm/litellm_core_utils/served_output_texts.py b/litellm/litellm_core_utils/served_output_texts.py new file mode 100644 index 00000000000..323561a3dd2 --- /dev/null +++ b/litellm/litellm_core_utils/served_output_texts.py @@ -0,0 +1,171 @@ +"""Assistant text the caller received, per choice, so the logging payload stores the response a +post-call guardrail rewrote rather than the provider response the proxy assembled before it ran.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Final, Literal + +from pydantic import BaseModel, TypeAdapter, ValidationError + +from litellm.types.utils import ModelResponse, ModelResponseStream + +SERVED_OUTPUT_TEXTS_KEY: Final = "served_output_texts" + +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_JSON_LIST: Final = TypeAdapter(list[object]) +_TEXTS: Final = TypeAdapter(tuple[str | None, ...]) + +ServedTexts = tuple[str | None, ...] + + +class _TextBlock(BaseModel): + type: str + text: str | None = None + + +class _AnthropicMessage(BaseModel): + type: Literal["message"] + content: list[_TextBlock] + + +class _ResponsesOutputItem(BaseModel): + type: str + content: list[_TextBlock] = [] + + +class _ResponsesResponse(BaseModel): + object: Literal["response"] + output: list[_ResponsesOutputItem] + + +class _ChatChoices(BaseModel): + choices: list[object] + + +def _as_json_object(response: object) -> dict[str, object] | None: + candidate: Final = response.model_dump() if isinstance(response, BaseModel) else response + try: + return _JSON_OBJECT.validate_python(candidate) + except ValidationError: + return None + + +def _joined_block_texts(blocks: Sequence[_TextBlock], *, text_type: str) -> str | None: + texts: Final = tuple(block.text for block in blocks if block.type == text_type and block.text is not None) + return "".join(texts) if texts else None + + +def _chat_texts(response: ModelResponse) -> ServedTexts | None: + texts: Final = tuple( + choice.message.content if isinstance(choice.message.content, str) else None for choice in response.choices + ) + return texts if any(text is not None for text in texts) else None + + +def _anthropic_message_text(response: dict[str, object]) -> str | None: + try: + message: Final = _AnthropicMessage.model_validate(response) + except ValidationError: + return None + return _joined_block_texts(message.content, text_type="text") + + +def _responses_api_text(response: dict[str, object]) -> str | None: + try: + parsed: Final = _ResponsesResponse.model_validate(response) + except ValidationError: + return None + texts: Final = tuple( + text + for item in parsed.output + if item.type == "message" and (text := _joined_block_texts(item.content, text_type="output_text")) is not None + ) + return "".join(texts) if texts else None + + +def _chat_dict_texts(response: dict[str, object]) -> ServedTexts | None: + try: + _ChatChoices.model_validate(response) + return _chat_texts(ModelResponse(**response)) + except (ValidationError, TypeError, ValueError): + return None + + +def served_output_texts(response: object) -> ServedTexts | None: + if isinstance(response, ModelResponse): + return _chat_texts(response) + mapping: Final = _as_json_object(response) + if mapping is None: + return None + chat_texts: Final = _chat_dict_texts(mapping) + if chat_texts is not None: + return chat_texts + anthropic_text: Final = _anthropic_message_text(mapping) + text: Final = anthropic_text if anthropic_text is not None else _responses_api_text(mapping) + return (text,) if text is not None else None + + +def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None: + if chunks and all(isinstance(chunk, ModelResponseStream) for chunk in chunks): + return _chat_stream_texts(tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream))) + from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_stream, is_anthropic_sse_stream + + if not is_anthropic_sse_stream(chunks): + return None + assembled: Final = assemble_anthropic_sse_stream(chunks) + return _chat_texts(assembled) if assembled is not None else None + + +def _chat_stream_choice_text(chunks: Sequence[ModelResponseStream], index: int) -> str | None: + contents: Final = tuple( + content + for chunk in chunks + for choice in chunk.choices + if choice.index == index and isinstance(content := choice.delta.content, str) + ) + return "".join(contents) if contents else None + + +def _chat_stream_texts(chunks: Sequence[ModelResponseStream]) -> ServedTexts | None: + choice_count: Final = max((choice.index + 1 for chunk in chunks for choice in chunk.choices), default=0) + texts: Final = tuple(_chat_stream_choice_text(chunks, index) for index in range(choice_count)) + return texts if any(text is not None for text in texts) else None + + +def record_served_output_texts(model_call_details: dict[str, object], texts: ServedTexts | None) -> None: + if texts is None: + return + model_call_details[SERVED_OUTPUT_TEXTS_KEY] = texts # rebind-ok: model_call_details is the shared kwargs bag + + +def overlay_served_output_texts( + response_obj: dict[str, object] | str | list[object] | None, served_texts: object +) -> dict[str, object] | str | list[object] | None: + if not isinstance(response_obj, dict): + return response_obj + logged: Final = _as_json_object(response_obj) + if logged is None: + return response_obj + try: + texts: Final = _TEXTS.validate_python(served_texts) + choices: Final = _JSON_LIST.validate_python(logged.get("choices")) + except ValidationError: + return response_obj + return { + **logged, + "choices": [ + _choice_with_text(choice, texts[index]) if index < len(texts) else choice + for index, choice in enumerate(choices) + ], + } + + +def _choice_with_text(choice: object, text: str | None) -> object: + choice_obj: Final = _as_json_object(choice) + if choice_obj is None or text is None: + return choice + message: Final = _as_json_object(choice_obj.get("message")) + if message is None or message.get("content") == text: + return choice + return {**choice_obj, "message": {**message, "content": text}} diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 0c0f2492e56..6a7b6970bb8 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -68,6 +68,10 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.served_output_texts import ( + record_served_output_texts, + served_output_texts, +) from litellm.litellm_core_utils.streaming_handler import ( backfill_missing_cache_usage_fields, ) @@ -2803,6 +2807,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, response=response, ) + record_served_output_texts(logging_obj.model_call_details, served_output_texts(response)) except Exception: _exception_raised = True raise diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index e1a92b4380e..ea3ecb7f637 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -144,6 +144,10 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.litellm_core_utils.served_output_texts import ( + record_served_output_texts, + served_stream_output_texts, +) from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.llms import load_guardrail_translation_mappings from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -3809,12 +3813,16 @@ class ProxyLogging: translation=pipeline_translation, ) + served_chunks: Final[list[object]] = [] # mutable-ok: accumulates while yielding to the client try: async for chunk in current_response: + served_chunks.append(chunk) yield chunk except (GeneratorExit, asyncio.CancelledError): + ProxyLogging._record_served_stream_output(request_data, served_chunks) raise except Exception as e: + ProxyLogging._record_served_stream_output(request_data, served_chunks) if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e): ProxyLogging._fire_deferred_stream_logging(request_data) raise @@ -3823,6 +3831,7 @@ class ProxyLogging: # completed. unified_guardrail writes guardrail_information during # its end-of-stream block (inside current_response), so by the time # we reach this point the metadata is fully populated. + ProxyLogging._record_served_stream_output(request_data, served_chunks) ProxyLogging._fire_deferred_stream_logging(request_data) async def _pipeline_gated_stream( @@ -3890,6 +3899,13 @@ class ProxyLogging: for buffered_item in buffered: yield buffered_item + @staticmethod + def _record_served_stream_output(request_data: Mapping[str, object], served_chunks: Sequence[object]) -> None: + logging_obj: Final = request_data.get("litellm_logging_obj") + if not isinstance(logging_obj, Logging): + return + record_served_output_texts(logging_obj.model_call_details, served_stream_output_texts(served_chunks)) + @staticmethod def _fire_deferred_stream_logging(request_data: dict) -> None: """ diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml index 4fe9f949fae..02b5a921add 100644 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ b/tests/e2e/coverage_registry/guardrail.yaml @@ -3,6 +3,7 @@ - {id: guardrail.presidio.pre_call.masks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "PII masking pre-call; data-leak blast radius"} - {id: guardrail.presidio.post_call.masks, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Mask PII in model output"} - {id: guardrail.presidio.post_call.masks_generated_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, anthropic_messages_stream], source: "guardrail_hooks/presidio.py", rationale: "Mask model-generated credit-card output with the UI default scope"} +- {id: guardrail.presidio.post_call.spend_log_stores_masked_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, messages, anthropic_messages_stream, responses], source: "guardrail_hooks/presidio.py", fail_before_fix: proven, rationale: "When an output guardrail masks the response, the spend log stores the masked text the caller received rather than the raw model output, on every endpoint and both stream modes (LIT-8325)"} - {id: guardrail.presidio.logging_only.masks, module: guardrail, tier: P0, hook_point: logging_only, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Redact in logs without blocking"} - {id: guardrail.presidio.pre_call.logs_masked_entities, module: guardrail, tier: P0, hook_point: pre_call, assertions: [logs_masked_entities], exercised_on: [chat_completions], source: "guardrail_hooks/presidio.py", rationale: "A masking run must record itself on the spend log: the dashboard's guardrail panel renders the masked-entity counts and per-entity scores straight off metadata.guardrail_information, so a run that masks but records nothing leaves an operator unable to audit it"} - {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"} diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 97ecac0290f..007e9392616 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -381,6 +381,26 @@ class GuardrailsClient: ), ) + def messages_raw( + self, + key: str, + model: str, + text: str, + *, + guardrails: list[str] | None = None, + max_tokens: int = 64, + ) -> StreamingResponse: + return self.proxy.transport.send( + "/v1/messages", + headers=self.proxy.transport.bearer(key), + json=AnthropicMessagesBody( + model=model, + messages=[ChatMessage(role="user", content=text)], + max_tokens=max_tokens, + guardrails=guardrails, + ), + ) + def messages_stream_raw( self, key: str, diff --git a/tests/e2e/guardrails/test_presidio_masking_e2e.py b/tests/e2e/guardrails/test_presidio_masking_e2e.py index d47d64be9e3..1fbc4a68bbd 100644 --- a/tests/e2e/guardrails/test_presidio_masking_e2e.py +++ b/tests/e2e/guardrails/test_presidio_masking_e2e.py @@ -29,6 +29,7 @@ this suite deliberately requires the detected-entity details to remain visible. from __future__ import annotations +import json import os import re import time @@ -481,6 +482,149 @@ class TestPresidioCreditCardOutputMasking: _assert_eventually_masks_generated_card(fetch) +def _wire_text(outcome: StreamingResponse) -> str: + return "\n".join(outcome.stream_events) if outcome.is_streaming else outcome.body + + +def _poll_until_generated_card_masked(fetch: Callable[[], StreamingResponse]) -> StreamingResponse: + """The raw HTTP outcome once the output masker is in effect on the serving + worker: whichever wire shape the endpoint speaks, a masked body carries the + CREDIT_CARD placeholder and no Luhn-valid card run. A raw card is a worker + that has not loaded the guardrail yet, so it is polled through like any + other unmasked answer.""" + deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS + last: str = "" + while True: + outcome = fetch() + if outcome.ok and not outcome.stream_error: + wire = _wire_text(outcome) + last = wire + if MASKED_CREDIT_CARD_TOKEN in wire and not _contains_card_number(wire): + return outcome + if time.monotonic() >= deadline: + pytest.fail( + "presidio post_call output masking never masked the generated card within " + f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}" + ) + time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) + + +def _spend_log_response_text(client: GuardrailsClient, key: str, call_id: str) -> str: + rows = client.proxy.poll_logs_for_key( + key, + predicate=lambda logged: any(row.litellm_call_id == call_id for row in logged), + ) + row = next((row for row in rows if row.litellm_call_id == call_id), None) + assert row is not None, f"no spend log row ever appeared for x-litellm-call-id {call_id}" + return json.dumps(row.response) + + +class TestPresidioSpendLogStoresMaskedOutput: + """The spend log stores the response the caller received, on every endpoint + and both stream modes, when an output-only post_call Presidio guardrail masks + a card number the model generated.""" + + _CELL: Final = "guardrail.presidio.post_call.spend_log_stores_masked_output" + + def _assert_spend_log_is_masked( + self, + client: GuardrailsClient, + resources: ResourceManager, + key: str, + *, + name: str, + fetch: Callable[[str, str], StreamingResponse], + ) -> None: + _register_presidio( + client, + resources, + name=name, + mode="post_call", + filter_scope="output", + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + outcome = _poll_until_generated_card_masked(lambda: fetch(prompt, name)) + assert outcome.call_id, f"the served response must carry x-litellm-call-id: {dict(outcome.headers)}" + + logged = _spend_log_response_text(client, key, outcome.call_id) + assert not _contains_card_number(logged), ( + "the caller got the masked response but the spend log stored the raw model output: " + f"{logged[:400]!r}" + ) + assert MASKED_CREDIT_CARD_TOKEN in logged, ( + f"the spend log response carries neither the card nor the placeholder: {logged[:400]!r}" + ) + + @pytest.mark.covers(_CELL, exercised_on=["chat_completions"]) + def test_spend_log_stores_masked_output_on_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + self._assert_spend_log_is_masked( + client, + resources, + scoped_key, + name=f"e2e-presidio-log-card-chat-{unique_marker()}", + fetch=lambda prompt, guardrail: client.chat_raw( + scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512 + ), + ) + + @pytest.mark.covers(_CELL, exercised_on=["chat_completions_stream"]) + def test_spend_log_stores_masked_output_on_streaming_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + self._assert_spend_log_is_masked( + client, + resources, + scoped_key, + name=f"e2e-presidio-log-card-chat-stream-{unique_marker()}", + fetch=lambda prompt, guardrail: client.chat_stream_raw( + scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512 + ), + ) + + @pytest.mark.covers(_CELL, exercised_on=["messages"]) + def test_spend_log_stores_masked_output_on_anthropic_messages( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + self._assert_spend_log_is_masked( + client, + resources, + scoped_key, + name=f"e2e-presidio-log-card-messages-{unique_marker()}", + fetch=lambda prompt, guardrail: client.messages_raw( + scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512 + ), + ) + + @pytest.mark.covers(_CELL, exercised_on=["anthropic_messages_stream"]) + def test_spend_log_stores_masked_output_on_streaming_anthropic_messages( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + self._assert_spend_log_is_masked( + client, + resources, + scoped_key, + name=f"e2e-presidio-log-card-messages-stream-{unique_marker()}", + fetch=lambda prompt, guardrail: client.messages_stream_raw( + scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512 + ), + ) + + @pytest.mark.covers(_CELL, exercised_on=["responses"]) + def test_spend_log_stores_masked_output_on_responses( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + self._assert_spend_log_is_masked( + client, + resources, + scoped_key, + name=f"e2e-presidio-log-card-responses-{unique_marker()}", + fetch=lambda prompt, guardrail: client.responses(scoped_key, MODEL, prompt, guardrails=[guardrail]), + ) + + _LOGGED_ENTITIES: dict[PiiEntity, PiiAction] = {"EMAIL_ADDRESS": "MASK", "PHONE_NUMBER": "MASK"} _ENTITY_LIST_ADAPTER: Final = TypeAdapter(list[GuardrailEntityMatch]) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index d0b1e8824da..53fbe03c025 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -874,6 +874,8 @@ class SpendLogRow(BaseModel): request_tags: list[str] | None = None metadata: SpendLogMetadata | None = None proxy_server_request: JsonValue = None + response: JsonValue = None + litellm_call_id: str | None = None class SpendLogs(RootModel[list[SpendLogRow]]): diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 021e012f29d..52eed9c9f95 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -1,6 +1,7 @@ import asyncio import contextlib import datetime +import json import logging import os import sys @@ -3013,6 +3014,54 @@ def test_get_final_response_obj_with_empty_response_obj_and_list_init(): assert result[1].name == "Object2" +def test_get_final_response_obj_stores_the_text_a_post_call_guardrail_served(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + + raw = { + "id": "x", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Card: 4111 1111 1111 1111"}, + } + ], + } + + logged = StandardLoggingPayloadSetup.get_final_response_obj( + response_obj=raw, init_response_obj=raw, kwargs={SERVED_OUTPUT_TEXTS_KEY: ("Card: ",)} + ) + untouched = StandardLoggingPayloadSetup.get_final_response_obj(response_obj=raw, init_response_obj=raw, kwargs={}) + + assert isinstance(logged, dict) + assert logged["choices"][0]["message"]["content"] == "Card: " + assert logged["choices"][0]["finish_reason"] == "stop" + assert untouched == raw + + +def test_get_final_response_obj_redacts_the_served_text_when_message_logging_is_off(monkeypatch: pytest.MonkeyPatch): + import litellm + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + raw = { + "id": "x", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Card: 4111"}}], + } + + logged = StandardLoggingPayloadSetup.get_final_response_obj( + response_obj=raw, + init_response_obj=raw, + kwargs={SERVED_OUTPUT_TEXTS_KEY: ("Card: ",), "litellm_params": {}}, + ) + + assert isinstance(logged, dict) + assert "" not in json.dumps(logged), logged + assert "4111" not in json.dumps(logged), logged + + def test_get_usage_as_dict(): """ Test get_usage_as_dict returns usage as plain dict from response_obj or combined_usage_object. diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index 22298c00219..76d037ce760 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -1032,3 +1032,11 @@ def test_a_callback_that_redacts_itself_keeps_its_messages_but_not_the_classifie assert "classifier_input" not in stored assert stored["messages"] == payload["messages"] assert stored["response"] == payload["response"] + + +def test_perform_redaction_drops_the_served_output_texts_from_the_callback_kwargs() -> None: + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + + details: Final = {"litellm_params": {}, SERVED_OUTPUT_TEXTS_KEY: ("Card: ",)} + perform_redaction(details, None) + assert SERVED_OUTPUT_TEXTS_KEY not in details diff --git a/tests/test_litellm/litellm_core_utils/test_served_output_texts.py b/tests/test_litellm/litellm_core_utils/test_served_output_texts.py new file mode 100644 index 00000000000..ade39b8fca9 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_served_output_texts.py @@ -0,0 +1,114 @@ +from litellm.litellm_core_utils.served_output_texts import ( + SERVED_OUTPUT_TEXTS_KEY, + overlay_served_output_texts, + record_served_output_texts, + served_output_texts, + served_stream_output_texts, +) +from litellm.types.utils import Choices, Delta, Message, ModelResponse, ModelResponseStream, StreamingChoices + +RAW = "Card: 4111 1111 1111 1111" +MASKED = "Card: " + + +def _chat_response(*texts: str) -> ModelResponse: + return ModelResponse( + choices=[Choices(index=i, message=Message(content=text, role="assistant")) for i, text in enumerate(texts)] + ) + + +def _chat_dict(*texts: str) -> dict[str, object]: + return { + "id": "x", + "choices": [ + {"index": i, "finish_reason": "stop", "message": {"role": "assistant", "content": text}} + for i, text in enumerate(texts) + ], + } + + +def _choice_texts(response: object) -> tuple[str | None, ...]: + texts = served_output_texts(response) + assert texts is not None, response + return texts + + +def _stream_chunk(text: str, index: int = 0) -> ModelResponseStream: + return ModelResponseStream(choices=[StreamingChoices(index=index, delta=Delta(content=text))]) + + +def test_served_output_texts_reads_each_response_shape(): + assert served_output_texts(_chat_response(MASKED, "second")) == (MASKED, "second") + assert served_output_texts(_chat_dict(MASKED)) == (MASKED,) + assert served_output_texts( + { + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}], + } + ) == ("ab",) + responses_api: dict[str, object] = { + "object": "response", + "output": [ + {"type": "reasoning", "content": []}, + {"type": "message", "content": [{"type": "output_text", "text": MASKED}]}, + ], + } + assert served_output_texts(responses_api) == (MASKED,) + assert served_output_texts({"data": [{"embedding": [0.1]}]}) is None + assert served_output_texts("plain") is None + + +def test_served_stream_output_texts_joins_chat_chunks_and_reads_anthropic_sse(): + assert served_stream_output_texts([_stream_chunk("Card: "), _stream_chunk("")]) == (MASKED,) + sse = ( + 'event: message_start\ndata: {"type":"message_start","message":{"id":"m","type":"message","role":"assistant",' + '"content":[],"model":"x","usage":{"input_tokens":1,"output_tokens":1}}}\n\n', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n\n', + f'event: content_block_delta\ndata: {{"type":"content_block_delta","index":0,"delta":{{"type":"text_delta","text":"{MASKED}"}}}}\n\n', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', + 'event: message_stop\ndata: {"type":"message_stop"}\n\n', + ) + assert served_stream_output_texts(tuple(chunk.encode() for chunk in sse)) == (MASKED,) + assert served_stream_output_texts([]) is None + assert served_stream_output_texts([b"not sse"]) is None + + +def test_served_stream_output_texts_keeps_every_choice_index_when_chunks_carry_one_choice_each(): + chunks = [_stream_chunk("first ", 0), _stream_chunk(MASKED, 1), _stream_chunk("choice", 0)] + assert served_stream_output_texts(chunks) == ("first choice", MASKED) + + +def test_blanked_output_is_served_as_empty_text_and_overlaid(): + assert served_output_texts(_chat_response("")) == ("",) + assert served_output_texts({"type": "message", "role": "assistant", "content": [{"type": "text", "text": ""}]}) == ( + "", + ) + assert served_stream_output_texts([_stream_chunk("")]) == ("",) + assert _choice_texts(overlay_served_output_texts(_chat_dict(RAW), ("",))) == ("",) + + +def test_overlay_replaces_logged_choice_text_with_served_text(): + logged = _chat_dict(RAW, RAW) + overlaid = overlay_served_output_texts(logged, (MASKED,)) + assert _choice_texts(overlaid) == (MASKED, RAW) + assert isinstance(overlaid, dict) + assert overlaid["id"] == "x" + assert _choice_texts(logged) == (RAW, RAW) + + +def test_overlay_leaves_unreadable_inputs_untouched(): + logged = _chat_dict(RAW) + assert overlay_served_output_texts(logged, None) is logged + assert overlay_served_output_texts(logged, "not a tuple") is logged + assert overlay_served_output_texts(logged, (None,)) == logged + assert overlay_served_output_texts("text", (MASKED,)) == "text" + assert overlay_served_output_texts({"data": []}, (MASKED,)) == {"data": []} + + +def test_record_served_output_texts_only_writes_readable_texts(): + details: dict[str, object] = {} + record_served_output_texts(details, None) + assert SERVED_OUTPUT_TEXTS_KEY not in details + record_served_output_texts(details, (MASKED,)) + assert details[SERVED_OUTPUT_TEXTS_KEY] == (MASKED,) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index f96142def06..360122cde25 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -9034,6 +9034,60 @@ class TestDetachedStreamFailureHook: assert [call["original_exception"] for call in recorder.calls] == [failure] +class TestPostCallMaskedOutputReachesDeferredLogging: + @pytest.mark.asyncio + async def test_non_streaming_records_the_masked_response_before_deferred_logging_fires(self, monkeypatch): + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + from litellm.types.utils import Choices, Message, ModelResponse + + logging_obj = MagicMock() + logging_obj.litellm_call_id = "lit-8325-call" + logging_obj._defer_async_logging = False + logging_obj._on_deferred_stream_complete = None + logging_obj.cost_breakdown = None + logging_obj.model_call_details = {} + recorded_at_enqueue: dict[str, object] = {} + logging_obj._enqueue_deferred_logging = lambda: recorded_at_enqueue.update(logging_obj.model_call_details) + + processor = ProxyBaseLLMRequestProcessing(data={"model": "oa", "litellm_logging_obj": logging_obj}) + + def mask(data, user_api_key_dict, response): + response.choices[0].message.content = "Card: " + return response + + proxy_logging_obj = MagicMock(spec=ProxyLogging) + proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + proxy_logging_obj.update_request_status = AsyncMock(return_value=None) + proxy_logging_obj.post_call_success_hook = AsyncMock(side_effect=mask) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value=None) + + async def fake_route_request(**kwargs): + async def call(): + return ModelResponse( + choices=[Choices(index=0, message=Message(content="Card: 4111 1111 1111 1111", role="assistant"))] + ) + + return call() + + monkeypatch.setattr(litellm.proxy.common_request_processing, "route_request", fake_route_request) + + result = await processor.base_process_llm_request( + request=Request(scope={"type": "http", "headers": []}), + fastapi_response=Response(), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + route_type="acompletion", + proxy_logging_obj=proxy_logging_obj, + general_settings={}, + proxy_config=MagicMock(spec=ProxyConfig), + select_data_generator=MagicMock(), + is_streaming_request=False, + skip_pre_call_logic=True, + ) + + assert result.choices[0].message.content == "Card: " + assert recorded_at_enqueue[SERVED_OUTPUT_TEXTS_KEY] == ("Card: ",) + + class TestStreamingResponseHeadersFollowFallback: """LIT-6767: the streaming branch has to publish the deployment that served the stream.""" diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index f45715953f9..9eb89b2f301 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -347,6 +347,60 @@ async def test_post_call_stream_guardrail_keeps_own_iterator_on_chat_completions assert delivered_text != "" +@pytest.mark.asyncio +async def test_post_call_stream_records_masked_text_for_deferred_logging(monkeypatch): + from litellm.caching.caching import DualCache + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + monkeypatch.setattr(litellm, "callbacks", [_content_filter_guardrail("MASK")]) + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + logging_obj = _streaming_logging_obj() + + async def fake_stream(): + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="the zebra runs"))]) + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")]) + + delivered_text = "" + async for chunk in proxy_logging.async_post_call_streaming_iterator_hook( + response=fake_stream(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/chat/completions"), + request_data={"model": "gpt-4o-mini", "metadata": {}, "litellm_logging_obj": logging_obj}, + ): + for choice in chunk.choices: + delivered_text += choice.delta.content or "" + + assert "zebra" not in delivered_text + assert logging_obj.model_call_details[SERVED_OUTPUT_TEXTS_KEY] == (delivered_text,) + + +@pytest.mark.asyncio +async def test_post_call_stream_records_the_served_text_when_the_client_disconnects(monkeypatch): + from litellm.caching.caching import DualCache + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + monkeypatch.setattr(litellm, "callbacks", [_content_filter_guardrail("MASK")]) + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + logging_obj = _streaming_logging_obj() + + async def fake_stream(): + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="the zebra runs"))]) + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content=" far"))]) + + stream = proxy_logging.async_post_call_streaming_iterator_hook( + response=fake_stream(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/chat/completions"), + request_data={"model": "gpt-4o-mini", "metadata": {}, "litellm_logging_obj": logging_obj}, + ) + first = await stream.__anext__() + await stream.aclose() + + delivered_text = "".join(choice.delta.content or "" for choice in first.choices) + assert "zebra" not in delivered_text + assert logging_obj.model_call_details[SERVED_OUTPUT_TEXTS_KEY] == (delivered_text,) + + @pytest.mark.asyncio async def test_unified_guardrail_iterator_accepts_explicit_guardrail(): """ From 071cb49d320c9b31fc1d359fca368b8bc5564523 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 02:22:00 -0700 Subject: [PATCH 17/28] fix(jwt): accept a team alias in x-litellm-team-id (#42445) * fix(jwt): accept a team alias in x-litellm-team-id The header only matched canonical team ids, so a JWT caller selecting one of their teams by its alias got a 403 even though they belonged to it. The header value is now resolved through the existing alias lookup before the JWT allowed-team check and the DB membership fallback, while a value that is already a team id never costs an alias lookup and denials keep naming the value the caller sent Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(jwt): only alias a header team id the database provably lacks Under fallback_to_db_teams a header value whose team row read fails for any reason other than TeamNotFoundError now keeps the membership denial instead of falling through to the alias lookup, so a degraded read cannot select a different team that carries the value as an alias. Drops the HeaderTeam docstring that only restated its fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: ryan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/handle_jwt.py | 164 ++++++-- tests/e2e/coverage_registry/other.yaml | 2 + tests/e2e/other/other_client.py | 24 +- tests/e2e/other/test_jwt_auth_e2e.py | 81 ++++ .../proxy/auth/test_handle_jwt.py | 392 ++++++++++++++++-- 5 files changed, 575 insertions(+), 88 deletions(-) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 07d8d00d202..63bbaae862e 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -70,6 +70,7 @@ from litellm.types.agents import AgentResponse from litellm.types.proxy.auth.auth_checks import UserNotFoundError from .auth_checks import ( + TeamNotFoundError, _allowed_routes_check, allowed_routes_check, get_actual_routes, @@ -147,6 +148,12 @@ class _JWTProvisioning: team_id_upsert: bool +@dataclass(frozen=True, slots=True) +class HeaderTeam: + header_value: str + team_id: str + + class AgentLookup(Protocol): """The registered-agent lookups a JWT agent claim is matched against.""" @@ -1871,48 +1878,104 @@ class JWTAuthManager: return True @staticmethod - def get_team_id_from_header( - request_headers: Mapping[str, str] | None, - allowed_team_ids: set[str], - fallback_to_db_teams: bool = False, - ) -> str | None: - """ - Extract team_id from x-litellm-team-id header if present. - Validates that the team is in the user's allowed teams from JWT. - - Args: - request_headers: Dictionary of request headers - allowed_team_ids: Set of team IDs the user is allowed to access (from JWT) - fallback_to_db_teams: When True and the JWT carries no team claims - (allowed_team_ids is empty), the header value is returned - provisionally and validated against DB memberships later in - auth_builder instead of being rejected here. - - Returns: - The team_id from header if valid, None otherwise - - Raises: - HTTPException: If team_id is provided but not in allowed_team_ids - """ + def _team_header_value(request_headers: Mapping[str, str] | None) -> str | None: if not request_headers: return None - - # Normalize headers to lowercase for case-insensitive lookup normalized_headers: Final = {k.lower(): v for k, v in request_headers.items()} - header_team_id: Final = normalized_headers.get("x-litellm-team-id") + return normalized_headers.get("x-litellm-team-id") - if not header_team_id: + @staticmethod + def _raise_header_team_not_allowed(header_value: str, allowed_team_ids: set[str]) -> NoReturn: + raise HTTPException( + status_code=403, + detail=f"Team '{header_value}' from x-litellm-team-id header is not in your JWT's allowed teams. Allowed teams: {list(allowed_team_ids)}", + ) + + @staticmethod + async def _team_id_by_alias( + team_alias: str, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + parent_otel_span: Span | None, + proxy_logging_obj: ProxyLogging, + ) -> str | None: + if prisma_client is None: + return None + try: + team: Final = await get_team_object_by_alias( + team_alias=team_alias, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except HTTPException as exc: + if exc.status_code >= 500: + raise + return None + return team.team_id + + @staticmethod + async def resolve_team_from_header( + request_headers: Mapping[str, str] | None, + allowed_team_ids: set[str], + fallback_to_db_teams: bool, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + parent_otel_span: Span | None, + proxy_logging_obj: ProxyLogging, + ) -> HeaderTeam | None: + """ + The team named by x-litellm-team-id, which may carry a team id or a team + alias. A value that is already an allowed team id (or, under the DB + fallback, an existing team id) never costs an alias lookup; an alias is + accepted only when the team it names would have been accepted by id. + Under the DB fallback only a team row that is provably absent falls + through to the alias lookup; a read that failed for any other reason + keeps the membership denial the id path already gives. + + Raises: + HTTPException: 403 when neither the value nor the team it aliases is + an allowed team, or the DB fallback's membership denial when the + value names no team at all; a 5xx from the alias lookup itself + is re-raised rather than reported as a denial + """ + header_value: Final = JWTAuthManager._team_header_value(request_headers) + if not header_value: return None - defer_to_db_membership: Final = fallback_to_db_teams and not allowed_team_ids - if not defer_to_db_membership and header_team_id not in allowed_team_ids: - raise HTTPException( - status_code=403, - detail=f"Team '{header_team_id}' from x-litellm-team-id header is not in your JWT's allowed teams. Allowed teams: {list(allowed_team_ids)}", - ) + if fallback_to_db_teams and not allowed_team_ids: + try: + await get_team_object( + team_id=header_value, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + team_id_upsert=False, + ) + except TeamNotFoundError: + aliased_team_id: Final = await JWTAuthManager._team_id_by_alias( + header_value, prisma_client, user_api_key_cache, parent_otel_span, proxy_logging_obj + ) + if aliased_team_id is None: + JWTAuthManager._raise_header_team_membership_denial(header_value) + return HeaderTeam(header_value=header_value, team_id=aliased_team_id) + except HTTPException: + JWTAuthManager._raise_header_team_membership_denial(header_value) + return HeaderTeam(header_value=header_value, team_id=header_value) - verbose_proxy_logger.debug("Using team_id from x-litellm-team-id header: %s", header_team_id) - return header_team_id + if header_value in allowed_team_ids: + verbose_proxy_logger.debug("Using team_id from x-litellm-team-id header: %s", header_value) + return HeaderTeam(header_value=header_value, team_id=header_value) + + team_id_by_alias: Final = await JWTAuthManager._team_id_by_alias( + header_value, prisma_client, user_api_key_cache, parent_otel_span, proxy_logging_obj + ) + if team_id_by_alias is None or team_id_by_alias not in allowed_team_ids: + JWTAuthManager._raise_header_team_not_allowed(header_value, allowed_team_ids) + verbose_proxy_logger.debug("Using team_id %s for x-litellm-team-id alias: %s", team_id_by_alias, header_value) + return HeaderTeam(header_value=header_value, team_id=team_id_by_alias) @staticmethod async def map_user_to_teams( @@ -2264,31 +2327,34 @@ class JWTAuthManager: ) @staticmethod - def _raise_header_team_membership_denial(team_id: str) -> NoReturn: + def _raise_header_team_membership_denial(header_value: str) -> NoReturn: """ The single denial shape for a provisional x-litellm-team-id header, raised identically for nonexistent teams and for teams the user is not - a member of, so the response does not reveal whether a team id exists. + a member of, and naming only the value the caller sent, so the response + reveals neither whether a team exists nor which id an alias maps to. """ raise HTTPException( status_code=403, - detail=(f"Team '{team_id}' (from x-litellm-team-id header) is not in your team memberships."), + detail=(f"Team '{header_value}' (from x-litellm-team-id header) is not in your team memberships."), ) @staticmethod def _validate_header_team_in_db_membership( team_id: str, user_object: LiteLLM_UserTable | None, + header_value: str, ) -> None: """ A provisional team_id from the x-litellm-team-id header (accepted without JWT-team validation when the JWT carries no team claims) must exist in the - user's DB team memberships before it becomes request context. + user's DB team memberships before it becomes request context. The denial + names `header_value`, the id or alias the caller sent, not `team_id`. """ user_team_ids: Final = user_object.teams if user_object else [] if team_id in user_team_ids: return - JWTAuthManager._raise_header_team_membership_denial(team_id) + JWTAuthManager._raise_header_team_membership_denial(header_value) @staticmethod async def auth_builder( @@ -2514,13 +2580,17 @@ class JWTAuthManager: if specific_team_id and not db_team_fallback: all_team_ids.add(specific_team_id) - header_team_id: Final = JWTAuthManager.get_team_id_from_header( + header_team: Final = await JWTAuthManager.resolve_team_from_header( request_headers=request_headers, allowed_team_ids=all_team_ids, fallback_to_db_teams=db_team_fallback, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, ) - if header_team_id: - team_id = header_team_id + if header_team: + team_id = header_team.team_id # A provisional header team (accepted only because the JWT carries no # team claims) is validated against DB membership further down; never # upsert it here or an attacker-supplied x-litellm-team-id would create @@ -2538,7 +2608,7 @@ class JWTAuthManager: except HTTPException: if not db_team_fallback: raise - JWTAuthManager._raise_header_team_membership_denial(team_id) + JWTAuthManager._raise_header_team_membership_denial(header_team.header_value) elif not team_id and not db_team_fallback: ## SPECIFIC TEAM ID ( @@ -2679,10 +2749,11 @@ class JWTAuthManager: proxy_logging_obj=proxy_logging_obj, team_id_upsert=team_id_upsert, ) - elif db_team_fallback and team_id == header_team_id: + elif db_team_fallback and header_team is not None and team_id == header_team.team_id: JWTAuthManager._validate_header_team_in_db_membership( team_id=team_id, user_object=user_object, + header_value=header_team.header_value, ) if not JWTAuthManager._is_team_route_allowed( route=route, @@ -2692,7 +2763,8 @@ class JWTAuthManager: raise HTTPException( status_code=403, detail=( - f"Team '{team_id}' (from x-litellm-team-id header) is not allowed to access route '{route}'." + f"Team '{header_team.header_value}' (from x-litellm-team-id header) " + f"is not allowed to access route '{route}'." ), ) diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 9292e5f07db..1706b2f8f25 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -15,6 +15,8 @@ - {id: other.auth.jwt.invalid_signature_denied, module: other, tier: P0, area: auth, assertions: [invalid_signature_denied], source: "handle_jwt.py:1158-1166 _decode_jwt_with_public_key", rationale: "A genuine token whose signature bytes were altered fails verification with 401"} - {id: other.auth.jwt.unknown_team_denied, module: other, tier: P0, area: auth, assertions: [unknown_team_denied], source: "handle_jwt.py:1549-1632 find_team_with_model_access", rationale: "A verified JWT whose groups claim resolves to no existing team is denied with 403 naming the unresolved team, never silently admitted without a team. The proxy words it as a model-access denial, the same body an existing team without model access gets"} - {id: other.auth.jwt.virtual_key_unaffected, module: other, tier: P0, area: auth, assertions: [virtual_key_unaffected], source: "handle_jwt.py:213 is_jwt / user_api_key_auth.py:1332-1333", rationale: "enable_jwt_auth only routes three-segment bearer tokens into the JWT branch, so sk- virtual keys keep working on the same proxy"} +- {id: other.auth.jwt.team_header_alias_binds_team, module: other, tier: P0, area: auth, assertions: [team_header_alias_binds_team], source: "handle_jwt.py JWTAuthManager.resolve_team_from_header / LIT-7181", fail_before_fix: proven, rationale: "x-litellm-team-id carrying the team alias binds and attributes the same team as the team id, so a managed client can pin a stable alias instead of a uuid"} +- {id: other.auth.jwt.team_header_non_member_alias_denied, module: other, tier: P0, area: auth, assertions: [team_header_non_member_alias_denied], source: "handle_jwt.py JWTAuthManager.resolve_team_from_header / LIT-7181", rationale: "x-litellm-team-id naming the alias of a team the JWT does not grant is denied 403 with the same body as an unknown value, so the response does not reveal whether that team exists"} - {id: other.auth.model_access_group.wildcard_bare_name_allowed, module: other, tier: P0, area: auth, assertions: [wildcard_bare_name_allowed], source: "auth_checks.py:3232 / LIT-5813", fail_before_fix: proven, rationale: "A grant of a group holding a wildcard deployment covers the bare model names callers actually send, not only the provider-prefixed spelling"} - {id: other.auth.model_access_group.member_allowed, module: other, tier: P0, area: auth, assertions: [member_allowed], source: "auth_checks.py:3232", rationale: "A key whose allow-list is a model access group can call the deployments in that group"} - {id: other.auth.model_access_group.non_member_denied, module: other, tier: P0, area: auth, assertions: [non_member_denied], source: "auth_checks.py:3232", rationale: "That same grant reaches nothing outside the group, including provider models the group's wildcard does not cover"} diff --git a/tests/e2e/other/other_client.py b/tests/e2e/other/other_client.py index 4313bbe4068..057a735e5e5 100644 --- a/tests/e2e/other/other_client.py +++ b/tests/e2e/other/other_client.py @@ -15,15 +15,25 @@ from __future__ import annotations from dataclasses import dataclass -from e2e_http import NoBody, ProbeResult, Result +from e2e_http import AuthHeaders, NoBody, ProbeResult, Result from idp import Keycloak, keycloak_from_env from models import ( + ChatBody, + ChatResponse, ReadinessDetailsResponse, ReadinessResponse, UserListParams, UserListResponse, ) from proxy_client import ProxyClient +from pydantic import Field + + +class TeamHeaders(AuthHeaders): + """Bearer auth plus ``x-litellm-team-id``, the header a JWT caller sends to + pick one of the teams it belongs to.""" + + x_litellm_team_id: str = Field(serialization_alias="x-litellm-team-id") @dataclass(frozen=True, slots=True) @@ -66,6 +76,18 @@ class OtherClient: response_type=ReadinessDetailsResponse, ) + def chat_as_team(self, token: str, team: str, body: ChatBody) -> Result[ChatResponse]: + """POST /chat/completions under `token` with `x-litellm-team-id: team`.""" + return self.proxy.transport.post( + "/chat/completions", + headers=TeamHeaders( + authorization=self.proxy.transport.bearer(token).authorization, + x_litellm_team_id=team, + ), + json=body, + response_type=ChatResponse, + ) + def list_users_as(self, key: str) -> Result[UserListResponse]: """GET /user/list under `key`. Admin-only, so it doubles as the master key's authorization proof: the master key (proxy admin) reads it, a diff --git a/tests/e2e/other/test_jwt_auth_e2e.py b/tests/e2e/other/test_jwt_auth_e2e.py index 2bed40f4d69..de06c376f76 100644 --- a/tests/e2e/other/test_jwt_auth_e2e.py +++ b/tests/e2e/other/test_jwt_auth_e2e.py @@ -4,6 +4,7 @@ from __future__ import annotations import base64 import time +from dataclasses import dataclass from typing import Final import pytest @@ -52,6 +53,31 @@ def identity(client: OtherClient, resources: ResourceManager) -> Identity: return provisioned +@dataclass(frozen=True, slots=True) +class BoundTeam: + identity: Identity + team_id: str + team_alias: str + + +def _team(client: OtherClient, resources: ResourceManager, *, marker: str, team_id: str) -> str: + """A litellm team whose alias differs from its id, so a header naming one + cannot accidentally match the other.""" + team_alias: Final = f"e2e-jwt-alias-{marker}" + created: Final = client.proxy.create_team(TeamNewBody(team_alias=team_alias, team_id=team_id)) + resources.defer(lambda: client.proxy.delete_team(created)) + return team_alias + + +@pytest.fixture +def bound_team(client: OtherClient, resources: ResourceManager) -> BoundTeam: + """An identity whose single group is a real team, plus that team's alias.""" + marker: Final = unique_marker() + provisioned: Final = _provision(client, resources, marker=marker) + team_alias: Final = _team(client, resources, marker=marker, team_id=provisioned.group) + return BoundTeam(identity=provisioned, team_id=provisioned.group, team_alias=team_alias) + + def _ping() -> ChatBody: return ChatBody( model=CHEAP_OPENAI_MODEL, @@ -155,3 +181,58 @@ class TestJwtAuth: def test_plain_virtual_key_still_works_with_jwt_auth_enabled(self, client: OtherClient, scoped_key: str) -> None: response: Final = unwrap(client.proxy.chat(scoped_key, _ping())) assert response.choices, f"an sk- key must keep working on a proxy with enable_jwt_auth, got {response}" + + +def _team_of_request(client: OtherClient, token: str, team: str) -> str | None: + response: Final = unwrap(client.chat_as_team(token, team, _ping())) + assert response.id is not None and response.choices, ( + f"chat with x-litellm-team-id={team!r} returned no completion: {response}" + ) + rows: Final = client.proxy.poll_logs_for_request_id(response.id) + assert rows, f"no spend log row for request {response.id} within the poll deadline" + return rows[0].team_id + + +def _denial(client: OtherClient, token: str, team: str) -> str: + result: Final = client.chat_as_team(token, team, _ping()) + assert isinstance(result, UnknownApiError) and result.status_code == 403, ( + f"x-litellm-team-id={team!r} names no team the caller is in, so it must be rejected with 403, got {result}" + ) + assert team in result.body, f"the 403 must name the header value it rejected ({team!r}), got {result.body[:300]}" + return result.body + + +class TestJwtTeamHeader: + @pytest.mark.covers("other.auth.jwt.team_header_alias_binds_team") + def test_team_header_with_the_team_alias_binds_the_same_team_as_the_team_id( + self, client: OtherClient, bound_team: BoundTeam + ) -> None: + token: Final = client.idp.access_token(bound_team.identity) + assert bound_team.team_alias != bound_team.team_id + + by_id: Final = _team_of_request(client, token, bound_team.team_id) + assert by_id == bound_team.team_id, ( + f"precondition: x-litellm-team-id with the team id must bind {bound_team.team_id!r}, got {by_id!r}" + ) + + by_alias: Final = _team_of_request(client, token, bound_team.team_alias) + assert by_alias == bound_team.team_id, ( + f"x-litellm-team-id={bound_team.team_alias!r} must bind the same team as its id " + f"{bound_team.team_id!r}, got {by_alias!r}" + ) + + @pytest.mark.covers("other.auth.jwt.team_header_non_member_alias_denied") + def test_team_header_with_the_alias_of_a_team_the_caller_is_not_in_is_rejected_like_an_unknown_value( + self, client: OtherClient, resources: ResourceManager, bound_team: BoundTeam + ) -> None: + token: Final = client.idp.access_token(bound_team.identity) + other_marker: Final = unique_marker() + other_alias: Final = _team(client, resources, marker=other_marker, team_id=f"e2e-jwt-other-{other_marker}") + unknown: Final = f"e2e-jwt-unknown-{unique_marker()}" + + for_other_alias: Final = _denial(client, token, other_alias) + for_unknown: Final = _denial(client, token, unknown) + assert for_other_alias.replace(other_alias, "") == for_unknown.replace(unknown, ""), ( + "a non-member alias and an unknown value must get the same denial body, so the response does not " + f"reveal whether the team exists; got {for_other_alias[:300]!r} vs {for_unknown[:300]!r}" + ) diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index cd14f630130..208af23f1c5 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -28,10 +28,12 @@ from litellm.proxy._types import ( ) from litellm.caching.dual_cache import DualCache from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry +from litellm.proxy.auth.auth_checks import TeamNotFoundError from litellm.proxy.auth.handle_jwt import ( JWKS_FETCH_ATTEMPTS, STALE_CACHE_KEY_PREFIX, STALE_WRITTEN_AT_CACHE_KEY_PREFIX, + HeaderTeam, JWKSUnreachableError, JWTAuthManager, JWTHandler, @@ -1993,29 +1995,41 @@ async def test_auth_builder_oidc_enabled_falls_back_to_jwt_auth_for_jwt_tokens() assert result["user_object"] == user_object -def test_get_team_id_from_header(): - """Test get_team_id_from_header returns team when valid, None when missing, raises on invalid.""" - from fastapi import HTTPException - - # Valid team in allowed list - result = JWTAuthManager.get_team_id_from_header( +@pytest.mark.asyncio +async def test_resolve_team_from_header_returns_allowed_id_none_without_header_and_403_on_invalid(): + """Without a DB, x-litellm-team-id resolves to the team when it names an allowed + team id, to None when the header is absent, and to a 403 for any other value.""" + allowed = await JWTAuthManager.resolve_team_from_header( request_headers={"x-litellm-team-id": "team-1"}, allowed_team_ids={"team-1", "team-2"}, + fallback_to_db_teams=False, + prisma_client=None, + user_api_key_cache=MagicMock(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), ) - assert result == "team-1" + assert allowed == HeaderTeam(header_value="team-1", team_id="team-1") - # No header returns None - result = JWTAuthManager.get_team_id_from_header( + absent = await JWTAuthManager.resolve_team_from_header( request_headers={"authorization": "Bearer token"}, allowed_team_ids={"team-1"}, + fallback_to_db_teams=False, + prisma_client=None, + user_api_key_cache=MagicMock(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), ) - assert result is None + assert absent is None - # Invalid team raises 403 with pytest.raises(HTTPException) as exc_info: - JWTAuthManager.get_team_id_from_header( + await JWTAuthManager.resolve_team_from_header( request_headers={"x-litellm-team-id": "invalid-team"}, allowed_team_ids={"team-1", "team-2"}, + fallback_to_db_teams=False, + prisma_client=None, + user_api_key_cache=MagicMock(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), ) assert exc_info.value.status_code == 403 @@ -5309,32 +5323,23 @@ def test_build_decode_kwargs_warns_for_unscoped_global_fallback_in_mixed_deploym # --------------------------------------------------------------------------- -def test_get_team_id_from_header_defers_to_db_membership_only_without_jwt_claims(): - """With fallback_to_db_teams=True, an x-litellm-team-id header is accepted - provisionally only when the JWT carries no team claims (allowed set empty). - When the JWT does carry team claims, the header must still be validated +@pytest.mark.asyncio +async def test_resolve_team_from_header_defers_to_db_membership_only_without_jwt_claims(): + """With fallback_to_db_teams=True, an x-litellm-team-id header naming an existing + team is accepted provisionally only when the JWT carries no team claims (allowed + set empty). When the JWT does carry team claims, the header must still be validated against them, and the flag-off behavior must keep rejecting unknown teams.""" - deferred = JWTAuthManager.get_team_id_from_header( - request_headers={"x-litellm-team-id": "team-from-db"}, - allowed_team_ids=set(), - fallback_to_db_teams=True, - ) - assert deferred == "team-from-db" + known_ids = frozenset({"team-from-db"}) + + deferred, _, _ = await _resolve_header("team-from-db", set(), True, _teams_by_id(known_ids), _team_alias_lookup_404) + assert deferred == HeaderTeam(header_value="team-from-db", team_id="team-from-db") with pytest.raises(HTTPException) as exc_info: - JWTAuthManager.get_team_id_from_header( - request_headers={"x-litellm-team-id": "team-x"}, - allowed_team_ids={"team-1", "team-2"}, - fallback_to_db_teams=True, - ) + await _resolve_header("team-x", {"team-1", "team-2"}, True, _teams_by_id(known_ids), _team_alias_lookup_404) assert exc_info.value.status_code == 403 with pytest.raises(HTTPException): - JWTAuthManager.get_team_id_from_header( - request_headers={"x-litellm-team-id": "team-from-db"}, - allowed_team_ids=set(), - fallback_to_db_teams=False, - ) + await _resolve_header("team-from-db", set(), False, _teams_by_id(known_ids), _team_alias_lookup_404) @pytest.mark.asyncio @@ -5788,6 +5793,7 @@ def test_validate_header_team_in_db_membership_does_not_leak_team_ids(): JWTAuthManager._validate_header_team_in_db_membership( team_id="outsider_team", user_object=user_object, + header_value="outsider_team", ) detail = exc_info.value.detail @@ -5797,6 +5803,43 @@ def test_validate_header_team_in_db_membership_does_not_leak_team_ids(): assert "outsider_team" in detail +async def _team_lookup_404(team_id, **kwargs): + raise HTTPException( + status_code=404, + detail=f"Team doesn't exist in db. Team={team_id}. Create team via `/team/new` call.", + ) + + +async def _team_alias_lookup_404(team_alias, **kwargs): + raise HTTPException( + status_code=404, + detail={"error": f"Team with alias '{team_alias}' doesn't exist in db. Create team via `/team/new` call."}, + ) + + +def _teams_by_alias(aliases: Mapping[str, str]): + """An alias lookup over `aliases` (alias -> team_id) that 404s like the real one otherwise.""" + + async def lookup(team_alias, **kwargs): + if team_alias not in aliases: + return await _team_alias_lookup_404(team_alias) + return LiteLLM_TeamTable(team_id=aliases[team_alias], team_alias=team_alias) + + return lookup + + +def _teams_by_id(team_ids: frozenset[str]): + """A team lookup that knows exactly `team_ids` and, like the real one, reports + any other id as provably absent.""" + + async def lookup(team_id, **kwargs): + if team_id not in team_ids: + raise TeamNotFoundError(team_id=team_id) + return LiteLLM_TeamTable(team_id=team_id) + + return lookup + + async def _run_auth_builder_with_header_team( jwt_auth_config: LiteLLM_JWTAuth, token: dict, @@ -5804,6 +5847,8 @@ async def _run_auth_builder_with_header_team( user_object: LiteLLM_UserTable, fake_get_team, allowed_team_ids: set, + fake_get_team_by_alias=_team_alias_lookup_404, + route: str = "/chat/completions", ): jwt_handler = JWTHandler() jwt_handler.litellm_jwtauth = jwt_auth_config @@ -5848,14 +5893,19 @@ async def _run_auth_builder_with_header_team( new_callable=AsyncMock, side_effect=fake_get_team, ), + patch( + "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", + new_callable=AsyncMock, + side_effect=fake_get_team_by_alias, + ), ): return await JWTAuthManager.auth_builder( api_key="test_jwt_token", jwt_handler=jwt_handler, request_data={"model": "gpt-4"}, general_settings={"enforce_rbac": False}, - route="/chat/completions", - prisma_client=None, + route=route, + prisma_client=MagicMock(), user_api_key_cache=None, parent_otel_span=None, proxy_logging_obj=None, @@ -5863,13 +5913,6 @@ async def _run_auth_builder_with_header_team( ) -async def _team_lookup_404(team_id, **kwargs): - raise HTTPException( - status_code=404, - detail=f"Team doesn't exist in db. Team={team_id}. Create team via `/team/new` call.", - ) - - @pytest.mark.asyncio async def test_auth_builder_header_team_not_found_matches_non_membership_denial() -> ( None @@ -6942,6 +6985,273 @@ async def test_auth_builder_header_team_enforces_team_allowed_routes_under_db_fa assert result["team_id"] == header_team +async def _resolve_header( + header_value: str, + allowed_team_ids: set[str], + fallback_to_db_teams: bool, + fake_get_team, + fake_get_team_by_alias, +) -> tuple[HeaderTeam | None, AsyncMock, AsyncMock]: + with ( + patch( + "litellm.proxy.auth.handle_jwt.get_team_object", + new_callable=AsyncMock, + side_effect=fake_get_team, + ) as by_id, + patch( + "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", + new_callable=AsyncMock, + side_effect=fake_get_team_by_alias, + ) as by_alias, + ): + resolved = await JWTAuthManager.resolve_team_from_header( + request_headers={"X-LiteLLM-Team-Id": header_value}, + allowed_team_ids=allowed_team_ids, + fallback_to_db_teams=fallback_to_db_teams, + prisma_client=MagicMock(), + user_api_key_cache=MagicMock(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + return resolved, by_id, by_alias + + +@pytest.mark.asyncio +async def test_resolve_team_from_header_accepts_the_alias_of_an_allowed_team(): + """x-litellm-team-id may carry the team alias instead of the team id (LIT-7181). + The alias resolves to its team id before the allowed-teams check, so a + caller whose JWT grants team_a gets team_a whether it sends the id or the + alias, and the id path never pays for an alias lookup.""" + aliases = {"alias_a": "team_a", "alias_b": "team_b"} + + by_alias_value, lookups_by_id, lookups_by_alias = await _resolve_header( + "alias_a", {"team_a"}, False, _team_lookup_404, _teams_by_alias(aliases) + ) + assert by_alias_value == HeaderTeam(header_value="alias_a", team_id="team_a") + lookups_by_alias.assert_awaited_once() + assert lookups_by_alias.await_args.kwargs["team_alias"] == "alias_a" + lookups_by_id.assert_not_awaited() + + by_id_value, lookups_by_id, lookups_by_alias = await _resolve_header( + "team_a", {"team_a"}, False, _team_lookup_404, _teams_by_alias(aliases) + ) + assert by_id_value == HeaderTeam(header_value="team_a", team_id="team_a") + lookups_by_alias.assert_not_awaited() + lookups_by_id.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_team_from_header_denies_aliases_of_teams_the_jwt_does_not_grant(): + """An alias that exists but names a team outside the JWT's allowed teams is + refused with the same 403 as an unknown value, and the detail names only + what the caller sent, so the response reveals neither that the alias exists + nor which team id it maps to.""" + aliases = {"alias_a": "team_a", "alias_b": "team_b"} + + with pytest.raises(HTTPException) as other_team: + await _resolve_header("alias_b", {"team_a"}, False, _team_lookup_404, _teams_by_alias(aliases)) + with pytest.raises(HTTPException) as unknown: + await _resolve_header("no_such", {"team_a"}, False, _team_lookup_404, _teams_by_alias(aliases)) + + assert other_team.value.status_code == 403 + assert unknown.value.status_code == 403 + assert "team_b" not in other_team.value.detail + assert other_team.value.detail.replace("alias_b", "") == unknown.value.detail.replace("no_such", "") + + +@pytest.mark.asyncio +async def test_resolve_team_from_header_under_db_fallback_tries_the_id_before_the_alias(): + """Under fallback_to_db_teams a claimless JWT's header is provisional: a value + that is an existing team id resolves to itself without an alias lookup, a + value that is only an alias resolves to that team's id, and a value that is + neither gets the membership denial the id path already uses.""" + known_ids = frozenset({"team_a"}) + aliases = {"alias_a": "team_a"} + + as_id, _, lookups_by_alias = await _resolve_header( + "team_a", set(), True, _teams_by_id(known_ids), _teams_by_alias(aliases) + ) + assert as_id == HeaderTeam(header_value="team_a", team_id="team_a") + lookups_by_alias.assert_not_awaited() + + as_alias, _, _ = await _resolve_header("alias_a", set(), True, _teams_by_id(known_ids), _teams_by_alias(aliases)) + assert as_alias == HeaderTeam(header_value="alias_a", team_id="team_a") + + with pytest.raises(HTTPException) as neither: + await _resolve_header("ghost", set(), True, _teams_by_id(known_ids), _teams_by_alias(aliases)) + assert neither.value.status_code == 403 + assert neither.value.detail == ("Team 'ghost' (from x-litellm-team-id header) is not in your team memberships.") + + +@pytest.mark.asyncio +async def test_resolve_team_from_header_under_db_fallback_never_aliases_a_team_id_it_could_not_read(): + """Only a team row the database provably lacks falls through to the alias + lookup. When the id read fails for any other reason (the generic 404 the + team lookup uses for an unreadable database) the value keeps the id path's + membership denial, so an outage can never turn a team id into the team + that happens to carry it as an alias.""" + aliases = {"team_a": "team_b"} + + with ( + patch("litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock, side_effect=_team_lookup_404), + patch( + "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", + new_callable=AsyncMock, + side_effect=_teams_by_alias(aliases), + ) as lookups_by_alias, + pytest.raises(HTTPException) as unreadable, + ): + await JWTAuthManager.resolve_team_from_header( + request_headers={"x-litellm-team-id": "team_a"}, + allowed_team_ids=set(), + fallback_to_db_teams=True, + prisma_client=MagicMock(), + user_api_key_cache=MagicMock(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + assert unreadable.value.status_code == 403 + assert unreadable.value.detail == ("Team 'team_a' (from x-litellm-team-id header) is not in your team memberships.") + lookups_by_alias.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_team_from_header_treats_a_duplicate_alias_as_no_match_but_surfaces_lookup_errors(): + """An alias two teams share cannot name one team, so it is refused like an + unknown value (a 4xx from the lookup is a miss), while a lookup failure + (5xx) is not disguised as a denial and propagates as is.""" + + async def duplicate_alias(team_alias, **kwargs): + raise HTTPException(status_code=400, detail={"error": f"Multiple teams found with alias '{team_alias}'."}) + + async def db_down(team_alias, **kwargs): + raise HTTPException(status_code=500, detail={"error": f"Error looking up team by alias '{team_alias}'"}) + + with pytest.raises(HTTPException) as duplicate: + await _resolve_header("shared_alias", {"team_a"}, False, _team_lookup_404, duplicate_alias) + assert duplicate.value.status_code == 403 + assert "Multiple teams" not in str(duplicate.value.detail) + + with pytest.raises(HTTPException) as failure: + await _resolve_header("alias_a", {"team_a"}, False, _team_lookup_404, db_down) + assert failure.value.status_code == 500 + + +@pytest.mark.asyncio +async def test_auth_builder_header_alias_binds_the_aliased_team_under_claims_and_db_fallback(): + """End to end through auth_builder, x-litellm-team-id carrying a team alias + binds the request to the aliased team (result team_id is the canonical id) + both when the JWT grants that team by claim and when a claimless JWT relies + on fallback_to_db_teams and DB membership.""" + aliases = {"alias_member": "team_member"} + user_object = LiteLLM_UserTable( + user_id="u_alias", + user_role=LitellmUserRoles.INTERNAL_USER, + teams=["team_member"], + ) + + claims_config = LiteLLM_JWTAuth(team_ids_jwt_field="team_ids") + by_claim = await _run_auth_builder_with_header_team( + claims_config, + {"sub": "u_alias", "scope": "", "team_ids": ["team_member"]}, + "alias_member", + user_object, + _teams_by_id(frozenset({"team_member"})), + {"team_member"}, + _teams_by_alias(aliases), + ) + assert by_claim["team_id"] == "team_member" + assert by_claim["team_object"].team_id == "team_member" + + fallback_config = LiteLLM_JWTAuth(fallback_to_db_teams=True) + by_membership = await _run_auth_builder_with_header_team( + fallback_config, + {"sub": "u_alias", "scope": ""}, + "alias_member", + user_object, + _teams_by_id(frozenset({"team_member"})), + set(), + _teams_by_alias(aliases), + ) + assert by_membership["team_id"] == "team_member" + assert by_membership["team_object"].team_id == "team_member" + + +@pytest.mark.asyncio +async def test_auth_builder_header_alias_of_a_non_member_team_is_denied_like_an_unknown_value_under_db_fallback(): + """Under fallback_to_db_teams, an alias naming a team the user is not a member + of is denied with the exact same 403 as an unknown value, naming the alias + the caller sent rather than the team id it resolved to.""" + aliases = {"alias_other": "team_other"} + known_ids = frozenset({"team_member", "team_other"}) + user_object = LiteLLM_UserTable( + user_id="u_alias_outsider", + user_role=LitellmUserRoles.INTERNAL_USER, + teams=["team_member"], + ) + config = LiteLLM_JWTAuth(fallback_to_db_teams=True) + token = {"sub": "u_alias_outsider", "scope": ""} + + with pytest.raises(HTTPException) as outsider_alias: + await _run_auth_builder_with_header_team( + config, token, "alias_other", user_object, _teams_by_id(known_ids), set(), _teams_by_alias(aliases) + ) + with pytest.raises(HTTPException) as unknown: + await _run_auth_builder_with_header_team( + config, token, "alias_ghost", user_object, _teams_by_id(known_ids), set(), _teams_by_alias(aliases) + ) + + assert outsider_alias.value.status_code == 403 + assert unknown.value.status_code == 403 + assert "team_other" not in outsider_alias.value.detail + assert outsider_alias.value.detail.replace("alias_other", "") == unknown.value.detail.replace( + "alias_ghost", "" + ) + + +@pytest.mark.asyncio +async def test_auth_builder_header_alias_under_db_fallback_keeps_the_team_allowed_routes_gate(): + """Under fallback_to_db_teams, a member team selected by alias is still held + to team_allowed_routes, and the denial names the alias the caller sent.""" + aliases = {"alias_member": "team_member"} + user_object = LiteLLM_UserTable( + user_id="u_alias_routes", + user_role=LitellmUserRoles.INTERNAL_USER, + teams=["team_member"], + ) + config = LiteLLM_JWTAuth(fallback_to_db_teams=True, team_allowed_routes=["openai_routes"]) + token = {"sub": "u_alias_routes", "scope": ""} + + with pytest.raises(HTTPException) as exc_info: + await _run_auth_builder_with_header_team( + config, + token, + "alias_member", + user_object, + _teams_by_id(frozenset({"team_member"})), + set(), + _teams_by_alias(aliases), + route="/key/info", + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail == ( + "Team 'alias_member' (from x-litellm-team-id header) is not allowed to access route '/key/info'." + ) + + allowed = await _run_auth_builder_with_header_team( + config, + token, + "alias_member", + user_object, + _teams_by_id(frozenset({"team_member"})), + set(), + _teams_by_alias(aliases), + route="/chat/completions", + ) + assert allowed["team_id"] == "team_member" + + @pytest.mark.asyncio async def test_sync_user_role_and_teams_singular_claim_only_recognized_under_flag(): """Reading the singular team claim during sync is scoped to fallback_to_db_teams. From 40825235968702047d1b02caba52c86d7d0de998 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:42:25 -0700 Subject: [PATCH 18/28] chore(prices): sync OpenRouter prices: 14 models, 1 deprecated [enrichment failed: OpenRouter, 5 held] (#42438) openrouter/~z-ai/glm-latest: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/aion-labs/aion-2.0: max_input_tokens openrouter/aion-labs/aion-3.0: max_input_tokens openrouter/aion-labs/aion-3.0-mini: max_input_tokens openrouter/deepseek/deepseek-v4-flash: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/deepseek/deepseek-v4-pro: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/dots-studio/dots-3-note-preview:free: deprecation_date openrouter/moonshotai/kimi-k2.7-code: output_cost_per_token openrouter/moonshotai/kimi-k3:batch: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/openai/gpt-oss-20b: max_tokens, max_output_tokens, supports_prompt_caching, input_cost_per_token, output_cost_per_token openrouter/qwen/qwen3-next-80b-a3b-thinking: max_tokens, max_output_tokens openrouter/z-ai/glm-5.3: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/z-ai/glm-5.3-flash:batch: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost openrouter/z-ai/glm-5.3:batch: max_tokens, max_output_tokens, input_cost_per_token, output_cost_per_token, cache_read_input_token_cost Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 78 +++++++++---------- model_prices_and_context_window.json | 78 +++++++++---------- 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b5106670a2b..3ce2d32613e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -43093,21 +43093,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.5526e-07, + "input_cost_per_token": 9.36294e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.91052e-06, + "output_cost_per_token": 1.872588e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9605e-08, + "cache_read_input_token_cost": 7.80245e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -44234,19 +44234,19 @@ }, "openrouter/openai/gpt-oss-20b": { "cache_read_input_token_cost": 3e-08, - "input_cost_per_token": 3e-08, + "input_cost_per_token": 1.8e-08, "litellm_provider": "openrouter", "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1.3e-07, + "output_cost_per_token": 9e-08, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": false, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -68315,9 +68315,9 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3": { - "input_cost_per_token": 8.4e-07, - "output_cost_per_token": 2.64e-06, - "cache_read_input_token_cost": 1.56e-07, + "input_cost_per_token": 6.538e-07, + "output_cost_per_token": 2.0548e-06, + "cache_read_input_token_cost": 1.2142e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, @@ -68683,7 +68683,7 @@ }, "openrouter/moonshotai/kimi-k2.7-code": { "input_cost_per_token": 7.062e-07, - "output_cost_per_token": 3.21e-06, + "output_cost_per_token": 3.3e-06, "cache_read_input_token_cost": 1.8e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, @@ -69004,9 +69004,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 8.8606e-08, - "output_cost_per_token": 1.77212e-07, - "cache_read_input_token_cost": 1.77212e-08, + "input_cost_per_token": 4.9e-08, + "output_cost_per_token": 9.8e-08, + "cache_read_input_token_cost": 9.8e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -69866,8 +69866,8 @@ "output_cost_per_token": 1.2e-06, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 235929, + "max_tokens": 235929, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -73345,14 +73345,14 @@ "supports_web_search": false }, "openrouter/~z-ai/glm-latest": { - "cache_read_input_token_cost": 1.56e-07, - "input_cost_per_token": 8.4e-07, + "cache_read_input_token_cost": 1.2142e-07, + "input_cost_per_token": 6.538e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.64e-06, + "output_cost_per_token": 2.0548e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73368,7 +73368,7 @@ "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 8e-07, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -73388,7 +73388,7 @@ "cache_read_input_token_cost": 7.5e-07, "input_cost_per_token": 3e-06, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -73408,7 +73408,7 @@ "cache_read_input_token_cost": 1.8e-07, "input_cost_per_token": 7e-07, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -74187,7 +74187,7 @@ "supports_web_search": false }, "openrouter/dots-studio/dots-3-note-preview:free": { - "deprecation_date": "2026-09-30", + "deprecation_date": "2026-12-31", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 512000, @@ -75098,14 +75098,14 @@ "supports_web_search": false }, "openrouter/moonshotai/kimi-k3:batch": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.28e-07, + "input_cost_per_token": 2.28e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.14e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -76853,14 +76853,14 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3-flash:batch": { - "cache_read_input_token_cost": 1.5e-08, - "input_cost_per_token": 7.5e-08, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 6e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -76873,14 +76873,14 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3:batch": { - "cache_read_input_token_cost": 1.3e-07, - "input_cost_per_token": 7e-07, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 7.2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.2e-06, + "output_cost_per_token": 2.4e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b5106670a2b..3ce2d32613e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -43093,21 +43093,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.5526e-07, + "input_cost_per_token": 9.36294e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.91052e-06, + "output_cost_per_token": 1.872588e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9605e-08, + "cache_read_input_token_cost": 7.80245e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -44234,19 +44234,19 @@ }, "openrouter/openai/gpt-oss-20b": { "cache_read_input_token_cost": 3e-08, - "input_cost_per_token": 3e-08, + "input_cost_per_token": 1.8e-08, "litellm_provider": "openrouter", "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1.3e-07, + "output_cost_per_token": 9e-08, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": false, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -68315,9 +68315,9 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3": { - "input_cost_per_token": 8.4e-07, - "output_cost_per_token": 2.64e-06, - "cache_read_input_token_cost": 1.56e-07, + "input_cost_per_token": 6.538e-07, + "output_cost_per_token": 2.0548e-06, + "cache_read_input_token_cost": 1.2142e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, @@ -68683,7 +68683,7 @@ }, "openrouter/moonshotai/kimi-k2.7-code": { "input_cost_per_token": 7.062e-07, - "output_cost_per_token": 3.21e-06, + "output_cost_per_token": 3.3e-06, "cache_read_input_token_cost": 1.8e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, @@ -69004,9 +69004,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 8.8606e-08, - "output_cost_per_token": 1.77212e-07, - "cache_read_input_token_cost": 1.77212e-08, + "input_cost_per_token": 4.9e-08, + "output_cost_per_token": 9.8e-08, + "cache_read_input_token_cost": 9.8e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -69866,8 +69866,8 @@ "output_cost_per_token": 1.2e-06, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 235929, + "max_tokens": 235929, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -73345,14 +73345,14 @@ "supports_web_search": false }, "openrouter/~z-ai/glm-latest": { - "cache_read_input_token_cost": 1.56e-07, - "input_cost_per_token": 8.4e-07, + "cache_read_input_token_cost": 1.2142e-07, + "input_cost_per_token": 6.538e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.64e-06, + "output_cost_per_token": 2.0548e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73368,7 +73368,7 @@ "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 8e-07, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -73388,7 +73388,7 @@ "cache_read_input_token_cost": 7.5e-07, "input_cost_per_token": 3e-06, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -73408,7 +73408,7 @@ "cache_read_input_token_cost": 1.8e-07, "input_cost_per_token": 7e-07, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -74187,7 +74187,7 @@ "supports_web_search": false }, "openrouter/dots-studio/dots-3-note-preview:free": { - "deprecation_date": "2026-09-30", + "deprecation_date": "2026-12-31", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 512000, @@ -75098,14 +75098,14 @@ "supports_web_search": false }, "openrouter/moonshotai/kimi-k3:batch": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.28e-07, + "input_cost_per_token": 2.28e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.14e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -76853,14 +76853,14 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3-flash:batch": { - "cache_read_input_token_cost": 1.5e-08, - "input_cost_per_token": 7.5e-08, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 6e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -76873,14 +76873,14 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3:batch": { - "cache_read_input_token_cost": 1.3e-07, - "input_cost_per_token": 7e-07, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 7.2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.2e-06, + "output_cost_per_token": 2.4e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, From d6ffc554ecf103c73eb47f9a3343bc325980c658 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 09:02:15 -0700 Subject: [PATCH 19/28] feat(rust): align secret manager operation contexts (#42480) * refactor(rust): simplify Python secret callbacks * feat(rust): align secret manager operation contexts * test(rust): port secret name validation cases * fix(rust): restore secret manager CI checks * fix(rust): validate Vault contexts and preserve rotation timeouts --------- Co-authored-by: Yujong Lee --- litellm-rust/Cargo.lock | 1 + litellm-rust/crates/python-bridge/Cargo.toml | 3 +- .../python-bridge/src/routes/ocr/host.rs | 2 +- .../python-bridge/src/secrets/callback.rs | 225 +++++----- .../crates/python-bridge/src/secrets/error.rs | 27 ++ .../crates/python-bridge/src/secrets/mod.rs | 3 + litellm-rust/crates/secrets-aws/src/error.rs | 4 + .../crates/secrets-aws/src/secret_manager.rs | 235 +++++++++-- litellm-rust/crates/secrets-aws/tests/kms.rs | 71 +++- .../secrets-aws/tests/secret_manager.rs | 202 ++++++++- .../crates/secrets-azure/src/key_vault.rs | 5 +- .../crates/secrets-azure/tests/key_vault.rs | 90 ++-- .../crates/secrets-azure/tests/live.rs | 8 +- .../secrets-cyberark/src/secret_manager.rs | 154 +++++-- .../secrets-cyberark/tests/secret_manager.rs | 105 +++-- .../crates/secrets-google/tests/kms.rs | 57 ++- .../secrets-google/tests/secret_manager.rs | 103 +++-- .../crates/secrets-hashicorp/src/error.rs | 4 + .../secrets-hashicorp/src/secret_manager.rs | 211 ++++++++-- .../secrets-hashicorp/tests/secret_manager.rs | 392 ++++++++++++++++-- .../secrets-types/src/base_secret_manager.rs | 32 +- .../crates/secrets-types/src/context.rs | 65 +++ litellm-rust/crates/secrets-types/src/lib.rs | 5 + .../crates/secrets-types/tests/config.rs | 56 ++- .../crates/secrets-types/tests/context.rs | 110 +++++ .../crates/secrets-types/tests/rotation.rs | 97 ++++- litellm-rust/crates/secrets/src/handler.rs | 18 +- .../crates/secrets/tests/resolution.rs | 40 +- 28 files changed, 1839 insertions(+), 486 deletions(-) create mode 100644 litellm-rust/crates/python-bridge/src/secrets/error.rs create mode 100644 litellm-rust/crates/secrets-types/src/context.rs create mode 100644 litellm-rust/crates/secrets-types/tests/context.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 37384cbfa53..76004c5d978 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3065,6 +3065,7 @@ dependencies = [ "serde_json", "serde_with", "sha2 0.10.9", + "thiserror 2.0.19", "tokio", "tokio-tungstenite", "url", diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 7846beef28a..dd0975219e4 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -52,8 +52,9 @@ pyo3-async-runtimes.workspace = true reqwest.workspace = true redis = { version = "1.7.0", features = ["tls-rustls"] } serde_json.workspace = true -url.workspace = true +thiserror.workspace = true tokio = { workspace = true, features = ["rt", "sync"] } +url.workspace = true [dev-dependencies] litellm-secrets-aws.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs index 325377e5285..8bf99cd355f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs @@ -131,7 +131,7 @@ impl RouteHost for OcrRouteHost { fn classify(&self, py: Python<'_>, error: Error) -> PyResult { if let Error::Secret(source) = &error - && let Some(original) = crate::secrets::callback::python_error(py, source) + && let Some(original) = crate::secrets::python_error(py, source) { return Ok(original); } diff --git a/litellm-rust/crates/python-bridge/src/secrets/callback.rs b/litellm-rust/crates/python-bridge/src/secrets/callback.rs index 2b98081acef..a8e5850fe3f 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/callback.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/callback.rs @@ -1,45 +1,20 @@ -use std::{fmt, future::Future, pin::Pin}; +use std::{future::Future, pin::Pin}; use litellm_core_utils::settings::Lookup; use litellm_secrets::{ Error, ExternalSecretManager, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue, }; -use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict}; +use pyo3::{prelude::*, types::PyDict}; + +use super::error::external_error; const HANDLER_MODULE: &str = "litellm.secret_managers.secret_manager_handler"; -struct PythonSecretError(Py); - -impl fmt::Debug for PythonSecretError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("PythonSecretError") - } -} - -impl fmt::Display for PythonSecretError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("Python secret manager failed") - } -} - -impl std::error::Error for PythonSecretError {} - -pub(crate) fn python_error(py: Python<'_>, error: &Error) -> Option { - let Error::ExternalManager(source) = error else { - return None; - }; - source - .downcast_ref::() - .map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any())) -} - /// A secret manager whose reads execute in Python: a custom manager, a legacy compatible /// client, or a manually assigned SDK client. pub(crate) struct PythonSecretManager { client: Py, system: Option, - /// The `key_manager` name Python's handler dispatches on. - key_manager: &'static str, settings: Option>, } @@ -52,7 +27,6 @@ impl PythonSecretManager { Self { client, system, - key_manager: system.map_or("local", python_name), settings, } } @@ -78,14 +52,12 @@ impl PythonSecretManager { } let kwargs = PyDict::new(py); kwargs.set_item("client", client)?; - kwargs.set_item("key_manager", self.key_manager)?; + kwargs.set_item("key_manager", self.system.map_or("local", python_name))?; kwargs.set_item("secret_name", name)?; - kwargs.set_item( - "key_management_settings", - self.settings - .as_ref() - .map_or_else(|| py.None(), |settings| settings.clone_ref(py)), - )?; + match &self.settings { + Some(settings) => kwargs.set_item("key_management_settings", settings.bind(py))?, + None => kwargs.set_item("key_management_settings", py.None())?, + } py.import(HANDLER_MODULE)? .getattr("get_secret_from_manager")? .call((), Some(&kwargs))? @@ -123,9 +95,7 @@ impl ExternalSecretManager for PythonSecretManager { Python::attach(|py| { self.read(py, name) .map(|value| value.map(SecretValue::new).map(Secret::String)) - .map_err(|error| { - Error::ExternalManager(Box::new(PythonSecretError(error.into_value(py)))) - }) + .map_err(|error| external_error(py, error)) }) }) } @@ -140,19 +110,27 @@ mod tests { SecretManagerState, SecretResolver, }; use pyo3::{prelude::*, types::PyDict}; + use rstest::rstest; - use super::{HANDLER_MODULE, PythonSecretManager, python_error, python_name}; + use super::{HANDLER_MODULE, PythonSecretManager, python_name}; + use crate::secrets::python_error; + #[rstest] + #[case::value_error("ValueError", None)] + #[case::value_error_with_fallback("ValueError", Some("environment-key"))] + #[case::cancelled("asyncio.CancelledError", None)] + #[case::cancelled_with_fallback("asyncio.CancelledError", Some("environment-key"))] #[tokio::test] - async fn callback_failures_preserve_python_exceptions_even_with_environment_fallback() { + async fn callback_failures_preserve_python_exceptions_even_with_environment_fallback( + #[case] failure_type: &str, + #[case] fallback: Option<&'static str>, + ) { Python::initialize(); - for failure_type in ["ValueError", "asyncio.CancelledError"] { - for fallback in [None, Some("environment-key")] { - let (reader, locals) = Python::attach(|py| { - let locals = PyDict::new(py); - locals.set_item("failure_type", failure_type).unwrap(); - py.run( - c" + let (reader, locals) = Python::attach(|py| { + let locals = PyDict::new(py); + locals.set_item("failure_type", failure_type).unwrap(); + py.run( + c" import asyncio failure = eval(failure_type)('secret manager failed') cause = RuntimeError('original cause') @@ -164,48 +142,46 @@ class Manager: raise failure manager = Manager() ", - Some(&locals), - Some(&locals), - ) - .unwrap(); - let reader = PythonSecretManager::new( - locals.get_item("manager").unwrap().unwrap().unbind(), - None, - None, - ); - (reader, locals.unbind()) - }); - let resolver = SecretResolver::new( - Arc::new(SecretManagerState::new( - SecretManager::External(Arc::new(reader)), - KeyManagementSettings::default(), - )), - Arc::new(move |_: &str| fallback.map(str::to_owned)), - OidcResolver::default(), - ) - .with_failure_policy(FailurePolicy::EnvironmentFallback); - let error = resolver.get_secret("API_KEY", None).await.unwrap_err(); - Python::attach(|py| { - let original = python_error(py, &error).unwrap(); - let locals = locals.bind(py); - assert!( - original - .value(py) - .is(locals.get_item("failure").unwrap().unwrap()) - ); - for (attribute, name) in [("__cause__", "cause"), ("__context__", "context")] { - assert!( - original - .value(py) - .getattr(attribute) - .unwrap() - .is(locals.get_item(name).unwrap().unwrap()) - ); - } - assert!(original.traceback(py).is_some()); - }); + Some(&locals), + Some(&locals), + ) + .unwrap(); + let reader = PythonSecretManager::new( + locals.get_item("manager").unwrap().unwrap().unbind(), + None, + None, + ); + (reader, locals.unbind()) + }); + let resolver = SecretResolver::new( + Arc::new(SecretManagerState::new( + SecretManager::External(Arc::new(reader)), + KeyManagementSettings::default(), + )), + Arc::new(move |_: &str| fallback.map(str::to_owned)), + OidcResolver::default(), + ) + .with_failure_policy(FailurePolicy::EnvironmentFallback); + let error = resolver.get_secret("API_KEY", None).await.unwrap_err(); + Python::attach(|py| { + let original = python_error(py, &error).unwrap(); + let locals = locals.bind(py); + assert!( + original + .value(py) + .is(locals.get_item("failure").unwrap().unwrap()) + ); + for (attribute, name) in [("__cause__", "cause"), ("__context__", "context")] { + assert!( + original + .value(py) + .getattr(attribute) + .unwrap() + .is(locals.get_item(name).unwrap().unwrap()) + ); } - } + assert!(original.traceback(py).is_some()); + }); } /// Installs a fake `get_secret_from_manager` that records its kwargs, runs `body`, and @@ -245,47 +221,57 @@ for name in installed: .unwrap(); } - #[test] - fn python_names_round_trip_through_serde() { - for system in [ - KeyManagementSystem::GoogleKms, - KeyManagementSystem::AzureKeyVault, - KeyManagementSystem::AwsSecretManager, - KeyManagementSystem::GoogleSecretManager, - KeyManagementSystem::HashicorpVault, - KeyManagementSystem::Cyberark, - KeyManagementSystem::Local, - KeyManagementSystem::AwsKms, - KeyManagementSystem::Custom, - ] { - assert_eq!( - serde_json::to_value(system).unwrap(), - serde_json::Value::String(python_name(system).to_owned()) - ); - } + #[rstest] + #[case::google_kms(KeyManagementSystem::GoogleKms)] + #[case::azure_key_vault(KeyManagementSystem::AzureKeyVault)] + #[case::aws_secret_manager(KeyManagementSystem::AwsSecretManager)] + #[case::google_secret_manager(KeyManagementSystem::GoogleSecretManager)] + #[case::hashicorp_vault(KeyManagementSystem::HashicorpVault)] + #[case::cyberark(KeyManagementSystem::Cyberark)] + #[case::local(KeyManagementSystem::Local)] + #[case::aws_kms(KeyManagementSystem::AwsKms)] + #[case::custom(KeyManagementSystem::Custom)] + fn python_names_round_trip_through_serde(#[case] system: KeyManagementSystem) { + assert_eq!( + serde_json::to_value(system).unwrap(), + serde_json::Value::String(python_name(system).to_owned()) + ); } - #[test] - fn custom_readers_without_a_system_are_called_directly() { + #[rstest] + #[case::legacy(None, false)] + #[case::custom(Some(KeyManagementSystem::Custom), true)] + fn direct_readers_receive_compatible_kwargs( + #[case] system: Option, + #[case] expects_optional_params: bool, + ) { Python::initialize(); Python::attach(|py| { let locals = PyDict::new(py); py.run( c" +class Settings: + def model_dump(self): + return {'scope': 'custom'} class Manager: def __init__(self): self.names = [] + self.optional_params = [] def sync_read_secret(self, secret_name, optional_params=None, timeout=None): self.names.append(secret_name) + self.optional_params.append(optional_params) return 'direct-' + secret_name manager = Manager() +settings = Settings() ", Some(&locals), Some(&locals), ) .unwrap(); let manager = locals.get_item("manager").unwrap().unwrap(); - let reader = PythonSecretManager::new(manager.clone().unbind(), None, None); + let settings = expects_optional_params + .then(|| locals.get_item("settings").unwrap().unwrap().unbind()); + let reader = PythonSecretManager::new(manager.clone().unbind(), system, settings); assert_eq!( reader.read(py, "API_KEY").unwrap().as_deref(), Some("direct-API_KEY") @@ -298,6 +284,23 @@ manager = Manager() .unwrap(), ["API_KEY"] ); + let optional_params = manager + .getattr("optional_params") + .unwrap() + .get_item(0) + .unwrap(); + if expects_optional_params { + assert_eq!( + optional_params + .get_item("scope") + .unwrap() + .extract::() + .unwrap(), + "custom" + ); + } else { + assert!(optional_params.is_none()); + } }); } diff --git a/litellm-rust/crates/python-bridge/src/secrets/error.rs b/litellm-rust/crates/python-bridge/src/secrets/error.rs new file mode 100644 index 00000000000..1bf9351ee64 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/secrets/error.rs @@ -0,0 +1,27 @@ +use std::fmt; + +use litellm_secrets::Error; +use pyo3::{exceptions::PyBaseException, prelude::*}; + +#[derive(thiserror::Error)] +#[error("Python secret manager failed")] +struct PythonSecretError(Py); + +impl fmt::Debug for PythonSecretError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("PythonSecretError") + } +} + +pub(super) fn external_error(py: Python<'_>, error: PyErr) -> Error { + Error::ExternalManager(Box::new(PythonSecretError(error.into_value(py)))) +} + +pub(crate) fn python_error(py: Python<'_>, error: &Error) -> Option { + let Error::ExternalManager(source) = error else { + return None; + }; + source + .downcast_ref::() + .map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any())) +} diff --git a/litellm-rust/crates/python-bridge/src/secrets/mod.rs b/litellm-rust/crates/python-bridge/src/secrets/mod.rs index f6ca57b08d1..c753015aa9e 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/mod.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/mod.rs @@ -1,3 +1,6 @@ pub(crate) mod callback; pub(crate) mod config; +mod error; pub(crate) mod resolved; + +pub(crate) use error::python_error; diff --git a/litellm-rust/crates/secrets-aws/src/error.rs b/litellm-rust/crates/secrets-aws/src/error.rs index 23595397a13..cc9e6b69786 100644 --- a/litellm-rust/crates/secrets-aws/src/error.rs +++ b/litellm-rust/crates/secrets-aws/src/error.rs @@ -6,6 +6,10 @@ pub enum Error { Auth(#[from] #[redact] litellm_auth_aws::Error), #[error("AWS region is not configured")] MissingRegion, + #[error("AWS Secrets Manager received a non-AWS operation context")] + InvalidOperationContext, + #[error("AWS Secrets Manager was constructed without context-aware configuration")] + OperationContextUnavailable, #[error("KMS response has no plaintext")] MissingPlaintext, #[error("AWS request timed out")] diff --git a/litellm-rust/crates/secrets-aws/src/secret_manager.rs b/litellm-rust/crates/secrets-aws/src/secret_manager.rs index 493cb1d2e8f..76c61534861 100644 --- a/litellm-rust/crates/secrets-aws/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-aws/src/secret_manager.rs @@ -16,7 +16,8 @@ use litellm_auth_aws::constants::{ }; use litellm_core_utils::settings::Lookup; use litellm_secrets_types::{ - BaseSecretManager, KeyManagementSettings, Secret, SecretValue, async_rotate_secret, + AwsOperationContext, BaseSecretManager, KeyManagementSettings, Secret, SecretOperationContext, + SecretValue, SecretWriteContext, async_rotate_secret, }; use serde_json::Value; @@ -25,9 +26,17 @@ use crate::{Error, auth}; #[derive(Clone)] pub struct AwsSecretsManagerV2 { client: Client, + context_client_factory: Option>, write_settings: AwsSecretWriteSettings, } +#[derive(Clone)] +struct ContextClientFactory { + settings: KeyManagementSettings, + environment: Arc, + endpoint_url: Option, +} + #[derive(Clone, Debug, Default)] pub struct AwsSecretWriteSettings { pub kms_key_id: Option, @@ -55,6 +64,19 @@ impl AwsSecretsManagerV2 { pub fn new(client: Client, write_settings: AwsSecretWriteSettings) -> Self { Self { client, + context_client_factory: None, + write_settings, + } + } + + fn with_context_client_factory( + client: Client, + write_settings: AwsSecretWriteSettings, + context_client_factory: ContextClientFactory, + ) -> Self { + Self { + client, + context_client_factory: Some(Box::new(context_client_factory)), write_settings, } } @@ -67,19 +89,18 @@ impl AwsSecretsManagerV2 { if use_aws_secret_manager != Some(true) { return Ok(None); } - let builder = aws_sdk_secretsmanager::Config::builder() - .behavior_version(BehaviorVersion::latest()) - .region(Region::new(auth::region(&settings, environment.as_ref())?)) - .credentials_provider(auth::Credentials::new(&settings, environment.clone())); - let config = match environment.get(AWS_BEDROCK_RUNTIME_ENDPOINT) { - Some(url) => builder - .endpoint_url(url.replace("bedrock-runtime", "secretsmanager")) - .build(), - None => builder.build(), + let context_client_factory = ContextClientFactory { + settings: settings.clone(), + environment: environment.clone(), + endpoint_url: environment + .get(AWS_BEDROCK_RUNTIME_ENDPOINT) + .map(|url| url.replace("bedrock-runtime", "secretsmanager")), }; - Ok(Some(Self::new( - Client::from_conf(config), + let client = context_client_factory.client(&AwsOperationContext::default())?; + Ok(Some(Self::with_context_client_factory( + client, (&settings).into(), + context_client_factory, ))) } @@ -118,7 +139,14 @@ impl AwsSecretsManagerV2 { } pub async fn async_read_secret(&self, name: &str) -> Result, Error> { - match self.client.get_secret_value().secret_id(name).send().await { + Self::async_read_secret_with_client(&self.client, name).await + } + + async fn async_read_secret_with_client( + client: &Client, + name: &str, + ) -> Result, Error> { + match client.get_secret_value().secret_id(name).send().await { Ok(response) => response .secret_string .map(SecretValue::new) @@ -149,8 +177,19 @@ impl AwsSecretsManagerV2 { value: &SecretValue, description: Option<&str>, ) -> Result { - let response = self - .client + self.async_write_secret_with_client_and_tags(&self.client, name, value, description, None) + .await + } + + async fn async_write_secret_with_client_and_tags( + &self, + client: &Client, + name: &str, + value: &SecretValue, + description: Option<&str>, + tags: Option<&BTreeMap>, + ) -> Result { + let response = client .create_secret() .name(name) .secret_string(value.expose()) @@ -161,7 +200,7 @@ impl AwsSecretsManagerV2 { .clone() .filter(|v| !v.is_empty()), ) - .set_tags(self.write_settings.tags.as_ref().map(|tags| { + .set_tags(tags.or(self.write_settings.tags.as_ref()).map(|tags| { tags.iter() .map(|(key, value)| Tag::builder().key(key).value(value).build()) .collect() @@ -171,7 +210,10 @@ impl AwsSecretsManagerV2 { .map_err(|error| Error::Create(Box::new(error)))?; if let Some(regions) = &self.write_settings.replica_regions && !regions.is_empty() - && self.async_replicate_secret(name, regions).await.is_err() + && self + .async_replicate_secret_with_client(client, name, regions) + .await + .is_err() { tracing::warn!("secret created but replication failed"); } @@ -182,11 +224,21 @@ impl AwsSecretsManagerV2 { &self, name: &str, regions: &[String], + ) -> Result, Error> { + self.async_replicate_secret_with_client(&self.client, name, regions) + .await + } + + async fn async_replicate_secret_with_client( + &self, + client: &Client, + name: &str, + regions: &[String], ) -> Result, Error> { if regions.is_empty() { return Ok(None); } - self.client + client .replicate_secret_to_regions() .secret_id(name) .set_add_replica_regions(Some( @@ -206,7 +258,17 @@ impl AwsSecretsManagerV2 { name: &str, value: &SecretValue, ) -> Result { - self.client + self.async_put_secret_value_with_client(&self.client, name, value) + .await + } + + async fn async_put_secret_value_with_client( + &self, + client: &Client, + name: &str, + value: &SecretValue, + ) -> Result { + client .put_secret_value() .secret_id(name) .secret_string(value.expose()) @@ -218,12 +280,22 @@ impl AwsSecretsManagerV2 { pub async fn async_delete_secret( &self, name: &str, - recovery_window_in_days: i64, + recovery_window_in_days: Option, ) -> Result { - self.client + self.async_delete_secret_with_client(&self.client, name, recovery_window_in_days) + .await + } + + async fn async_delete_secret_with_client( + &self, + client: &Client, + name: &str, + recovery_window_in_days: Option, + ) -> Result { + client .delete_secret() .secret_id(name) - .recovery_window_in_days(recovery_window_in_days) + .set_recovery_window_in_days(recovery_window_in_days.map(i64::from)) .send() .await .map_err(|error| Error::Delete(Box::new(error))) @@ -234,17 +306,105 @@ impl AwsSecretsManagerV2 { current_name: &str, new_name: &str, value: &SecretValue, + ) -> Result { + self.async_rotate_secret_with_context( + current_name, + new_name, + value, + &SecretOperationContext::default(), + ) + .await + } + + pub async fn async_rotate_secret_with_context( + &self, + current_name: &str, + new_name: &str, + value: &SecretValue, + context: &SecretOperationContext, ) -> Result { if current_name == new_name { + let client = self.client_for_context(context)?; return self - .async_put_secret_value(current_name, value) + .async_put_secret_value_with_client(&client, current_name, value) .await .map(RotationResponse::Updated); } - async_rotate_secret(self, current_name, new_name, value) + async_rotate_secret(self, current_name, new_name, value, context) .await .map(RotationResponse::Created) } + + fn client_for_context(&self, context: &SecretOperationContext) -> Result { + match context { + SecretOperationContext::Default => Ok(self.client.clone()), + SecretOperationContext::Aws(context) if context == &AwsOperationContext::default() => { + Ok(self.client.clone()) + } + SecretOperationContext::Aws(context) => self + .context_client_factory + .as_ref() + .ok_or(Error::OperationContextUnavailable)? + .client(context), + _ => Err(Error::InvalidOperationContext), + } + } +} + +impl ContextClientFactory { + fn client(&self, context: &AwsOperationContext) -> Result { + let settings = KeyManagementSettings { + aws_region_name: context + .region_name + .clone() + .or_else(|| self.settings.aws_region_name.clone()), + aws_role_name: context + .role_name + .clone() + .or_else(|| self.settings.aws_role_name.clone()), + aws_session_name: context + .session_name + .clone() + .or_else(|| self.settings.aws_session_name.clone()), + aws_external_id: context + .external_id + .clone() + .or_else(|| self.settings.aws_external_id.clone()), + aws_profile_name: context + .profile_name + .clone() + .or_else(|| self.settings.aws_profile_name.clone()), + aws_web_identity_token: context + .web_identity_token + .clone() + .or_else(|| self.settings.aws_web_identity_token.clone()), + aws_sts_endpoint: context + .sts_endpoint + .clone() + .or_else(|| self.settings.aws_sts_endpoint.clone()), + ..self.settings.clone() + }; + let builder = aws_sdk_secretsmanager::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new(auth::region( + &settings, + self.environment.as_ref(), + )?)) + .credentials_provider(auth::Credentials::new(&settings, self.environment.clone())); + let builder = match context.timeout { + Some(timeout) => builder.timeout_config( + aws_sdk_secretsmanager::config::timeout::TimeoutConfig::builder() + .operation_timeout(timeout) + .build(), + ), + None => builder, + }; + let config = match &self.endpoint_url { + Some(endpoint_url) => builder.endpoint_url(endpoint_url.clone()).build(), + None => builder.build(), + }; + Ok(Client::from_conf(config)) + } } impl BaseSecretManager for AwsSecretsManagerV2 { @@ -252,25 +412,40 @@ impl BaseSecretManager for AwsSecretsManagerV2 { type WriteResponse = CreateSecretOutput; type DeleteResponse = DeleteSecretOutput; - async fn async_read_secret(&self, name: &str) -> Result, Error> { - self.async_read_secret(name).await + async fn async_read_secret( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { + let client = self.client_for_context(context)?; + Self::async_read_secret_with_client(&client, name).await } async fn async_write_secret( &self, name: &str, value: &SecretValue, - description: Option<&str>, + context: &SecretWriteContext, ) -> Result { - self.async_write_secret(name, value, description).await + let client = self.client_for_context(&context.operation)?; + self.async_write_secret_with_client_and_tags( + &client, + name, + value, + context.description.as_deref(), + (!context.tags.is_empty()).then_some(&context.tags), + ) + .await } async fn async_delete_secret( &self, name: &str, - recovery_window_in_days: i64, + recovery_window_in_days: Option, + context: &SecretOperationContext, ) -> Result { - self.async_delete_secret(name, recovery_window_in_days) + let client = self.client_for_context(context)?; + self.async_delete_secret_with_client(&client, name, recovery_window_in_days) .await } } diff --git a/litellm-rust/crates/secrets-aws/tests/kms.rs b/litellm-rust/crates/secrets-aws/tests/kms.rs index 39a50297551..687cca104e0 100644 --- a/litellm-rust/crates/secrets-aws/tests/kms.rs +++ b/litellm-rust/crates/secrets-aws/tests/kms.rs @@ -3,12 +3,15 @@ use aws_sdk_kms::{ config::{BehaviorVersion, Credentials, Region, retry::RetryConfig}, }; use base64::{Engine, engine::general_purpose::STANDARD}; -use litellm_secrets_aws::AwsKms; +use litellm_secrets_aws::{AwsKms, Error, load_aws_kms}; +use litellm_secrets_types::KeyManagementSettings; +use rstest::rstest; use wiremock::{ Mock, MockServer, ResponseTemplate, matchers::{body_json, header}, }; +#[rstest] #[tokio::test] async fn kms_decrypt_calls_the_sdk_without_applying_lookup_policy() { let server = MockServer::start().await; @@ -40,20 +43,56 @@ async fn kms_decrypt_calls_the_sdk_without_applying_lookup_policy() { ); } -#[test] -fn disabled_kms_loader_does_not_require_environment_configuration() { - use litellm_secrets_aws::load_aws_kms; - use litellm_secrets_types::KeyManagementSettings; +#[rstest] +#[case::unset(None)] +#[case::disabled(Some(false))] +fn disabled_kms_loader_does_not_require_environment_configuration(#[case] enabled: Option) { use std::sync::Arc; - for enabled in [None, Some(false)] { - assert!( - load_aws_kms( - enabled, - &KeyManagementSettings::default(), - Arc::new(|_: &str| None) - ) - .unwrap() - .is_none() - ); - } + assert!( + load_aws_kms( + enabled, + &KeyManagementSettings::default(), + Arc::new(|_: &str| None) + ) + .unwrap() + .is_none() + ); +} + +#[rstest] +#[case::settings(Some("configured-region"), None)] +#[case::environment(None, Some("environment-region"))] +fn enabled_kms_loader_accepts_either_region_source( + #[case] configured_region: Option<&'static str>, + #[case] environment_region: Option<&'static str>, +) { + use std::sync::Arc; + let settings = KeyManagementSettings { + aws_region_name: configured_region.map(str::to_owned), + ..KeyManagementSettings::default() + }; + let environment = Arc::new(move |name: &str| { + (name == "AWS_REGION_NAME") + .then(|| environment_region.map(str::to_owned)) + .flatten() + }); + + assert!( + load_aws_kms(Some(true), &settings, environment) + .unwrap() + .is_some() + ); +} + +#[rstest] +fn enabled_kms_loader_rejects_missing_region() { + use std::sync::Arc; + assert!(matches!( + load_aws_kms( + Some(true), + &KeyManagementSettings::default(), + Arc::new(|_: &str| None), + ), + Err(Error::MissingRegion) + )); } diff --git a/litellm-rust/crates/secrets-aws/tests/secret_manager.rs b/litellm-rust/crates/secrets-aws/tests/secret_manager.rs index a410767cb5a..7dbc8b61374 100644 --- a/litellm-rust/crates/secrets-aws/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-aws/tests/secret_manager.rs @@ -1,11 +1,21 @@ -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; use aws_sdk_secretsmanager::{ Client, config::{BehaviorVersion, Credentials, Region, retry::RetryConfig}, }; use litellm_secrets_aws::{AwsSecretsManagerV2, Error, RotationResponse}; -use litellm_secrets_types::{KeyManagementSettings, SecretValue}; +use litellm_secrets_types::{ + AwsOperationContext, BaseSecretManager, KeyManagementSettings, SecretOperationContext, + SecretValue, SecretWriteContext, +}; +use rstest::{fixture, rstest}; use serde_json::json; use wiremock::{ Mock, MockServer, ResponseTemplate, @@ -25,12 +35,39 @@ fn manager(server: &MockServer, settings: KeyManagementSettings) -> AwsSecretsMa AwsSecretsManagerV2::new(client, (&settings).into()) } -#[rstest::rstest] +fn loaded_manager(server: &MockServer) -> AwsSecretsManagerV2 { + let endpoint_url = server.uri(); + let environment: Arc = + Arc::new(move |name: &str| match name { + "AWS_BEDROCK_RUNTIME_ENDPOINT" => Some(endpoint_url.clone()), + "AWS_ACCESS_KEY_ID" => Some("test".into()), + "AWS_SECRET_ACCESS_KEY" => Some("test".into()), + _ => None, + }); + AwsSecretsManagerV2::load_aws_secret_manager( + Some(true), + KeyManagementSettings { + aws_region_name: Some("us-east-1".into()), + ..Default::default() + }, + environment, + ) + .unwrap() + .unwrap() +} + +#[fixture] +fn default_settings() -> KeyManagementSettings { + KeyManagementSettings::default() +} + +#[rstest] #[case::string_value("KEY", Some("value"))] #[case::missing_value("missing", None)] #[case::non_string_value("BOOL", None)] #[tokio::test] async fn primary_lookup_preserves_read_semantics( + default_settings: KeyManagementSettings, #[case] name: &str, #[case] expected: Option<&str>, ) { @@ -45,7 +82,7 @@ async fn primary_lookup_preserves_read_semantics( .expect(1) .mount(&server) .await; - let manager = manager(&server, KeyManagementSettings::default()); + let manager = manager(&server, default_settings); assert_eq!( manager .read_secret_for_resolver(name, Some("primary"), &|_: &str| None) @@ -57,16 +94,19 @@ async fn primary_lookup_preserves_read_semantics( ); } -#[rstest::rstest] +#[rstest] #[case::access_key("AWS_ACCESS_KEY_ID")] #[case::secret_access_key("AWS_SECRET_ACCESS_KEY")] #[case::region_name("AWS_REGION_NAME")] #[case::region("AWS_REGION")] #[case::bedrock_endpoint("AWS_BEDROCK_RUNTIME_ENDPOINT")] #[tokio::test] -async fn bootstrap_keys_bypass_primary_lookup(#[case] name: &str) { +async fn bootstrap_keys_bypass_primary_lookup( + default_settings: KeyManagementSettings, + #[case] name: &str, +) { let server = MockServer::start().await; - let manager = manager(&server, KeyManagementSettings::default()); + let manager = manager(&server, default_settings); assert_eq!( manager .read_secret_for_resolver(name, Some("primary"), &|_: &str| Some("bootstrap".into())) @@ -79,8 +119,11 @@ async fn bootstrap_keys_bypass_primary_lookup(#[case] name: &str) { ); } +#[rstest] #[tokio::test] -async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() { +async fn failed_read_returns_none_but_invalid_primary_json_is_an_error( + default_settings: KeyManagementSettings, +) { let server = MockServer::start().await; Mock::given(body_partial_json(json!({"SecretId":"missing"}))) .respond_with( @@ -92,7 +135,11 @@ async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() { .respond_with(ResponseTemplate::new(200).set_body_json(json!({"SecretString":"not-json"}))) .mount(&server) .await; - let manager = manager(&server, KeyManagementSettings::default()); + Mock::given(body_partial_json(json!({"SecretId":"no-string"}))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name":"no-string"}))) + .mount(&server) + .await; + let manager = manager(&server, default_settings); assert!( manager .async_read_secret("missing") @@ -106,10 +153,17 @@ async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() { .await, Err(Error::PrimarySecret) )); + assert!(matches!( + manager.async_read_secret("no-string").await, + Err(Error::MissingString) + )); } +#[rstest] #[tokio::test] -async fn same_name_rotation_uses_put_and_returns_its_response() { +async fn same_name_rotation_uses_put_and_returns_its_response( + default_settings: KeyManagementSettings, +) { let server = MockServer::start().await; Mock::given(header("x-amz-target", "secretsmanager.PutSecretValue")) .and(body_partial_json( @@ -121,7 +175,7 @@ async fn same_name_rotation_uses_put_and_returns_its_response() { .expect(1) .mount(&server) .await; - let response = manager(&server, KeyManagementSettings::default()) + let response = manager(&server, default_settings) .async_rotate_secret("key", "key", &SecretValue::new("replacement")) .await .unwrap(); @@ -132,8 +186,11 @@ async fn same_name_rotation_uses_put_and_returns_its_response() { assert_eq!(server.received_requests().await.unwrap().len(), 1); } +#[rstest] #[tokio::test] -async fn renamed_rotation_reads_creates_verifies_then_deletes() { +async fn renamed_rotation_reads_creates_verifies_then_deletes( + default_settings: KeyManagementSettings, +) { let server = MockServer::start().await; let step = AtomicUsize::new(0); Mock::given(wiremock::matchers::method("POST")) @@ -176,7 +233,7 @@ async fn renamed_rotation_reads_creates_verifies_then_deletes() { .mount(&server) .await; assert!(matches!( - manager(&server, KeyManagementSettings::default()) + manager(&server, default_settings) .async_rotate_secret("old", "new", &SecretValue::new("replacement")) .await .unwrap(), @@ -184,6 +241,7 @@ async fn renamed_rotation_reads_creates_verifies_then_deletes() { )); } +#[rstest] #[tokio::test] async fn creation_passes_tags_and_kms_and_survives_replication_failure() { let server = MockServer::start().await; @@ -230,6 +288,112 @@ async fn creation_passes_tags_and_kms_and_survives_replication_failure() { ); } +#[rstest] +#[tokio::test] +async fn trait_write_uses_typed_write_context(default_settings: KeyManagementSettings) { + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.CreateSecret")) + .and(body_partial_json(json!({ + "Name": "key", + "SecretString": "value", + "Description": "created by caller", + "Tags": [{"Key": "stage", "Value": "test"}], + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name": "key"}))) + .expect(1) + .mount(&server) + .await; + let context = SecretWriteContext { + description: Some("created by caller".into()), + tags: std::collections::BTreeMap::from([("stage".into(), "test".into())]), + ..Default::default() + }; + let response = BaseSecretManager::async_write_secret( + &manager(&server, default_settings), + "key", + &SecretValue::new("value"), + &context, + ) + .await + .unwrap(); + assert_eq!(response.name(), Some("key")); +} + +#[rstest] +#[tokio::test] +async fn trait_delete_accepts_an_unspecified_recovery_window( + default_settings: KeyManagementSettings, +) { + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.DeleteSecret")) + .and(body_partial_json(json!({"SecretId": "key"}))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name": "key"}))) + .expect(1) + .mount(&server) + .await; + let response = BaseSecretManager::async_delete_secret( + &manager(&server, default_settings), + "key", + None, + &SecretOperationContext::default(), + ) + .await + .unwrap(); + assert_eq!(response.name(), Some("key")); +} + +#[rstest] +#[tokio::test] +async fn trait_read_uses_the_aws_region_from_its_operation_context() { + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .respond_with(|request: &wiremock::Request| { + let authorization = request + .headers + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + assert!(authorization.contains("/us-west-2/secretsmanager/aws4_request")); + ResponseTemplate::new(200).set_body_json(json!({"SecretString": "value"})) + }) + .expect(1) + .mount(&server) + .await; + let context = SecretOperationContext::Aws(AwsOperationContext { + region_name: Some("us-west-2".into()), + ..Default::default() + }); + let value = BaseSecretManager::async_read_secret(&loaded_manager(&server), "key", &context) + .await + .unwrap(); + assert_eq!(value.unwrap().expose(), "value"); +} + +#[rstest] +#[tokio::test] +async fn trait_read_applies_the_aws_operation_timeout() { + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .respond_with( + ResponseTemplate::new(200) + .set_delay(Duration::from_secs(1)) + .set_body_json(json!({"SecretString": "late"})), + ) + .expect(1) + .mount(&server) + .await; + let context = SecretOperationContext::Aws(AwsOperationContext { + timeout: Some(Duration::from_millis(30)), + ..Default::default() + }); + assert!(matches!( + BaseSecretManager::async_read_secret(&loaded_manager(&server), "key", &context).await, + Err(Error::Timeout) + )); +} + +#[rstest] #[tokio::test] async fn credential_failures_are_not_swallowed_as_missing_secrets() { use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future}; @@ -260,9 +424,9 @@ async fn credential_failures_are_not_swallowed_as_missing_secrets() { assert!(server.received_requests().await.unwrap().is_empty()); } +#[rstest] #[tokio::test] async fn read_timeout_is_an_error_and_cannot_be_mistaken_for_missing() { - use std::time::Duration; let server = MockServer::start().await; Mock::given(wiremock::matchers::method("POST")) .respond_with( @@ -291,12 +455,16 @@ async fn read_timeout_is_an_error_and_cannot_be_mistaken_for_missing() { )); } -#[rstest::rstest] +#[rstest] #[case::denied(400, "AccessDeniedException")] #[case::throttled(400, "ThrottlingException")] #[case::unavailable(503, "ServiceUnavailableException")] #[tokio::test] -async fn service_failures_remain_errors(#[case] status: u16, #[case] code: &str) { +async fn service_failures_remain_errors( + default_settings: KeyManagementSettings, + #[case] status: u16, + #[case] code: &str, +) { let server = MockServer::start().await; Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) .respond_with(ResponseTemplate::new(status).set_body_json(json!({"__type":code}))) @@ -304,7 +472,7 @@ async fn service_failures_remain_errors(#[case] status: u16, #[case] code: &str) .mount(&server) .await; assert!(matches!( - manager(&server, KeyManagementSettings::default()) + manager(&server, default_settings) .async_read_secret("key") .await, Err(Error::Read(_)) diff --git a/litellm-rust/crates/secrets-azure/src/key_vault.rs b/litellm-rust/crates/secrets-azure/src/key_vault.rs index e12289b83f5..81ad0331ca7 100644 --- a/litellm-rust/crates/secrets-azure/src/key_vault.rs +++ b/litellm-rust/crates/secrets-azure/src/key_vault.rs @@ -76,10 +76,7 @@ impl AzureKeyVault { .unwrap_or_default() } - pub async fn get_secret_from_azure_key_vault( - &self, - name: &str, - ) -> Result, Error> { + pub async fn get_secret(&self, name: &str) -> Result, Error> { let token = self .auth .get_azure_ad_token(&self.inputs, &|key| self.environment.get(key)) diff --git a/litellm-rust/crates/secrets-azure/tests/key_vault.rs b/litellm-rust/crates/secrets-azure/tests/key_vault.rs index cf9102d0b45..3b199f81264 100644 --- a/litellm-rust/crates/secrets-azure/tests/key_vault.rs +++ b/litellm-rust/crates/secrets-azure/tests/key_vault.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use litellm_secrets_azure::{AzureKeyVault, Error}; use litellm_secrets_types::{Secret, SecretValue}; +use rstest::{fixture, rstest}; use serde::Deserialize; use wiremock::{ Mock, MockServer, ResponseTemplate, @@ -17,6 +18,7 @@ fn manager(server: &MockServer) -> AzureKeyVault { .unwrap() } +#[rstest] #[tokio::test] async fn reads_secret_with_bearer_token_and_api_version() { let server = MockServer::start().await; @@ -32,7 +34,7 @@ async fn reads_secret_with_bearer_token_and_api_version() { .await; let secret = manager(&server) - .get_secret_from_azure_key_vault("OPENAI-API-KEY") + .get_secret("OPENAI-API-KEY") .await .unwrap() .unwrap(); @@ -40,6 +42,7 @@ async fn reads_secret_with_bearer_token_and_api_version() { assert_eq!(secret, Secret::String(SecretValue::new("s3cret"))); } +#[rstest] #[tokio::test] async fn percent_encodes_secret_name_path_segment() { let server = MockServer::start().await; @@ -53,7 +56,7 @@ async fn percent_encodes_secret_name_path_segment() { .await; let secret = manager(&server) - .get_secret_from_azure_key_vault("name/with spaces") + .get_secret("name/with spaces") .await .unwrap() .unwrap(); @@ -61,9 +64,12 @@ async fn percent_encodes_secret_name_path_segment() { assert_eq!(secret.as_str(), Some("value")); } -#[rstest::rstest] +#[rstest] #[case::not_found(404, None)] +#[case::unauthorized(401, Some(401))] #[case::forbidden(403, Some(403))] +#[case::throttled(429, Some(429))] +#[case::server_error(500, Some(500))] #[tokio::test] async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option) { let server = MockServer::start().await; @@ -73,9 +79,7 @@ async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option assert_eq!(result.unwrap(), None), @@ -83,6 +87,7 @@ async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option, + #[case] missing_environment: bool, +) { + let result = AzureKeyVault::new(Arc::new(move |name: &str| { + (name == "AZURE_KEY_VAULT_URI") + .then(|| uri.map(str::to_owned)) + .flatten() + })); + + if missing_environment { + assert!(matches!( + result, + Err(Error::MissingEnvironment("AZURE_KEY_VAULT_URI")) + )); + } else { + assert!(matches!(result, Err(Error::VaultUri))); + } } -#[rstest::rstest] -#[case("https://myvault.vault.azure.net", "https://vault.azure.net/.default")] -#[case( +#[rstest] +#[case::public_cloud("https://myvault.vault.azure.net", "https://vault.azure.net/.default")] +#[case::government_cloud( "https://v.vault.usgovcloudapi.net/", "https://vault.usgovcloudapi.net/.default" )] -#[case("http://localhost:8080", "https://localhost/.default")] -#[test] +#[case::local("http://localhost:8080", "https://localhost/.default")] fn derives_scope_from_vault_host(#[case] uri: &str, #[case] expected: &str) { let manager = AzureKeyVault::with_client( reqwest::Client::new(), @@ -139,6 +146,7 @@ fn derives_scope_from_vault_host(#[case] uri: &str, #[case] expected: &str) { assert_eq!(manager.scope(), expected); } +#[rstest] #[tokio::test] async fn missing_credentials_do_not_request_vault() { let server = MockServer::start().await; @@ -150,7 +158,7 @@ async fn missing_credentials_do_not_request_vault() { assert!( manager_without_credentials(&server) - .get_secret_from_azure_key_vault("NAME") + .get_secret("NAME") .await .is_err() ); @@ -192,11 +200,15 @@ struct FixtureExpected { error: Option, } +#[fixture] +fn parity_fixture() -> Fixture { + serde_json::from_str(include_str!("fixtures/key_vault_parity.json")).unwrap() +} + +#[rstest] #[tokio::test] -async fn parity_fixture_matches_python_backend_contract() { - let fixture: Fixture = - serde_json::from_str(include_str!("fixtures/key_vault_parity.json")).unwrap(); - for case in fixture.cases { +async fn parity_fixture_matches_python_backend_contract(parity_fixture: Fixture) { + for case in parity_fixture.cases { let server = MockServer::start().await; Mock::given(path(format!("/secrets/{}", case.secret_name))) .respond_with( @@ -205,9 +217,7 @@ async fn parity_fixture_matches_python_backend_contract() { .expect(1) .mount(&server) .await; - let result = manager(&server) - .get_secret_from_azure_key_vault(&case.secret_name) - .await; + let result = manager(&server).get_secret(&case.secret_name).await; if case.expected.missing == Some(true) { assert_eq!(result.unwrap(), None); } else if case.expected.error == Some(true) { diff --git a/litellm-rust/crates/secrets-azure/tests/live.rs b/litellm-rust/crates/secrets-azure/tests/live.rs index a062ba95070..18306382613 100644 --- a/litellm-rust/crates/secrets-azure/tests/live.rs +++ b/litellm-rust/crates/secrets-azure/tests/live.rs @@ -3,18 +3,16 @@ use std::sync::Arc; use litellm_core_utils::settings::ProcessEnvironment; use litellm_secrets_azure::AzureKeyVault; use litellm_secrets_types::Secret; +use rstest::rstest; +#[rstest] #[tokio::test] #[ignore] async fn reads_a_real_secret() { let environment = Arc::new(ProcessEnvironment); let manager = AzureKeyVault::new(environment).unwrap(); let name = std::env::var("AZURE_KEY_VAULT_LIVE_SECRET_NAME").unwrap(); - let secret = manager - .get_secret_from_azure_key_vault(&name) - .await - .unwrap() - .unwrap(); + let secret = manager.get_secret(&name).await.unwrap().unwrap(); assert!(matches!(&secret, Secret::String(_))); let host = std::env::var("AZURE_KEY_VAULT_URI") .unwrap() diff --git a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs index 9d6eaaf1c4e..80e6fb12f9e 100644 --- a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs @@ -2,7 +2,10 @@ use std::{fs, sync::Arc, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_core_utils::settings::Lookup; -use litellm_secrets_types::{BaseSecretManager, SecretValue, validate_secret_name}; +use litellm_secrets_types::{ + BaseSecretManager, SecretOperationContext, SecretValue, SecretWriteContext, + validate_secret_name, +}; use moka::future::Cache; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; @@ -140,7 +143,7 @@ impl CyberArkSecretManager { .map_err(|_| Error::Endpoint) } - async fn authenticate(&self) -> Result { + async fn authenticate(&self, context: &SecretOperationContext) -> Result { if let Some(token) = self.token.get(&()).await { return Ok(token); } @@ -155,12 +158,12 @@ impl CyberArkSecretManager { self.account, self.username )) .map_err(|_| Error::Endpoint)?; - let response = self - .client - .post(url) - .body(self.api_key.expose().to_owned()) - .send() - .await?; + let response = with_timeout( + self.client.post(url).body(self.api_key.expose().to_owned()), + context, + ) + .send() + .await?; if !response.status().is_success() { return Err(Error::AuthStatus(response.status().as_u16())); } @@ -169,23 +172,37 @@ impl CyberArkSecretManager { Ok(token) } - async fn authorization_header(&self) -> Result { + async fn authorization_header( + &self, + context: &SecretOperationContext, + ) -> Result { Ok(format!( "Token token=\"{}\"", - self.authenticate().await?.expose() + self.authenticate(context).await?.expose() )) } pub async fn async_read_secret(&self, name: &str) -> Result, Error> { + self.async_read_secret_with_context(name, &SecretOperationContext::default()) + .await + } + + pub async fn async_read_secret_with_context( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { if let Some(value) = self.secrets.get(name).await { return Ok(Some(value)); } - let response = self - .client - .get(self.secret_url(name)?) - .header("Authorization", self.authorization_header().await?) - .send() - .await?; + let response = with_timeout( + self.client + .get(self.secret_url(name)?) + .header("Authorization", self.authorization_header(context).await?), + context, + ) + .send() + .await?; if response.status() == reqwest::StatusCode::NOT_FOUND { return Ok(None); } @@ -198,20 +215,38 @@ impl CyberArkSecretManager { } pub async fn async_write_secret( + &self, + name: &str, + value: &SecretValue, + description: Option<&str>, + ) -> Result<(), Error> { + self.async_write_secret_with_context( + name, + value, + description, + &SecretOperationContext::default(), + ) + .await + } + + pub async fn async_write_secret_with_context( &self, name: &str, value: &SecretValue, _description: Option<&str>, + context: &SecretOperationContext, ) -> Result<(), Error> { validate_secret_name(name)?; - self.ensure_variable_exists(name).await; - let response = self - .client - .post(self.secret_url(name)?) - .header("Authorization", self.authorization_header().await?) - .body(value.expose().to_owned()) - .send() - .await?; + self.ensure_variable_exists(name, context).await; + let response = with_timeout( + self.client + .post(self.secret_url(name)?) + .header("Authorization", self.authorization_header(context).await?) + .body(value.expose().to_owned()), + context, + ) + .send() + .await?; if !response.status().is_success() { return Err(Error::Status(response.status().as_u16())); } @@ -219,7 +254,7 @@ impl CyberArkSecretManager { Ok(()) } - async fn ensure_variable_exists(&self, name: &str) { + async fn ensure_variable_exists(&self, name: &str, context: &SecretOperationContext) { let policy_url = self .endpoint .join(&format!("policies/{}/policy/root", self.account)); @@ -227,7 +262,7 @@ impl CyberArkSecretManager { tracing::warn!("Could not build CyberArk policy endpoint"); return; }; - let Ok(authorization) = self.authorization_header().await else { + let Ok(authorization) = self.authorization_header(context).await else { tracing::warn!("Could not authenticate while ensuring CyberArk variable exists"); return; }; @@ -235,14 +270,16 @@ impl CyberArkSecretManager { "- !variable {}\n", serde_json::to_string(name).expect("serializing a string cannot fail") ); - let response = self - .client - .post(policy_url) - .header("Authorization", authorization) - .header("Content-Type", "application/x-yaml") - .body(body) - .send() - .await; + let response = with_timeout( + self.client + .post(policy_url) + .header("Authorization", authorization) + .header("Content-Type", "application/x-yaml") + .body(body), + context, + ) + .send() + .await; match response { Ok(response) if response.status().is_success() => {} Ok(response) @@ -271,7 +308,21 @@ impl CyberArkSecretManager { pub async fn async_delete_secret( &self, name: &str, - _recovery_window_in_days: i64, + recovery_window_in_days: Option, + ) -> Result { + self.async_delete_secret_with_context( + name, + recovery_window_in_days, + &SecretOperationContext::default(), + ) + .await + } + + pub async fn async_delete_secret_with_context( + &self, + name: &str, + _recovery_window_in_days: Option, + _context: &SecretOperationContext, ) -> Result { tracing::warn!( "CyberArk Conjur does not support direct secret deletion. Secrets must be removed through policy updates." @@ -286,29 +337,50 @@ impl BaseSecretManager for CyberArkSecretManager { type WriteResponse = (); type DeleteResponse = DeleteOutcome; - async fn async_read_secret(&self, name: &str) -> Result, Error> { - self.async_read_secret(name).await + async fn async_read_secret( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { + self.async_read_secret_with_context(name, context).await } async fn async_write_secret( &self, name: &str, value: &SecretValue, - description: Option<&str>, + context: &SecretWriteContext, ) -> Result<(), Error> { - self.async_write_secret(name, value, description).await + self.async_write_secret_with_context( + name, + value, + context.description.as_deref(), + &context.operation, + ) + .await } async fn async_delete_secret( &self, name: &str, - recovery_window_in_days: i64, + recovery_window_in_days: Option, + context: &SecretOperationContext, ) -> Result { - self.async_delete_secret(name, recovery_window_in_days) + self.async_delete_secret_with_context(name, recovery_window_in_days, context) .await } } +fn with_timeout( + request: reqwest::RequestBuilder, + context: &SecretOperationContext, +) -> reqwest::RequestBuilder { + match context.timeout() { + Some(timeout) => request.timeout(timeout), + None => request, + } +} + fn normalize_endpoint(mut endpoint: reqwest::Url) -> reqwest::Url { if !endpoint.path().ends_with('/') { endpoint.set_path(&format!("{}/", endpoint.path())); diff --git a/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs b/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs index fd7198b70fb..59964d90d50 100644 --- a/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs @@ -2,7 +2,10 @@ use std::{sync::Arc, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_secrets_cyberark::{CyberArkSecretManager, DeleteOutcome, Error}; -use litellm_secrets_types::SecretValue; +use litellm_secrets_types::{ + BaseSecretManager, CyberarkOperationContext, SecretOperationContext, SecretValue, +}; +use rstest::{fixture, rstest}; use serde::Deserialize; use wiremock::{ Match, Mock, MockServer, Request, ResponseTemplate, @@ -40,7 +43,8 @@ impl Match for RawPath { } } -fn fixture() -> ParityFixture { +#[fixture] +fn parity_fixture() -> ParityFixture { serde_json::from_str(include_str!("fixtures/parity.json")).unwrap() } @@ -65,6 +69,7 @@ async fn mount_auth(server: &MockServer, expected: u64) { .await; } +#[rstest] #[tokio::test] async fn successful_reads_cache_auth_secret_and_redact_values() { let server = MockServer::start().await; @@ -89,6 +94,7 @@ async fn successful_reads_cache_auth_secret_and_redact_values() { } } +#[rstest] #[tokio::test] async fn concurrent_reads_share_authentication_request() { let server = MockServer::start().await; @@ -122,7 +128,7 @@ async fn concurrent_reads_share_authentication_request() { assert_eq!(second.unwrap().unwrap().expose(), "value"); } -#[rstest::rstest] +#[rstest] #[case::not_found(404)] #[case::unauthorized(401)] #[case::forbidden(403)] @@ -162,6 +168,7 @@ async fn failed_reads_are_not_cached(#[case] status: u16) { } } +#[rstest] #[tokio::test] async fn failed_authentication_is_not_cached_and_does_not_read_secret() { let server = MockServer::start().await; @@ -199,6 +206,29 @@ async fn failed_authentication_is_not_cached_and_does_not_read_secret() { ); } +#[rstest] +#[tokio::test] +async fn trait_read_applies_cyberark_operation_timeout_to_authentication() { + let server = MockServer::start().await; + Mock::given(path("/authn/acct/admin/authenticate")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(TOKEN_JSON) + .set_delay(Duration::from_millis(50)), + ) + .mount(&server) + .await; + let manager = manager(&server, Duration::from_secs(60)); + let context = SecretOperationContext::Cyberark(CyberarkOperationContext { + timeout: Some(Duration::from_millis(10)), + }); + + let result = BaseSecretManager::async_read_secret(&manager, "key", &context).await; + + assert!(matches!(result, Err(Error::Http(error)) if error.is_timeout())); +} + +#[rstest] #[tokio::test] async fn expired_tokens_and_secrets_are_fetched_again() { let server = MockServer::start().await; @@ -215,13 +245,14 @@ async fn expired_tokens_and_secrets_are_fetched_again() { } } -#[rstest::rstest] +#[rstest] +#[case::plain("OPENAI_API_KEY")] +#[case::path("team/app/key")] +#[case::punctuation("a b+c.d-e_f~g")] +#[case::quote("needs \"quote\"")] #[tokio::test] -async fn secret_names_use_python_quote_encoding( - #[values("OPENAI_API_KEY", "team/app/key", "a b+c.d-e_f~g", "needs \"quote\"")] name: &str, -) { - let fixture = fixture(); - let secret = fixture +async fn secret_names_use_python_quote_encoding(parity_fixture: ParityFixture, #[case] name: &str) { + let secret = parity_fixture .secrets .iter() .find(|secret| secret.name == name) @@ -244,11 +275,11 @@ async fn secret_names_use_python_quote_encoding( ); } -#[rstest::rstest] -#[case(201)] -#[case(409)] -#[case(422)] -#[case(500)] +#[rstest] +#[case::created(201)] +#[case::already_exists(409)] +#[case::unprocessable(422)] +#[case::server_error(500)] #[tokio::test] async fn writes_tolerate_policy_status_and_cache_value(#[case] policy_status: u16) { let server = MockServer::start().await; @@ -282,6 +313,7 @@ async fn writes_tolerate_policy_status_and_cache_value(#[case] policy_status: u1 ); } +#[rstest] #[tokio::test] async fn failed_value_write_is_not_cached() { let server = MockServer::start().await; @@ -319,13 +351,17 @@ async fn failed_value_write_is_not_cached() { ); } +#[rstest] +#[case::parent("../etc")] +#[case::embedded_parent("team/../etc")] +#[case::control("key\n")] #[tokio::test] -async fn unsafe_names_fail_before_http_calls() { +async fn unsafe_names_fail_before_http_calls(#[case] name: &str) { let server = MockServer::start().await; let manager = manager(&server, Duration::from_secs(60)); assert!(matches!( manager - .async_write_secret("../etc", &SecretValue::new("v"), None) + .async_write_secret(name, &SecretValue::new("v"), None) .await, Err(Error::Operation( litellm_secrets_types::Error::UnsafeSecretName @@ -333,6 +369,7 @@ async fn unsafe_names_fail_before_http_calls() { )); } +#[rstest] #[tokio::test] async fn delete_invalidates_cache_and_reports_not_supported() { let server = MockServer::start().await; @@ -353,7 +390,7 @@ async fn delete_invalidates_cache_and_reports_not_supported() { "v" ); assert_eq!( - manager.async_delete_secret("key", 7).await.unwrap(), + manager.async_delete_secret("key", Some(7)).await.unwrap(), DeleteOutcome::NotSupported ); assert_eq!( @@ -367,7 +404,7 @@ async fn delete_invalidates_cache_and_reports_not_supported() { ); } -#[test] +#[rstest] fn new_validates_credentials_before_license_and_configuration() { let empty: Arc = Arc::new(|_: &str| None); @@ -413,6 +450,7 @@ fn new_validates_credentials_before_license_and_configuration() { )); } +#[rstest] #[tokio::test] async fn new_reads_environment_defaults_end_to_end() { let server = MockServer::start().await; @@ -446,7 +484,7 @@ async fn new_reads_environment_defaults_end_to_end() { ); } -#[test] +#[rstest] fn new_reports_missing_client_certificate_files() { assert!(matches!( CyberArkSecretManager::new( @@ -461,6 +499,7 @@ fn new_reports_missing_client_certificate_files() { )); } +#[rstest] #[tokio::test] async fn trailing_slash_endpoint_preserves_base_path() { let server = MockServer::start().await; @@ -494,23 +533,25 @@ async fn trailing_slash_endpoint_preserves_base_path() { ); } -#[test] -fn parity_fixture_matches_authentication_contract() { - let fixture = fixture(); - assert_eq!(fixture.endpoint, "http://conjur.test:8080"); - assert_eq!(fixture.account, "acct"); - assert_eq!(fixture.username, "admin"); - assert_eq!(fixture.api_key, "k3y"); - assert_eq!(fixture.authenticate_path, "/authn/acct/admin/authenticate"); - assert_eq!(fixture.token_json, TOKEN_JSON); +#[rstest] +fn parity_fixture_matches_authentication_contract(parity_fixture: ParityFixture) { + assert_eq!(parity_fixture.endpoint, "http://conjur.test:8080"); + assert_eq!(parity_fixture.account, "acct"); + assert_eq!(parity_fixture.username, "admin"); + assert_eq!(parity_fixture.api_key, "k3y"); assert_eq!( - fixture.authorization_header, + parity_fixture.authenticate_path, + "/authn/acct/admin/authenticate" + ); + assert_eq!(parity_fixture.token_json, TOKEN_JSON); + assert_eq!( + parity_fixture.authorization_header, format!("Token token=\"{}\"", STANDARD.encode(TOKEN_JSON)) ); - assert_eq!(fixture.policy_path, "/policies/acct/policy/root"); - assert_eq!(fixture.secrets.len(), 4); + assert_eq!(parity_fixture.policy_path, "/policies/acct/policy/root"); + assert_eq!(parity_fixture.secrets.len(), 4); assert_eq!( - fixture.secrets[1].policy_body, + parity_fixture.secrets[1].policy_body, "- !variable \"team/app/key\"\n" ); } diff --git a/litellm-rust/crates/secrets-google/tests/kms.rs b/litellm-rust/crates/secrets-google/tests/kms.rs index 667ecd268c8..9739a46b10c 100644 --- a/litellm-rust/crates/secrets-google/tests/kms.rs +++ b/litellm-rust/crates/secrets-google/tests/kms.rs @@ -1,11 +1,14 @@ use base64::{Engine, engine::general_purpose::STANDARD}; use google_cloud_kms_v1::client::KeyManagementService; -use litellm_secrets_google::GoogleKms; +use litellm_core_utils::settings::Lookup; +use litellm_secrets_google::{Error, GoogleKms, kms::validate_environment}; +use rstest::rstest; use wiremock::{ Mock, MockServer, ResponseTemplate, matchers::{body_json, path}, }; +#[rstest] #[tokio::test] async fn google_kms_decrypts_using_the_configured_resource() { let server = MockServer::start().await; @@ -35,15 +38,49 @@ async fn google_kms_decrypts_using_the_configured_resource() { ); } +#[rstest] +#[case::unset(None)] +#[case::disabled(Some(false))] #[tokio::test] -async fn disabled_google_kms_loader_does_not_require_environment_configuration() { +async fn disabled_google_kms_loader_does_not_require_environment_configuration( + #[case] enabled: Option, +) { use std::sync::Arc; - for enabled in [None, Some(false)] { - assert!( - litellm_secrets_google::load_google_kms(enabled, Arc::new(|_: &str| None)) - .await - .unwrap() - .is_none() - ); - } + assert!( + litellm_secrets_google::load_google_kms(enabled, Arc::new(|_: &str| None)) + .await + .unwrap() + .is_none() + ); +} + +#[rstest] +#[case::credentials_missing(None, None, "GOOGLE_APPLICATION_CREDENTIALS")] +#[case::resource_missing(Some("credentials"), None, "GOOGLE_KMS_RESOURCE_NAME")] +fn enabled_google_kms_requires_all_environment_values( + #[case] credentials: Option<&str>, + #[case] resource: Option<&str>, + #[case] missing: &'static str, +) { + let environment = move |name: &str| match name { + "GOOGLE_APPLICATION_CREDENTIALS" => credentials.map(str::to_owned), + "GOOGLE_KMS_RESOURCE_NAME" => resource.map(str::to_owned), + _ => None, + }; + + assert!(matches!( + validate_environment(&environment as &dyn Lookup), + Err(Error::MissingEnvironment(name)) if name == missing + )); +} + +#[rstest] +fn complete_google_kms_environment_is_valid() { + let environment = |name: &str| match name { + "GOOGLE_APPLICATION_CREDENTIALS" => Some("credentials".to_owned()), + "GOOGLE_KMS_RESOURCE_NAME" => Some("resource".to_owned()), + _ => None, + }; + + assert!(validate_environment(&environment as &dyn Lookup).is_ok()); } diff --git a/litellm-rust/crates/secrets-google/tests/secret_manager.rs b/litellm-rust/crates/secrets-google/tests/secret_manager.rs index b3b1d29e62c..867dcf935c1 100644 --- a/litellm-rust/crates/secrets-google/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-google/tests/secret_manager.rs @@ -2,6 +2,7 @@ use std::{sync::Arc, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_secrets_google::{Error, GoogleSecretManager}; +use rstest::{fixture, rstest}; use wiremock::{ Mock, MockServer, ResponseTemplate, @@ -20,11 +21,17 @@ fn manager(server: &MockServer, always_read: bool, ttl: Duration) -> GoogleSecre .unwrap() } -#[rstest::rstest] +#[fixture] +fn default_ttl() -> Duration { + Duration::from_secs(60) +} + +#[rstest] #[case::nonempty("private-value")] #[case::empty("")] #[tokio::test] async fn successful_reads_use_auth_latest_version_and_cache_including_empty_values( + default_ttl: Duration, #[case] value: &str, ) { let server = MockServer::start().await; @@ -39,7 +46,7 @@ async fn successful_reads_use_auth_latest_version_and_cache_including_empty_valu .expect(1) .mount(&server) .await; - let manager = manager(&server, false, Duration::from_secs(60)); + let manager = manager(&server, false, default_ttl); for _ in 0..2 { assert_eq!( manager @@ -54,21 +61,44 @@ async fn successful_reads_use_auth_latest_version_and_cache_including_empty_valu } } -#[rstest::rstest] -#[case::not_found(404, serde_json::json!({}))] -#[case::unauthorized(401, serde_json::json!({}))] -#[case::forbidden(403, serde_json::json!({}))] -#[case::throttled(429, serde_json::json!({}))] -#[case::unavailable(503, serde_json::json!({}))] -#[case::missing_payload(200, serde_json::json!({"payload":{}}))] -#[case::invalid_base64(200, serde_json::json!({"payload":{"data":"%%%"}}))] +enum ExpectedReadFailure { + Missing, + Status(u16), + MissingPayload, + Base64, + Utf8, +} + +#[rstest] +#[case::not_found(404, serde_json::json!({}), ExpectedReadFailure::Missing)] +#[case::unauthorized(401, serde_json::json!({}), ExpectedReadFailure::Status(401))] +#[case::forbidden(403, serde_json::json!({}), ExpectedReadFailure::Status(403))] +#[case::throttled(429, serde_json::json!({}), ExpectedReadFailure::Status(429))] +#[case::unavailable(503, serde_json::json!({}), ExpectedReadFailure::Status(503))] +#[case::missing_payload( + 200, + serde_json::json!({"payload":{}}), + ExpectedReadFailure::MissingPayload +)] +#[case::invalid_base64( + 200, + serde_json::json!({"payload":{"data":"%%%"}}), + ExpectedReadFailure::Base64 +)] +#[case::invalid_utf8( + 200, + serde_json::json!({"payload":{"data":STANDARD.encode([0xff])}}), + ExpectedReadFailure::Utf8 +)] #[tokio::test] async fn failed_or_missing_reads_are_not_cached( + default_ttl: Duration, #[case] status: u16, #[case] body: serde_json::Value, + #[case] expected: ExpectedReadFailure, ) { let server = MockServer::start().await; - let manager = manager(&server, false, Duration::from_secs(60)); + let manager = manager(&server, false, default_ttl); let failing = Mock::given(path( "/v1/projects/project/secrets/key/versions/latest:access", )) @@ -77,13 +107,16 @@ async fn failed_or_missing_reads_are_not_cached( .mount_as_scoped(&server) .await; let result = manager.get_secret_from_google_secret_manager("key").await; - match status { - 404 => assert_eq!(result.unwrap(), None), - 200 => assert!(matches!( - result, - Err(Error::MissingPayload | Error::Base64(_)) - )), - status => assert!(matches!(result, Err(Error::Status(actual)) if actual == status)), + match expected { + ExpectedReadFailure::Missing => assert_eq!(result.unwrap(), None), + ExpectedReadFailure::Status(expected) => { + assert!(matches!(result, Err(Error::Status(actual)) if actual == expected)); + } + ExpectedReadFailure::MissingPayload => { + assert!(matches!(result, Err(Error::MissingPayload))); + } + ExpectedReadFailure::Base64 => assert!(matches!(result, Err(Error::Base64(_)))), + ExpectedReadFailure::Utf8 => assert!(matches!(result, Err(Error::Utf8))), } drop(failing); Mock::given(path( @@ -109,7 +142,7 @@ async fn failed_or_missing_reads_are_not_cached( } } -#[rstest::rstest] +#[rstest] #[case::always_read(true, Duration::from_secs(60))] #[case::expired_cache(false, Duration::from_millis(1))] #[tokio::test] @@ -141,7 +174,7 @@ async fn always_read_and_expired_cache_fetch_again( } } -#[test] +#[rstest] fn google_manager_requires_host_license_and_project_configuration() { assert!(matches!( GoogleSecretManager::new(Arc::new(|_: &str| None), false), @@ -155,13 +188,29 @@ fn google_manager_requires_host_license_and_project_configuration() { )); } -#[rstest::rstest] -#[case("true")] -#[case("null")] -#[case("\"text\"")] -#[case("{\"key\":1}")] +#[rstest] +#[case::provider_specific("GOOGLE_SECRET_MANAGER_REFRESH_INTERVAL")] +#[case::shared("SECRET_MANAGER_REFRESH_INTERVAL")] +fn google_manager_rejects_invalid_refresh_intervals(#[case] variable: &'static str) { + let environment = Arc::new(move |name: &str| match name { + "GOOGLE_SECRET_MANAGER_PROJECT_ID" => Some("project".to_owned()), + name if name == variable => Some("not-a-number".to_owned()), + _ => None, + }); + + assert!(matches!( + GoogleSecretManager::new(environment, true), + Err(Error::RefreshInterval) + )); +} + +#[rstest] +#[case::boolean("true")] +#[case::null("null")] +#[case::string("\"text\"")] +#[case::object("{\"key\":1}")] #[tokio::test] -async fn cache_preserves_raw_values(#[case] raw: &str) { +async fn cache_preserves_raw_values(default_ttl: Duration, #[case] raw: &str) { let server = MockServer::start().await; Mock::given(path( "/v1/projects/project/secrets/key/versions/latest:access", @@ -173,7 +222,7 @@ async fn cache_preserves_raw_values(#[case] raw: &str) { .expect(1) .mount(&server) .await; - let manager = manager(&server, false, Duration::from_secs(60)); + let manager = manager(&server, false, default_ttl); for _ in 0..2 { assert_eq!( manager diff --git a/litellm-rust/crates/secrets-hashicorp/src/error.rs b/litellm-rust/crates/secrets-hashicorp/src/error.rs index e26033af085..664f9ab18dd 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/error.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/error.rs @@ -4,6 +4,8 @@ pub enum Error { EnterpriseRequired, #[error("invalid secret name")] InvalidSecretName(#[from] litellm_secrets_types::Error), + #[error("HashiCorp Vault received an incompatible operation context")] + InvalidOperationContext, #[error("HashiCorp Vault client failed")] Client( #[from] @@ -29,6 +31,8 @@ pub enum Error { MalformedPayload, #[error("HashiCorp Vault secret value is not a string")] NonStringValue, + #[error("HashiCorp Vault operation timed out")] + Timeout, #[error("invalid HashiCorp Vault refresh interval")] RefreshInterval, } diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs index 3ad9a549438..a4d1efb4ff6 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs @@ -1,13 +1,15 @@ use std::{ collections::HashMap, fmt, + future::Future, sync::Arc, time::{Duration, Instant}, }; use litellm_core_utils::settings::Lookup; use litellm_secrets_types::{ - BaseSecretManager, SecretValue, async_rotate_secret, validate_secret_name, + BaseSecretManager, HashicorpOperationContext, SecretOperationContext, SecretValue, + SecretWriteContext, async_rotate_secret, validate_secret_name, }; use moka::future::Cache; use rustify::errors::ClientError as RustifyClientError; @@ -31,17 +33,23 @@ struct CachedClient { expires_at: Option, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, Hash, PartialEq, Eq)] pub struct SecretLocation { pub namespace: Option, pub mount: String, pub path: String, } +#[derive(Clone, Hash, PartialEq, Eq)] +struct CacheKey { + location: SecretLocation, + data_key: String, +} + #[derive(Clone)] pub struct HashicorpVault { config: HashicorpVaultConfig, - cache: Cache, + cache: Cache, auth_client: Arc>>, } @@ -71,7 +79,7 @@ impl HashicorpVault { if !enterprise_enabled { return Err(Error::EnterpriseRequired); } - let cache: Cache = Cache::builder() + let cache: Cache = Cache::builder() .max_capacity(CACHE_CAPACITY) .time_to_live(config.refresh_interval) .build(); @@ -83,9 +91,21 @@ impl HashicorpVault { } pub fn secret_location(&self, secret_name: &str) -> Result { + self.secret_location_with_context(secret_name, &SecretOperationContext::default()) + } + + pub fn secret_location_with_context( + &self, + secret_name: &str, + context: &SecretOperationContext, + ) -> Result { validate_secret_name(secret_name).map_err(Error::InvalidSecretName)?; + let operation: Option<&HashicorpOperationContext> = hashicorp_context(context)?; let path: String = [ - self.config.path_prefix.clone(), + operation + .and_then(|operation| operation.path_prefix.as_deref()) + .and_then(path_component) + .or_else(|| self.config.path_prefix.clone()), Some(secret_name.to_owned()), ] .into_iter() @@ -94,7 +114,10 @@ impl HashicorpVault { .join("/"); Ok(SecretLocation { namespace: self.config.secret_namespace().map(str::to_owned), - mount: self.config.mount.clone(), + mount: operation + .and_then(|operation| operation.mount.as_deref()) + .and_then(path_component) + .unwrap_or_else(|| self.config.mount.clone()), path, }) } @@ -104,19 +127,37 @@ impl HashicorpVault { } pub async fn async_read_secret(&self, secret_name: &str) -> Result, Error> { - let location: SecretLocation = self.secret_location(secret_name)?; - let cache_key: String = cache_key(&location); + self.async_read_secret_with_context(secret_name, &SecretOperationContext::default()) + .await + } + + pub async fn async_read_secret_with_context( + &self, + secret_name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { + let location: SecretLocation = self.secret_location_with_context(secret_name, context)?; + let data_key: String = data_key(context)?; + let cache_key = CacheKey { + location: location.clone(), + data_key: data_key.clone(), + }; if let Some(value) = self.cache.get(&cache_key).await { return Ok(Some(value)); } - let client: Arc = self.vault_client().await?; - let data: HashMap = + let data: Option> = with_timeout(context, async { + let client: Arc = self.vault_client().await?; match kv2::read(client.as_ref(), &location.mount, &location.path).await { - Ok(data) => data, - Err(error) if api_status(&error) == Some(404) => return Ok(None), - Err(error) => return Err(map_api_error(error, ErrorContext::Read)), - }; - let Some(value) = data.get("key") else { + Ok(data) => Ok(Some(data)), + Err(error) if api_status(&error) == Some(404) => Ok(None), + Err(error) => Err(map_api_error(error, ErrorContext::Read)), + } + }) + .await?; + let Some(data) = data else { + return Ok(None); + }; + let Some(value) = data.get(&data_key) else { return Ok(None); }; let value: &str = value.as_str().ok_or(Error::NonStringValue)?; @@ -131,11 +172,29 @@ impl HashicorpVault { value: SecretValue, description: Option<&str>, ) -> Result { - let location: SecretLocation = self.secret_location(secret_name)?; - let cache_key: String = cache_key(&location); - let data: HashMap = match description { + self.async_write_secret_with_context( + secret_name, + &value, + &SecretWriteContext { + description: description.map(str::to_owned), + ..SecretWriteContext::default() + }, + ) + .await + } + + pub async fn async_write_secret_with_context( + &self, + secret_name: &str, + value: &SecretValue, + context: &SecretWriteContext, + ) -> Result { + let location: SecretLocation = + self.secret_location_with_context(secret_name, &context.operation)?; + let data_key: String = data_key(&context.operation)?; + let data: HashMap = match context.description.as_deref() { Some(description) => [ - ("key".to_owned(), Value::String(value.expose().to_owned())), + (data_key, Value::String(value.expose().to_owned())), ( "description".to_owned(), Value::String(description.to_owned()), @@ -143,27 +202,41 @@ impl HashicorpVault { ] .into_iter() .collect(), - None => [("key".to_owned(), Value::String(value.expose().to_owned()))] + None => [(data_key, Value::String(value.expose().to_owned()))] .into_iter() .collect(), }; - let client: Arc = self.vault_client().await?; - let metadata = kv2::set(client.as_ref(), &location.mount, &location.path, &data) - .await - .map_err(|error| map_api_error(error, ErrorContext::Secret))?; - self.cache.invalidate(&cache_key).await; + let metadata = with_timeout(&context.operation, async { + let client: Arc = self.vault_client().await?; + kv2::set(client.as_ref(), &location.mount, &location.path, &data) + .await + .map_err(|error| map_api_error(error, ErrorContext::Secret)) + }) + .await?; + self.cache.invalidate_all(); serde_json::to_value(metadata) .map_err(|source| Error::Client(ClientError::JsonParseError { source })) } pub async fn async_delete_secret(&self, secret_name: &str) -> Result<(), Error> { - let location: SecretLocation = self.secret_location(secret_name)?; - let cache_key: String = cache_key(&location); - let client: Arc = self.vault_client().await?; - kv2::delete_latest(client.as_ref(), &location.mount, &location.path) + self.async_delete_secret_with_context(secret_name, &SecretOperationContext::default()) .await - .map_err(|error| map_api_error(error, ErrorContext::Secret))?; - self.cache.invalidate(&cache_key).await; + } + + pub async fn async_delete_secret_with_context( + &self, + secret_name: &str, + context: &SecretOperationContext, + ) -> Result<(), Error> { + let location: SecretLocation = self.secret_location_with_context(secret_name, context)?; + with_timeout(context, async { + let client: Arc = self.vault_client().await?; + kv2::delete_latest(client.as_ref(), &location.mount, &location.path) + .await + .map_err(|error| map_api_error(error, ErrorContext::Secret)) + }) + .await?; + self.cache.invalidate_all(); Ok(()) } @@ -173,7 +246,23 @@ impl HashicorpVault { new_name: &str, value: &SecretValue, ) -> Result { - async_rotate_secret(self, current_name, new_name, value).await + self.async_rotate_secret_with_context( + current_name, + new_name, + value, + &SecretOperationContext::default(), + ) + .await + } + + pub async fn async_rotate_secret_with_context( + &self, + current_name: &str, + new_name: &str, + value: &SecretValue, + context: &SecretOperationContext, + ) -> Result { + async_rotate_secret(self, current_name, new_name, value, context).await } async fn vault_client(&self) -> Result, Error> { @@ -254,25 +343,30 @@ impl BaseSecretManager for HashicorpVault { type WriteResponse = Value; type DeleteResponse = (); - async fn async_read_secret(&self, name: &str) -> Result, Error> { - HashicorpVault::async_read_secret(self, name).await + async fn async_read_secret( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { + HashicorpVault::async_read_secret_with_context(self, name, context).await } async fn async_write_secret( &self, name: &str, value: &SecretValue, - description: Option<&str>, + context: &SecretWriteContext, ) -> Result { - HashicorpVault::async_write_secret(self, name, value.clone(), description).await + HashicorpVault::async_write_secret_with_context(self, name, value, context).await } async fn async_delete_secret( &self, name: &str, - _recovery_window_in_days: i64, + _recovery_window_in_days: Option, + context: &SecretOperationContext, ) -> Result<(), Error> { - HashicorpVault::async_delete_secret(self, name).await + HashicorpVault::async_delete_secret_with_context(self, name, context).await } } @@ -283,11 +377,42 @@ enum ErrorContext { Secret, } -fn cache_key(location: &SecretLocation) -> String { - format!( - "{:?}/{}/{}", - location.namespace, location.mount, location.path - ) +fn hashicorp_context( + context: &SecretOperationContext, +) -> Result, Error> { + match context { + SecretOperationContext::Hashicorp(context) => Ok(Some(context)), + SecretOperationContext::Default => Ok(None), + SecretOperationContext::Aws(_) | SecretOperationContext::Cyberark(_) => { + Err(Error::InvalidOperationContext) + } + } +} + +fn path_component(value: &str) -> Option { + let value: &str = value.trim().trim_matches('/'); + (!value.is_empty()).then(|| value.to_owned()) +} + +fn data_key(context: &SecretOperationContext) -> Result { + Ok(hashicorp_context(context)? + .and_then(|context| context.data_key.as_deref()) + .map(str::trim) + .filter(|data_key| !data_key.is_empty()) + .map(str::to_owned) + .unwrap_or_else(|| "key".to_owned())) +} + +async fn with_timeout( + context: &SecretOperationContext, + operation: impl Future>, +) -> Result { + match context.timeout() { + Some(timeout) => tokio::time::timeout(timeout, operation) + .await + .map_err(|_| Error::Timeout)?, + None => operation.await, + } } fn identity_for(tls: Option<&TlsCertAuth>) -> Result, Error> { diff --git a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs index c52db46e41e..6b322de4d8d 100644 --- a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs @@ -2,7 +2,11 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use litellm_core_utils::settings::Lookup; use litellm_secrets_hashicorp::{Error, HashicorpVault, HashicorpVaultConfig}; -use litellm_secrets_types::SecretValue; +use litellm_secrets_types::{ + AwsOperationContext, BaseSecretManager, CyberarkOperationContext, HashicorpOperationContext, + SecretOperationContext, SecretValue, SecretWriteContext, +}; +use rstest::{fixture, rstest}; use serde::Deserialize; use serde_json::json; use wiremock::{ @@ -69,8 +73,14 @@ fn read_response(data: serde_json::Value) -> serde_json::Value { }) } +#[fixture] +fn token_values() -> Vec<(&'static str, &'static str)> { + vec![("HCP_VAULT_TOKEN", "token")] +} + +#[rstest] #[tokio::test] -async fn token_reads_use_vault_headers_and_cache_values() { +async fn token_reads_use_vault_headers_and_cache_values(token_values: Vec<(&str, &str)>) { let server: MockServer = MockServer::start().await; Mock::given(method("GET")) .and(path("/v1/secret/data/name")) @@ -81,7 +91,7 @@ async fn token_reads_use_vault_headers_and_cache_values() { .expect(1) .mount(&server) .await; - let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]); + let manager: HashicorpVault = manager(&server, &token_values); assert_eq!( manager @@ -109,6 +119,7 @@ async fn token_reads_use_vault_headers_and_cache_values() { ); } +#[rstest] #[tokio::test] async fn namespace_mount_and_prefix_are_sanitized_in_the_url() { let server: MockServer = MockServer::start().await; @@ -138,7 +149,7 @@ async fn namespace_mount_and_prefix_are_sanitized_in_the_url() { assert!(manager.async_read_secret("name").await.unwrap().is_some()); } -#[test] +#[rstest] fn trailing_address_slashes_are_removed() { let environment: Arc = Arc::new(|name: &str| match name { "HCP_VAULT_ADDR" => Some("http://vault.test:8200///".to_owned()), @@ -159,9 +170,9 @@ fn trailing_address_slashes_are_removed() { ); } -#[rstest::rstest] -#[case("-1")] -#[case("not-a-number")] +#[rstest] +#[case::negative("-1")] +#[case::not_a_number("not-a-number")] fn invalid_refresh_intervals_are_rejected(#[case] value: &str) { let environment: Arc = Arc::new(move |name: &str| match name { "HCP_VAULT_REFRESH_INTERVAL" => Some(value.to_owned()), @@ -174,6 +185,7 @@ fn invalid_refresh_intervals_are_rejected(#[case] value: &str) { )); } +#[rstest] #[tokio::test] async fn approle_login_uses_namespace_and_reuses_the_token() { let server: MockServer = MockServer::start().await; @@ -216,6 +228,7 @@ async fn approle_login_uses_namespace_and_reuses_the_token() { assert!(manager.async_read_secret("name-2").await.unwrap().is_none()); } +#[rstest] #[tokio::test] async fn approle_tokens_expire_after_the_vault_lease() { let server: MockServer = MockServer::start().await; @@ -246,6 +259,7 @@ async fn approle_tokens_expire_after_the_vault_lease() { assert!(manager.async_read_secret("second").await.unwrap().is_some()); } +#[rstest] #[tokio::test] async fn tls_login_posts_the_role_and_uses_the_client_identity() { let server: MockServer = MockServer::start().await; @@ -343,21 +357,29 @@ async fn tls_login_posts_the_role_and_uses_the_client_identity() { assert!(login_bodies.contains(&json!({}))); } -#[rstest::rstest] -#[case::missing(404, json!({"errors": ["missing"]}), 0)] -#[case::malformed(200, json!({"data": "invalid"}), 1)] -#[case::missing_key(200, json!({}), 0)] -#[case::non_string(200, json!({"key": 1}), 2)] +#[derive(Clone, Copy)] +enum ExpectedRead { + Missing, + Malformed, + NonString, +} + +#[rstest] +#[case::missing(404, json!({"errors": ["missing"]}), ExpectedRead::Missing)] +#[case::malformed(200, json!({"data": "invalid"}), ExpectedRead::Malformed)] +#[case::missing_key(200, json!({}), ExpectedRead::Missing)] +#[case::non_string(200, json!({"key": 1}), ExpectedRead::NonString)] #[tokio::test] async fn read_responses_distinguish_absence_and_malformed_payloads( + token_values: Vec<(&str, &str)>, #[case] status: u16, #[case] body: serde_json::Value, - #[case] expected: u8, + #[case] expected: ExpectedRead, ) { let server: MockServer = MockServer::start().await; Mock::given(method("GET")) .respond_with(ResponseTemplate::new(status).set_body_json( - if status == 200 && expected != 1 { + if status == 200 && !matches!(expected, ExpectedRead::Malformed) { read_response(body) } else { body @@ -366,20 +388,19 @@ async fn read_responses_distinguish_absence_and_malformed_payloads( .expect(1) .mount(&server) .await; - let result: Result, Error> = - manager(&server, &[("HCP_VAULT_TOKEN", "token")]) - .async_read_secret("name") - .await; + let result: Result, Error> = manager(&server, &token_values) + .async_read_secret("name") + .await; match expected { - 0 => assert!(result.unwrap().is_none()), - 1 => assert!(matches!(result, Err(Error::MalformedPayload))), - 2 => assert!(matches!(result, Err(Error::NonStringValue))), - _ => unreachable!(), + ExpectedRead::Missing => assert!(result.unwrap().is_none()), + ExpectedRead::Malformed => assert!(matches!(result, Err(Error::MalformedPayload))), + ExpectedRead::NonString => assert!(matches!(result, Err(Error::NonStringValue))), } } +#[rstest] #[tokio::test] -async fn write_and_delete_invalidate_the_read_cache() { +async fn write_and_delete_invalidate_the_read_cache(token_values: Vec<(&str, &str)>) { let server: MockServer = MockServer::start().await; Mock::given(method("GET")) .and(path("/v1/secret/data/name")) @@ -418,7 +439,7 @@ async fn write_and_delete_invalidate_the_read_cache() { .expect(1) .mount(&server) .await; - let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]); + let manager: HashicorpVault = manager(&server, &token_values); assert!(manager.async_read_secret("name").await.unwrap().is_some()); assert!( @@ -431,6 +452,313 @@ async fn write_and_delete_invalidate_the_read_cache() { manager.async_delete_secret("name").await.unwrap(); } +#[rstest] +#[tokio::test] +async fn base_manager_context_overrides_vault_location_and_data_key( + token_values: Vec<(&str, &str)>, +) { + let server: MockServer = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/alternate/data/managed/name")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({"api_token": "value"}))), + ) + .expect(2) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/alternate/data/managed/name")) + .and(body_json(json!({ + "data": {"api_token": "updated", "description": "Managed key"} + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "data": { + "created_time": "", + "deletion_time": "", + "custom_metadata": null, + "destroyed": false, + "version": 2 + }, + "lease_id": "", + "lease_duration": 0, + "renewable": false, + "request_id": "", + "warnings": null, + "wrap_info": null + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("DELETE")) + .and(path("/v1/alternate/data/managed/name")) + .respond_with(ResponseTemplate::new(204)) + .expect(1) + .mount(&server) + .await; + let manager: HashicorpVault = manager(&server, &token_values); + let operation = SecretOperationContext::Hashicorp(HashicorpOperationContext { + mount: Some(" /alternate/ ".to_owned()), + path_prefix: Some(" /managed/ ".to_owned()), + data_key: Some("api_token".to_owned()), + ..HashicorpOperationContext::default() + }); + let write_context = SecretWriteContext { + description: Some("Managed key".to_owned()), + operation: operation.clone(), + ..SecretWriteContext::default() + }; + + assert_eq!( + BaseSecretManager::async_read_secret(&manager, "name", &operation) + .await + .unwrap() + .unwrap() + .expose(), + "value" + ); + BaseSecretManager::async_write_secret( + &manager, + "name", + &SecretValue::new("updated"), + &write_context, + ) + .await + .unwrap(); + assert!( + BaseSecretManager::async_read_secret(&manager, "name", &operation) + .await + .unwrap() + .is_some() + ); + BaseSecretManager::async_delete_secret(&manager, "name", None, &operation) + .await + .unwrap(); +} + +#[rstest] +#[tokio::test] +async fn reads_cache_each_data_key_for_the_same_vault_path(token_values: Vec<(&str, &str)>) { + let server: MockServer = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/name")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({ + "key": "primary", + "alternate": "secondary" + }))), + ) + .expect(2) + .mount(&server) + .await; + let manager: HashicorpVault = manager(&server, &token_values); + let alternate = SecretOperationContext::Hashicorp(HashicorpOperationContext { + data_key: Some("alternate".to_owned()), + ..HashicorpOperationContext::default() + }); + + assert_eq!( + manager + .async_read_secret("name") + .await + .unwrap() + .unwrap() + .expose(), + "primary" + ); + assert_eq!( + BaseSecretManager::async_read_secret(&manager, "name", &alternate) + .await + .unwrap() + .unwrap() + .expose(), + "secondary" + ); + assert_eq!( + manager + .async_read_secret("name") + .await + .unwrap() + .unwrap() + .expose(), + "primary" + ); +} + +#[rstest] +#[tokio::test] +async fn base_manager_context_timeout_limits_vault_io(token_values: Vec<(&str, &str)>) { + let server: MockServer = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/name")) + .respond_with( + ResponseTemplate::new(200) + .set_delay(Duration::from_millis(100)) + .set_body_json(read_response(json!({"key": "value"}))), + ) + .expect(1) + .mount(&server) + .await; + let manager: HashicorpVault = manager(&server, &token_values); + let context = SecretOperationContext::Hashicorp(HashicorpOperationContext { + timeout: Some(Duration::from_millis(10)), + ..HashicorpOperationContext::default() + }); + + assert!(matches!( + BaseSecretManager::async_read_secret(&manager, "name", &context).await, + Err(Error::Timeout) + )); +} + +#[rstest] +#[case::aws(SecretOperationContext::Aws(AwsOperationContext::default()))] +#[case::cyberark(SecretOperationContext::Cyberark(CyberarkOperationContext::default()))] +#[tokio::test] +async fn foreign_contexts_cannot_access_vault_secrets( + token_values: Vec<(&str, &str)>, + #[case] context: SecretOperationContext, + #[values(false, true)] cached: bool, +) { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/name")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))), + ) + .expect(u64::from(cached)) + .mount(&server) + .await; + let manager = manager(&server, &token_values); + if cached { + assert!(manager.async_read_secret("name").await.unwrap().is_some()); + } + assert!(matches!( + BaseSecretManager::async_read_secret(&manager, "name", &context).await, + Err(Error::InvalidOperationContext) + )); + assert!(matches!( + BaseSecretManager::async_write_secret( + &manager, + "name", + &SecretValue::new("replacement"), + &SecretWriteContext { + operation: context.clone(), + ..SecretWriteContext::default() + }, + ) + .await, + Err(Error::InvalidOperationContext) + )); + assert!(matches!( + BaseSecretManager::async_delete_secret(&manager, "name", None, &context).await, + Err(Error::InvalidOperationContext) + )); + assert!(matches!( + manager + .async_rotate_secret_with_context( + "name", + "new", + &SecretValue::new("replacement"), + &context + ) + .await, + Err(Error::InvalidOperationContext) + )); + assert_eq!( + server.received_requests().await.unwrap().len(), + usize::from(cached) + ); +} + +#[rstest] +#[tokio::test] +async fn rotation_applies_timeout_to_each_request(token_values: Vec<(&str, &str)>) { + let server = MockServer::start().await; + let timeout = Duration::from_secs(1); + let delay = timeout / 2; + Mock::given(method("GET")) + .and(path("/v1/alternate/data/managed/current")) + .respond_with( + ResponseTemplate::new(200) + .set_delay(delay) + .set_body_json(read_response(json!({"api_token": "original"}))), + ) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/alternate/data/managed/new")) + .and(body_json(json!({ + "data": {"api_token": "replacement", "description": "Rotated from current"} + }))) + .respond_with( + ResponseTemplate::new(200) + .set_delay(delay) + .set_body_json(json!({ + "data": { + "created_time": "", + "deletion_time": "", + "custom_metadata": null, + "destroyed": false, + "version": 1 + }, + "lease_id": "", + "lease_duration": 0, + "renewable": false, + "request_id": "", + "warnings": null, + "wrap_info": null + })), + ) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/alternate/data/managed/new")) + .respond_with( + ResponseTemplate::new(200) + .set_delay(delay) + .set_body_json(read_response(json!({"api_token": "replacement"}))), + ) + .mount(&server) + .await; + Mock::given(method("DELETE")) + .and(path("/v1/alternate/data/managed/current")) + .respond_with(ResponseTemplate::new(204).set_delay(delay)) + .mount(&server) + .await; + let manager = manager(&server, &token_values); + let context = SecretOperationContext::Hashicorp(HashicorpOperationContext { + timeout: Some(timeout), + mount: Some("alternate".to_owned()), + path_prefix: Some("managed".to_owned()), + data_key: Some("api_token".to_owned()), + }); + + manager + .async_rotate_secret_with_context( + "current", + "new", + &SecretValue::new("replacement"), + &context, + ) + .await + .unwrap(); + let requests = server.received_requests().await.unwrap(); + let operations: Vec<_> = requests + .iter() + .map(|request| (request.method.as_str(), request.url.path())) + .collect(); + assert_eq!( + operations, + [ + ("GET", "/v1/alternate/data/managed/current"), + ("POST", "/v1/alternate/data/managed/new"), + ("GET", "/v1/alternate/data/managed/new"), + ("DELETE", "/v1/alternate/data/managed/current"), + ] + ); +} + +#[rstest] #[tokio::test] async fn no_auth_and_invalid_names_fail_without_requests() { let server: MockServer = MockServer::start().await; @@ -447,6 +775,7 @@ async fn no_auth_and_invalid_names_fail_without_requests() { assert!(server.received_requests().await.unwrap().is_empty()); } +#[rstest] #[tokio::test] async fn debug_output_redacts_authentication_values() { let server: MockServer = MockServer::start().await; @@ -468,14 +797,18 @@ struct ParityCase { secret_name: String, } -#[test] -fn configuration_matches_python_parity_fixture() { - let cases: Vec = serde_json::from_str(include_str!(concat!( +#[fixture] +fn parity_cases() -> Vec { + serde_json::from_str(include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/../../../tests/test_litellm/secret_managers/hashicorp_vault_parity.json" ))) - .unwrap(); - for case in cases { + .unwrap() +} + +#[rstest] +fn configuration_matches_python_parity_fixture(parity_cases: Vec) { + for case in parity_cases { let values: HashMap = case.env.clone(); let environment: Arc = Arc::new(move |name: &str| values.get(name).cloned()); @@ -521,6 +854,7 @@ fn configuration_matches_python_parity_fixture() { } } +#[rstest] #[tokio::test] #[ignore] async fn live_vault_round_trip() { diff --git a/litellm-rust/crates/secrets-types/src/base_secret_manager.rs b/litellm-rust/crates/secrets-types/src/base_secret_manager.rs index d71bce64221..e8aed9f280e 100644 --- a/litellm-rust/crates/secrets-types/src/base_secret_manager.rs +++ b/litellm-rust/crates/secrets-types/src/base_secret_manager.rs @@ -1,4 +1,4 @@ -use crate::{Error, SecretValue}; +use crate::{Error, SecretOperationContext, SecretValue, SecretWriteContext}; pub fn validate_secret_name(name: &str) -> Result<(), Error> { if name.split('/').any(|segment| segment == "..") @@ -20,17 +20,22 @@ pub trait BaseSecretManager { type WriteResponse; type DeleteResponse; - async fn async_read_secret(&self, name: &str) -> Result, Self::Error>; + async fn async_read_secret( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Self::Error>; async fn async_write_secret( &self, name: &str, value: &SecretValue, - description: Option<&str>, + context: &SecretWriteContext, ) -> Result; async fn async_delete_secret( &self, name: &str, - recovery_window_in_days: i64, + recovery_window_in_days: Option, + context: &SecretOperationContext, ) -> Result; } @@ -39,20 +44,31 @@ pub async fn async_rotate_secret( current_name: &str, new_name: &str, value: &SecretValue, + context: &SecretOperationContext, ) -> Result { - if manager.async_read_secret(current_name).await?.is_none() { + if manager + .async_read_secret(current_name, context) + .await? + .is_none() + { return Err(Error::CurrentSecretMissing.into()); } let response = manager .async_write_secret( new_name, value, - Some(&format!("Rotated from {current_name}")), + &SecretWriteContext::rotated_from(current_name, context.clone()), ) .await?; - if manager.async_read_secret(new_name).await?.is_none() { + if manager + .async_read_secret(new_name, context) + .await? + .is_none() + { return Err(Error::NewSecretMissing.into()); } - manager.async_delete_secret(current_name, 7).await?; + manager + .async_delete_secret(current_name, Some(7), context) + .await?; Ok(response) } diff --git a/litellm-rust/crates/secrets-types/src/context.rs b/litellm-rust/crates/secrets-types/src/context.rs new file mode 100644 index 00000000000..126815ab32b --- /dev/null +++ b/litellm-rust/crates/secrets-types/src/context.rs @@ -0,0 +1,65 @@ +use std::{collections::BTreeMap, time::Duration}; + +use crate::SecretValue; + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub enum SecretOperationContext { + #[default] + Default, + Aws(AwsOperationContext), + Hashicorp(HashicorpOperationContext), + Cyberark(CyberarkOperationContext), +} + +impl SecretOperationContext { + pub fn timeout(&self) -> Option { + match self { + Self::Default => None, + Self::Aws(context) => context.timeout, + Self::Hashicorp(context) => context.timeout, + Self::Cyberark(context) => context.timeout, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct AwsOperationContext { + pub timeout: Option, + pub region_name: Option, + pub role_name: Option, + pub session_name: Option, + pub external_id: Option, + pub profile_name: Option, + pub web_identity_token: Option, + pub sts_endpoint: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct HashicorpOperationContext { + pub timeout: Option, + pub mount: Option, + pub path_prefix: Option, + pub data_key: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct CyberarkOperationContext { + pub timeout: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct SecretWriteContext { + pub description: Option, + pub tags: BTreeMap, + pub operation: SecretOperationContext, +} + +impl SecretWriteContext { + pub fn rotated_from(current_name: &str, operation: SecretOperationContext) -> Self { + Self { + description: Some(format!("Rotated from {current_name}")), + tags: BTreeMap::new(), + operation, + } + } +} diff --git a/litellm-rust/crates/secrets-types/src/lib.rs b/litellm-rust/crates/secrets-types/src/lib.rs index 0823ed13c06..d330fc7bbea 100644 --- a/litellm-rust/crates/secrets-types/src/lib.rs +++ b/litellm-rust/crates/secrets-types/src/lib.rs @@ -2,11 +2,16 @@ mod base_secret_manager; mod config; +mod context; mod error; mod value; pub use base_secret_manager::{BaseSecretManager, async_rotate_secret, validate_secret_name}; pub use config::{AccessMode, KeyManagementSettings, KeyManagementSystem}; +pub use context::{ + AwsOperationContext, CyberarkOperationContext, HashicorpOperationContext, + SecretOperationContext, SecretWriteContext, +}; pub use error::Error; pub use litellm_auth_types::SecretValue; pub use value::Secret; diff --git a/litellm-rust/crates/secrets-types/tests/config.rs b/litellm-rust/crates/secrets-types/tests/config.rs index 4a5f17bc68a..45fd9d0ee03 100644 --- a/litellm-rust/crates/secrets-types/tests/config.rs +++ b/litellm-rust/crates/secrets-types/tests/config.rs @@ -1,15 +1,20 @@ use litellm_secrets_types::{ AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue, }; +use rstest::{fixture, rstest}; use serde_json::json; -#[test] -fn config_preserves_defaults_nulls_and_serialized_names() { - let empty: KeyManagementSettings = serde_json::from_value(json!({})).unwrap(); - assert_eq!(empty, KeyManagementSettings::default()); - assert_eq!(empty.access_mode, AccessMode::ReadOnly); - assert_eq!(empty.store_virtual_keys, Some(false)); - assert_eq!(empty.prefix_for_stored_virtual_keys, "litellm/"); +#[fixture] +fn default_settings() -> KeyManagementSettings { + serde_json::from_value(json!({})).unwrap() +} + +#[rstest] +fn config_preserves_defaults_nulls_and_serialized_names(default_settings: KeyManagementSettings) { + assert_eq!(default_settings, KeyManagementSettings::default()); + assert_eq!(default_settings.access_mode, AccessMode::ReadOnly); + assert_eq!(default_settings.store_virtual_keys, Some(false)); + assert_eq!(default_settings.prefix_for_stored_virtual_keys, "litellm/"); let configured: KeyManagementSettings = serde_json::from_value(json!({ "hosted_keys": [], "store_virtual_keys": null, "access_mode": "write_only", "aws_web_identity_token": "private-token", "aws_external_id": "private-id", @@ -29,7 +34,7 @@ fn config_preserves_defaults_nulls_and_serialized_names() { ); } -#[rstest::rstest] +#[rstest] #[case::aws_kms("aws_kms", KeyManagementSystem::AwsKms)] #[case::aws_secret_manager("aws_secret_manager", KeyManagementSystem::AwsSecretManager)] #[case::google_kms("google_kms", KeyManagementSystem::GoogleKms)] @@ -50,11 +55,32 @@ fn key_management_system_serialization_round_trips( assert_eq!(serde_json::to_value(system).unwrap(), name); } -#[test] -fn secret_debug_never_exposes_values() { - assert!( - !format!("{:?}", Secret::String(SecretValue::new("sensitive-value"))) - .contains("sensitive-value") - ); - assert!(!format!("{:?}", Secret::Bool(true)).contains("true")); +#[rstest] +#[case::read_only(AccessMode::ReadOnly, true)] +#[case::write_only(AccessMode::WriteOnly, false)] +#[case::read_and_write(AccessMode::ReadAndWrite, true)] +fn access_mode_reports_readability(#[case] mode: AccessMode, #[case] expected: bool) { + assert_eq!(mode.readable(), expected); +} + +#[rstest] +#[case::string(json!("value"), Secret::String(SecretValue::new("value")))] +#[case::boolean(json!(true), Secret::Bool(true))] +#[case::number(json!(7), Secret::Json(json!(7)))] +#[case::null(json!(null), Secret::Json(json!(null)))] +#[case::array(json!(["value"]), Secret::Json(json!(["value"])))] +#[case::object(json!({"key": "value"}), Secret::Json(json!({"key": "value"})))] +fn secret_conversion_preserves_json_types( + #[case] value: serde_json::Value, + #[case] expected: Secret, +) { + assert_eq!(Secret::from_json(value), expected); +} + +#[rstest] +#[case::string(Secret::String(SecretValue::new("sensitive-value")), "sensitive-value")] +#[case::boolean(Secret::Bool(true), "true")] +#[case::json(Secret::Json(json!({"private": "value"})), "private")] +fn secret_debug_never_exposes_values(#[case] secret: Secret, #[case] sensitive: &str) { + assert!(!format!("{secret:?}").contains(sensitive)); } diff --git a/litellm-rust/crates/secrets-types/tests/context.rs b/litellm-rust/crates/secrets-types/tests/context.rs new file mode 100644 index 00000000000..7c9e91b3a9e --- /dev/null +++ b/litellm-rust/crates/secrets-types/tests/context.rs @@ -0,0 +1,110 @@ +use std::{collections::BTreeMap, time::Duration}; + +use litellm_secrets_types::{ + AwsOperationContext, CyberarkOperationContext, HashicorpOperationContext, + SecretOperationContext, SecretValue, SecretWriteContext, +}; +use rstest::{fixture, rstest}; + +#[fixture] +fn timeout() -> Duration { + Duration::from_secs(30) +} + +#[fixture] +fn aws_context(timeout: Duration) -> SecretOperationContext { + SecretOperationContext::Aws(AwsOperationContext { + timeout: Some(timeout), + region_name: Some("us-west-2".into()), + role_name: Some("role".into()), + external_id: Some(SecretValue::new("external-id")), + web_identity_token: Some(SecretValue::new("web-identity-token")), + ..AwsOperationContext::default() + }) +} + +#[rstest] +fn operation_context_preserves_backend_specific_values_and_redacts_secrets( + aws_context: SecretOperationContext, + timeout: Duration, +) { + assert_eq!(aws_context.timeout(), Some(timeout)); + assert_eq!( + aws_context, + SecretOperationContext::Aws(AwsOperationContext { + timeout: Some(timeout), + region_name: Some("us-west-2".into()), + role_name: Some("role".into()), + external_id: Some(SecretValue::new("external-id")), + web_identity_token: Some(SecretValue::new("web-identity-token")), + ..AwsOperationContext::default() + }) + ); + let debug = format!("{aws_context:?}"); + assert!(!debug.contains("external-id")); + assert!(!debug.contains("web-identity-token")); +} + +#[rstest] +#[case::default(SecretOperationContext::Default, None)] +#[case::aws( + SecretOperationContext::Aws(AwsOperationContext { + timeout: Some(Duration::from_secs(1)), + ..AwsOperationContext::default() + }), + Some(Duration::from_secs(1)) +)] +#[case::hashicorp( + SecretOperationContext::Hashicorp(HashicorpOperationContext { + timeout: Some(Duration::from_secs(2)), + ..HashicorpOperationContext::default() + }), + Some(Duration::from_secs(2)) +)] +#[case::cyberark( + SecretOperationContext::Cyberark(CyberarkOperationContext { + timeout: Some(Duration::from_secs(3)), + }), + Some(Duration::from_secs(3)) +)] +fn operation_context_reports_each_backend_timeout( + #[case] context: SecretOperationContext, + #[case] expected: Option, +) { + assert_eq!(context.timeout(), expected); +} + +#[rstest] +fn write_context_keeps_tags_separate_from_the_operation_context() { + let context = SecretWriteContext { + description: Some("Managed virtual key".into()), + tags: BTreeMap::from([("team".into(), "team-id".into())]), + operation: SecretOperationContext::Hashicorp(HashicorpOperationContext { + mount: Some("secret".into()), + path_prefix: Some("teams".into()), + data_key: Some("api_key".into()), + ..HashicorpOperationContext::default() + }), + }; + + assert_eq!(context.tags.get("team"), Some(&"team-id".into())); + assert_eq!(context.operation.timeout(), None); + assert_eq!( + context.operation, + SecretOperationContext::Hashicorp(HashicorpOperationContext { + mount: Some("secret".into()), + path_prefix: Some("teams".into()), + data_key: Some("api_key".into()), + ..HashicorpOperationContext::default() + }) + ); +} + +#[rstest] +fn rotation_write_context_preserves_the_operation_context(aws_context: SecretOperationContext) { + let context = SecretWriteContext::rotated_from("current", aws_context.clone()); + + assert_eq!(context.description.as_deref(), Some("Rotated from current")); + assert!(context.tags.is_empty()); + assert_eq!(context.operation, aws_context); +} diff --git a/litellm-rust/crates/secrets-types/tests/rotation.rs b/litellm-rust/crates/secrets-types/tests/rotation.rs index 48a5304ece8..1bc5e0a9eeb 100644 --- a/litellm-rust/crates/secrets-types/tests/rotation.rs +++ b/litellm-rust/crates/secrets-types/tests/rotation.rs @@ -1,12 +1,15 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use litellm_secrets_types::{ - BaseSecretManager, Error, SecretValue, async_rotate_secret, validate_secret_name, + BaseSecretManager, Error, HashicorpOperationContext, SecretOperationContext, SecretValue, + SecretWriteContext, async_rotate_secret, validate_secret_name, }; +use rstest::{fixture, rstest}; struct Manager { step: AtomicUsize, absent_at: Option, + operation: SecretOperationContext, } impl BaseSecretManager for Manager { @@ -14,7 +17,12 @@ impl BaseSecretManager for Manager { type WriteResponse = &'static str; type DeleteResponse = (); - async fn async_read_secret(&self, name: &str) -> Result, Error> { + async fn async_read_secret( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { + assert_eq!(context, &self.operation); let step = self.step.fetch_add(1, Ordering::SeqCst); assert_eq!(name, if step == 0 { "old" } else { "new" }); Ok((self.absent_at != Some(step)).then(|| SecretValue::new("value"))) @@ -24,35 +32,54 @@ impl BaseSecretManager for Manager { &self, name: &str, value: &SecretValue, - description: Option<&str>, + context: &SecretWriteContext, ) -> Result { assert_eq!(self.step.fetch_add(1, Ordering::SeqCst), 1); assert_eq!(name, "new"); assert_eq!(value.expose(), "replacement"); - assert_eq!(description, Some("Rotated from old")); + assert_eq!(context.description.as_deref(), Some("Rotated from old")); + assert!(context.tags.is_empty()); + assert_eq!(context.operation, self.operation); Ok("provider-response") } async fn async_delete_secret( &self, name: &str, - recovery_window_in_days: i64, + recovery_window_in_days: Option, + context: &SecretOperationContext, ) -> Result<(), Error> { assert_eq!(self.step.fetch_add(1, Ordering::SeqCst), 3); assert_eq!(name, "old"); - assert_eq!(recovery_window_in_days, 7); + assert_eq!(recovery_window_in_days, Some(7)); + assert_eq!(context, &self.operation); Ok(()) } } +#[fixture] +fn replacement() -> SecretValue { + SecretValue::new("replacement") +} + +#[rstest] +#[case::default(SecretOperationContext::Default)] +#[case::backend_specific(SecretOperationContext::Hashicorp(HashicorpOperationContext { + mount: Some("alternate".to_owned()), + ..HashicorpOperationContext::default() +}))] #[tokio::test] -async fn rotation_verifies_before_deleting_and_returns_provider_response() { +async fn rotation_verifies_before_deleting_and_returns_provider_response( + replacement: SecretValue, + #[case] operation: SecretOperationContext, +) { let manager = Manager { step: AtomicUsize::new(0), absent_at: None, + operation: operation.clone(), }; assert_eq!( - async_rotate_secret(&manager, "old", "new", &SecretValue::new("replacement")) + async_rotate_secret(&manager, "old", "new", &replacement, &operation,) .await .unwrap(), "provider-response" @@ -60,11 +87,12 @@ async fn rotation_verifies_before_deleting_and_returns_provider_response() { assert_eq!(manager.step.load(Ordering::SeqCst), 4); } -#[rstest::rstest] +#[rstest] #[case::current_secret_missing(0, Error::CurrentSecretMissing, 1)] #[case::new_secret_missing(2, Error::NewSecretMissing, 3)] #[tokio::test] async fn missing_old_or_new_value_stops_rotation_before_deletion( + replacement: SecretValue, #[case] absent_at: usize, #[case] expected: Error, #[case] calls: usize, @@ -72,32 +100,57 @@ async fn missing_old_or_new_value_stops_rotation_before_deletion( let manager = Manager { step: AtomicUsize::new(0), absent_at: Some(absent_at), + operation: SecretOperationContext::default(), }; assert_eq!( - async_rotate_secret(&manager, "old", "new", &SecretValue::new("replacement")) - .await - .unwrap_err(), + async_rotate_secret( + &manager, + "old", + "new", + &replacement, + &SecretOperationContext::default(), + ) + .await + .unwrap_err(), expected ); assert_eq!(manager.step.load(Ordering::SeqCst), calls); } -#[rstest::rstest] +#[rstest] #[case::parent("..")] -#[case::parent_prefix("../x")] -#[case::parent_segment("x/../y")] -#[case::parent_suffix("x/..")] -#[case::line_feed("line\n")] -#[case::next_line("\u{85}")] -#[case::line_separator("\u{2028}")] -#[case::paragraph_separator("\u{2029}")] +#[case::parent_prefix("../../../other-app/creds")] +#[case::nested_parent_prefix("litellm/../../secret")] +#[case::parent_segment("foo/../bar")] +#[case::parent_suffix("foo/..")] +#[case::single_parent_prefix("../foo")] +#[case::line_feed("foo\nbar")] +#[case::carriage_return("foo\rbar")] +#[case::tab("foo\tbar")] +#[case::null("foo\0bar")] +#[case::delete("foo\u{7f}bar")] +#[case::next_line("foo\u{85}bar")] +#[case::line_separator("foo\u{2028}bar")] +#[case::paragraph_separator("foo\u{2029}bar")] fn names_reject_path_traversal_and_control_characters(#[case] name: &str) { assert_eq!(validate_secret_name(name), Err(Error::UnsafeSecretName)); } -#[rstest::rstest] +#[rstest] +#[case::plain_alias("plain-alias")] +#[case::key_with_digits("my-key-123")] +#[case::service_path("prod/my-service-key")] +#[case::email_path("team/user@example.com")] +#[case::colon("foo: bar")] +#[case::spaced_fragment("foo # bar")] +#[case::query("foo?evil=1")] +#[case::fragment("foo#bar")] +#[case::long_alias(&"a".repeat(500))] #[case::embedded_double_dot("release-1.0..2")] -#[case::path_separator("folder/key")] +#[case::middle_double_dot("my..key")] +#[case::leading_double_dot("..foo")] +#[case::trailing_double_dot("foo..")] +#[case::version_double_dot("v2.0..1-beta")] #[case::empty("")] #[case::three_dots("...")] fn names_allow_safe_values(#[case] name: &str) { diff --git a/litellm-rust/crates/secrets/src/handler.rs b/litellm-rust/crates/secrets/src/handler.rs index 8762b8e7405..0ae84821883 100644 --- a/litellm-rust/crates/secrets/src/handler.rs +++ b/litellm-rust/crates/secrets/src/handler.rs @@ -2,7 +2,10 @@ use std::{future::Future, pin::Pin, sync::Arc}; use litellm_core_utils::settings::Lookup; -use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue}; +use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret}; + +#[cfg(any(feature = "aws", feature = "google"))] +use crate::SecretValue; pub trait ExternalSecretManager: Send + Sync { fn system(&self) -> KeyManagementSystem; @@ -17,7 +20,6 @@ pub trait ExternalSecretManager: Send + Sync { #[derive(Clone)] pub enum SecretManager { - Local, External(Arc), #[cfg(feature = "aws")] AwsKms(crate::aws::AwsKms), @@ -38,7 +40,6 @@ pub enum SecretManager { impl SecretManager { pub fn system(&self) -> KeyManagementSystem { match self { - Self::Local => KeyManagementSystem::Local, Self::External(manager) => manager.system(), #[cfg(feature = "aws")] Self::AwsKms(_) => KeyManagementSystem::AwsKms, @@ -65,10 +66,6 @@ pub async fn get_secret_from_manager( environment: &(dyn Lookup + Send + Sync), ) -> Result, Error> { match client { - SecretManager::Local => Ok(environment - .get(secret_name) - .map(SecretValue::new) - .map(Secret::String)), SecretManager::External(manager) => { manager .read_secret(secret_name, _settings, environment) @@ -117,10 +114,9 @@ pub async fn get_secret_from_manager( .map(|value| value.map(Secret::String)) .map_err(Error::from), #[cfg(feature = "azure")] - SecretManager::AzureKeyVault(client) => client - .get_secret_from_azure_key_vault(secret_name) - .await - .map_err(Error::from), + SecretManager::AzureKeyVault(client) => { + client.get_secret(secret_name).await.map_err(Error::from) + } #[cfg(feature = "cyberark")] SecretManager::Cyberark(client) => client .async_read_secret(secret_name) diff --git a/litellm-rust/crates/secrets/tests/resolution.rs b/litellm-rust/crates/secrets/tests/resolution.rs index 3a826092d72..98b34a2c72e 100644 --- a/litellm-rust/crates/secrets/tests/resolution.rs +++ b/litellm-rust/crates/secrets/tests/resolution.rs @@ -1,19 +1,14 @@ use std::sync::Arc; use litellm_secrets::{ - Error, KeyManagementSettings, OidcResolver, Secret, SecretManager, SecretManagerState, - SecretResolver, SecretValue, secret_manager_would_be_consulted, + Error, OidcResolver, Secret, SecretManagerState, SecretResolver, SecretValue, + secret_manager_would_be_consulted, }; -fn resolver(value: Option<&str>, configured: bool) -> SecretResolver { - let state = if configured { - SecretManagerState::new(SecretManager::Local, KeyManagementSettings::default()) - } else { - SecretManagerState::default() - }; +fn resolver(value: Option<&str>) -> SecretResolver { let value = value.map(str::to_owned); SecretResolver::new( - Arc::new(state), + Arc::new(SecretManagerState::default()), Arc::new(move |_: &str| value.clone()), OidcResolver::default(), ) @@ -30,9 +25,8 @@ fn resolver(value: Option<&str>, configured: bool) -> SecretResolver { async fn conversion_is_explicit_and_independent_of_manager_configuration( #[case] input: &str, #[case] boolean: Option, - #[values(false, true)] configured: bool, ) { - let resolver = resolver(Some(input), configured); + let resolver = resolver(Some(input)); assert_eq!( resolver.get_secret("key", None).await.unwrap(), Some(Secret::String(SecretValue::new(input))) @@ -62,8 +56,8 @@ async fn conversion_is_explicit_and_independent_of_manager_configuration( #[rstest::rstest] #[tokio::test] -async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool) { - let missing = resolver(None, configured); +async fn defaults_apply_only_to_absence() { + let missing = resolver(None); assert_eq!(missing.get_secret("key", None).await.unwrap(), None); assert_eq!( missing.get_secret_bool("key", Some(false)).await.unwrap(), @@ -92,7 +86,7 @@ async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool) ); } assert_eq!( - resolver(Some(""), configured) + resolver(Some("")) .get_secret_str("key", Some(SecretValue::new("default"))) .await .unwrap() @@ -103,12 +97,8 @@ async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool) } #[tokio::test] -async fn prefix_is_removed_once_and_local_manager_is_not_consulted() { - let state = SecretManagerState::new(SecretManager::Local, KeyManagementSettings::default()); - assert_eq!( - state.system(), - Some(litellm_secrets::KeyManagementSystem::Local) - ); +async fn prefix_is_removed_once_and_resolved_from_environment() { + let state = SecretManagerState::default(); assert!(!secret_manager_would_be_consulted( &state, "os.environ/os.environ/KEY" @@ -131,7 +121,7 @@ async fn prefix_is_removed_once_and_local_manager_is_not_consulted() { #[tokio::test] async fn resolver_future_can_run_on_a_tokio_worker() { - let resolver = resolver(Some("worker-value"), false); + let resolver = resolver(Some("worker-value")); let result = tokio::spawn(async move { resolver.get_secret_str("KEY", None).await }) .await .unwrap() @@ -142,7 +132,9 @@ async fn resolver_future_can_run_on_a_tokio_worker() { #[cfg(feature = "aws")] mod aws { use super::*; - use litellm_secrets::{AccessMode, FailurePolicy, aws::AwsSecretsManagerV2}; + use litellm_secrets::{ + AccessMode, FailurePolicy, KeyManagementSettings, SecretManager, aws::AwsSecretsManagerV2, + }; use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; fn state(server: &MockServer, settings: KeyManagementSettings) -> SecretManagerState { @@ -319,7 +311,9 @@ mod aws { #[case::failure(503)] #[tokio::test] async fn google_resolver_distinguishes_absence_from_failure(#[case] status: u16) { - use litellm_secrets::{FailurePolicy, google::GoogleSecretManager}; + use litellm_secrets::{ + FailurePolicy, KeyManagementSettings, SecretManager, google::GoogleSecretManager, + }; use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; let server = MockServer::start().await; Mock::given(method("GET")) From 226fa7845adec05dcffb1cdec12840529bd1b2f4 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 09:09:52 -0700 Subject: [PATCH 20/28] chore(prices): sync OpenRouter prices: 1 model [enrichment failed: OpenRouter, 5 held] (#42485) openrouter/deepseek/deepseek-v4-pro: input_cost_per_token, output_cost_per_token, cache_read_input_token_cost Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3ce2d32613e..5847d8d5e68 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -43093,21 +43093,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.36294e-07, + "input_cost_per_token": 9.31074e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.872588e-06, + "output_cost_per_token": 1.862148e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.80245e-08, + "cache_read_input_token_cost": 7.75895e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3ce2d32613e..5847d8d5e68 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -43093,21 +43093,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.36294e-07, + "input_cost_per_token": 9.31074e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.872588e-06, + "output_cost_per_token": 1.862148e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.80245e-08, + "cache_read_input_token_cost": 7.75895e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, From b673ee61e739e41a349888da1e0e665ce83b88b1 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:21:40 -0500 Subject: [PATCH 21/28] feat(arize): per-team success and error sampling rates for the Arize AX callback (#42383) * feat(arize): per-team success and error sampling rates for the Arize AX callback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(arize): fail open on invalid sampling rates and type the new sampling code Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(arize): type the sampling test helpers and parametrized fixture Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/arize/arize.py | 95 ++++++++++- litellm/integrations/opentelemetry.py | 34 ++-- .../initialize_dynamic_callback_params.py | 2 + .../callback_config_validation.py | 26 ++- litellm/types/utils.py | 2 + .../integrations/arize/test_arize.py | 148 ++++++++++++++--- ...test_initialize_dynamic_callback_params.py | 21 ++- .../test_callback_config_validation.py | 54 +++++- .../test_team_callback_endpoints.py | 154 ++++++++++++++---- .../src/components/callback_info_helpers.tsx | 2 + .../components/team/LoggingSettings.test.tsx | 27 +++ 11 files changed, 487 insertions(+), 78 deletions(-) diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 2e5b17185f2..4ab8d9796b2 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -4,13 +4,17 @@ arize AI is OTEL compatible this file has Arize ai specific helper functions """ +import math import os +import random +from collections.abc import Callable, Mapping from datetime import datetime from typing import TYPE_CHECKING, Any, Final +from litellm._logging import verbose_logger from litellm.integrations.arize import _utils from litellm.integrations.arize._utils import ArizeOTELAttributes -from litellm.integrations.opentelemetry import OpenTelemetry +from litellm.integrations.opentelemetry import _MAX_DYNAMIC_TRACER_PROVIDERS, OpenTelemetry, OpenTelemetryConfig from litellm.types.integrations.arize import ArizeConfig from litellm.types.services import ServiceLoggerPayload from litellm.types.utils import StandardCallbackDynamicParams @@ -26,6 +30,9 @@ else: Protocol = Any Span = Any +_SUCCESS_SAMPLING_RATE_VAR: Final = "arize_success_sampling_rate" +_ERROR_SAMPLING_RATE_VAR: Final = "arize_error_sampling_rate" + class ArizeLogger(OpenTelemetry): """ @@ -36,6 +43,26 @@ class ArizeLogger(OpenTelemetry): fighting over the global ``opentelemetry.trace`` TracerProvider singleton. """ + def __init__( + self, + config: OpenTelemetryConfig | None = None, + callback_name: str | None = None, + tracer_provider: object | None = None, + logger_provider: object | None = None, + meter_provider: object | None = None, + max_dynamic_tracer_providers: int = _MAX_DYNAMIC_TRACER_PROVIDERS, + random_draw: Callable[[], float] | None = None, + ) -> None: + super().__init__( + config=config, + callback_name=callback_name, + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + max_dynamic_tracer_providers=max_dynamic_tracer_providers, + ) + self._random_draw: Final[Callable[[], float]] = random_draw if random_draw is not None else random.random + def _init_tracing(self, tracer_provider): """ Override to always create a *private* TracerProvider for Arize. @@ -55,6 +82,72 @@ class ArizeLogger(OpenTelemetry): self.tracer = provider.get_tracer("litellm") self.span_kind = SpanKind + def _handle_success( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime, + end_time: datetime, + ) -> None: + if not self._should_export(kwargs, _SUCCESS_SAMPLING_RATE_VAR): + return + super()._handle_success(kwargs, response_obj, start_time, end_time) + + def _handle_failure( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime, + end_time: datetime, + ) -> None: + if not self._should_export(kwargs, _ERROR_SAMPLING_RATE_VAR): + return + super()._handle_failure(kwargs, response_obj, start_time, end_time) + + def _sampling_rate_for_request(self, kwargs: Mapping[str, object], var: str) -> float | None: + dynamic_params: Final = kwargs.get("standard_callback_dynamic_params") + if not isinstance(dynamic_params, Mapping): + return None + value: Final = dynamic_params.get(var) + if value is None or value in ("", "None"): + return None + try: + rate = float(value) + except (TypeError, ValueError): + verbose_logger.warning( + "ArizeLogger: %s value %r is not a number; exporting the request", + var, + value, + ) + return None + if not math.isfinite(rate) or not 0.0 <= rate <= 1.0: + verbose_logger.warning( + "ArizeLogger: %s value %r is outside 0.0..1.0; exporting the request", + var, + value, + ) + return None + return rate + + def _should_export(self, kwargs: dict[str, object], var: str) -> bool: + rate: Final = self._sampling_rate_for_request(kwargs, var) + if rate is None: + return True + otel_internal: Final = self._otel_internal_state(kwargs) + key: Final = f"arize_sampled:{var}" + cached: Final = otel_internal.get(key) + if isinstance(cached, bool): + return cached + sampled: Final = rate > 0.0 and self._random_draw() <= rate + otel_internal[key] = sampled + if not sampled: + verbose_logger.debug( + "ArizeLogger: dropping request, %s rate %r rejected the draw", + var, + rate, + ) + return sampled + def _init_otel_logger_on_litellm_proxy(self): """ Override: Arize should NOT overwrite the proxy's diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 180929bcfd4..feaeaec27a3 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1240,6 +1240,25 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # End of Team/Key Based Logging Control Flow ######################################################### + def _otel_internal_state(self, kwargs: dict[str, object]) -> dict[str, object]: + """Return the request-local ``_otel_internal`` marker dict, creating it if absent.""" + litellm_params = kwargs.get("litellm_params") + if not isinstance(litellm_params, dict): + litellm_params = {} + kwargs["litellm_params"] = litellm_params + + _metadata = litellm_params.get("metadata") + if not isinstance(_metadata, dict): + _metadata = {} + litellm_params["metadata"] = _metadata + + _otel_internal = _metadata.get("_otel_internal") + if not isinstance(_otel_internal, dict): + _otel_internal = {} + _metadata["_otel_internal"] = _otel_internal + + return _otel_internal + def _emit_once(self, kwargs: dict, *scope: object) -> bool: """Return True the first time this handler is asked to emit a span for the given (handler, scope) on this kwargs; False on repeats. @@ -1264,20 +1283,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): request-local (kwargs is shared across the sync/async callbacks and lifecycle hooks for one request). """ - litellm_params = kwargs.get("litellm_params") - if not isinstance(litellm_params, dict): - litellm_params = {} - kwargs["litellm_params"] = litellm_params - - _metadata = litellm_params.get("metadata") - if not isinstance(_metadata, dict): - _metadata = {} - litellm_params["metadata"] = _metadata - - _otel_internal = _metadata.get("_otel_internal") - if not isinstance(_otel_internal, dict): - _otel_internal = {} - _metadata["_otel_internal"] = _otel_internal + _otel_internal = self._otel_internal_state(kwargs) spans_logged = _otel_internal.get("spans_logged") if not isinstance(spans_logged, dict): diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 34d0ded618c..00ab05aba77 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -98,6 +98,8 @@ _supported_callback_params: Final[tuple[str, ...]] = ( "arize_api_key", "arize_space_key", "arize_space_id", + "arize_success_sampling_rate", + "arize_error_sampling_rate", "posthog_api_key", "posthog_host", "braintrust_api_key", diff --git a/litellm/proxy/common_utils/callback_config_validation.py b/litellm/proxy/common_utils/callback_config_validation.py index 30c4ab31d6f..6a0fcbe0bb3 100644 --- a/litellm/proxy/common_utils/callback_config_validation.py +++ b/litellm/proxy/common_utils/callback_config_validation.py @@ -5,6 +5,7 @@ Team callbacks arrive as a single ``AddTeamCallback``, key callbacks arrive as a per-integration checks here. """ +import math from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Final @@ -13,11 +14,18 @@ _NEWRELIC_CALLBACK: Final = "newrelic" _NEWRELIC_VAR_PREFIX: Final = "newrelic_" _LANGFUSE_OTEL_CALLBACK: Final = "langfuse_otel" _LANGFUSE_SPAN_SCOPE_VAR: Final = "langfuse_span_scope" +_ARIZE_CALLBACK: Final = "arize" +_ARIZE_SAMPLING_RATE_VARS: Final[frozenset[str]] = frozenset( + {"arize_success_sampling_rate", "arize_error_sampling_rate"} +) def callback_config_error(callback_name: str | None, callback_vars: Mapping[str, str] | None) -> str | None: if not callback_vars: return None + arize_error: Final = _arize_sampling_rate_error(callback_name, callback_vars) + if arize_error is not None: + return arize_error langfuse_error: Final = _langfuse_environment_error(callback_vars) or _langfuse_span_scope_error( callback_name, callback_vars ) @@ -86,7 +94,7 @@ _VAR_FAMILIES: Final[Mapping[str, str]] = MappingProxyType( } ) -_FAMILY_OPTION_VARS: Final[frozenset[str]] = frozenset({_LANGFUSE_SPAN_SCOPE_VAR}) +_FAMILY_OPTION_VARS: Final[frozenset[str]] = frozenset({_LANGFUSE_SPAN_SCOPE_VAR, *_ARIZE_SAMPLING_RATE_VARS}) def _family_of(var: str) -> str | None: @@ -212,6 +220,22 @@ def _logging_entry_error(entry: object) -> str | None: return callback_config_error(callback_name, _entry_callback_vars(entry)) +def _arize_sampling_rate_error(callback_name: str | None, callback_vars: Mapping[str, str]) -> str | None: + for var in sorted(_ARIZE_SAMPLING_RATE_VARS): + value = callback_vars.get(var) + if value is None or value in ("", "None"): + continue + if callback_name != _ARIZE_CALLBACK: + return f"{var} applies to the {_ARIZE_CALLBACK} callback only, not {callback_name!r}" + try: + rate = float(value) + except (TypeError, ValueError): + return f"{var} must be a number between 0.0 and 1.0 (inclusive), got {value!r}" + if not math.isfinite(rate) or not 0.0 <= rate <= 1.0: + return f"{var} must be a number between 0.0 and 1.0 (inclusive), got {value!r}" + return None + + def _newrelic_config_error(callback_vars: Mapping[str, str]) -> str | None: """Per-team New Relic routing runs on the OTel v2 path only. diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 346181d9d9e..bb0b3d2a687 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3638,6 +3638,8 @@ class StandardCallbackDynamicParams(TypedDict, total=False): arize_api_key: str | None arize_space_key: str | None arize_space_id: str | None + arize_success_sampling_rate: ReadOnly[float | None] + arize_error_sampling_rate: ReadOnly[float | None] # PostHog dynamic params posthog_api_key: str | None diff --git a/tests/test_litellm/integrations/arize/test_arize.py b/tests/test_litellm/integrations/arize/test_arize.py index cdafd856b49..5fde627680d 100644 --- a/tests/test_litellm/integrations/arize/test_arize.py +++ b/tests/test_litellm/integrations/arize/test_arize.py @@ -5,6 +5,8 @@ from unittest.mock import MagicMock, Mock, patch # Adds the grandparent directory to sys.path to allow importing project modules import asyncio +import datetime +from collections.abc import Callable import pytest from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -64,9 +66,7 @@ async def test_arize_dynamic_params(): print(f"Tracer calls: {len(tracer_calls)}") # We should have captured calls for both requests - assert ( - len(tracer_calls) >= 2 - ), f"Expected at least 2 tracer calls, got {len(tracer_calls)}" + assert len(tracer_calls) >= 2, f"Expected at least 2 tracer calls, got {len(tracer_calls)}" # Check that we have the expected dynamic params in the kwargs team1_found = False @@ -87,9 +87,7 @@ async def test_arize_dynamic_params(): assert team1_found, "team1 dynamic params not found" assert team2_found, "team2 dynamic params not found" - print( - "✅ All assertions passed - OpenTelemetry logger correctly received dynamic params" - ) + print("✅ All assertions passed - OpenTelemetry logger correctly received dynamic params") @pytest.mark.asyncio @@ -114,11 +112,8 @@ async def test_arize_dynamic_headers_in_grpc_requests(): "opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter", mock_otlp_http_exporter, ): - # Create ArizeLogger with HTTP configuration - config = OpenTelemetryConfig( - exporter="otlp_http", endpoint="https://otlp.arize.com/v1" - ) + config = OpenTelemetryConfig(exporter="otlp_http", endpoint="https://otlp.arize.com/v1") arize_logger = ArizeLogger(config=config) litellm.callbacks = [arize_logger] @@ -147,25 +142,17 @@ async def test_arize_dynamic_headers_in_grpc_requests(): print(f"Captured exporter headers: {exporter_headers}") # Should have multiple exporter calls (default + dynamic) - assert ( - len(exporter_headers) >= 2 - ), f"Expected at least 2 exporter calls, got {len(exporter_headers)}" + assert len(exporter_headers) >= 2, f"Expected at least 2 exporter calls, got {len(exporter_headers)}" # Find team1 and team2 headers team1_found = False team2_found = False for headers in exporter_headers: - if ( - headers.get("api_key") == "team1_api_key" - and headers.get("arize-space-id") == "team1_space_id" - ): + if headers.get("api_key") == "team1_api_key" and headers.get("arize-space-id") == "team1_space_id": team1_found = True print(f"✅ Found team1 headers: {headers}") - elif ( - headers.get("api_key") == "team2_api_key" - and headers.get("arize-space-id") == "team2_space_id" - ): + elif headers.get("api_key") == "team2_api_key" and headers.get("arize-space-id") == "team2_space_id": team2_found = True print(f"✅ Found team2 headers: {headers}") @@ -173,6 +160,119 @@ async def test_arize_dynamic_headers_in_grpc_requests(): assert team1_found, "team1 dynamic headers not found in exporter calls" assert team2_found, "team2 dynamic headers not found in exporter calls" - print( - "✅ Test passed - Dynamic Arize params correctly passed to gRPC/HTTP exporter" - ) + print("✅ Test passed - Dynamic Arize params correctly passed to gRPC/HTTP exporter") + + +_START = datetime.datetime.now() +_END = datetime.datetime.now() + + +def _sampled_arize_logger( + random_draw: Callable[[], float] | None = None, +) -> tuple[ArizeLogger, InMemorySpanExporter]: + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + logger = ArizeLogger(tracer_provider=provider, random_draw=random_draw) + return logger, exporter + + +def _request_spans(exporter: InMemorySpanExporter) -> int: + return sum(1 for span in exporter.get_finished_spans() if span.name == "litellm_request") + + +def _arize_kwargs(callback_vars: dict[str, str] | None = None) -> dict[str, object]: + kwargs: dict[str, object] = { + "model": "gpt-4", + "litellm_params": {"metadata": {}}, + "standard_logging_object": { + "id": "call-1", + "call_type": "completion", + "model": "gpt-4", + "metadata": {}, + "messages": [{"role": "user", "content": "hi"}], + }, + } + if callback_vars is not None: + kwargs["standard_callback_dynamic_params"] = callback_vars + return kwargs + + +@pytest.mark.asyncio +async def test_success_sampling_rate_zero_exports_no_spans(): + logger, exporter = _sampled_arize_logger(random_draw=lambda: 0.0) + await logger.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": "0.0"}), None, _START, _END) + assert _request_spans(exporter) == 0 + + +@pytest.mark.asyncio +async def test_success_sampling_rate_one_exports_a_span(): + logger, exporter = _sampled_arize_logger() + await logger.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": "1.0"}), None, _START, _END) + assert _request_spans(exporter) == 1 + + +@pytest.mark.asyncio +async def test_unset_sampling_rate_exports_everything(): + logger, exporter = _sampled_arize_logger() + await logger.async_log_success_event(_arize_kwargs(), None, _START, _END) + assert _request_spans(exporter) == 1 + + +@pytest.mark.asyncio +async def test_draw_above_rate_is_dropped_draw_at_rate_is_exported(): + dropped, dropped_exporter = _sampled_arize_logger(random_draw=lambda: 0.3) + await dropped.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": "0.2"}), None, _START, _END) + assert _request_spans(dropped_exporter) == 0 + + kept, kept_exporter = _sampled_arize_logger(random_draw=lambda: 0.2) + await kept.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": "0.2"}), None, _START, _END) + assert _request_spans(kept_exporter) == 1 + + +@pytest.mark.asyncio +async def test_success_and_error_rates_are_independent(): + logger, exporter = _sampled_arize_logger() + kwargs = _arize_kwargs({"arize_success_sampling_rate": "0.0", "arize_error_sampling_rate": "1.0"}) + await logger.async_log_success_event(kwargs, None, _START, _END) + await logger.async_log_failure_event(kwargs, ValueError("boom"), _START, _END) + assert _request_spans(exporter) == 1 + + logger2, exporter2 = _sampled_arize_logger() + kwargs2 = _arize_kwargs({"arize_success_sampling_rate": "1.0", "arize_error_sampling_rate": "0.0"}) + await logger2.async_log_success_event(kwargs2, None, _START, _END) + await logger2.async_log_failure_event(kwargs2, ValueError("boom"), _START, _END) + assert _request_spans(exporter2) == 1 + + +@pytest.mark.asyncio +async def test_one_draw_per_request_across_sync_and_async_handlers(): + draws: list[int] = [] + + def counting_draw() -> float: + draws.append(1) + return 0.5 + + logger, _ = _sampled_arize_logger(random_draw=counting_draw) + kwargs = _arize_kwargs({"arize_success_sampling_rate": "1.0"}) + logger.log_success_event(kwargs, None, _START, _END) + await logger.async_log_success_event(kwargs, None, _START, _END) + assert len(draws) == 1 + + +@pytest.mark.asyncio +async def test_unparsable_sampling_rate_exports_rather_than_dropping(): + logger, exporter = _sampled_arize_logger(random_draw=lambda: 0.99) + await logger.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": "abc"}), None, _START, _END) + assert _request_spans(exporter) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("bad", ["nan", "inf", "-inf", "1.5", "-0.1"]) +async def test_out_of_range_sampling_rate_exports_rather_than_dropping(bad: str): + logger, exporter = _sampled_arize_logger(random_draw=lambda: 0.99) + await logger.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": bad}), None, _START, _END) + assert _request_spans(exporter) == 1 diff --git a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py index 521244c6ede..a9a6509f6dd 100644 --- a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py +++ b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py @@ -1,4 +1,3 @@ - from types import MappingProxyType from typing import Final @@ -246,9 +245,7 @@ def test_trusted_vars_overlay_uses_shared_parser_semantics(): # datadog handler consumes, so values are str()-coerced identically. from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD - params = initialize_standard_callback_dynamic_params( - {TRUSTED_CALLBACK_VARS_FIELD: {"newrelic_api_key": 12345}} - ) + params = initialize_standard_callback_dynamic_params({TRUSTED_CALLBACK_VARS_FIELD: {"newrelic_api_key": 12345}}) assert params.get("newrelic_api_key") == "12345" @@ -266,3 +263,19 @@ def test_validate_langfuse_environment_value(): for bad in ["Production", "langfuse-eu", "", "team a"]: with pytest.raises(ValueError, match="langfuse_environment"): validate_langfuse_environment_value(bad) + + +def test_arize_sampling_rates_are_picked_up_from_metadata(): + kwargs = { + "litellm_params": { + "metadata": { + "arize_success_sampling_rate": "0.5", + "arize_error_sampling_rate": "0.1", + } + } + } + + params = initialize_standard_callback_dynamic_params(kwargs) + + assert params.get("arize_success_sampling_rate") == "0.5" + assert params.get("arize_error_sampling_rate") == "0.1" diff --git a/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py b/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py index a707c92dc15..33ae986a511 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py @@ -3,6 +3,7 @@ import pytest from litellm.proxy.common_utils.callback_config_validation import ( callback_config_error, conflicting_span_scope_error, + cross_entry_family_error, logging_metadata_config_error, ) @@ -67,8 +68,16 @@ def test_one_span_scope_per_team(new_vars, stored, rejected): def test_key_logging_entries_may_not_disagree_on_the_span_scope(): disagreeing = { "logging": [ - {"callback_name": "langfuse_otel", "callback_type": "success", "callback_vars": {"langfuse_span_scope": "full"}}, - {"callback_name": "langfuse_otel", "callback_type": "failure", "callback_vars": {"langfuse_span_scope": "llm_only"}}, + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"langfuse_span_scope": "full"}, + }, + { + "callback_name": "langfuse_otel", + "callback_type": "failure", + "callback_vars": {"langfuse_span_scope": "llm_only"}, + }, ] } error = logging_metadata_config_error(disagreeing) @@ -76,9 +85,46 @@ def test_key_logging_entries_may_not_disagree_on_the_span_scope(): agreeing = { "logging": [ - {"callback_name": "langfuse_otel", "callback_type": "success", "callback_vars": {"langfuse_span_scope": "llm_only"}}, - {"callback_name": "langfuse_otel", "callback_type": "failure", "callback_vars": {"langfuse_span_scope": "llm_only"}}, + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"langfuse_span_scope": "llm_only"}, + }, + { + "callback_name": "langfuse_otel", + "callback_type": "failure", + "callback_vars": {"langfuse_span_scope": "llm_only"}, + }, {"callback_name": "otel", "callback_type": "success", "callback_vars": {}}, ] } assert logging_metadata_config_error(agreeing) is None + + +@pytest.mark.parametrize("var", ["arize_success_sampling_rate", "arize_error_sampling_rate"]) +@pytest.mark.parametrize("bad", ["1.5", "-0.1", "abc", "nan", "inf"]) +def test_callback_config_error_rejects_out_of_range_arize_sampling_rate(var, bad): + error = callback_config_error("arize", {var: bad}) + assert error is not None and var in error and repr(bad) in error + + +@pytest.mark.parametrize("var", ["arize_success_sampling_rate", "arize_error_sampling_rate"]) +@pytest.mark.parametrize("good", ["0", "1", "0.25", "", "None"]) +def test_callback_config_error_accepts_in_range_arize_sampling_rate(var, good): + assert callback_config_error("arize", {var: good}) is None + + +def test_arize_sampling_rate_rejected_on_non_arize_callback(): + error = callback_config_error("langfuse", {"arize_success_sampling_rate": "0.5"}) + assert error is not None + assert "applies to the arize callback only" in error + assert callback_config_error("arize", {"arize_success_sampling_rate": "0.5"}) is None + + +def test_arize_sampling_rates_are_not_family_credentials(): + """The rates choose what the Arize family exports, not where it sends, so an + entry that repeats or adds a rate next to a stored Arize entry is not the + credential-redirect shape cross_entry_family_error rejects.""" + stored = [{"arize_api_key": "k1", "arize_success_sampling_rate": "0.5"}] + assert cross_entry_family_error({"arize_success_sampling_rate": "0.1"}, stored) is None + assert cross_entry_family_error({"arize_error_sampling_rate": "0.5"}, stored) is None diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py index acc7c8ca21a..b6eebcb2ef3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py @@ -110,9 +110,7 @@ def patched_prisma(): @pytest.mark.asyncio -async def test_add_team_callbacks_rejects_unauthorized_caller( - patched_prisma, unauthorized_caller -): +async def test_add_team_callbacks_rejects_unauthorized_caller(patched_prisma, unauthorized_caller): data = AddTeamCallback( callback_name="langfuse", callback_type="success", @@ -133,9 +131,7 @@ async def test_add_team_callbacks_rejects_unauthorized_caller( @pytest.mark.asyncio -async def test_disable_team_logging_rejects_unauthorized_caller( - patched_prisma, unauthorized_caller -): +async def test_disable_team_logging_rejects_unauthorized_caller(patched_prisma, unauthorized_caller): with pytest.raises(HTTPException) as exc: await disable_team_logging( http_request=Mock(spec=Request), @@ -147,9 +143,7 @@ async def test_disable_team_logging_rejects_unauthorized_caller( @pytest.mark.asyncio -async def test_get_team_callbacks_rejects_unauthorized_caller( - patched_prisma, unauthorized_caller -): +async def test_get_team_callbacks_rejects_unauthorized_caller(patched_prisma, unauthorized_caller): with pytest.raises(HTTPException) as exc: await get_team_callbacks( http_request=Mock(spec=Request), @@ -180,9 +174,7 @@ async def test_proxy_admin_can_add_team_callbacks(patched_prisma): @pytest.mark.asyncio async def test_team_admin_of_target_team_can_add_callbacks(patched_prisma): - patched_prisma.get_data = AsyncMock( - return_value=_team_row(admin_user_id="team_admin_user") - ) + patched_prisma.get_data = AsyncMock(return_value=_team_row(admin_user_id="team_admin_user")) team_admin = UserAPIKeyAuth( user_role=LitellmUserRoles.INTERNAL_USER, @@ -472,9 +464,7 @@ async def test_add_team_callbacks_writes_encrypted_callback_vars(monkeypatch): litellm_changed_by=None, ) - written = json.loads( - mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"] - ) + written = json.loads(mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) cv = written["logging"][0]["callback_vars"] assert cv["langfuse_secret_key"] != "sk-lf-real-secret" assert cv["langfuse_public_key"] != "pk-lf-real-public" @@ -1192,8 +1182,12 @@ async def test_add_team_callbacks_rejects_team_deleted_before_write(): ) with ( - patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: proxy_server module global is the endpoint's only injection point - patch("litellm.proxy.proxy_server.master_key", None), # test-quality-ok: proxy_server module global is the endpoint's only injection point + patch( + "litellm.proxy.proxy_server.prisma_client", mock_prisma + ), # test-quality-ok: proxy_server module global is the endpoint's only injection point + patch( + "litellm.proxy.proxy_server.master_key", None + ), # test-quality-ok: proxy_server module global is the endpoint's only injection point ): with pytest.raises(HTTPException) as exc: await add_team_callbacks( @@ -1482,12 +1476,16 @@ async def test_unknown_team_is_indistinguishable_from_no_access(call_handler, un a probe for valid team ids. The unknown-team response has to match the no-access one exactly, status and body. """ - with patch("litellm.proxy.proxy_server.prisma_client") as mock_client: # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_client + ): # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through mock_client.get_data = AsyncMock(return_value=None) with pytest.raises(HTTPException) as unknown_team: await call_handler(unauthorized_caller) - with patch("litellm.proxy.proxy_server.prisma_client") as mock_client: # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_client + ): # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through mock_client.get_data = AsyncMock(return_value=_team_row()) mock_client.db.litellm_teamtable.update = AsyncMock() with patch( # test-quality-ok: _verify_team_access calls this module-level helper directly, so there is no seam to inject through @@ -1508,7 +1506,9 @@ async def test_proxy_admin_still_told_the_team_is_unknown(): """The masking is only for callers who could not have managed the team; a proxy admin keeps the diagnosable error.""" admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin", api_key="sk-admin") - with patch("litellm.proxy.proxy_server.prisma_client") as mock_client: # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_client + ): # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through mock_client.get_data = AsyncMock(return_value=None) with pytest.raises(HTTPException) as exc: await get_team_callbacks( @@ -1526,13 +1526,29 @@ async def test_proxy_admin_still_told_the_team_is_unknown(): [ # the redirect, in every carrier a caller could pick: an entry naming # only a host, pairing with a key pair written on another entry - ({"langfuse_host": "http://attacker.invalid"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], True), + ( + {"langfuse_host": "http://attacker.invalid"}, + [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], + True, + ), # the sibling carrier -- langfuse and langfuse_otel are one account - ({"langfuse_host": "http://attacker.invalid"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_secret_key": "sk"}], True), + ( + {"langfuse_host": "http://attacker.invalid"}, + [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_secret_key": "sk"}], + True, + ), # a destination variable no integration registry lists ({"dd_agent_host": "attacker.invalid"}, [{"dd_api_key": "k", "dd_site": "us5.datadoghq.com"}], True), # one entry owning its family end to end is the feature - ({"langfuse_host": "https://eu.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, [], False), + ( + { + "langfuse_host": "https://eu.cloud.langfuse.com", + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + }, + [], + False, + ), # a different family alongside an existing one stays fine ({"gcs_bucket_name": "bucket"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], False), ({"langsmith_api_key": "k"}, [{"dd_api_key": "k"}], False), @@ -1541,19 +1557,55 @@ async def test_proxy_admin_still_told_the_team_is_unknown(): # the span scope picks what the family exports, not where to, so a second # entry may set either legal value next to the family's credentials ({"langfuse_span_scope": "llm_only"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], False), - ({"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "full"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "llm_only"}], False), + ( + {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "full"}, + [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "llm_only"}], + False, + ), # the scope on the stored entry must not shield a redirect riding next to it - ({"langfuse_host": "http://attacker.invalid", "langfuse_span_scope": "llm_only"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "llm_only"}], True), + ( + {"langfuse_host": "http://attacker.invalid", "langfuse_span_scope": "llm_only"}, + [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "llm_only"}], + True, + ), # the same integration registered for a second event: identical values # flatten to the identical dict, so there is nothing to redirect - ({"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], False), + ( + { + "langfuse_host": "https://us.cloud.langfuse.com", + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + }, + [ + { + "langfuse_host": "https://us.cloud.langfuse.com", + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + } + ], + False, + ), # the same credential under its other spelling is the same credential ({"langfuse_secret": "sk"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], False), # a value the family already holds cannot be moved into another of its # variables either; the exporter would address or authenticate with it - ({"langfuse_host": "pk"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk"}], True), + ( + {"langfuse_host": "pk"}, + [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk"}], + True, + ), # the same shape with one value moved is the redirect again - ({"langfuse_host": "http://attacker.invalid", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], True), + ( + {"langfuse_host": "http://attacker.invalid", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, + [ + { + "langfuse_host": "https://us.cloud.langfuse.com", + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + } + ], + True, + ), ], ) def test_one_entry_owns_a_credential_family(new_vars, stored, rejected): @@ -1568,7 +1620,13 @@ def test_one_entry_owns_a_credential_family(new_vars, stored, rejected): @pytest.mark.asyncio -@pytest.mark.parametrize("caller", [_admin_auth(), UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="victim_admin", api_key="sk-team-admin")]) +@pytest.mark.parametrize( + "caller", + [ + _admin_auth(), + UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="victim_admin", api_key="sk-team-admin"), + ], +) async def test_a_second_entry_may_not_flip_the_span_scope(patched_prisma, caller): """The entries flatten last-wins at request time, so a failure entry saying llm_only next to a success entry saying full would export whichever is stored @@ -1580,7 +1638,11 @@ async def test_a_second_entry_may_not_flip_the_span_scope(patched_prisma, caller { "callback_name": "langfuse_otel", "callback_type": "success", - "callback_vars": {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "full"}, + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "langfuse_span_scope": "full", + }, } ] } @@ -1610,3 +1672,35 @@ async def test_a_second_entry_may_not_flip_the_span_scope(patched_prisma, caller user_api_key_dict=caller, ) patched_prisma.db.litellm_teamtable.update.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_add_team_callbacks_rejects_out_of_range_arize_sampling_rate(patched_prisma): + data = AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={"arize_success_sampling_rate": "1.5"}, + ) + with pytest.raises(HTTPException) as exc: + await add_team_callbacks( + data=data, + http_request=Mock(spec=Request), + team_id="team-victim", + user_api_key_dict=_admin_auth(), + ) + assert exc.value.status_code == 400 + assert "arize_success_sampling_rate" in str(exc.value.detail) + patched_prisma.db.litellm_teamtable.update.assert_not_called() + + +def test_add_team_callback_accepts_arize_sampling_rate_vars(): + data = AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={ + "arize_success_sampling_rate": "0.5", + "arize_error_sampling_rate": "1.0", + }, + ) + assert data.callback_vars["arize_success_sampling_rate"] == "0.5" + assert data.callback_vars["arize_error_sampling_rate"] == "1.0" diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index b43a85d18c5..f5138d55b5d 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -30,6 +30,8 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ dynamic_params: { arize_api_key: "password", arize_space_id: "password", + arize_success_sampling_rate: "number", + arize_error_sampling_rate: "number", }, description: "Arize Logging Integration", }, diff --git a/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx b/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx index f7ca6d516f5..5ce8ff71267 100644 --- a/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx @@ -239,6 +239,33 @@ describe("LoggingSettings", () => { ]); }); + it("renders sampling rate inputs for the Arize callback and records changes", () => { + const mockOnChange = vi.fn(); + + const initialValue = [ + { + callback_name: "arize", + callback_type: "success", + callback_vars: {}, + }, + ]; + + renderWithProviders(); + + const successInput = screen.getByPlaceholderText("os.environ/ARIZE_SUCCESS_SAMPLING_RATE"); + const errorInput = screen.getByPlaceholderText("os.environ/ARIZE_ERROR_SAMPLING_RATE"); + expect(successInput).toBeInTheDocument(); + expect(errorInput).toBeInTheDocument(); + + fireEvent.change(successInput, { target: { value: "0.4" } }); + let lastCall = mockOnChange.mock.calls[mockOnChange.mock.calls.length - 1]; + expect(lastCall[0][0].callback_vars.arize_success_sampling_rate).toBe("0.4"); + + fireEvent.change(errorInput, { target: { value: "0.9" } }); + lastCall = mockOnChange.mock.calls[mockOnChange.mock.calls.length - 1]; + expect(lastCall[0][0].callback_vars.arize_error_sampling_rate).toBe("0.9"); + }); + it("correctly handles numerical input with decimal values", () => { const mockOnChange = vi.fn(); From 691c0d6596b3c5c5b82679033a7d6bd11c411880 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 09:30:28 -0700 Subject: [PATCH 22/28] test(unit): make bedrock collector and secret scan timing tests deterministic (#42405) * test(unit): make bedrock collector and secret scan timing tests deterministic Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(unit): count interpreter calls instead of wall clock in the secret scan scaling test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(unit): profile the secret scan with cProfile, restore the outer profiler and tighten the scaling bound Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_secret_detection.py | 66 +++++-- ...test_bedrock_passthrough_transformation.py | 169 +++++++++--------- 2 files changed, 136 insertions(+), 99 deletions(-) diff --git a/tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py b/tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py index bf9eeeb9968..405136924e7 100644 --- a/tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py +++ b/tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py @@ -10,16 +10,20 @@ Covers the three defects from the ticket: handling live only on the native path). """ +import cProfile +import sys import tempfile import time +from collections.abc import Callable +from typing import Final import pytest - from litellm_enterprise.enterprise_callbacks.secret_detection import ( - _ENTERPRISE_SecretDetection, _default_detect_secrets_config, + _ENTERPRISE_SecretDetection, _masked_entity_count, ) + from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth @@ -665,14 +669,47 @@ def test_scan_message_stays_linear_on_repeated_sk_separators(): assert time.perf_counter() - started < 2.0 +_SCALE: Final = 4 + + +def _value_run(n: int) -> str: + return f"api_key: '{OPENAI_KEY}'\npassword=" + "a-" * n + "!" + + +def _quote_run(n: int) -> str: + return f"api_key: '{OPENAI_KEY}'\npassword:" + '"' * n + + +def _keyword_run(n: int) -> str: + return f"api_key: '{OPENAI_KEY}'\n" + "api_key:" * n + + +def _value_repeat(n: int) -> str: + return f"api_key: '{OPENAI_KEY}'\nsecret=" + "aB3dE6gH9jK2mN5p " * n + + +def _assignment_flood(n: int) -> str: + return f"api_key: '{OPENAI_KEY}'\n" + "\n".join(f"password{i}=aB3dE6gH9jK2mN5p{i}" for i in range(n)) + + +def _calls_to_redact(guardrail: _ENTERPRISE_SecretDetection, content: str) -> int: + previous_profiler: Final = sys.getprofile() + profile: Final = cProfile.Profile() + try: + profile.runcall(guardrail.redact_text, content) + finally: + sys.setprofile(previous_profiler) + return sum(entry.callcount for entry in profile.getstats()) + + @pytest.mark.parametrize( - "content", + ("adversarial_content", "size"), [ - f"api_key: '{OPENAI_KEY}'\npassword=" + "a-" * 10_000 + "!", - f"api_key: '{OPENAI_KEY}'\npassword:" + '"' * 20_000, - f"api_key: '{OPENAI_KEY}'\n" + "api_key:" * 10_000, - f"api_key: '{OPENAI_KEY}'\nsecret=" + "aB3dE6gH9jK2mN5p " * 2_000, - f"api_key: '{OPENAI_KEY}'\n" + "\n".join(f"password{i}=aB3dE6gH9jK2mN5p{i}" for i in range(3_000)), + (_value_run, 10_000), + (_quote_run, 20_000), + (_keyword_run, 10_000), + (_value_repeat, 2_000), + (_assignment_flood, 3_000), ], ids=[ "value-run", @@ -682,12 +719,15 @@ def test_scan_message_stays_linear_on_repeated_sk_separators(): "assignment-flood", ], ) -def test_scan_message_stays_linear_on_adversarial_credential_lines(content): - guardrail = _guardrail() +def test_scan_message_stays_linear_on_adversarial_credential_lines( + adversarial_content: Callable[[int], str], size: int +) -> None: + guardrail: Final = _guardrail() - started = time.perf_counter() - guardrail.redact_text(content) - assert time.perf_counter() - started < 10.0 + small: Final = _calls_to_redact(guardrail, adversarial_content(size // _SCALE)) + large: Final = _calls_to_redact(guardrail, adversarial_content(size)) + + assert large <= 1.25 * _SCALE * small, (small, large) def test_scan_message_redacts_whole_stripe_live_key(): diff --git a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index d1d636a15f7..fbcbda183bc 100644 --- a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -1,9 +1,13 @@ import base64 +import gc import json import struct -import tracemalloc +import sys +import types from binascii import crc32 +from collections.abc import Callable from datetime import datetime +from typing import Final from unittest.mock import patch from litellm.litellm_core_utils.litellm_logging import Logging @@ -50,10 +54,7 @@ def test_bedrock_passthrough_get_complete_url_default_endpoint(): ) # Verify URL construction - assert ( - str(url) - == "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet/invoke" - ) + assert str(url) == "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet/invoke" assert api_base == "https://bedrock-runtime.us-east-1.amazonaws.com" @@ -112,9 +113,7 @@ def test_bedrock_passthrough_get_complete_url_custom_endpoint_with_path(): model="anthropic.claude-3-sonnet", endpoint="/model/anthropic.claude-3-sonnet/invoke", request_query_params=None, - litellm_params={ - "aws_bedrock_runtime_endpoint": "http://proxy.com/bedrockproxy" - }, + litellm_params={"aws_bedrock_runtime_endpoint": "http://proxy.com/bedrockproxy"}, ) # Verify get_runtime_endpoint was called with correct parameters @@ -126,10 +125,7 @@ def test_bedrock_passthrough_get_complete_url_custom_endpoint_with_path(): ) # Verify URL construction preserves the proxy path - assert ( - str(url) - == "http://proxy.com/bedrockproxy/model/anthropic.claude-3-sonnet/invoke" - ) + assert str(url) == "http://proxy.com/bedrockproxy/model/anthropic.claude-3-sonnet/invoke" assert api_base == "http://proxy.com/bedrockproxy" @@ -211,9 +207,7 @@ def test_bedrock_passthrough_with_application_inference_profile(): config = BedrockPassthroughConfig() model = "anthropic.claude-sonnet-4-20250514-v1:0" - model_id = ( - "arn:aws:bedrock:eu-west-1:123456789:application-inference-profile/abcdefgh1234" - ) + model_id = "arn:aws:bedrock:eu-west-1:123456789:application-inference-profile/abcdefgh1234" endpoint = f"model/{model}/invoke" with ( @@ -239,12 +233,8 @@ def test_bedrock_passthrough_with_application_inference_profile(): # Verify that the URL contains the encoded model_id (ARN) instead of the model name url_str = str(url) # The ARN slash should be encoded as %2F - assert ( - "application-inference-profile%2F" in url_str - ), f"Expected encoded ARN in URL, but got: {url_str}" - assert ( - model not in url_str - ), f"Model name should be replaced by model_id, but got: {url_str}" + assert "application-inference-profile%2F" in url_str, f"Expected encoded ARN in URL, but got: {url_str}" + assert model not in url_str, f"Model name should be replaced by model_id, but got: {url_str}" assert "/invoke" in url_str, "Expected /invoke action in URL" # Verify the complete URL structure with encoded ARN @@ -258,9 +248,7 @@ def test_bedrock_passthrough_with_inference_profile_converse_endpoint(): config = BedrockPassthroughConfig() model = "anthropic.claude-sonnet-4-20250514-v1:0" - model_id = ( - "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz123" - ) + model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz123" endpoint = f"model/{model}/converse" with ( @@ -323,12 +311,8 @@ def test_bedrock_passthrough_without_model_id_backward_compatibility(): # Verify that the URL contains the model name (not replaced) url_str = str(url) - assert ( - model in url_str - ), f"Expected model name in URL when model_id not provided, but got: {url_str}" - expected_url = ( - f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model}/invoke" - ) + assert model in url_str, f"Expected model name in URL when model_id not provided, but got: {url_str}" + expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model}/invoke" assert url_str == expected_url @@ -338,9 +322,7 @@ def test_bedrock_passthrough_region_extraction_from_inference_profile_arn(): model = "anthropic.claude-sonnet-4-20250514-v1:0" # ARN contains us-west-2 region - model_id = ( - "arn:aws:bedrock:us-west-2:123456789:application-inference-profile/test123" - ) + model_id = "arn:aws:bedrock:us-west-2:123456789:application-inference-profile/test123" endpoint = f"model/{model}/invoke" # Don't provide aws_region_name in litellm_params to test ARN extraction @@ -358,15 +340,13 @@ def test_bedrock_passthrough_region_extraction_from_inference_profile_arn(): model=model, endpoint=endpoint, request_query_params=None, - litellm_params={ - "model_id": model_id - }, # Region should be extracted from ARN + litellm_params={"model_id": model_id}, # Region should be extracted from ARN ) # Verify that the region from ARN is used in the base URL - assert ( - "us-west-2" in api_base - ), f"Expected region 'us-west-2' from ARN in base URL, but got: {api_base}" + assert "us-west-2" in api_base, f"Expected region 'us-west-2' from ARN in base URL, but got: {api_base}" + + def test_bedrock_passthrough_model_id_arn_encoding(): """ Test that model_id ARNs are properly URL-encoded when used in endpoints. @@ -409,14 +389,14 @@ def test_bedrock_passthrough_model_id_arn_encoding(): url_str = str(url) # The slash in the ARN after application-inference-profile should be encoded as %2F - assert ( - "application-inference-profile%2F" in url_str - ), f"Expected encoded ARN with %2F in URL, but got: {url_str}" + assert "application-inference-profile%2F" in url_str, ( + f"Expected encoded ARN with %2F in URL, but got: {url_str}" + ) # The unencoded version should NOT be in the URL - assert ( - "application-inference-profile/" not in url_str - ), f"ARN slash should be encoded, but found unencoded version in: {url_str}" + assert "application-inference-profile/" not in url_str, ( + f"ARN slash should be encoded, but found unencoded version in: {url_str}" + ) # Verify the complete expected URL structure expected_encoded_model_id = ( @@ -433,9 +413,7 @@ def test_bedrock_passthrough_model_id_arn_encoding_invoke_endpoint(): config = BedrockPassthroughConfig() model = "anthropic.claude-sonnet-4-5-20250929-v1:0" - model_id = ( - "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz789" - ) + model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz789" endpoint = f"/model/{model}/invoke" with ( @@ -464,9 +442,7 @@ def test_bedrock_passthrough_model_id_arn_encoding_invoke_endpoint(): assert "application-inference-profile%2F" in url_str assert "/invoke" in url_str - expected_encoded_model_id = ( - "arn:aws:bedrock:us-east-1:123456789:application-inference-profile%2Fxyz789" - ) + expected_encoded_model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile%2Fxyz789" expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{expected_encoded_model_id}/invoke" assert url_str == expected_url @@ -508,9 +484,7 @@ def test_bedrock_passthrough_model_id_without_arn(): assert model_id in url_str assert "%2F" not in url_str, "Non-ARN model IDs should not be encoded" - expected_url = ( - f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse" - ) + expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse" assert url_str == expected_url @@ -518,8 +492,7 @@ def _event_frame(event_type: str, payload: dict) -> bytes: def header(name: str, value: str) -> bytes: name_b, value_b = name.encode(), value.encode() return ( - struct.pack("!B", len(name_b)) + name_b - + struct.pack("!B", 7) + struct.pack("!H", len(value_b)) + value_b + struct.pack("!B", len(name_b)) + name_b + struct.pack("!B", 7) + struct.pack("!H", len(value_b)) + value_b ) payload_b = json.dumps(payload, separators=(",", ":")).encode() @@ -592,29 +565,57 @@ def _feed(collector: PassthroughStreamCollector, stream: bytes, chunk_size: int collector.add(stream[offset : offset + chunk_size]) -def test_converse_stream_collector_keeps_usage_without_retaining_the_stream(): - texts = [f"tok{i} " for i in range(4000)] - stream = ( - _event_frame("messageStart", {"role": "assistant"}) + _text_block(0, texts) + _stream_tail("end_turn", 4000) +_SHARED_OBJECT_TYPES: Final = (types.ModuleType, type, types.FunctionType, types.MethodType, types.CodeType, property) +_SMALL_TOKENS: Final = 4000 +_LARGE_TOKENS: Final = 8000 + + +def _bytes_reachable_from(root: object) -> int: + seen: Final[set[int]] = set() + pending: Final[list[object]] = [root] + sizes: Final[list[int]] = [] + while pending: + obj = pending.pop() + if id(obj) in seen or isinstance(obj, _SHARED_OBJECT_TYPES): + continue + seen.add(id(obj)) + sizes.append(sys.getsizeof(obj)) + pending.extend(gc.get_referents(obj)) + return sum(sizes) + + +def _converse_text_stream(texts: list[str]) -> bytes: + return ( + _event_frame("messageStart", {"role": "assistant"}) + + _text_block(0, texts) + + _stream_tail("end_turn", len(texts)) ) - _feed(_converse_stream_collector(), stream) - tracemalloc.start() - try: - base = tracemalloc.get_traced_memory()[0] - collector = _converse_stream_collector() - _feed(collector, stream) - retained = tracemalloc.get_traced_memory()[0] - base - finally: - tracemalloc.stop() - assert retained < len(stream) // 4 +def _retained_growth(build_stream: Callable[[list[str]], bytes], endpoint: str) -> tuple[int, int, list[str]]: + small_texts: Final = [f"tok{i} " for i in range(_SMALL_TOKENS)] + large_texts: Final = [f"tok{i} " for i in range(_LARGE_TOKENS)] + small_stream, large_stream = build_stream(small_texts), build_stream(large_texts) + small, large = _stream_collector(endpoint), _stream_collector(endpoint) + _feed(small, small_stream) + _feed(large, large_stream) + growth: Final = _bytes_reachable_from(large) - _bytes_reachable_from(small) + return growth, len(large_stream) - len(small_stream), large_texts + +def test_converse_stream_collector_keeps_usage_without_retaining_the_stream(): + growth, extra_stream_bytes, texts = _retained_growth(_converse_text_stream, CONVERSE_STREAM_ENDPOINT) + extra_text_bytes = sum(len(text) for text in texts[_SMALL_TOKENS:]) + + assert extra_text_bytes <= growth < extra_stream_bytes // 4, (growth, extra_text_bytes, extra_stream_bytes) + + collector = _converse_stream_collector() + _feed(collector, _converse_text_stream(texts)) response = collector.build_logged_response(_converse_stream_logging_obj()) assert isinstance(response, ModelResponse) assert response.choices[0].message.content == "".join(texts) assert response.choices[0].finish_reason == "stop" - assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (25, 4000) + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (25, _LARGE_TOKENS) def test_converse_stream_collector_keeps_tool_calls_between_text_runs(): @@ -645,9 +646,8 @@ def test_converse_stream_collector_keeps_tool_calls_between_text_runs(): assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (25, 12) -def test_invoke_stream_collector_keeps_usage_without_retaining_the_stream(): - texts = [f"tok{i} " for i in range(4000)] - stream = ( +def _invoke_text_stream(texts: list[str]) -> bytes: + return ( _invoke_chunk( { "type": "message_start", @@ -669,28 +669,25 @@ def test_invoke_stream_collector_keeps_usage_without_retaining_the_stream(): ) + _invoke_chunk({"type": "content_block_stop", "index": 0}) + _invoke_chunk( - {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 4000}} + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": len(texts)}} ) + _invoke_chunk({"type": "message_stop"}) ) - _feed(_stream_collector(INVOKE_STREAM_ENDPOINT), stream) - tracemalloc.start() - try: - base = tracemalloc.get_traced_memory()[0] - collector = _stream_collector(INVOKE_STREAM_ENDPOINT) - _feed(collector, stream) - retained = tracemalloc.get_traced_memory()[0] - base - finally: - tracemalloc.stop() - assert retained < len(stream) // 4 +def test_invoke_stream_collector_keeps_usage_without_retaining_the_stream(): + growth, extra_stream_bytes, texts = _retained_growth(_invoke_text_stream, INVOKE_STREAM_ENDPOINT) + extra_text_bytes = sum(len(text) for text in texts[_SMALL_TOKENS:]) + assert extra_text_bytes <= growth < extra_stream_bytes // 4, (growth, extra_text_bytes, extra_stream_bytes) + + collector = _stream_collector(INVOKE_STREAM_ENDPOINT) + _feed(collector, _invoke_text_stream(texts)) response = collector.build_logged_response(_stream_logging_obj(INVOKE_STREAM_ENDPOINT)) assert isinstance(response, ModelResponse) assert response.choices[0].message.content == "".join(texts) assert response.choices[0].finish_reason == "stop" - assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (25, 4000) + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (25, _LARGE_TOKENS) def test_stream_collector_logs_nothing_for_an_unrecognized_endpoint(): From c835a1a9823670ba96c7a0b9bab80f7e4aff8ec6 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 09:52:35 -0700 Subject: [PATCH 23/28] feat(anthropic): add Claude Opus 5.5 (#42489) Adds the anthropic cost map entry for claude-opus-5-5 at $4/$20 per MTok with $5 per MTok 5m cache writes, $8 per MTok 1h cache writes, $0.20 per MTok cache reads (0.05x base), and fast mode at 2x. The entry sets thinking_always_on (Opus 5.5 cannot turn thinking off) and supports_forced_tool_use false (tool_choice required/named 400s, same as Fable 5.1), mirrors that model by omitting thinking cache preservation, and registers the model in the setup wizard provider list Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 43 +++++++++++++++++++ litellm/setup_wizard.py | 3 +- model_prices_and_context_window.json | 43 +++++++++++++++++++ .../test_litellm/test_claude_opus_5_config.py | 28 ++++++++++++ 4 files changed, 116 insertions(+), 1 deletion(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5847d8d5e68..468d86d049d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15008,6 +15008,49 @@ "supports_native_structured_output": true, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-opus-5-5": { + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "fast": 2.0 + }, + "supports_output_config": true, + "supports_speed": true, + "supports_fast_mode": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, "claude-opus-5": { "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, diff --git a/litellm/setup_wizard.py b/litellm/setup_wizard.py index dd147aaccee..8b946923cd2 100644 --- a/litellm/setup_wizard.py +++ b/litellm/setup_wizard.py @@ -53,13 +53,14 @@ PROVIDERS: Final[list[dict]] = [ { "id": "anthropic", "name": "Anthropic", - "description": "Claude Fable 5.1, Fable 5, Opus 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 5, Sonnet 4.6, Haiku 4.5", + "description": "Claude Fable 5.1, Fable 5, Opus 5.5, Opus 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 5, Sonnet 4.6, Haiku 4.5", "env_key": "ANTHROPIC_API_KEY", "key_hint": "sk-ant-...", "test_model": "claude-haiku-4-5-20251001", "models": [ "claude-fable-5-1", "claude-fable-5", + "claude-opus-5-5", "claude-opus-5", "claude-sonnet-5", "claude-opus-4-8", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5847d8d5e68..468d86d049d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15008,6 +15008,49 @@ "supports_native_structured_output": true, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-opus-5-5": { + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "fast": 2.0 + }, + "supports_output_config": true, + "supports_speed": true, + "supports_fast_mode": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, "claude-opus-5": { "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, diff --git a/tests/test_litellm/test_claude_opus_5_config.py b/tests/test_litellm/test_claude_opus_5_config.py index aaf179e0216..2e0a359b79b 100644 --- a/tests/test_litellm/test_claude_opus_5_config.py +++ b/tests/test_litellm/test_claude_opus_5_config.py @@ -1,6 +1,8 @@ """ Validate Claude Opus 5 model configuration entries. +Opus 5.5 (``claude-opus-5-5``) is covered here too. + Opus 5 carries Opus 4.8's pricing ($5 / $25 per MTok) and the gen-5 adaptive thinking profile, but differs from 4.8 in two ways that are behavior-bearing in LiteLLM: the cacheable-prefix minimum drops to 512 tokens, and Bedrock's Opus 5 @@ -12,14 +14,23 @@ validator accepts the full effort ladder, so the entries must not carry the ``anthropic/*`` wildcard deployment). """ +import json import os import pytest from litellm.constants import BEDROCK_CONVERSE_MODELS +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") + +def _load_root_cost_map() -> dict: + json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json") + with open(json_path) as f: + return json.load(f) + + ALL_OPUS_5_VARIANTS = ( "claude-opus-5", "anthropic.claude-opus-5", @@ -58,3 +69,20 @@ def test_opus_5_registered_for_bedrock_converse(): assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS +def test_opus_5_5_present_in_bundled_backup(): + backup = GetModelCostMap.load_local_model_cost_map() + root = _load_root_cost_map() + assert "claude-opus-5-5" in backup + assert "claude-opus-5-5" in root + assert backup["claude-opus-5-5"] == root["claude-opus-5-5"] + + +@pytest.mark.parametrize("model", ["claude-opus-5-5", "anthropic/claude-opus-5-5"]) +def test_opus_5_5_thinking_profile(local_model_cost_map, model): + """Opus 5.5 has thinking always on with the adaptive thinking surface, and + no forced tool use, same as Fable 5.1.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True + assert AnthropicModelInfo._is_always_on_thinking_model(model, "anthropic") is True + assert AnthropicModelInfo.forced_tool_use_unsupported(model.removeprefix("anthropic/")) is True From 7056151b91c1bdf21f494f778afd8f52648a9ddd Mon Sep 17 00:00:00 2001 From: Philipp Schmid <32632186+philschmid@users.noreply.github.com> Date: Tue, 22 Sep 2026 18:53:22 +0200 Subject: [PATCH 24/28] fix(gemini): simplify model version check (#42465) --- litellm/llms/gemini/chat/transformation.py | 2 +- .../vertex_and_google_ai_studio_gemini.py | 36 +++++------- .../test_fallback_generalizations.py | 2 +- ...test_vertex_and_google_ai_studio_gemini.py | 56 +++++++++++++++++-- 4 files changed, 68 insertions(+), 28 deletions(-) diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index 42c9ef13730..285350aecba 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -102,7 +102,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): "include_server_side_tool_invocations", "service_tier", ] - if supports_reasoning(model, custom_llm_provider="gemini"): + if supports_reasoning(model, custom_llm_provider="gemini") or self._is_gemini_3_or_newer(model): supported_params.append("reasoning_effort") supported_params.append("thinking") if self.is_model_gemini_audio_model(model): diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 46f1b948026..204b1ddd5ea 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -2,6 +2,7 @@ ## httpx client for vertex ai calls ## Initial implementation - covers gemini + image gen calls import json +import re import time from collections.abc import Callable, Mapping, Sequence from copy import deepcopy @@ -283,20 +284,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @staticmethod def _is_gemini_3_or_newer(model: str) -> bool: """ - Check if the model is Gemini 3 Pro or newer. - - Gemini 3 models include: - - gemini-3-pro-preview - - gemini-3-flash - - gemini-3-flash-preview (Gemini 3 Flash) - - gemini-3.1-pro-preview, gemini-3.1-flash, gemini-3.1-flash-lite-preview - - gemini-3.5-flash - - Any future Gemini 3.x models + Check if the model is Gemini 3 or newer. """ - # Check for Gemini 3 models - if "gemini-3" in model: - return True - return False + model_name = model.split("/")[-1].lower() + if not model_name: + return False + # Pre-Gemini 3 models: gemini-1.x, gemini-2.x, gemini-pro, gemini-flash, gemini-exp + if re.match(r"^gemini-(?:[12](?:\.\d+)?|exp|(?:pro|flash)(?!-(?:lite-)?latest$))(?:-|$)", model_name): + return False + return True @staticmethod def _forward_gemini_function_call_id(model: str) -> bool: @@ -347,9 +343,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if self._supports_penalty_parameters(model): supported_params.extend(["frequency_penalty", "presence_penalty"]) - if supports_reasoning(model): + if supports_reasoning(model) or self._is_gemini_3_or_newer(model): supported_params.append("reasoning_effort") supported_params.append("thinking") + return supported_params def map_tool_choice_values(self, model: str, tool_choice: str | dict) -> ToolConfig | None: @@ -871,8 +868,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @staticmethod def _supports_minimal_thinking_level(model: str) -> bool: lowered: Final = model.lower() - is_gemini3flash: Final = "gemini-3" in lowered and "flash" in lowered - return is_gemini3flash and not is_explicitly_disabled_factory( + is_gemini3_or_newer_flash: Final = VertexGeminiConfig._is_gemini_3_or_newer(model) and "flash" in lowered + return is_gemini3_or_newer_flash and not is_explicitly_disabled_factory( model=model, custom_llm_provider=None, key="supports_minimal_reasoning_effort" ) @@ -890,9 +887,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Returns: GeminiThinkingConfig with thinkingLevel and includeThoughts """ - is_gemini3flash: Final = model and ("flash" in model.lower() and "gemini-3" in model.lower()) supports_minimal: Final = bool(model) and VertexGeminiConfig._supports_minimal_thinking_level(model) - is_gemini31pro: Final = model and ("gemini-3.1-pro-preview" in model.lower()) if reasoning_effort == "minimal": if supports_minimal: return {"thinkingLevel": "minimal", "includeThoughts": True} @@ -901,10 +896,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif reasoning_effort == "low": return {"thinkingLevel": "low", "includeThoughts": True} elif reasoning_effort == "medium": - if is_gemini31pro or is_gemini3flash: - return {"thinkingLevel": "medium", "includeThoughts": True} - else: - return {"thinkingLevel": "high", "includeThoughts": True} + return {"thinkingLevel": "medium", "includeThoughts": True} elif reasoning_effort == "high": return {"thinkingLevel": "high", "includeThoughts": True} elif reasoning_effort in ("disable", "none"): diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 25a12bebf9a..7928768b3bd 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -517,7 +517,7 @@ def test_shipped_gemini_chat_baseline_keeps_reasoning_effort_on_unmapped_model(s drop_params=False, ) assert isinstance(optional_params, dict) - assert optional_params["thinkingConfig"]["thinkingBudget"] > 0 + assert optional_params["thinkingConfig"]["thinkingLevel"] == "medium" assert optional_params["thinkingConfig"]["includeThoughts"] is True diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 6c818016c87..a796e2ac607 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -2435,10 +2435,16 @@ def test_is_gemini_3_or_newer(): VertexGeminiConfig, ) - # Gemini 3 models + # Gemini 3+ models assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-3-pro-preview") == True assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-3-flash") == True assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-3-pro") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-3.1-pro-preview") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-test-id-bla") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("test-id-bla") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-flash-latest") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-flash-lite-latest") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-pro-latest") == True assert ( VertexGeminiConfig._is_gemini_3_or_newer("vertex_ai/gemini-3-pro-preview") == True @@ -2453,11 +2459,53 @@ def test_is_gemini_3_or_newer(): assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-2.0-flash") == False assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-1.5-pro") == False assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-pro") == False + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-flash") == False # Edge cases assert VertexGeminiConfig._is_gemini_3_or_newer("") == False +@pytest.mark.parametrize( + "model", + [ + "gemini-3.1-pro-preview", + "gemini-3-flash", + "gemini-test-id-bla", + "test-id-bla", + ], +) +def test_gemini_3_reasoning_effort_maps_to_thinking_level(model: str): + """Test that reasoning_effort maps to thinkingLevel and default temperature=1.0""" + from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + vertex_cfg = VertexGeminiConfig() + studio_cfg = GoogleAIStudioGeminiConfig() + + for cfg in (vertex_cfg, studio_cfg): + supported = cfg.get_supported_openai_params(model) + assert "reasoning_effort" in supported + assert "thinking" in supported + + for effort in ("low", "medium", "high"): + mapped = vertex_cfg.map_openai_params( + non_default_params={"reasoning_effort": effort}, + optional_params={}, + model=model, + drop_params=False, + ) + assert mapped["thinkingConfig"] == { + "thinkingLevel": effort, + "includeThoughts": True, + } + assert mapped["temperature"] == 1.0 + assert "thinkingBudget" not in mapped["thinkingConfig"] + + + + def _tool_call_messages(tool_call_id: str): return [ {"role": "user", "content": "hi"}, @@ -2643,7 +2691,7 @@ def test_reasoning_effort_maps_to_thinking_level_gemini_3(): assert result["thinkingConfig"]["thinkingLevel"] == "low" assert result["thinkingConfig"]["includeThoughts"] is True - # Test medium -> high + includeThoughts=True (medium not available yet) + # Test medium -> medium + includeThoughts=True optional_params = {} non_default_params = {"reasoning_effort": "medium"} result = v.map_openai_params( @@ -2652,7 +2700,7 @@ def test_reasoning_effort_maps_to_thinking_level_gemini_3(): model=model, drop_params=False, ) - assert result["thinkingConfig"]["thinkingLevel"] == "high" + assert result["thinkingConfig"]["thinkingLevel"] == "medium" assert result["thinkingConfig"]["includeThoughts"] is True # Test high -> high + includeThoughts=True @@ -2853,7 +2901,7 @@ def test_reasoning_effort_dict_format_gemini_3(): model=model, drop_params=False, ) - assert result["thinkingConfig"]["thinkingLevel"] == "high" + assert result["thinkingConfig"]["thinkingLevel"] == "medium" assert result["thinkingConfig"]["includeThoughts"] is True # Test dict format without effort key - no thinkingConfig should be set From f3c920a06bfc02c91e8dec8c48e9528bc49d7464 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 22 Sep 2026 09:57:05 -0700 Subject: [PATCH 25/28] fix(bedrock/claude_platform): strip body params the AWS endpoint rejects (#31203) * fix(bedrock/claude_platform): strip body params the AWS endpoint rejects Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(bedrock/claude_platform): assert exact bodies through a strict fake gateway for every workspace alias and auth mode Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): block Claude Platform workspace id aliases in request bodies without admin opt-in Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Venkata Donavalli Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: shrey kharbanda --- .../bedrock/claude_platform/common_utils.py | 61 ++ .../messages_transformation.py | 41 +- .../bedrock/claude_platform/transformation.py | 25 +- litellm/proxy/auth/auth_utils.py | 4 + .../bedrock/test_claude_platform_provider.py | 536 +++++++++++++++--- .../proxy/auth/test_auth_utils.py | 24 + 6 files changed, 597 insertions(+), 94 deletions(-) diff --git a/litellm/llms/bedrock/claude_platform/common_utils.py b/litellm/llms/bedrock/claude_platform/common_utils.py index fb7f2185ec5..e25e58056f5 100644 --- a/litellm/llms/bedrock/claude_platform/common_utils.py +++ b/litellm/llms/bedrock/claude_platform/common_utils.py @@ -1,15 +1,76 @@ +from collections.abc import Mapping from typing import Final import httpx import litellm +from litellm._logging import verbose_logger from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import BedrockError from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams CLAUDE_PLATFORM_SERVICE_NAME: Final = "aws-external-anthropic" CLAUDE_PLATFORM_BEDROCK_ROUTE: Final = "claude_platform/" +CLAUDE_PLATFORM_UNSUPPORTED_PARAMS_OVERRIDE_KEY: Final = "claude_platform_unsupported_params" +CLAUDE_PLATFORM_ON_AWS_NON_REQUEST_PARAMS: Final = frozenset( + { + "workspace_id", + "aws_workspace_id", + "anthropic_workspace_id", + "anthropic-workspace-id", + CLAUDE_PLATFORM_UNSUPPORTED_PARAMS_OVERRIDE_KEY, + } +) +CLAUDE_PLATFORM_ON_AWS_UNSUPPORTED_REQUEST_PARAMS: Final = frozenset({"context_management"}) + + +def filter_claude_platform_request_body( + params: Mapping[str, object], + unsupported_override: frozenset[str] | None = None, + log_dropped: bool = True, +) -> dict[str, object]: + unsupported: Final = ( + unsupported_override if unsupported_override is not None else CLAUDE_PLATFORM_ON_AWS_UNSUPPORTED_REQUEST_PARAMS + ) + dropped_unsupported: Final = tuple(k for k in params if k in unsupported) + if dropped_unsupported and log_dropped: + verbose_logger.warning( + "bedrock/claude_platform: dropping unsupported Messages API param(s) %s from the request body; " + "the Claude Platform on AWS endpoint rejects unknown fields. The request will proceed without them.", + dropped_unsupported, + ) + return { + k: v + for k, v in params.items() + if k not in CLAUDE_PLATFORM_ON_AWS_NON_REQUEST_PARAMS and k not in unsupported and not k.startswith("aws_") + } + + +def resolve_unsupported_override( + litellm_params: Mapping[str, object] | GenericLiteLLMParams, + optional_params: Mapping[str, object] | None = None, + log_invalid: bool = True, +) -> frozenset[str] | None: + from_optional: Final = (optional_params or {}).get(CLAUDE_PLATFORM_UNSUPPORTED_PARAMS_OVERRIDE_KEY) + raw: Final = ( + from_optional + if from_optional is not None + else litellm_params.get(CLAUDE_PLATFORM_UNSUPPORTED_PARAMS_OVERRIDE_KEY) + ) + if raw is None: + return None + if isinstance(raw, (list, set, frozenset, tuple)): + return frozenset(str(item) for item in raw) + if log_invalid: + verbose_logger.warning( + "bedrock/claude_platform: ignoring claude_platform_unsupported_params of type %s; " + "expected a list of param names. Using the default unsupported-param set.", + type(raw).__name__, + ) + return None + def strip_claude_platform_route(model: str) -> str: if model.startswith(CLAUDE_PLATFORM_BEDROCK_ROUTE): diff --git a/litellm/llms/bedrock/claude_platform/messages_transformation.py b/litellm/llms/bedrock/claude_platform/messages_transformation.py index 1e3eea075f3..f423d22589b 100644 --- a/litellm/llms/bedrock/claude_platform/messages_transformation.py +++ b/litellm/llms/bedrock/claude_platform/messages_transformation.py @@ -8,7 +8,12 @@ from litellm.llms.anthropic.experimental_pass_through.messages.transformation im from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams -from .common_utils import BedrockClaudePlatformMixin, strip_claude_platform_route +from .common_utils import ( + BedrockClaudePlatformMixin, + filter_claude_platform_request_body, + resolve_unsupported_override, + strip_claude_platform_route, +) class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicMessagesConfig): @@ -46,26 +51,38 @@ class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicM if resolved_api_key and "x-api-key" not in headers: headers["x-api-key"] = resolved_api_key - headers = self._update_headers_with_anthropic_beta( - headers=headers, - optional_params=optional_params, - messages=messages, + return ( + self._update_headers_with_anthropic_beta( + headers=headers, + optional_params=filter_claude_platform_request_body( + optional_params, + unsupported_override=resolve_unsupported_override( + litellm_params, optional_params=optional_params, log_invalid=False + ), + log_dropped=False, + ), + messages=messages, + ), + api_base, ) - return headers, api_base - def transform_anthropic_messages_request( self, model: str, - messages: list[dict], - anthropic_messages_optional_request_params: dict, + messages: list[dict[str, object]], + anthropic_messages_optional_request_params: dict[str, object], litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> dict: + headers: dict[str, str], + ) -> dict[str, object]: return super().transform_anthropic_messages_request( model=strip_claude_platform_route(model), messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + anthropic_messages_optional_request_params=filter_claude_platform_request_body( + anthropic_messages_optional_request_params, + unsupported_override=resolve_unsupported_override( + litellm_params, optional_params=anthropic_messages_optional_request_params + ), + ), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py index a57f309b605..96a6ee4a701 100644 --- a/litellm/llms/bedrock/claude_platform/transformation.py +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -5,7 +5,11 @@ from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues -from .common_utils import BedrockClaudePlatformMixin +from .common_utils import ( + BedrockClaudePlatformMixin, + filter_claude_platform_request_body, + resolve_unsupported_override, +) class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): @@ -66,6 +70,25 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): anthropic_headers["anthropic-workspace-id"] = workspace_id return {**headers, **anthropic_headers} + def transform_request( + self, + model: str, + messages: list[AllMessageValues], + optional_params: dict[str, object], + litellm_params: dict[str, object], + headers: dict[str, str], + ) -> dict[str, object]: + return super().transform_request( + model=model, + messages=messages, + optional_params=filter_claude_platform_request_body( + optional_params, + unsupported_override=resolve_unsupported_override(litellm_params, optional_params=optional_params), + ), + litellm_params=litellm_params, + headers=headers, + ) + def get_model_response_iterator( self, streaming_response: Any, diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index ee012e65ab1..c002b2b9508 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -338,6 +338,10 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( # re-route the request's retention and accounting to any project # reachable with the deployment's shared AWS credentials. "aws_bedrock_project_id", + "workspace_id", + "aws_workspace_id", + "anthropic_workspace_id", + "anthropic-workspace-id", "bedrock_tags", # Provider-specific endpoint overrides that flow into the outbound # request via ``optional_params``. Same threat as ``api_base``: diff --git a/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py b/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py index 40f78c84ca3..7b42d1f5a35 100644 --- a/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py +++ b/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py @@ -1,9 +1,62 @@ import json +from collections.abc import Callable, Mapping +from typing import Final from unittest.mock import MagicMock, patch import httpx import pytest from botocore.credentials import Credentials +from pydantic import BaseModel, ConfigDict, ValidationError + +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.router import GenericLiteLLMParams + +WORKSPACE_ALIASES: Final = ("workspace_id", "aws_workspace_id", "anthropic_workspace_id", "anthropic-workspace-id") + + +class ClaudePlatformMessagesBody(BaseModel): + """Messages API fields per https://docs.anthropic.com/en/api/messages (2026-09) minus context_management, which + the AWS endpoint rejects with 400""" + + model_config = ConfigDict(extra="forbid") + + model: str + messages: list[dict] + max_tokens: int + system: str | list[dict] | None = None + metadata: dict | None = None + stop_sequences: list[str] | None = None + stream: bool | None = None + temperature: float | None = None + top_k: int | None = None + top_p: float | None = None + tools: list[dict] | None = None + tool_choice: dict | None = None + thinking: dict | None = None + service_tier: str | None = None + mcp_servers: list[dict] | None = None + output_format: dict | None = None + container: str | dict | None = None + + +def _gateway_reject(url: str, message: str) -> httpx.Response: + return httpx.Response( + status_code=400, + json={"type": "error", "error": {"type": "invalid_request_error", "message": message}}, + request=httpx.Request("POST", url), + ) + + +def _fake_claude_platform_gateway(url: str, headers: Mapping[str, str], data: bytes | str | None) -> httpx.Response: + if "anthropic-workspace-id" not in headers: + return _gateway_reject(url, "missing anthropic-workspace-id header") + if "x-api-key" not in headers and not headers.get("Authorization", "").startswith("AWS4-HMAC-SHA256 "): + return _gateway_reject(url, "missing x-api-key or SigV4 Authorization") + try: + ClaudePlatformMessagesBody.model_validate_json(data or "{}") + except ValidationError as exc: + return _gateway_reject(url, "; ".join(f"{'.'.join(map(str, e['loc']))}: {e['msg']}" for e in exc.errors())) + return _anthropic_response(url) def _anthropic_response(url: str) -> httpx.Response: @@ -23,15 +76,39 @@ def _anthropic_response(url: str) -> httpx.Response: ) -def _capture_request(url: str, headers: dict, data: bytes | str | None) -> dict: +def _capture_request(url: str, headers: Mapping[str, str], data: bytes | str | None) -> dict: raw_body = data.decode("utf-8") if isinstance(data, bytes) else data or "{}" return { "path": httpx.URL(url).path, - "headers": headers, + "headers": httpx.Headers(dict(headers)), "body": json.loads(raw_body), } +GatewayResponder = Callable[[str, Mapping[str, str], bytes], httpx.Response] + + +def _gateway_transport(requests: list[dict], respond: GatewayResponder) -> httpx.MockTransport: + def handle(request: httpx.Request) -> httpx.Response: + requests.append(_capture_request(url=str(request.url), headers=request.headers, data=request.content)) + return respond(str(request.url), request.headers, request.content) + + return httpx.MockTransport(handle) + + +def _sync_gateway_client( + requests: list[dict], + respond: GatewayResponder = _fake_claude_platform_gateway, +) -> HTTPHandler: + return HTTPHandler(client=httpx.Client(transport=_gateway_transport(requests, respond))) + + +def _async_gateway_client(requests: list[dict]) -> AsyncHTTPHandler: + handler: Final = AsyncHTTPHandler() + handler.client = httpx.AsyncClient(transport=_gateway_transport(requests, _fake_claude_platform_gateway)) + return handler + + def test_claude_platform_builds_default_messages_url_from_region(): from litellm.llms.bedrock.claude_platform.transformation import ( BedrockClaudePlatformConfig, @@ -77,9 +154,7 @@ def test_claude_platform_uses_bedrock_subroute(): import litellm from litellm.llms.bedrock.common_utils import BedrockModelInfo - model, provider, _, _ = litellm.get_llm_provider( - model="bedrock/claude_platform/claude-sonnet-4-6" - ) + model, provider, _, _ = litellm.get_llm_provider(model="bedrock/claude_platform/claude-sonnet-4-6") assert provider == "bedrock" assert model == "claude_platform/claude-sonnet-4-6" @@ -177,9 +252,7 @@ def test_claude_platform_sigv4_signs_transformed_request_body(): assert signed_body == json.dumps(request_body).encode() assert headers["Authorization"] == "signed" mock_sign_request.assert_called_once() - assert ( - mock_sign_request.call_args.kwargs["service_name"] == "aws-external-anthropic" - ) + assert mock_sign_request.call_args.kwargs["service_name"] == "aws-external-anthropic" assert mock_sign_request.call_args.kwargs["request_data"] == request_body @@ -237,7 +310,7 @@ def test_bedrock_claude_platform_messages_config_round_trips_native_body(): model="claude_platform/claude-sonnet-4-6", messages=[{"role": "user", "content": "hello"}], anthropic_messages_optional_request_params={"max_tokens": 10}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers=headers, ) @@ -250,69 +323,6 @@ def test_bedrock_claude_platform_messages_config_round_trips_native_body(): } -def test_chat_completion_routes_bedrock_claude_platform_to_messages_api(): - import litellm - - requests = [] - - def mock_post(self, url, data=None, headers=None, **kwargs): - requests.append(_capture_request(url=url, headers=headers or {}, data=data)) - return _anthropic_response(url) - - with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post): - response = litellm.completion( - model="bedrock/claude_platform/claude-sonnet-4-6", - messages=[{"role": "user", "content": "hello"}], - max_tokens=10, - api_base="https://aws-external-anthropic.us-west-2.api.aws", - api_key="fake-platform-key", - workspace_id="wrkspc_test", - ) - - assert response.choices[0].message.content == "ok" - assert len(requests) == 1 - assert requests[0]["path"] == "/v1/messages" - assert requests[0]["headers"]["x-api-key"] == "fake-platform-key" - assert requests[0]["headers"]["anthropic-workspace-id"] == "wrkspc_test" - assert requests[0]["body"]["model"] == "claude-sonnet-4-6" - - -@pytest.mark.asyncio -async def test_anthropic_messages_routes_bedrock_claude_platform_to_messages_api(): - import litellm - - requests = [] - - async def mock_post(self, url, data=None, headers=None, **kwargs): - requests.append(_capture_request(url=url, headers=headers or {}, data=data)) - return _anthropic_response(url) - - try: - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new=mock_post, - ): - response = await litellm.anthropic_messages( - model="bedrock/claude_platform/claude-sonnet-4-6", - messages=[{"role": "user", "content": "hello"}], - max_tokens=10, - api_base="https://aws-external-anthropic.us-west-2.api.aws", - api_key="fake-platform-key", - workspace_id="wrkspc_test", - ) - finally: - await litellm.close_litellm_async_clients() - - assert response["content"][0]["text"] == "ok" - assert len(requests) == 1 - assert requests[0]["path"] == "/v1/messages" - assert requests[0]["headers"]["x-api-key"] == "fake-platform-key" - assert requests[0]["headers"]["anthropic-workspace-id"] == "wrkspc_test" - assert requests[0]["body"]["messages"] == [{"role": "user", "content": "hello"}] - assert requests[0]["body"]["max_tokens"] == 10 - assert requests[0]["body"]["model"] == "claude-sonnet-4-6" - - @pytest.mark.asyncio async def test_anthropic_messages_bedrock_claude_platform_forwards_anthropic_beta_verbatim(): import litellm @@ -348,16 +358,380 @@ async def test_anthropic_messages_bedrock_claude_platform_forwards_anthropic_bet ] -def test_sigv4_no_duplicate_content_type_when_caller_sets_lowercase(): - """ - Regression: get_anthropic_headers() supplies "content-type" (lowercase). - _sign_request() used to prepend "Content-Type" (uppercase), leaving both - keys in the dict. botocore joins them into "application/json, application/json" - in the canonical string, while the wire request sends only one value → 401. +def test_claude_platform_strips_auth_params_from_request_body(): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) - Fix: prepend with lowercase "content-type" so **headers overwrites it when - the caller already set it. - """ + config = BedrockClaudePlatformConfig() + optional_params = { + "workspace_id": "wrkspc_test", + "aws_region_name": "us-west-2", + "max_tokens": 10, + } + + request_body = config.transform_request( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request_body == EXPECTED_CHAT_BODY + assert optional_params == {"workspace_id": "wrkspc_test", "aws_region_name": "us-west-2", "max_tokens": 10} + + +def test_claude_platform_messages_strips_auth_params_from_request_body(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + assert config is not None + + input_params = { + "workspace_id": "wrkspc_test", + "aws_region_name": "us-west-2", + "max_tokens": 10, + } + request_body = config.transform_anthropic_messages_request( + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + anthropic_messages_optional_request_params=input_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert request_body == { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + } + assert input_params == {"workspace_id": "wrkspc_test", "aws_region_name": "us-west-2", "max_tokens": 10} + + +def test_claude_platform_strips_unsupported_context_management_param(caplog): + import logging + + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + optional_params = { + "workspace_id": "wrkspc_test", + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + "max_tokens": 10, + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + request_body = config.transform_request( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "context_management" not in request_body + assert request_body["max_tokens"] == 10 + assert "context_management" in optional_params + assert any( + "context_management" in record.message and record.levelno == logging.WARNING for record in caplog.records + ) + + +def test_claude_platform_messages_strips_unsupported_context_management_param(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + assert config is not None + + input_params = { + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + "max_tokens": 10, + } + request_body = config.transform_anthropic_messages_request( + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + anthropic_messages_optional_request_params=input_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert "context_management" not in request_body + assert request_body["max_tokens"] == 10 + assert "context_management" in input_params + + +@pytest.mark.parametrize("override_in", ["litellm_params", "optional_params"]) +def test_claude_platform_unsupported_override_allows_context_management(override_in): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + override = {"claude_platform_unsupported_params": []} + context_management = {"edits": [{"type": "clear_tool_uses_20250919"}]} + + request_body = config.transform_request( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={ + "context_management": context_management, + "max_tokens": 10, + **(override if override_in == "optional_params" else {}), + }, + litellm_params=override if override_in == "litellm_params" else {}, + headers={}, + ) + + assert request_body == { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}], + "max_tokens": 10, + "context_management": context_management, + } + + +def test_claude_platform_unsupported_override_ignores_invalid_type(): + from litellm.llms.bedrock.claude_platform import common_utils + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + with patch.object(common_utils.verbose_logger, "warning") as mock_warning: + request_body = config.transform_request( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={ + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + "max_tokens": 10, + }, + litellm_params={"claude_platform_unsupported_params": "not_a_list"}, + headers={}, + ) + + assert "context_management" not in request_body + assert request_body["max_tokens"] == 10 + warned = [call.args[0] for call in mock_warning.call_args_list] + assert any("claude_platform_unsupported_params" in message for message in warned) + + +def test_claude_platform_messages_does_not_advertise_beta_for_stripped_context_management(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + assert config is not None + + headers, _ = config.validate_anthropic_messages_environment( + api_key="fake-platform-key", + headers={}, + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={ + "max_tokens": 10, + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + }, + litellm_params={"workspace_id": "wrkspc_test"}, + ) + + assert "context-management-2025-06-27" not in headers.get("anthropic-beta", "") + + +def test_claude_platform_messages_override_keeps_beta_for_context_management(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + assert config is not None + + headers, _ = config.validate_anthropic_messages_environment( + api_key="fake-platform-key", + headers={}, + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={ + "max_tokens": 10, + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + }, + litellm_params={ + "workspace_id": "wrkspc_test", + "claude_platform_unsupported_params": [], + }, + ) + + assert "context-management-2025-06-27" in headers.get("anthropic-beta", "") + + +def test_claude_platform_messages_unsupported_override_allows_context_management(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + assert config is not None + + request_body = config.transform_anthropic_messages_request( + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + anthropic_messages_optional_request_params={ + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + "max_tokens": 10, + }, + litellm_params=GenericLiteLLMParams.model_validate({"claude_platform_unsupported_params": []}), + headers={}, + ) + + assert "context_management" in request_body + assert request_body["max_tokens"] == 10 + + +SIGV4_KWARGS: Final = { + "aws_region_name": "us-west-2", + "aws_access_key_id": "AKIATEST", + "aws_secret_access_key": "test-secret", + "aws_session_token": "test-token", +} +API_KEY_KWARGS: Final = {"api_key": "fake-platform-key", "aws_region_name": "us-west-2"} +EXPECTED_CHAT_BODY: Final = { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}], + "max_tokens": 10, +} +EXPECTED_NATIVE_BODY: Final = { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + "stream": False, +} + + +def _assert_gateway_accepted(request: dict, auth_kwargs: dict) -> None: + assert request["path"] == "/v1/messages" + assert request["headers"]["anthropic-workspace-id"] == "wrkspc_test" + if "api_key" in auth_kwargs: + assert request["headers"]["x-api-key"] == auth_kwargs["api_key"] + assert "Authorization" not in request["headers"] + else: + assert request["headers"]["Authorization"].startswith("AWS4-HMAC-SHA256 Credential=AKIATEST/") + assert "/us-west-2/aws-external-anthropic/aws4_request" in request["headers"]["Authorization"] + + +@pytest.mark.parametrize("auth_kwargs", [API_KEY_KWARGS, SIGV4_KWARGS], ids=["api_key", "sigv4"]) +@pytest.mark.parametrize("workspace_alias", WORKSPACE_ALIASES) +def test_chat_completion_claude_platform_sends_exact_body_through_strict_gateway(auth_kwargs, workspace_alias): + import litellm + + requests: list[dict] = [] + + response = litellm.completion( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + client=_sync_gateway_client(requests), + **{workspace_alias: "wrkspc_test"}, + **auth_kwargs, + ) + + assert response.choices[0].message.content == "ok" + assert len(requests) == 1, requests + assert requests[0]["body"] == EXPECTED_CHAT_BODY + _assert_gateway_accepted(requests[0], auth_kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_kwargs", [API_KEY_KWARGS, SIGV4_KWARGS], ids=["api_key", "sigv4"]) +@pytest.mark.parametrize("workspace_alias", WORKSPACE_ALIASES) +async def test_anthropic_messages_claude_platform_sends_exact_body_through_strict_gateway(auth_kwargs, workspace_alias): + import litellm + + requests: list[dict] = [] + + response = await litellm.anthropic_messages( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + client=_async_gateway_client(requests), + **{workspace_alias: "wrkspc_test"}, + **auth_kwargs, + ) + + assert response["content"][0]["text"] == "ok" + assert len(requests) == 1, requests + assert requests[0]["body"] == EXPECTED_NATIVE_BODY + _assert_gateway_accepted(requests[0], auth_kwargs) + + +def test_chat_completion_claude_platform_drops_context_management_and_gateway_accepts(): + import litellm + + requests: list[dict] = [] + + litellm.completion( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + workspace_id="wrkspc_test", + context_management={"edits": [{"type": "clear_tool_uses_20250919"}]}, + client=_sync_gateway_client(requests), + **API_KEY_KWARGS, + ) + + assert requests[0]["body"] == EXPECTED_CHAT_BODY + + +def test_chat_completion_claude_platform_override_kwarg_is_honoured_and_not_sent(): + import litellm + + requests: list[dict] = [] + context_management = {"edits": [{"type": "clear_tool_uses_20250919"}]} + + litellm.completion( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + workspace_id="wrkspc_test", + context_management=context_management, + claude_platform_unsupported_params=[], + client=_sync_gateway_client(requests, respond=lambda url, headers, data: _anthropic_response(url)), + **API_KEY_KWARGS, + ) + + assert requests[0]["body"] == {**EXPECTED_CHAT_BODY, "context_management": context_management} + + +def test_fake_claude_platform_gateway_rejects_leaked_internal_fields(): + leaked = json.dumps({**EXPECTED_NATIVE_BODY, "workspace_id": "wrkspc_test", "aws_region_name": "us-west-2"}) + response = _fake_claude_platform_gateway( + url="https://aws-external-anthropic.us-west-2.api.aws/v1/messages", + headers={"anthropic-workspace-id": "wrkspc_test", "x-api-key": "k"}, + data=leaked, + ) + assert response.status_code == 400 + assert response.json()["error"]["message"] == ( + "workspace_id: Extra inputs are not permitted; aws_region_name: Extra inputs are not permitted" + ) + + +def test_sigv4_no_duplicate_content_type_when_caller_sets_lowercase(): from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM llm = BaseAWSLLM() diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 965acd57bf3..83ac56c4c85 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -2727,6 +2727,30 @@ class TestIsRequestBodySafeBlocksBedrockProjectOverride: ) +class TestIsRequestBodySafeBlocksClaudePlatformWorkspaceOverride: + @pytest.mark.parametrize( + "alias", ["workspace_id", "aws_workspace_id", "anthropic_workspace_id", "anthropic-workspace-id"] + ) + def test_workspace_alias_in_request_body_is_rejected(self, alias): + with pytest.raises(ValueError, match=alias): + is_request_body_safe( + request_body={"model": "gpt-4", alias: "wrkspc_attacker"}, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + def test_admin_opt_in_proxy_wide_allows_workspace_id(self): + assert ( + is_request_body_safe( + request_body={"model": "gpt-4", "workspace_id": "wrkspc_byok"}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="gpt-4", + ) + is True + ) + class TestIsRequestBodySafeBlocksRustOptIn: """``rust`` hands the whole call to the Rust core, which signs and sends with its own HTTP client rather than the one the deployment configured, and From f275be5fac43c4f30c1593c3f0aa6995c6a9c90c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 10:17:19 -0700 Subject: [PATCH 26/28] fix(proxy): never render credential-bearing config keys in the bug report (#42493) Co-authored-by: ryan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/bug_report_config.py | 40 ++++++++++++++--- .../proxy/test_bug_report_config.py | 44 +++++++++++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/bug_report_config.py b/litellm/proxy/bug_report_config.py index fa4a2604209..527be65c9a7 100644 --- a/litellm/proxy/bug_report_config.py +++ b/litellm/proxy/bug_report_config.py @@ -21,6 +21,30 @@ from litellm.types.secret_managers.main import KeyManagementSystem _OBJECT_MAP: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _OBJECT_LIST: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) _JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +CREDENTIAL_KEY_PARTS: Final = frozenset( + { + "key", + "keys", + "secret", + "secrets", + "token", + "password", + "passwd", + "credential", + "credentials", + "url", + "uri", + "dsn", + "host", + "hosts", + "base", + "endpoint", + "cert", + "pem", + "salt", + } +) +ENUM_KEYS_WITH_CREDENTIAL_PARTS: Final = frozenset({"key_management_system"}) def _object_map(value: object) -> Mapping[str, object]: @@ -87,22 +111,26 @@ def _cache_params_keys() -> frozenset[str]: return frozenset(name for name in inspect.signature(Cache.__init__).parameters if name != "self") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped params, only names are read -def _render_json(value: JsonValue) -> str | None: +def _is_credential_key(key: str) -> bool: + return key not in ENUM_KEYS_WITH_CREDENTIAL_PARTS and not CREDENTIAL_KEY_PARTS.isdisjoint(key.lower().split("_")) + + +def _render_json(key: str, value: JsonValue) -> str | None: match value: case bool(): return str(value).lower() case str(): - return value if value in _known_values() else None + return value if value in _known_values() and not _is_credential_key(key) else None case list(): - known_items: Final = tuple(rendered for item in value if (rendered := _render_json(item)) is not None) + known_items: Final = tuple(rendered for item in value if (rendered := _render_json(key, item)) is not None) return f"[{', '.join(known_items)}]" if known_items else None case _: return None -def _render(value: object) -> str | None: +def _render(key: str, value: object) -> str | None: try: - return _render_json(_JSON.validate_python(value)) + return _render_json(key, _JSON.validate_python(value)) except ValidationError: return None @@ -111,7 +139,7 @@ def _section_lines(section: str, values: Mapping[str, object], known_keys: froze return tuple( f"{section}.{key} = {rendered}" for key, value in values.items() - if key in known_keys and (rendered := _render(value)) is not None + if key in known_keys and (rendered := _render(key, value)) is not None ) diff --git a/tests/test_litellm/proxy/test_bug_report_config.py b/tests/test_litellm/proxy/test_bug_report_config.py index ce4baa946f9..06bca8c66fb 100644 --- a/tests/test_litellm/proxy/test_bug_report_config.py +++ b/tests/test_litellm/proxy/test_bug_report_config.py @@ -124,6 +124,50 @@ def test_secrets_numbers_and_unknown_values_leave_no_line(): assert safe_config_lines({}, general_settings) == ("general_settings.background_health_checks = false",) +def test_credential_keys_never_render_even_when_the_secret_equals_a_litellm_token(): + config: Mapping[str, object] = { + "litellm_settings": { + "openai_key": "openai", + "token": "langfuse", + "api_base": "azure", + "callbacks": ["langfuse"], + "cache_params": {"type": "redis", "password": "redis", "host": "openai", "qdrant_api_key": "qdrant"}, + }, + "router_settings": { + "routing_strategy": "simple-shuffle", + "redis_password": "simple-shuffle", + "redis_url": "redis", + }, + "guardrails": [ + { + "litellm_params": { + "guardrail": "presidio", + "api_key": "presidio", + "auth_token": "pre_call", + "client_secret": "openai", + "credentials": ["openai", "azure"], + } + } + ], + } + general_settings: Mapping[str, object] = { + "master_key": "redis", + "database_url": "openai", + "alert_to_webhook_url": "langfuse", + "key_management_system": "aws_secret_manager", + "use_azure_key_vault": True, + } + + assert safe_config_lines(config, general_settings) == ( + "general_settings.key_management_system = aws_secret_manager", + "general_settings.use_azure_key_vault = true", + "litellm_settings.callbacks = [langfuse]", + "litellm_settings.cache_params.type = redis", + "router_settings.routing_strategy = simple-shuffle", + "guardrails[0].litellm_params.guardrail = presidio", + ) + + def test_malformed_sections_produce_no_lines(): config: Mapping[str, object] = { "litellm_settings": "acme", From deba47382188582e14a46cb3cd61c522b346937e Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 22 Sep 2026 10:22:41 -0700 Subject: [PATCH 27/28] fix(cost): bill batch prompts above 272K at OpenAI's long-context batch tier (#39861) * fix(cost): bill batch prompts above 272K at OpenAI's long-context batch tier * fix(cost): mirror batch long-context keys on custom pricing params Register the two *_above_272k_tokens_batches keys on CustomPricingLiteLLMParams so a per-deployment override stays out of the shared backend key, add them to the inline model-info schema and alias-count tests, and build LiteLLM_Params and GenericLiteLLMParams through model_validate at the two dict-splat call sites so basedpyright's reportArgumentType budget ratchets down instead of blocking the new fields. * fix(cost): add the gpt-5.5-pro batch long-context tier and ignore malformed batch tier keys * fix(cost): bill cached batch tokens at OpenAI's cached batch rate Adds cache_read_input_token_cost_batches and cache_read_input_token_cost_above_272k_tokens_batches for the tiered OpenAI entries at half the standard cached rate, bills cached batch tokens at that rate per output line, and parses string-valued batch rates in deployment-level model_info. * fix(cost): bill batch cache writes at the batch cache-write rate and carry published batch rates for one-sided deployments OpenAI's Batch table prices cache writes for gpt-6-astra, gpt-5.6, gpt-5.6-sol, gpt-5.6-terra and gpt-5.6-luna at half the standard cache-write rate, so the cost map gains cache_creation_input_token_cost_batches and its above_272k tier for those entries and batch cost pulls written tokens out of the input bucket at that rate; models without the key keep billing writes at the batch input rate. A deployment declaring only one side of its batch pricing now carries every published batch rate of the other side (tier, cached, cache write), its own keys win, and a lone tier, cached or cache-write batch key counts as declared pricing instead of being ignored. * fix(cost): select the batch long-context tier from any batch tier key A deployment that declares its own flat standard input rate keeps every published batch rate of the output direction, including the 272K output tier, but the tier was only ever selected when an input tier key was also present. Detect the crossed tier from any of the four batch tier keys so the carried output, cache-read, and cache-write tiers bill at their tier rate above 272K tokens. * chore(proxy): keep the OpenAPI snapshot as CI generates it * fix(cost): pick each batch price component's tier from its own keys The batch rate picker crossed one threshold for every component, so a deployment declaring only an output tier also moved its input, cached, and cache-write rates to that cutoff. Each component now crosses its own *_above_k_tokens_batches keys and falls back to its flat key. The JSON schema is regenerated with the generator as it is on main: cost-map-guard renders the PR's cost map with the base branch's generator, so the descriptions for the new batch cache keys move to a follow-up. * chore(proxy): restore the lazy OpenAPI snapshot to what CI's Python 3.12 generates The merge commit carried a snapshot regenerated on a Python 3.14 venv, which dedents docstrings at compile time, so one description line differed from the file CI regenerates on 3.12 and the schema.d.ts sync check went red. The snapshot is byte-identical to main again --- litellm/cost_calculator.py | 62 ++-- litellm/litellm_core_utils/litellm_logging.py | 67 ++++- .../litellm_core_utils/llm_cost_calc/utils.py | 90 +++++- .../llms/azure/passthrough/transformation.py | 2 +- ...odel_prices_and_context_window_backup.json | 54 ++++ litellm/types/utils.py | 12 + litellm/utils.py | 14 + model_prices_and_context_window.json | 54 ++++ model_prices_and_context_window.schema.json | 28 ++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 145 ++++++++++ .../test_litellm_logging.py | 136 +++++++++ tests/test_litellm/test_cost_calculator.py | 270 +++++++++++++++++- tests/test_litellm/test_utils.py | 6 + tests/unit/batches/test_batch_utils.py | 68 ++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 24 ++ 15 files changed, 985 insertions(+), 47 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 7b02a5ede00..ae54b91c5a0 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( _get_service_tier_cost_key, calculate_cost_component, generic_cost_per_token, + get_batch_cost_rates, get_billable_input_tokens, get_token_type_cost_breakdown, parse_prompt_tokens_details, @@ -2557,35 +2558,14 @@ def batch_cost_calculator( if not model_info: return 0.0, 0.0 - input_cost_per_token_batches: Final = model_info.get("input_cost_per_token_batches") + batch_rates: Final = get_batch_cost_rates(model_info, usage, custom_llm_provider) input_cost_per_token: Final = model_info.get("input_cost_per_token") - output_cost_per_token_batches: Final = model_info.get("output_cost_per_token_batches") output_cost_per_token: Final = model_info.get("output_cost_per_token") total_prompt_cost = 0.0 total_completion_cost = 0.0 - if input_cost_per_token_batches is not None: - batch_details: Final = parse_prompt_tokens_details(usage) - audio_tokens, image_tokens, video_tokens = ( - batch_details["audio_tokens"], - batch_details["image_tokens"], - batch_details["video_tokens"], - ) - modality_rates: Final = ( - _batch_rate(model_info, "input_cost_per_audio_token_batches", input_cost_per_token_batches), - _batch_rate(model_info, "input_cost_per_image_token_batches", input_cost_per_token_batches), - _batch_rate(model_info, "input_cost_per_video_token_batches", input_cost_per_token_batches), - ) - total_prompt_cost = sum( - tokens * rate - for tokens, rate in zip( - ( - max((usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens, 0), - audio_tokens, - image_tokens, - video_tokens, - ), - (input_cost_per_token_batches, *modality_rates), - ) + if batch_rates.input is not None: + total_prompt_cost = _batch_prompt_cost( + usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation ) elif input_cost_per_token: details: Final = parse_prompt_tokens_details(usage) @@ -2605,8 +2585,8 @@ def batch_cost_calculator( cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2 - if output_cost_per_token_batches is not None: - total_completion_cost = usage.completion_tokens * output_cost_per_token_batches + if batch_rates.output is not None: + total_completion_cost = usage.completion_tokens * batch_rates.output elif output_cost_per_token: total_completion_cost = ( usage.completion_tokens * (output_cost_per_token) / 2 @@ -2620,6 +2600,34 @@ def batch_cost_calculator( return total_prompt_cost, total_completion_cost +def _batch_prompt_cost( + usage: Usage, + model_info: ModelInfo, + input_rate: float, + cache_read_rate: float | None, + cache_creation_rate: float | None, +) -> float: + details: Final = parse_prompt_tokens_details(usage) + cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 + written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0 + audio_tokens, image_tokens, video_tokens = ( + details["audio_tokens"], + details["image_tokens"], + details["video_tokens"], + ) + text_tokens: Final = max( + (usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens - cached_tokens - written_tokens, 0 + ) + return ( + text_tokens * input_rate + + audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate) + + image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate) + + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) + + cached_tokens * (cache_read_rate or 0.0) + + written_tokens * (cache_creation_rate or 0.0) + ) + + def _attribute_value(obj: object, name: str) -> object: return getattr(obj, name) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 306812cca0a..357ee51a44a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -387,11 +387,40 @@ _DEPLOYMENT_PRICING_KEYS: Final = ( "output_cost_per_token", "input_cost_per_token_batches", "output_cost_per_token_batches", + "input_cost_per_token_above_272k_tokens_batches", + "output_cost_per_token_above_272k_tokens_batches", + "cache_read_input_token_cost_batches", + "cache_read_input_token_cost_above_272k_tokens_batches", + "cache_creation_input_token_cost_batches", + "cache_creation_input_token_cost_above_272k_tokens_batches", "ocr_cost_per_page", "ocr_cost_per_page_batches", "annotation_cost_per_page", "annotation_cost_per_page_batches", ) +_INPUT_PRICING_KEY_PREFIXES: Final = ( + "input_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", +) +_OUTPUT_PRICING_KEY_PREFIXES: Final = ("output_cost_per_token",) +_BATCH_PRICING_KEY_SUFFIX: Final = "_batches" + + +_NO_CARRIED_RATES: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _published_direction( + published: ModelInfo, registered: Mapping[str, object], flat_key: str, prefixes: tuple[str, ...] +) -> Mapping[str, object]: + return MappingProxyType( + { + key: value + for key, value in published.items() + if registered.get(key) is None + and (key == flat_key or (key.startswith(prefixes) and key.endswith(_BATCH_PRICING_KEY_SUFFIX))) + } + ) def deployment_pricing_model_info(model_id: str | None, deployment_model: str | None) -> ModelInfo | None: @@ -403,12 +432,15 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str | get_model_info fills absent costs with 0, so asking it directly cannot tell "configured as free" apart from "no pricing configured". A deployment may declare only one side of its pricing, so the side it leaves out keeps - the model's published rates instead of billing as zero. Ownership is per - token direction: declaring either rate for a direction takes that whole - direction, so a published batch rate can never displace a standard rate - the deployment configured itself. OCR per-page rates count as declared - pricing too; they pass through as registered and ``ocr_batch_cost`` layers - the published rate under each per-page family the deployment leaves out. + the model's published standard rate and every published batch rate for + that direction (flat, long-context tier, cached, cache write) instead of + billing as zero. Ownership is per token direction: declaring the flat + standard or flat batch rate for a direction takes that whole direction, so + a published batch rate can never displace a standard rate the deployment + configured itself. A tier-only override keeps every published rate it left + out. OCR per-page rates count as declared pricing too; they pass through as + registered and ``ocr_batch_cost`` layers the published rate under each + per-page family the deployment leaves out. """ if model_id is None: return None @@ -429,13 +461,22 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str | registered.get("output_cost_per_token") is not None or registered.get("output_cost_per_token_batches") is not None ) - if not declares_input: - merged["input_cost_per_token"] = published.get("input_cost_per_token") - merged["input_cost_per_token_batches"] = published.get("input_cost_per_token_batches") - if not declares_output: - merged["output_cost_per_token"] = published.get("output_cost_per_token") - merged["output_cost_per_token_batches"] = published.get("output_cost_per_token_batches") - return merged + carried_input: Final = ( + _NO_CARRIED_RATES + if declares_input + else _published_direction(published, registered, "input_cost_per_token", _INPUT_PRICING_KEY_PREFIXES) + ) + carried_output: Final = ( + _NO_CARRIED_RATES + if declares_output + else _published_direction(published, registered, "output_cost_per_token", _OUTPUT_PRICING_KEY_PREFIXES) + ) + priced: Final[ModelInfo] = { # pyright: ignore[reportAssignmentType] # carried keys are ModelInfo rates + **merged, + **carried_input, + **carried_output, + } + return priced def _published_pricing(deployment_model: str | None) -> ModelInfo | None: diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index c60e3089816..3845f50af79 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -67,6 +67,15 @@ _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType( ) _INCLUSIVE_THRESHOLD_PROVIDERS: Final = frozenset({"xai"}) +_BATCH_KEY_SUFFIX: Final = "_batches" +_BATCH_RATE_PREFIXES: Final = ( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", +) +_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_RATE_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$") +_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX) def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool: @@ -248,9 +257,80 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str: return f"{base_key}_{suffix}" +def _parse_token_threshold(threshold: str) -> float: + return float(threshold.replace("k", "")) * (1000 if "k" in threshold else 1) + + def _parse_above_token_threshold(key: str) -> float: - threshold_str: Final = key.split("_above_")[1].split("_tokens")[0] - return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1) + return _parse_token_threshold(key.split("_above_")[1].split("_tokens")[0]) + + +def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool: + return prompt_tokens > threshold or (inclusive and prompt_tokens == threshold) + + +@dataclass(frozen=True, slots=True) +class BatchCostRates: + input: float | None + output: float | None + cache_read: float | None + cache_creation: float | None + + +def _batch_rate(model_info: ModelInfo, key: str) -> float | None: + value: Final = model_info.get(key) + if isinstance(value, (int, float)): + return float(value) + if not isinstance(value, str): + return None + try: + return float(value) + except ValueError: + return None + + +def _batch_tier_rate(model_info: ModelInfo, tier_key: str, flat_key: str) -> float | None: + tier_rate: Final = _batch_rate(model_info, tier_key) + return _batch_rate(model_info, flat_key) if tier_rate is None else tier_rate + + +def _batch_tier_thresholds(model_info: ModelInfo, prefix: str) -> frozenset[str]: + return frozenset( + tier.group(2) + for key, value in model_info.items() + if value is not None and (tier := _BATCH_TIER_KEY.match(key)) is not None and tier.group(1) == prefix + ) + + +def _crossed_batch_tier(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> str | None: + return next( + ( + threshold + for threshold in sorted( + _batch_tier_thresholds(model_info, prefix), key=_parse_token_threshold, reverse=True + ) + if _prompt_exceeds_threshold(usage.prompt_tokens, _parse_token_threshold(threshold), inclusive) + ), + None, + ) + + +def _batch_rate_for_prefix(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> float | None: + flat_key: Final = f"{prefix}{_BATCH_KEY_SUFFIX}" + threshold: Final = _crossed_batch_tier(model_info, prefix, usage, inclusive) + if threshold is None: + return _batch_rate(model_info, flat_key) + return _batch_tier_rate(model_info, f"{prefix}_above_{threshold}_tokens{_BATCH_KEY_SUFFIX}", flat_key) + + +def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provider: str | None) -> BatchCostRates: + inclusive: Final = _uses_inclusive_token_thresholds(custom_llm_provider) + return BatchCostRates( + input=_batch_rate_for_prefix(model_info, "input_cost_per_token", usage, inclusive), + output=_batch_rate_for_prefix(model_info, "output_cost_per_token", usage, inclusive), + cache_read=_batch_rate_for_prefix(model_info, "cache_read_input_token_cost", usage, inclusive), + cache_creation=_batch_rate_for_prefix(model_info, "cache_creation_input_token_cost", usage, inclusive), + ) def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict | None: @@ -576,7 +656,9 @@ def _get_token_base_cost( # so that the threshold detection loop only processes standard keys. The # service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key. threshold_keys: Final = [ - k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES) + k + for k in model_info + if k.startswith("input_cost_per_token_above_") and not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) ] # Only sort the threshold keys (typically 1-2 keys instead of 66+) @@ -588,7 +670,7 @@ def _get_token_base_cost( # Handle both formats: _above_128k_tokens and _above_128_tokens threshold_str = key.split("_above_")[1].split("_tokens")[0] threshold = _parse_above_token_threshold(key) - if usage.prompt_tokens > threshold or (threshold_is_inclusive and usage.prompt_tokens == threshold): + if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive): # Prefer a service_tier-specific above-threshold key when available, # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini # ON_DEMAND_PRIORITY. Falls back to the standard key automatically diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index 1b5a4083ebe..c40cefecdd0 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -143,7 +143,7 @@ class AzurePassthroughConfig(BasePassthroughConfig): ) -> dict: return BaseAzureLLM._base_validate_azure_environment( headers=headers, - litellm_params=GenericLiteLLMParams(**{**litellm_params, "api_key": api_key}), + litellm_params=GenericLiteLLMParams.model_validate({**litellm_params, "api_key": api_key}), ) @staticmethod diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 468d86d049d..7cde9a1162e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -34129,6 +34129,10 @@ "cache_read_input_token_cost_above_272k_tokens": 2e-06, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, + "cache_read_input_token_cost_batches": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 1e-06, + "cache_creation_input_token_cost_batches": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-05, "cache_read_input_token_cost_flex": 5e-07, "cache_read_input_token_cost_priority": 2e-06, "input_cost_per_token": 1e-05, @@ -34136,6 +34140,7 @@ "input_cost_per_token_above_272k_tokens_flex": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 4e-05, "input_cost_per_token_batches": 5e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-05, "input_cost_per_token_flex": 5e-06, "input_cost_per_token_priority": 2e-05, "litellm_provider": "openai", @@ -34148,6 +34153,7 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, "output_cost_per_token_above_272k_tokens_priority": 0.00015, "output_cost_per_token_batches": 2.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-05, "output_cost_per_token_flex": 2.5e-05, "output_cost_per_token_priority": 0.0001, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34199,6 +34205,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34206,6 +34216,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34218,6 +34229,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34266,6 +34278,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34273,6 +34289,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34285,6 +34302,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34335,6 +34353,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, @@ -34342,6 +34364,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, "litellm_provider": "openai", @@ -34354,6 +34377,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-06, "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, + "output_cost_per_token_above_272k_tokens_batches": 9e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34403,6 +34427,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_read_input_token_cost_batches": 1e-08, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-08, + "cache_creation_input_token_cost_batches": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -34410,6 +34438,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-07, "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_above_272k_tokens_batches": 2e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, "litellm_provider": "openai", @@ -34422,6 +34451,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-07, "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_above_272k_tokens_batches": 9e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34692,12 +34722,15 @@ "gpt-5.5": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34708,6 +34741,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34750,12 +34784,15 @@ "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34766,6 +34803,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34810,6 +34848,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34819,6 +34858,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34859,6 +34899,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34868,6 +34909,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34906,12 +34948,15 @@ "gpt-5.4": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34922,6 +34967,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34959,12 +35005,15 @@ "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34975,6 +35024,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -35014,6 +35064,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -35023,6 +35074,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -35064,6 +35116,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -35073,6 +35126,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { diff --git a/litellm/types/utils.py b/litellm/types/utils.py index bb0b3d2a687..f10f102d8ec 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -291,6 +291,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost_above_272k_tokens_priority: float | None cache_read_input_token_cost_above_272k_tokens_flex: float | None cache_read_input_token_cost_above_512k_tokens: float | None + cache_read_input_token_cost_batches: ReadOnly[float | None] + cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] + cache_creation_input_token_cost_batches: ReadOnly[float | None] + cache_creation_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] # Smallest prefix this model will actually cache, whatever caching mechanism its provider uses. # Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT. prompt_cache_min_tokens: int | None @@ -316,7 +320,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_second: float | None # for OpenAI Speech models input_cost_per_token_batches: float | None input_cost_per_video_token_batches: ReadOnly[float | None] + input_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token_batches: float | None + output_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token: Required[float | None] output_cost_per_token_flex: float | None # OpenAI flex service tier pricing output_cost_per_token_priority: float | None # OpenAI priority service tier pricing @@ -3718,6 +3724,10 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_read_input_token_cost_above_200k_tokens_priority: float | None = None cache_read_input_token_cost_above_272k_tokens_priority: float | None = None cache_read_input_token_cost_above_272k_tokens_flex: float | None = None + cache_read_input_token_cost_batches: float | None = None + cache_read_input_token_cost_above_272k_tokens_batches: float | None = None + cache_creation_input_token_cost_batches: float | None = None + cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None cache_read_input_audio_token_cost: float | None = None cache_read_input_image_token_cost: float | None = None input_cost_per_character_above_128k_tokens: float | None = None @@ -3728,6 +3738,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): input_cost_per_token_above_200k_tokens_priority: float | None = None input_cost_per_token_above_272k_tokens_priority: float | None = None input_cost_per_token_above_272k_tokens_flex: float | None = None + input_cost_per_token_above_272k_tokens_batches: float | None = None input_cost_per_query: float | None = None input_cost_per_image: float | None = None input_cost_per_image_above_128k_tokens: float | None = None @@ -3751,6 +3762,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_token_above_200k_tokens_priority: float | None = None output_cost_per_token_above_272k_tokens_priority: float | None = None output_cost_per_token_above_272k_tokens_flex: float | None = None + output_cost_per_token_above_272k_tokens_batches: float | None = None output_cost_per_character_above_128k_tokens: float | None = None output_cost_per_image: float | None = None output_cost_per_image_token: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index df2a6adbe1f..8ade6022e3e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6042,6 +6042,14 @@ def _get_model_info_helper( cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None), cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None), cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None), + cache_read_input_token_cost_batches=_model_info.get("cache_read_input_token_cost_batches"), + cache_read_input_token_cost_above_272k_tokens_batches=_model_info.get( + "cache_read_input_token_cost_above_272k_tokens_batches" + ), + cache_creation_input_token_cost_batches=_model_info.get("cache_creation_input_token_cost_batches"), + cache_creation_input_token_cost_above_272k_tokens_batches=_model_info.get( + "cache_creation_input_token_cost_above_272k_tokens_batches" + ), cache_creation_input_token_cost_above_1hr=_model_info.get( "cache_creation_input_token_cost_above_1hr", None ), @@ -6072,7 +6080,13 @@ def _get_model_info_helper( input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None), input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"), input_cost_per_video_token_batches=_model_info.get("input_cost_per_video_token_batches", None), + input_cost_per_token_above_272k_tokens_batches=_model_info.get( + "input_cost_per_token_above_272k_tokens_batches" + ), output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"), + output_cost_per_token_above_272k_tokens_batches=_model_info.get( + "output_cost_per_token_above_272k_tokens_batches" + ), output_cost_per_token=_output_cost_per_token, output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None), output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 468d86d049d..7cde9a1162e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -34129,6 +34129,10 @@ "cache_read_input_token_cost_above_272k_tokens": 2e-06, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, + "cache_read_input_token_cost_batches": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 1e-06, + "cache_creation_input_token_cost_batches": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-05, "cache_read_input_token_cost_flex": 5e-07, "cache_read_input_token_cost_priority": 2e-06, "input_cost_per_token": 1e-05, @@ -34136,6 +34140,7 @@ "input_cost_per_token_above_272k_tokens_flex": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 4e-05, "input_cost_per_token_batches": 5e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-05, "input_cost_per_token_flex": 5e-06, "input_cost_per_token_priority": 2e-05, "litellm_provider": "openai", @@ -34148,6 +34153,7 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, "output_cost_per_token_above_272k_tokens_priority": 0.00015, "output_cost_per_token_batches": 2.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-05, "output_cost_per_token_flex": 2.5e-05, "output_cost_per_token_priority": 0.0001, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34199,6 +34205,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34206,6 +34216,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34218,6 +34229,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34266,6 +34278,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34273,6 +34289,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34285,6 +34302,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34335,6 +34353,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, @@ -34342,6 +34364,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, "litellm_provider": "openai", @@ -34354,6 +34377,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-06, "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, + "output_cost_per_token_above_272k_tokens_batches": 9e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34403,6 +34427,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_read_input_token_cost_batches": 1e-08, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-08, + "cache_creation_input_token_cost_batches": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -34410,6 +34438,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-07, "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_above_272k_tokens_batches": 2e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, "litellm_provider": "openai", @@ -34422,6 +34451,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-07, "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_above_272k_tokens_batches": 9e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34692,12 +34722,15 @@ "gpt-5.5": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34708,6 +34741,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34750,12 +34784,15 @@ "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34766,6 +34803,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34810,6 +34848,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34819,6 +34858,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34859,6 +34899,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34868,6 +34909,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34906,12 +34948,15 @@ "gpt-5.4": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34922,6 +34967,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34959,12 +35005,15 @@ "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34975,6 +35024,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -35014,6 +35064,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -35023,6 +35074,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -35064,6 +35116,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -35073,6 +35126,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 16b762f7803..737a9b7fa60 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -113,6 +113,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_creation_input_token_cost_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_creation_input_token_cost_above_272k_tokens_flex": { "type": "number", "minimum": 0, @@ -123,6 +128,10 @@ "minimum": 0, "description": "Priority service-tier rate for the same-named base field." }, + "cache_creation_input_token_cost_batches": { + "type": "number", + "minimum": 0 + }, "cache_creation_input_token_cost_flex": { "type": "number", "minimum": 0, @@ -171,6 +180,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_read_input_token_cost_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_read_input_token_cost_above_272k_tokens_flex": { "type": "number", "minimum": 0, @@ -186,6 +200,10 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_read_input_token_cost_batches": { + "type": "number", + "minimum": 0 + }, "cache_read_input_token_cost_flex": { "type": "number", "minimum": 0, @@ -338,6 +356,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "input_cost_per_token_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "input_cost_per_token_above_272k_tokens_flex": { "type": "number", "minimum": 0, @@ -669,6 +692,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "output_cost_per_token_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "output_cost_per_token_above_272k_tokens_flex": { "type": "number", "minimum": 0, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 4fe3d410ef8..4b25c87d70f 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,5 +1,6 @@ from collections.abc import Mapping from datetime import datetime, timezone +from typing import cast import pytest @@ -3634,3 +3635,147 @@ def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_ assert creation == pytest.approx(expected_creation) assert creation_1h == pytest.approx(expected_creation_1h) + + +def _batch_rates_model_info(**rates: object) -> ModelInfo: + return cast(ModelInfo, dict(rates)) + + +def test_get_batch_cost_rates_parses_string_rates(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches="1e-06", + output_cost_per_token_batches="4e-06", + cache_read_input_token_cost_batches="1e-07", + input_cost_per_token_above_272k_tokens_batches="2e-06", + output_cost_per_token_above_272k_tokens_batches="6e-06", + cache_read_input_token_cost_above_272k_tokens_batches="2e-07", + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert (rates.input, rates.output, rates.cache_read) == (2e-6, 6e-6, 2e-7) + + +def test_get_batch_cost_rates_falls_back_to_the_flat_rates_when_tier_rates_are_unparsable(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost_batches=1e-7, + input_cost_per_token_above_272k_tokens_batches="two", + output_cost_per_token_above_272k_tokens_batches="six", + cache_read_input_token_cost_above_272k_tokens_batches="none", + cache_creation_input_token_cost_batches=1.25e-7, + cache_creation_input_token_cost_above_272k_tokens_batches="nope", + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert (rates.input, rates.output, rates.cache_read, rates.cache_creation) == (1e-6, 4e-6, 1e-7, 1.25e-7) + + +def test_get_batch_cost_rates_has_no_cached_rate_without_a_cached_batch_key(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost=2e-7, + input_cost_per_token_above_272k_tokens_batches=2e-6, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert (rates.input, rates.output, rates.cache_read) == (2e-6, 4e-6, None) + + +@pytest.mark.parametrize(("prompt_tokens", "expected"), [(1_000, 1.25e-7), (272_000, 1.25e-7), (300_000, 2.5e-7)]) +def test_get_batch_cost_rates_reads_the_batch_cache_write_rate_for_the_crossed_tier(prompt_tokens, expected): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-7, + input_cost_per_token_above_272k_tokens_batches=2e-7, + cache_creation_input_token_cost_batches=1.25e-7, + cache_creation_input_token_cost_above_272k_tokens_batches=2.5e-7, + ), + Usage(prompt_tokens=prompt_tokens, completion_tokens=1, total_tokens=prompt_tokens + 1), + "openai", + ) + + assert rates.cache_creation == expected + + +@pytest.mark.parametrize( + ("tier_key", "attribute"), + [ + ("output_cost_per_token_above_272k_tokens_batches", "output"), + ("cache_read_input_token_cost_above_272k_tokens_batches", "cache_read"), + ("cache_creation_input_token_cost_above_272k_tokens_batches", "cache_creation"), + ], +) +def test_get_batch_cost_rates_crosses_a_tier_declared_without_an_input_tier_key(tier_key, attribute): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost_batches=1e-7, + cache_creation_input_token_cost_batches=1.25e-7, + **{tier_key: 9e-6}, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert getattr(rates, attribute) == 9e-6 + assert rates.input == 1e-6 + + +@pytest.mark.parametrize( + ("prompt_tokens", "expected_input", "expected_output"), + [(200_000, 1e-6, 4e-6), (250_000, 1e-6, 5e-6), (300_000, 2e-6, 5e-6)], +) +def test_get_batch_cost_rates_crosses_each_components_own_tier(prompt_tokens, expected_input, expected_output): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + input_cost_per_token_above_272k_tokens_batches=2e-6, + output_cost_per_token_batches=4e-6, + output_cost_per_token_above_200k_tokens_batches=5e-6, + ), + Usage(prompt_tokens=prompt_tokens, completion_tokens=1, total_tokens=prompt_tokens + 1), + "openai", + ) + + assert (rates.input, rates.output) == (expected_input, expected_output) + + +def test_get_batch_cost_rates_has_no_cache_write_rate_without_a_cache_write_batch_key(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-7, + input_cost_per_token_above_272k_tokens_batches=2e-7, + cache_creation_input_token_cost=2.5e-7, + cache_creation_input_token_cost_above_272k_tokens=5e-7, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert rates.cache_creation is None diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 52eed9c9f95..b0fc725b50a 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6,6 +6,7 @@ import logging import os import sys from collections.abc import Callable, Iterator, Mapping +from types import MappingProxyType from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -7478,6 +7479,141 @@ def test_passthrough_embeddings_result_swapped_for_callbacks(): assert swapped_result.data[0]["embedding"] == [0.1, 0.2, 0.3] +_PUBLISHED_BATCH_MODEL: Final = "lit-published-batch-tier-model" +_PUBLISHED_BATCH_DEPLOYMENT: Final = f"openai/{_PUBLISHED_BATCH_MODEL}" +_PUBLISHED_BATCH_RATES: Final = MappingProxyType( + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "input_cost_per_token_batches": 1.1e-6, + "output_cost_per_token_batches": 4.1e-6, + "cache_read_input_token_cost_batches": 1.2e-7, + "cache_creation_input_token_cost_batches": 1.3e-6, + "input_cost_per_token_above_272k_tokens_batches": 3.1e-6, + "output_cost_per_token_above_272k_tokens_batches": 7.1e-6, + "cache_read_input_token_cost_above_272k_tokens_batches": 3.2e-7, + "cache_creation_input_token_cost_above_272k_tokens_batches": 3.3e-6, + } +) +_PUBLISHED_INPUT_BATCH_KEYS: Final = ( + "input_cost_per_token_batches", + "input_cost_per_token_above_272k_tokens_batches", + "cache_read_input_token_cost_batches", + "cache_read_input_token_cost_above_272k_tokens_batches", + "cache_creation_input_token_cost_batches", + "cache_creation_input_token_cost_above_272k_tokens_batches", +) +_PUBLISHED_OUTPUT_BATCH_KEYS: Final = ( + "output_cost_per_token_batches", + "output_cost_per_token_above_272k_tokens_batches", +) + + +@pytest.fixture +def _published_batch_model(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.register_model( + model_cost={_PUBLISHED_BATCH_MODEL: {**_PUBLISHED_BATCH_RATES}}, persist_across_reloads=False + ) + + +def _batch_deployment_id(custom_pricing: dict[str, float]) -> str: + from litellm import Router + + router: Final = Router( + model_list=[ + { + "model_name": "published-batch", + "litellm_params": {"model": _PUBLISHED_BATCH_DEPLOYMENT, "api_key": "sk-test", **custom_pricing}, + } + ] + ) + return router.model_list[0]["model_info"]["id"] + + +def test_deployment_pricing_model_info_carries_every_published_input_batch_rate_when_only_output_is_declared( + _published_batch_model: None, +) -> None: + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"output_cost_per_token_batches": 4e-6}), _PUBLISHED_BATCH_DEPLOYMENT + ) + + assert info is not None + assert {key: info[key] for key in _PUBLISHED_INPUT_BATCH_KEYS} == { + key: _PUBLISHED_BATCH_RATES[key] for key in _PUBLISHED_INPUT_BATCH_KEYS + } + assert info["output_cost_per_token_batches"] == 4e-6 + assert info["output_cost_per_token_above_272k_tokens_batches"] is None + + +def test_deployment_pricing_model_info_carries_the_published_output_batch_tier_when_only_input_is_declared( + _published_batch_model: None, +) -> None: + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"input_cost_per_token_batches": 1e-6}), _PUBLISHED_BATCH_DEPLOYMENT + ) + + assert info is not None + assert {key: info[key] for key in _PUBLISHED_OUTPUT_BATCH_KEYS} == { + key: _PUBLISHED_BATCH_RATES[key] for key in _PUBLISHED_OUTPUT_BATCH_KEYS + } + assert info["input_cost_per_token_batches"] == 1e-6 + assert info["input_cost_per_token_above_272k_tokens_batches"] is None + assert info["cache_read_input_token_cost_batches"] is None + assert info["cache_creation_input_token_cost_batches"] is None + + +def test_batch_cost_calculator_bills_the_carried_output_tier_when_the_deployment_declares_its_own_input_rate( + _published_batch_model: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + from litellm.types.utils import Usage + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"input_cost_per_token": 5e-6}), _PUBLISHED_BATCH_DEPLOYMENT + ) + assert info is not None + + prompt_cost, completion_cost = batch_cost_calculator( + usage=Usage(prompt_tokens=300_000, completion_tokens=10, total_tokens=300_010), + model=_PUBLISHED_BATCH_DEPLOYMENT, + custom_llm_provider="openai", + model_info=info, + ) + + assert prompt_cost == pytest.approx(300_000 * 5e-6 / 2) + assert completion_cost == pytest.approx( + 10 * _PUBLISHED_BATCH_RATES["output_cost_per_token_above_272k_tokens_batches"] + ) + + +def test_deployment_pricing_model_info_honors_a_tier_only_batch_override_over_the_published_flat_rates( + _published_batch_model: None, +) -> None: + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"input_cost_per_token_above_272k_tokens_batches": 1e-3}), _PUBLISHED_BATCH_DEPLOYMENT + ) + carried_keys: Final = tuple( + key + for key in (*_PUBLISHED_INPUT_BATCH_KEYS, *_PUBLISHED_OUTPUT_BATCH_KEYS) + if key != "input_cost_per_token_above_272k_tokens_batches" + ) + + assert info is not None + assert info["input_cost_per_token_above_272k_tokens_batches"] == 1e-3 + assert {key: info[key] for key in carried_keys} == {key: _PUBLISHED_BATCH_RATES[key] for key in carried_keys} + + def test_get_status_fields_ranks_guardrail_flagged_between_success_and_intervened(): """LIT-6894: a non-blocking flagged verdict must outrank success in the request-level guardrail_status but never mask an intervention.""" diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index bdeedb5f38c..a33701cd1bf 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,6 +1,7 @@ import datetime import time -from typing import Final +from types import MappingProxyType +from typing import Final, cast import pytest from pydantic import BaseModel @@ -4464,6 +4465,273 @@ def test_completion_cost_prices_responses_websocket_turns_per_service_tier(): assert ws_cost != pytest.approx(_http_cost(160, 50, "priority")) +_TIERED_BATCH_MODEL: Final = "lit-tiered-batch-model" +_FLAT_CACHE_BATCH_MODEL: Final = "lit-tiered-batch-model-without-cache-batch-rates" +_TIERED_BATCH_ENTRY: Final = MappingProxyType( + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "cache_read_input_token_cost": 2e-7, + "cache_creation_input_token_cost": 2.5e-6, + "input_cost_per_token_above_272k_tokens": 4e-6, + "output_cost_per_token_above_272k_tokens": 1.2e-5, + "input_cost_per_token_batches": 1e-6, + "output_cost_per_token_batches": 4e-6, + "cache_read_input_token_cost_batches": 1e-7, + "cache_creation_input_token_cost_batches": 1.25e-6, + "input_cost_per_token_above_272k_tokens_batches": 3e-6, + "output_cost_per_token_above_272k_tokens_batches": 7e-6, + "cache_read_input_token_cost_above_272k_tokens_batches": 3e-7, + "cache_creation_input_token_cost_above_272k_tokens_batches": 3.75e-6, + } +) +_BATCH_RATE_PREFIXES: Final = ( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", +) + + +@pytest.fixture +def _tiered_batch_models(_local_model_cost_map: None) -> None: + litellm.register_model( + model_cost={ + _TIERED_BATCH_MODEL: {**_TIERED_BATCH_ENTRY}, + _FLAT_CACHE_BATCH_MODEL: { + key: rate + for key, rate in _TIERED_BATCH_ENTRY.items() + if not (key.startswith("cache_") and key.endswith("_batches")) + }, + }, + persist_across_reloads=False, + ) + + +def test_batch_cost_calculator_bills_the_long_context_batch_tier_above_272k(_tiered_batch_models: None) -> None: + from litellm.cost_calculator import batch_cost_calculator + + usage: Final = Usage(prompt_tokens=300_035, completion_tokens=64, total_tokens=300_099) + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(300_035 * 3e-6) + assert completion_cost_value == pytest.approx(64 * 7e-6) + + +@pytest.mark.parametrize("prompt_tokens", [272_000, 1_000]) +def test_batch_cost_calculator_bills_the_flat_batch_rate_at_or_below_272k( + _tiered_batch_models: None, prompt_tokens: int +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=64, total_tokens=prompt_tokens + 64) + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(prompt_tokens * 1e-6) + assert completion_cost_value == pytest.approx(64 * 4e-6) + + +def test_get_model_info_exposes_every_registered_batch_rate(_tiered_batch_models: None) -> None: + info: Final = litellm.get_model_info(_TIERED_BATCH_MODEL, custom_llm_provider="openai") + batch_keys: Final = tuple(key for key in _TIERED_BATCH_ENTRY if key.endswith("_batches")) + + assert len(batch_keys) == 8 + assert {key: info[key] for key in batch_keys} == {key: _TIERED_BATCH_ENTRY[key] for key in batch_keys} + + +def test_regular_path_never_bills_the_batch_tier_keys(_local_model_cost_map, monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "lit-batch-tier-guard", + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "input_cost_per_token_batches": 1e-6, + "output_cost_per_token_batches": 4e-6, + "input_cost_per_token_above_272k_tokens_batches": 5e-6, + "output_cost_per_token_above_272k_tokens_batches": 9e-6, + }, + ) + + prompt_cost, completion_cost = litellm.cost_per_token( + model="lit-batch-tier-guard", custom_llm_provider="openai", prompt_tokens=300_035, completion_tokens=64 + ) + + assert prompt_cost == pytest.approx(300_035 * 2e-6) + assert completion_cost == pytest.approx(64 * 8e-6) + + +@pytest.mark.parametrize("prefix", _BATCH_RATE_PREFIXES) +def test_every_openai_entry_with_a_long_context_rate_and_a_batch_rate_declares_the_batch_tier( + _local_model_cost_map: None, prefix: str +) -> None: + undeclared: Final = [ + name + for name, entry in litellm.model_cost.items() + if isinstance(entry, dict) + and entry.get("litellm_provider") == "openai" + and entry.get(f"{prefix}_above_272k_tokens") is not None + and entry.get(f"{prefix}_batches") is not None + and entry.get(f"{prefix}_above_272k_tokens_batches") is None + ] + + assert undeclared == [] + + +def test_batch_cost_calculator_ignores_malformed_batch_tier_keys(): + from litellm.cost_calculator import batch_cost_calculator + + usage = Usage(prompt_tokens=300_035, completion_tokens=64, total_tokens=300_099) + model_info = cast( + ModelInfo, + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "input_cost_per_token_batches": 1e-6, + "output_cost_per_token_batches": 4e-6, + "input_cost_per_token_above_272k_tokens_batches": 2e-6, + "output_cost_per_token_above_272k_tokens_batches": 6e-6, + "input_cost_per_token_above_lots_tokens_batches": 1.0, + }, + ) + + prompt_cost, completion_cost = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(300_035 * 2e-6) + assert completion_cost == pytest.approx(64 * 6e-6) + + +def _cached_usage(prompt_tokens: int, cached_tokens: int, completion_tokens: int) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), + ) + + +def test_batch_cost_calculator_bills_cached_tokens_at_the_long_context_batch_cached_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=_cached_usage(300_048, 300_045, 11), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(3 * 3e-6 + 300_045 * 3e-7) + assert completion_cost_value == pytest.approx(11 * 7e-6) + + +def test_batch_cost_calculator_bills_cached_tokens_at_the_flat_batch_cached_rate_at_or_below_272k( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cached_usage(1_000, 900, 4), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(100 * 1e-6 + 900 * 1e-7) + + +def test_batch_cost_calculator_bills_cached_tokens_at_the_batch_input_rate_without_a_cached_batch_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cached_usage(300_048, 300_045, 11), model=_FLAT_CACHE_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(300_048 * 3e-6) + + +def _cache_write_usage(prompt_tokens: int, cache_write_tokens: int, completion_tokens: int) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0, cache_write_tokens=cache_write_tokens), + ) + + +def test_batch_cost_calculator_bills_cache_write_tokens_at_the_long_context_batch_cache_write_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=_cache_write_usage(300_048, 300_045, 4), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(3 * 3e-6 + 300_045 * 3.75e-6) + assert completion_cost_value == pytest.approx(4 * 7e-6) + + +def test_batch_cost_calculator_bills_cache_write_tokens_at_the_flat_batch_cache_write_rate_at_or_below_272k( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cache_write_usage(1_000, 900, 4), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(100 * 1e-6 + 900 * 1.25e-6) + + +def test_batch_cost_calculator_bills_cache_write_tokens_at_the_batch_input_rate_without_a_cache_write_batch_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cache_write_usage(300_048, 300_045, 4), model=_FLAT_CACHE_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(300_048 * 3e-6) + + +def test_batch_cost_calculator_prices_modalities_and_cached_tokens_together_in_the_crossed_tier() -> None: + from litellm.cost_calculator import batch_cost_calculator + + model_info: Final = cast( + ModelInfo, + { + "input_cost_per_token_batches": 1e-6, + "input_cost_per_token_above_272k_tokens_batches": 3e-6, + "input_cost_per_audio_token_batches": 5e-6, + "cache_read_input_token_cost_batches": 1e-7, + "cache_read_input_token_cost_above_272k_tokens_batches": 3e-7, + }, + ) + usage: Final = Usage( + prompt_tokens=300_000, + completion_tokens=0, + total_tokens=300_000, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=64, image_tokens=10, cached_tokens=1_000), + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(298_926 * 3e-6 + 64 * 5e-6 + 10 * 3e-6 + 1_000 * 3e-7) + + QWEN3_NEXT_REGIONS: Final = ("ap-northeast-1", "ap-south-1", "ap-southeast-2", "eu-west-1", "eu-west-2", "sa-east-1") diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 38aad3ace1c..96c420299e7 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -740,6 +740,8 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_token_cost_above_272k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_272k_tokens_flex": {"type": "number"}, "cache_creation_input_token_cost_above_272k_tokens_priority": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_batches": {"type": "number"}, "cache_creation_input_token_cost_flex": {"type": "number"}, "cache_creation_input_token_cost_priority": {"type": "number"}, "cache_read_input_token_cost": {"type": "number"}, @@ -749,6 +751,8 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_read_input_token_cost_above_272k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_272k_tokens_flex": {"type": "number"}, "cache_read_input_token_cost_above_512k_tokens": {"type": "number"}, + "cache_read_input_token_cost_batches": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_batches": {"type": "number"}, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, "cache_read_input_image_token_cost": {"type": "number"}, @@ -776,12 +780,14 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_token_priority": {"type": "number"}, "input_cost_per_token_above_200k_tokens_priority": {"type": "number"}, "input_cost_per_token_above_272k_tokens_priority": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_batches": {"type": "number"}, "input_cost_per_token_above_272k_tokens_flex": {"type": "number"}, "input_cost_per_audio_token_priority": {"type": "number"}, "output_cost_per_token_flex": {"type": "number"}, "output_cost_per_token_priority": {"type": "number"}, "output_cost_per_token_above_200k_tokens_priority": {"type": "number"}, "output_cost_per_token_above_272k_tokens_priority": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_batches": {"type": "number"}, "output_cost_per_token_above_272k_tokens_flex": {"type": "number"}, "regional_endpoint_uplift_multiplier": {"type": "number"}, "regional_processing_uplift_multiplier_eu": {"type": "number"}, diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index 708c472939e..a4de8eee23c 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -26,7 +26,7 @@ from openai.types.batch import BatchRequestCounts import litellm import litellm.batches.batch_utils as bu -from litellm.types.utils import LiteLLMBatch, Usage +from litellm.types.utils import LiteLLMBatch, ModelInfo, Usage # --------------------------------------------------------------------------- # # Builders for batch OUTPUT file rows. @@ -437,6 +437,33 @@ def test_total_usage_and_cost_normalize_mixed_responses_and_chat(): assert result.cost == pytest.approx((30 * 0.00125) + (12 * 0.005)) +def test_total_cost_applies_the_long_context_batch_tier_per_line(): + long_row = _success_row(usage=_usage(300_000, 10)) + short_row = _success_row(usage=_usage(100, 10)) + + result = bu._aggregate_batch_cost_usage_models( + entries=[long_row, short_row], + custom_llm_provider="openai", + model_info=ModelInfo( + key="lit-batch-tier", + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + input_cost_per_token=2e-6, + output_cost_per_token=8e-6, + litellm_provider="openai", + mode="chat", + supported_openai_params=None, + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + input_cost_per_token_above_272k_tokens_batches=2e-6, + output_cost_per_token_above_272k_tokens_batches=6e-6, + ), + ) + + assert result.cost == pytest.approx((300_000 * 2e-6) + (10 * 6e-6) + (100 * 1e-6) + (10 * 4e-6)) + + def test_total_usage_empty_is_zero(): result = bu._aggregate_batch_cost_usage_models(entries=[], custom_llm_provider="openai") assert result.cost == 0.0 @@ -1824,6 +1851,45 @@ def test_unparsable_bedrock_batch_usage_warns(caplog): assert "inputTextTokenCount" in caplog.text +def test_total_cost_bills_cached_tokens_per_line_at_the_batch_cached_rate(): + responses_row = _success_row( + usage={ + "input_tokens": 300_000, + "output_tokens": 10, + "total_tokens": 300_010, + "input_tokens_details": {"cached_tokens": 299_000}, + } + ) + chat_row = _success_row(usage={**_usage(100, 10), "prompt_tokens_details": {"cached_tokens": 60}}) + + result = bu._aggregate_batch_cost_usage_models( + entries=[responses_row, chat_row], + custom_llm_provider="openai", + model_info=ModelInfo( + key="lit-batch-cached-tier", + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + input_cost_per_token=2e-6, + output_cost_per_token=8e-6, + cache_read_input_token_cost=1e-6, + litellm_provider="openai", + mode="chat", + supported_openai_params=None, + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost_batches=5e-7, + input_cost_per_token_above_272k_tokens_batches=2e-6, + output_cost_per_token_above_272k_tokens_batches=6e-6, + cache_read_input_token_cost_above_272k_tokens_batches=1e-6, + ), + ) + + long_line = 1_000 * 2e-6 + 299_000 * 1e-6 + 10 * 6e-6 + short_line = 40 * 1e-6 + 60 * 5e-7 + 10 * 4e-6 + assert result.cost == pytest.approx(long_line + short_line) + + # --------------------------------------------------------------------------- # # batch_cost_is_final # --------------------------------------------------------------------------- # diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 38cfdc9666b..38f8ac365ff 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -31186,10 +31186,14 @@ export interface components { cache_creation_input_token_cost_above_200k_tokens?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens */ cache_creation_input_token_cost_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 272K Tokens Batches */ + cache_creation_input_token_cost_above_272k_tokens_batches?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens Flex */ cache_creation_input_token_cost_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens Priority */ cache_creation_input_token_cost_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Batches */ + cache_creation_input_token_cost_batches?: number | null; /** Cache Creation Input Token Cost Flex */ cache_creation_input_token_cost_flex?: number | null; /** Cache Creation Input Token Cost Priority */ @@ -31208,12 +31212,16 @@ export interface components { cache_read_input_token_cost_above_200k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 272K Tokens */ cache_read_input_token_cost_above_272k_tokens?: number | null; + /** Cache Read Input Token Cost Above 272K Tokens Batches */ + cache_read_input_token_cost_above_272k_tokens_batches?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Flex */ cache_read_input_token_cost_above_272k_tokens_flex?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Priority */ cache_read_input_token_cost_above_272k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 512K Tokens */ cache_read_input_token_cost_above_512k_tokens?: number | null; + /** Cache Read Input Token Cost Batches */ + cache_read_input_token_cost_batches?: number | null; /** Cache Read Input Token Cost Flex */ cache_read_input_token_cost_flex?: number | null; /** Cache Read Input Token Cost Priority */ @@ -31282,6 +31290,8 @@ export interface components { input_cost_per_token_above_200k_tokens_priority?: number | null; /** Input Cost Per Token Above 272K Tokens */ input_cost_per_token_above_272k_tokens?: number | null; + /** Input Cost Per Token Above 272K Tokens Batches */ + input_cost_per_token_above_272k_tokens_batches?: number | null; /** Input Cost Per Token Above 272K Tokens Flex */ input_cost_per_token_above_272k_tokens_flex?: number | null; /** Input Cost Per Token Above 272K Tokens Priority */ @@ -31403,6 +31413,8 @@ export interface components { output_cost_per_token_above_200k_tokens_priority?: number | null; /** Output Cost Per Token Above 272K Tokens */ output_cost_per_token_above_272k_tokens?: number | null; + /** Output Cost Per Token Above 272K Tokens Batches */ + output_cost_per_token_above_272k_tokens_batches?: number | null; /** Output Cost Per Token Above 272K Tokens Flex */ output_cost_per_token_above_272k_tokens_flex?: number | null; /** Output Cost Per Token Above 272K Tokens Priority */ @@ -42026,10 +42038,14 @@ export interface components { cache_creation_input_token_cost_above_200k_tokens?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens */ cache_creation_input_token_cost_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 272K Tokens Batches */ + cache_creation_input_token_cost_above_272k_tokens_batches?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens Flex */ cache_creation_input_token_cost_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens Priority */ cache_creation_input_token_cost_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Batches */ + cache_creation_input_token_cost_batches?: number | null; /** Cache Creation Input Token Cost Flex */ cache_creation_input_token_cost_flex?: number | null; /** Cache Creation Input Token Cost Priority */ @@ -42048,12 +42064,16 @@ export interface components { cache_read_input_token_cost_above_200k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 272K Tokens */ cache_read_input_token_cost_above_272k_tokens?: number | null; + /** Cache Read Input Token Cost Above 272K Tokens Batches */ + cache_read_input_token_cost_above_272k_tokens_batches?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Flex */ cache_read_input_token_cost_above_272k_tokens_flex?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Priority */ cache_read_input_token_cost_above_272k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 512K Tokens */ cache_read_input_token_cost_above_512k_tokens?: number | null; + /** Cache Read Input Token Cost Batches */ + cache_read_input_token_cost_batches?: number | null; /** Cache Read Input Token Cost Flex */ cache_read_input_token_cost_flex?: number | null; /** Cache Read Input Token Cost Priority */ @@ -42122,6 +42142,8 @@ export interface components { input_cost_per_token_above_200k_tokens_priority?: number | null; /** Input Cost Per Token Above 272K Tokens */ input_cost_per_token_above_272k_tokens?: number | null; + /** Input Cost Per Token Above 272K Tokens Batches */ + input_cost_per_token_above_272k_tokens_batches?: number | null; /** Input Cost Per Token Above 272K Tokens Flex */ input_cost_per_token_above_272k_tokens_flex?: number | null; /** Input Cost Per Token Above 272K Tokens Priority */ @@ -42243,6 +42265,8 @@ export interface components { output_cost_per_token_above_200k_tokens_priority?: number | null; /** Output Cost Per Token Above 272K Tokens */ output_cost_per_token_above_272k_tokens?: number | null; + /** Output Cost Per Token Above 272K Tokens Batches */ + output_cost_per_token_above_272k_tokens_batches?: number | null; /** Output Cost Per Token Above 272K Tokens Flex */ output_cost_per_token_above_272k_tokens_flex?: number | null; /** Output Cost Per Token Above 272K Tokens Priority */ From 88a4cbdd7b9188f6fbc68b29bacdde4371b4ef22 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 10:31:51 -0700 Subject: [PATCH 28/28] fix(otel v2): map rerank and search output and the OCR, image edit and search input onto the Langfuse generation (#42444) * fix(otel v2): map rerank and search output and the OCR, image edit and search input onto the Langfuse generation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel v2): summarize OCR data URIs by media type and size and log an empty document URL as empty Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel v2): keep URL-less search results, name OCR file streams and skip non-str query parts when logging Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(otel v2): drop the unused typing imports and the decorative section divider Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/otel/model/payloads.py | 28 +++ litellm/utils.py | 40 +++- ..._otel_v2_langfuse_generation_output_e2e.py | 221 +++++++++++++++++- tests/e2e/models.py | 51 ++++ tests/e2e/proxy_client.py | 26 +++ .../otel/test_otel_v2_sources_of_truth.py | 88 +++++++ tests/test_litellm/test_utils.py | 129 +++++++++- 7 files changed, 565 insertions(+), 18 deletions(-) diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index 7ecbeea255c..240107251c5 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -760,6 +760,8 @@ def _output_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object or _ocr_choices(response) or _transcription_choices(response) or _moderation_choices(response) + or _rerank_choices(response) + or _search_choices(response) or _image_choices(response) or _binary_choices(response) ) @@ -815,6 +817,32 @@ def _moderation_verdict(flagged: bool, categories: object) -> str: return f"flagged: {', '.join(hits)}" if hits else "flagged" +def _rerank_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple( + _rerank_line(index, score, result.get("document")) + for result in _dicts(response.get("results")) + if (index := as_int(result.get("index"))) is not None + if (score := as_float(result.get("relevance_score"))) is not None + ) + ) + + +def _rerank_line(index: int, score: float, document: object) -> str: + text: Final = as_str((as_str_mapping(document) or {}).get("text")) + return f"[{index}] {score}\n{text}" if text else f"[{index}] {score}" + + +def _search_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple( + line + for result in _dicts(response.get("results")) + if (line := "\n".join(part for key in ("title", "url", "snippet") if (part := as_str(result.get(key))))) + ) + ) + + def _image_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: return _joined_choice( tuple(summary for item in _dicts(response.get("data")) if (summary := _image_summary(item)) is not None) diff --git a/litellm/utils.py b/litellm/utils.py index 8ade6022e3e..8ce83f475a4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -35,6 +35,7 @@ from importlib import resources from inspect import iscoroutine from io import StringIO from os.path import abspath, dirname, join +from pathlib import PurePath from types import MappingProxyType import dotenv @@ -288,7 +289,7 @@ except (ImportError, AttributeError, TypeError): claude_json_str = json.dumps(json_data) import importlib.metadata from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast, get_args +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_checkable from typing_extensions import assert_never @@ -886,6 +887,32 @@ async def _run_success_deployment_hook_on_converted_chat_stream( ) +@runtime_checkable +class _NamedFile(Protocol): + @property + def name(self) -> object: ... + + +def _ocr_document_summary(document: object) -> str: + if not isinstance(document, Mapping): + return "default-message-value" + doc: Final = cast(Mapping[str, object], document) # cast-ok: ocr()/aocr() type the document as Mapping[str, object] + location: Final = doc.get("document_url", doc.get("image_url")) + if isinstance(location, str): + header, separator, payload = location.partition(",") + return f"{header} ({len(payload)} chars)" if separator and header.startswith("data:") else location + file_input: Final = doc.get("file") + mime_type: Final = doc.get("mime_type") + kind: Final = f"file ({mime_type})" if isinstance(mime_type, str) else "file" + if isinstance(file_input, PurePath): + return f"{kind} {file_input.name}" + if isinstance(file_input, bytes): + return f"{kind} {len(file_input)} bytes" + if isinstance(file_input, _NamedFile) and isinstance(file_input.name, str): + return f"{kind} {PurePath(file_input.name).name}" + return kind + + # Runs once per call to check if the user wants to send their data anywhere - PostHog/Sentry/Slack/etc. def function_setup( original_function: str, @@ -1126,6 +1153,17 @@ def function_setup( messages = args[0] if len(args) > 0 else kwargs["prompt"] elif call_type == CallTypes.rerank.value or call_type == CallTypes.arerank.value: messages = kwargs.get("query") + elif call_type in (CallTypes.search.value, CallTypes.asearch.value): + search_query: Final = args[0] if len(args) > 0 else kwargs.get("query") + messages = ( + "\n".join(part for part in search_query if isinstance(part, str)) + if isinstance(search_query, list) + else search_query + ) + elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): + messages = args[1] if len(args) > 1 else kwargs.get("prompt") + elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value): + messages = _ocr_document_summary(args[1] if len(args) > 1 else kwargs.get("document")) elif call_type == CallTypes.atranscription.value or call_type == CallTypes.transcription.value: _file_obj: Final[FileTypes] = args[1] if len(args) > 1 else kwargs["file"] # Lazy import audio_utils.utils only when needed for transcription calls diff --git a/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py index 71008d94557..9e08cf21da0 100644 --- a/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py +++ b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py @@ -1,13 +1,17 @@ -"""Live e2e: the OTel v2 Langfuse generation carries output for every non-chat endpoint (LIT-8309). +"""Live e2e: the OTel v2 Langfuse generation carries input and output for every non-chat endpoint. With LITELLM_OTEL_V2=true the proxy exports one generation per request to the -team's Langfuse destination. Chat, Responses, embeddings and OCR already fill -its output; this file pins the remaining five families. Each test registers a -real OpenAI deployment, drives the endpoint through the shared transport, then -reads the generation back from Langfuse and asserts its output reflects what -the caller received: the completion text, the transcript, the moderation -verdict, and for images and speech a bounded summary that never carries the -raw base64 or audio bytes. +team's Langfuse destination. Chat, Responses and embeddings already fill both +panels; this file pins the other families. LIT-8309 covers the output of +completions, images, speech, transcription and moderation; LIT-8326 covers the +rerank and search output and the OCR, image-edit and search input, which used +to read the `default-message-value` placeholder. Each test registers a real +deployment, drives the endpoint through the shared transport, then reads the +generation back from Langfuse and asserts it reflects what the caller sent and +received: the completion text, the transcript, the moderation verdict, the +ranked rerank indices and scores, the search results, the OCR document URL, +the edit prompt, and for images, speech and uploaded documents a bounded +summary that never carries the raw base64 or audio bytes. """ from __future__ import annotations @@ -25,11 +29,22 @@ from logging_client import LangfuseCreds, LangfuseObservation, LoggingClient, lo from models import ( CompletionBody, CompletionResponse, + ImageEditForm, ImageGenerationBody, ImageGenerationResponse, LiteLLMParamsBody, ModerationBody, ModerationResponse, + OcrBody, + OcrDocument, + OcrForm, + OcrResponse, + RerankBody, + SearchBody, + SearchResponse, + SearchToolBody, + SearchToolCreateBody, + SearchToolLiteLLMParamsBody, SpeechBody, TranscriptionForm, TranscriptionResponse, @@ -41,7 +56,19 @@ pytestmark = [pytest.mark.e2e, pytest.mark.otel_v2] WEATHER_WAV: Final = ( Path(__file__).resolve().parent.parent / "llm_translation" / "realtime" / "fixtures" / "weather_question_24k.wav" ) +DUMMY_PDF: Final = Path(__file__).resolve().parent.parent.parent / "llm_translation" / "fixtures" / "dummy.pdf" +DUMMY_PDF_URL: Final = ( + "https://cdn.jsdelivr.net/gh/BerriAI/litellm" + "@d769e81c90d453240c61fc572cdb27fae06a89d0" + "/tests/llm_translation/fixtures/dummy.pdf" +) +RED_SQUARE_PNG: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3" + "3UrYvQQckD4XAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB" + "AYHLAMpT0sIcNbcEAAAAAElFTkSuQmCC" +) BOUNDED_OUTPUT_CHARS: Final = 1024 +PLACEHOLDER_INPUT: Final = "default-message-value" class _OutputMessage(BaseModel): @@ -50,7 +77,14 @@ class _OutputMessage(BaseModel): content: str = "" +class _InputMessage(BaseModel): + """One user message of the Langfuse generation input; only the text is read.""" + + content: str = "" + + _OUTPUT_MESSAGES: Final = TypeAdapter(list[_OutputMessage]) +_INPUT_MESSAGES: Final = TypeAdapter(list[_InputMessage]) @pytest.fixture(scope="session") @@ -91,10 +125,50 @@ def _output_text(observation: LangfuseObservation) -> str: return "\n".join(message.content for message in messages) +def _input_text(observation: LangfuseObservation) -> str: + assert observation.input not in (None, "", [], {}), f"generation input is empty: {observation!r}" + try: + messages: Final = _INPUT_MESSAGES.validate_python(observation.input) + except ValidationError: + pytest.fail(f"generation input is not a list of user messages: {observation!r}") + assert messages, f"generation input is empty: {observation!r}" + text: Final = "\n".join(message.content for message in messages) + assert text != PLACEHOLDER_INPUT, f"generation input is the placeholder, not the request: {observation!r}" + return text + + def _openai(model: str) -> LiteLLMParamsBody: return LiteLLMParamsBody(model=model, api_key="os.environ/OPENAI_API_KEY") +def _mistral_ocr() -> LiteLLMParamsBody: + return LiteLLMParamsBody(model="mistral/mistral-ocr-latest", api_key="os.environ/MISTRAL_API_KEY") + + +def _langfuse_search_tool( + client: LoggingClient, creds: LangfuseCreds, resources: ResourceManager +) -> tuple[str, str, str]: + """A keyless DuckDuckGo search tool registered for this run, plus a key on a team whose Langfuse callback is + `creds`.""" + tool: Final = f"e2e-otel-search-{unique_marker()}" + tool_id: Final = client.proxy.create_search_tool( + SearchToolCreateBody( + search_tool=SearchToolBody( + search_tool_name=tool, + litellm_params=SearchToolLiteLLMParamsBody(search_provider="duckduckgo"), + ) + ) + ) + resources.defer(lambda: client.proxy.delete_search_tool(tool_id)) + team_id: Final = client.create_team(f"otel-search-team-{unique_marker()}", models=[tool]) + resources.defer(lambda: client.delete_team(team_id)) + client.add_team_langfuse_callback(team_id, creds) + alias: Final = f"otel-search-key-{unique_marker()}" + key: Final = client.key_with_alias(alias, models=[tool], team_id=team_id) + resources.defer(lambda: client.delete_key(key)) + return tool, key, alias + + class TestOtelV2LangfuseGenerationOutput: @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["completions"]) def test_completions_output_is_the_completion_text( @@ -208,3 +282,134 @@ class TestOtelV2LangfuseGenerationOutput: assert output.startswith(verdict), ( f"generation output does not carry the moderation verdict {verdict!r}: {output!r}" ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["rerank"]) + def test_rerank_output_is_the_ranked_indices_and_scores( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key( + client, + langfuse_creds, + resources, + LiteLLMParamsBody(model="cohere/rerank-v4.0-fast", api_key="os.environ/COHERE_API_KEY"), + ) + query: Final = f"What is the capital of France? {unique_marker()}" + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.rerank( + key, + RerankBody( + model=model, + query=query, + documents=["Paris is the capital of France.", "Berlin is in Germany.", "Bananas are yellow."], + top_n=2, + ), + ) + ) + ranked: Final = tuple(f"[{item.index}] {item.relevance_score}" for item in response.results) + assert len(ranked) == 2 and all(item.index is not None for item in response.results), ( + f"/v1/rerank returned no ranked results: {response!r}" + ) + + generation: Final = _generation(client, langfuse_creds, alias=alias, started=started) + assert _input_text(generation) == query + output: Final = _output_text(generation) + assert output == "\n\n".join(ranked), f"generation output is not the ranked indices and scores: {output!r}" + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["ocr"]) + def test_ocr_input_is_the_document_url( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _mistral_ocr()) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.ocr( + key, OcrBody(model=model, document=OcrDocument(type="document_url", document_url=DUMMY_PDF_URL)) + ) + ) + assert response.pages and response.pages[0].markdown, f"/v1/ocr returned no page markdown: {response!r}" + + generation: Final = _generation(client, langfuse_creds, alias=alias, started=started) + assert _input_text(generation) == DUMMY_PDF_URL + assert response.pages[0].markdown in _output_text(generation) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["ocr"]) + def test_ocr_upload_input_is_a_bounded_document_summary_without_base64( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _mistral_ocr()) + pdf: Final = DUMMY_PDF.read_bytes() + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.upload( + "/v1/ocr", + headers=client.proxy.transport.bearer(key), + form=OcrForm(model=model), + filename=DUMMY_PDF.name, + content=pdf, + file_content_type="application/pdf", + response_type=OcrResponse, + ) + ) + assert response.pages and response.pages[0].markdown, f"/v1/ocr returned no page markdown: {response!r}" + + text: Final = _input_text(_generation(client, langfuse_creds, alias=alias, started=started)) + encoded: Final = base64.b64encode(pdf).decode() + assert text == f"data:application/pdf;base64 ({len(encoded)} chars)", ( + f"OCR upload input is not the bounded document summary: {text!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["images_edits"]) + def test_image_edit_input_is_the_edit_prompt( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-image-1-mini")) + prompt: Final = f"make the square blue {unique_marker()}" + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.upload( + "/v1/images/edits", + headers=client.proxy.transport.bearer(key), + form=ImageEditForm(model=model, prompt=prompt), + filename="red_square.png", + content=RED_SQUARE_PNG, + file_content_type="image/png", + file_field="image", + response_type=ImageGenerationResponse, + timeout=180.0, + ) + ) + assert response.data and (response.data[0].b64_json or response.data[0].url), ( + f"/v1/images/edits returned no image: {response!r}" + ) + + generation: Final = _generation(client, langfuse_creds, alias=alias, started=started) + assert _input_text(generation) == prompt + assert _output_text(generation).startswith("b64_json image (") + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["search"]) + def test_search_input_is_the_query_and_output_the_results( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + tool, key, alias = _langfuse_search_tool(client, langfuse_creds, resources) + query: Final = "Eiffel Tower" + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + f"/v1/search/{tool}", + headers=client.proxy.transport.bearer(key), + json=SearchBody(query=query, max_results=2), + response_type=SearchResponse, + ) + ) + assert response.results and all(result.url for result in response.results), ( + f"/v1/search returned no results with a url: {response!r}" + ) + + generation: Final = _generation(client, langfuse_creds, alias=alias, started=started) + assert _input_text(generation) == query + output: Final = _output_text(generation) + assert output == "\n\n".join( + "\n".join(part for part in (result.title, result.url, result.snippet) if part) + for result in response.results + ), f"generation output is not the search results the caller received: {output!r}" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 53fbe03c025..5278dbd287a 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -749,6 +749,12 @@ class OcrBody(BaseModel): document: OcrDocument +class OcrForm(BaseModel): + """Multipart /v1/ocr form fields; the document travels as the `file` part.""" + + model: str + + class OcrPage(BaseModel): index: int markdown: str @@ -798,6 +804,51 @@ class ImageGenerationResponse(BaseModel): data: list[ImageDatum] = [] +class ImageEditForm(BaseModel): + """POST /v1/images/edits form fields; the image travels as the `image` multipart part.""" + + model: str + prompt: str + size: str = "1024x1024" + quality: str = "low" + + +class SearchBody(BaseModel): + """POST /v1/search/{search_tool_name} body (Perplexity-compatible).""" + + query: str + max_results: int = 2 + + +class SearchResultItem(BaseModel): + title: str = "" + url: str = "" + snippet: str = "" + + +class SearchResponse(BaseModel): + results: list[SearchResultItem] = [] + + +class SearchToolLiteLLMParamsBody(BaseModel): + search_provider: str + + +class SearchToolBody(BaseModel): + search_tool_name: str + litellm_params: SearchToolLiteLLMParamsBody + + +class SearchToolCreateBody(BaseModel): + """POST /search_tools body: the tool as it would sit under `search_tools:` in the config.""" + + search_tool: SearchToolBody + + +class SearchToolCreateResponse(BaseModel): + search_tool_id: str + + # ---------- audio ---------- diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 76c57452c69..f1981e0aa5e 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -85,6 +85,8 @@ from models import ( RerankResponse, RouterCurrentValues, RouterSettingsResponse, + SearchToolCreateBody, + SearchToolCreateResponse, SpendLogRow, SpendLogs, SpendLogsPage, @@ -859,6 +861,30 @@ class ProxyClient: response_type=NoBody, ) + def create_search_tool(self, body: SearchToolCreateBody) -> str: + """POST /search_tools: register a search tool on the running proxy and return its id + once every worker has had a config-reload window to pick it up from the DB.""" + search_tool_id: Final = unwrap( + self.transport.post( + "/search_tools", + headers=self.management_headers(), + json=body, + response_type=SearchToolCreateResponse, + ) + ).search_tool_id + settle_propagation(time.monotonic()) + return search_tool_id + + def delete_search_tool(self, search_tool_id: str) -> None: + result = self.transport.delete( + f"/search_tools/{search_tool_id}", + headers=self.management_headers(), + json=NoBody(), + response_type=NoBody, + ) + if not is_ok(result): + warnings.warn(f"delete_search_tool({search_tool_id!r}) failed: {result}", stacklevel=2) + def create_credential(self, body: CredentialCreateBody) -> None: unwrap( self.transport.post( diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 70ab4fe07de..b9ec625fdfc 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -1024,6 +1024,94 @@ def test_moderation_results_without_a_verdict_produce_no_output() -> None: assert data.choices_out == () +def test_rerank_results_become_ranked_indices_and_scores_with_the_document_text() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "arerank", + "rerank-v4.0-fast", + { + "id": "rr-1", + "results": [ + {"index": 2, "relevance_score": 0.91, "document": {"text": "Paris is the capital of France."}}, + {"index": 0, "relevance_score": 0.07}, + {"index": 1, "relevance_score": 0.02, "document": "not-a-document"}, + ], + "meta": {"billed_units": {"search_units": 1}}, + }, + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("[2] 0.91\nParis is the capital of France.\n\n[0] 0.07\n\n[1] 0.02"),) + assert data.finish_reasons == () + assert data.response_id == "rr-1" + + +def test_rerank_output_follows_the_content_capture_gate() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("arerank", "rerank-v4.0-fast", {"results": [{"index": 0, "relevance_score": 0.5}]}) + ) + + assert data.choices_out == () + + +def test_rerank_results_without_an_index_and_score_produce_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "arerank", + "rerank-v4.0-fast", + {"results": [{"index": 0, "document": {"text": "x"}}, {"relevance_score": 0.5}, "not-a-result"]}, + ), + capture_content=True, + ) + + assert data.choices_out == () + + +def test_search_results_become_title_url_and_snippet_blocks_in_result_order() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "asearch", + "exa-search", + { + "object": "search", + "results": [ + {"title": "Eiffel Tower", "url": "https://example.com/eiffel", "snippet": "A lattice tower."}, + {"url": "https://example.com/bare", "date": "2024-01-01"}, + {"title": "no url", "snippet": "kept"}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == ( + _assistant_choice( + "Eiffel Tower\nhttps://example.com/eiffel\nA lattice tower.\n\nhttps://example.com/bare\n\nno url\nkept" + ), + ) + assert data.finish_reasons == () + + +def test_search_output_follows_the_content_capture_gate() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("asearch", "exa-search", {"results": [{"url": "https://example.com"}]}) + ) + + assert data.choices_out == () + + +def test_search_results_without_any_text_field_produce_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "asearch", "exa-search", {"results": [{"date": "2024-01-01"}, {"title": "", "url": None}, "not-a-result"]} + ), + capture_content=True, + ) + + assert data.choices_out == () + + def test_image_data_becomes_a_size_summary_and_never_carries_the_base64_payload() -> None: encoded: Final = "QUJDRA==" data: Final = LLMCallSpanData.from_standard_logging_payload( diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 96c420299e7..62aae7aed34 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1,6 +1,8 @@ import asyncio +import base64 import contextlib import contextvars +import io import json import logging import os @@ -9,7 +11,8 @@ import threading from collections.abc import Callable, Iterator from concurrent.futures import Future, ThreadPoolExecutor from datetime import datetime, timedelta, timezone -from typing import Final +from pathlib import PurePath +from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -19,9 +22,6 @@ from jsonschema import validate import litellm from litellm._internal_context import is_internal_call -from litellm.caching.caching import Cache -from litellm.caching.caching_handler import _PENDING_CACHE_WRITES -from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT from litellm._logging import ( CorrelationContextFilter, JsonFormatter, @@ -29,14 +29,18 @@ from litellm._logging import ( trace_id_var, verbose_logger, ) +from litellm.caching.caching import Cache +from litellm.caching.caching_handler import _PENDING_CACHE_WRITES +from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.thread_pool_executor import executor as logging_executor from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.proxy.utils import is_valid_api_key -from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams from litellm.types.integrations.custom_logger import HEADROOM_CONVERTED_STREAM_KEY +from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams from litellm.types.utils import ( + ADDRESSED_RESPONSE_ID_FIELD, CallTypes, Choices, Delta, @@ -46,7 +50,6 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, StreamingChoices, Usage, - ADDRESSED_RESPONSE_ID_FIELD, all_litellm_params, bedrock_batch_litellm_params, ) @@ -1638,7 +1641,6 @@ class TestProxyFunctionCalling: # For now, we expect False (current behavior), but document the limitation assert proxy_result is False, f"Current limitation: {proxy_model_with_hints} returns False without inference" - def test_litellm_utils_supports_function_calling_import(self): """Test that supports_function_calling can be imported from litellm.utils.""" try: @@ -1658,7 +1660,6 @@ class TestProxyFunctionCalling: except Exception as e: pytest.fail(f"Failed to access litellm.supports_function_calling: {e}") - def test_edge_cases_and_malformed_proxy_models(self): """Test edge cases and malformed proxy model names.""" test_cases = [ @@ -5004,7 +5005,9 @@ def _budget_reservation(callback_bound: bool = False) -> dict: _BUDGET_RESERVATION_CALL_KWARGS: Final = {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]} -_BUDGET_RESERVATION_REFUSAL: Final = litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o") +_BUDGET_RESERVATION_REFUSAL: Final = litellm.AuthenticationError( + message="bad key", llm_provider="openai", model="gpt-4o" +) @pytest.mark.asyncio @@ -6004,3 +6007,111 @@ def test_calculate_max_parallel_requests_precedence( ) == expected ) + + +class _NamedStream(io.BytesIO): + def __init__(self, name: str | int) -> None: + super().__init__(b"%PDF-1.4 secret document body") + self.name = name + + +def _logged_request_messages(original_function: str, *args: object, **kwargs: object) -> object: + logging_obj, _ = litellm.utils.function_setup( + original_function, + litellm.utils.Rules(), + datetime.now(), + *args, + litellm_call_id="request-text-call", + **kwargs, + ) + return logging_obj.messages + + +@pytest.mark.parametrize( + ("original_function", "args", "kwargs", "expected"), + [ + ("search", (), {"query": "Eiffel Tower"}, "Eiffel Tower"), + ("asearch", ("Eiffel Tower",), {}, "Eiffel Tower"), + ("asearch", (), {"query": ["Eiffel Tower", "Louvre"]}, "Eiffel Tower\nLouvre"), + ("asearch", (), {"query": ["Eiffel Tower", 7, None]}, "Eiffel Tower"), + ("image_edit", (), {"prompt": "make it blue", "image": b"png"}, "make it blue"), + ("aimage_edit", (b"png", "make it blue"), {}, "make it blue"), + ( + "aocr", + (), + {"document": {"type": "document_url", "document_url": "https://x.test/a.pdf"}}, + "https://x.test/a.pdf", + ), + ( + "ocr", + ("mistral-ocr-latest", {"type": "image_url", "image_url": "https://x.test/a.png"}), + {}, + "https://x.test/a.png", + ), + ( + "aocr", + (), + {"document": {"type": "document_url", "document_url": "data:application/pdf;base64,JVBERi0xLjQ="}}, + "data:application/pdf;base64 (12 chars)", + ), + ( + "aocr", + (), + {"document": {"type": "image_url", "image_url": "https://x.test/a,b.png"}}, + "https://x.test/a,b.png", + ), + ( + "aocr", + (), + {"document": {"type": "file", "file": PurePath("/tmp/hello.pdf"), "mime_type": "application/pdf"}}, + "file (application/pdf) hello.pdf", + ), + ("aocr", (), {"document": {"type": "document_url", "document_url": ""}}, ""), + ("aocr", (), {"document": {"type": "file", "file": b"%PDF"}}, "file 4 bytes"), + ("aocr", (), {"document": {"type": "file", "file": io.BytesIO(b"%PDF")}}, "file"), + ("aocr", (), {"document": {"type": "file", "file": _NamedStream("/tmp/scan.pdf")}}, "file scan.pdf"), + ( + "aocr", + (), + {"document": {"type": "file", "file": _NamedStream(3), "mime_type": "application/pdf"}}, + "file (application/pdf)", + ), + ("aocr", (), {"document": "not-a-document"}, "default-message-value"), + ], +) +def test_function_setup_logs_the_search_query_edit_prompt_and_ocr_document_summary_as_the_request( + original_function: str, args: tuple[object, ...], kwargs: dict[str, object], expected: str +) -> None: + assert _logged_request_messages(original_function, *args, **kwargs) == [{"role": "user", "content": expected}] + + +def test_search_with_a_mixed_type_query_list_still_reaches_its_own_validation_error() -> None: + mixed_query: Final = cast(list[str], ["Eiffel Tower", 7]) # cast-ok: the invalid list is the point of the test + + with pytest.raises(litellm.APIConnectionError, match="All items in query list must be strings"): + litellm.search(query=mixed_query, search_provider="duckduckgo") + + +def test_function_setup_never_logs_the_ocr_file_bytes() -> None: + content: Final = b"%PDF-1.4 secret document body" + logged: Final = _logged_request_messages("aocr", document={"type": "file", "file": content}) + + assert logged == [{"role": "user", "content": "file 29 bytes"}] + + +def test_function_setup_leaves_the_ocr_file_stream_unread_and_never_logs_its_bytes() -> None: + stream: Final = _NamedStream("/tmp/scan.pdf") + logged: Final = _logged_request_messages("aocr", document={"type": "file", "file": stream}) + + assert logged == [{"role": "user", "content": "file scan.pdf"}] + assert stream.tell() == 0 + + +def test_function_setup_never_logs_the_ocr_data_uri_payload() -> None: + payload: Final = base64.b64encode(b"%PDF-1.4 secret document body").decode() + logged: Final = _logged_request_messages( + "aocr", document={"type": "document_url", "document_url": f"data:application/pdf;base64,{payload}"} + ) + + assert logged == [{"role": "user", "content": f"data:application/pdf;base64 ({len(payload)} chars)"}] + assert payload not in str(logged)