diff --git a/.circleci/config.yml b/.circleci/config.yml index 55fa9410845..dfc539fb80e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,10 +112,10 @@ commands: node --version npm --version install_rust: - description: "Install pinned rustup (1.28.2) and Rust toolchain (1.97.1) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself." + description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself." steps: - run: - name: Install Rust (rustup 1.28.2, toolchain 1.97.1) + name: Install Rust (rustup 1.28.2, toolchain 1.98.0) command: | case "$(uname -m)" in x86_64) @@ -135,7 +135,7 @@ commands: "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init" echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - chmod +x /tmp/rustup-init - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.97.1 + /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0 rm -f /tmp/rustup-init echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.cargo/bin:$PATH" @@ -300,7 +300,7 @@ jobs: if ($rustupActual -ne $rustupExpected) { throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual" } - & $rustupInit -y --profile minimal --default-toolchain stable + & $rustupInit -y --profile minimal --default-toolchain 1.98.0 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/.github/scripts/smoke_test_native_wheel.py b/.github/scripts/smoke_test_native_wheel.py new file mode 100644 index 00000000000..577bb32fcf0 --- /dev/null +++ b/.github/scripts/smoke_test_native_wheel.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path +from typing import Final + +CHILD_SCRIPT: Final = """ +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +import sys + +native_path = Path(sys.argv[1]) +spec = spec_from_file_location("litellm.rust_bridge._native", native_path) +if spec is None or spec.loader is None: + raise RuntimeError("cannot create native extension import specification") +module = module_from_spec(spec) +spec.loader.exec_module(module) + +before = module.gil_stats() +if not isinstance(before.get("releases"), int): + raise AssertionError(f"unexpected gil_stats result: {before!r}") + +try: + module._panic_for_test() +except BaseException as error: + if type(error).__name__ != "PanicException": + raise AssertionError(f"expected PanicException, got {type(error).__name__}") from error +else: + raise AssertionError("Rust panic returned without raising") + +after = module.gil_stats() +if not isinstance(after.get("releases"), int): + raise AssertionError(f"native module unusable after panic: {after!r}") +""" + + +def main() -> int: + if len(sys.argv) != 2: + sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") + return 2 + + wheel: Final = Path(sys.argv[1]) + with tempfile.TemporaryDirectory() as temporary_directory, zipfile.ZipFile(wheel) as archive: + native_members: Final = tuple( + member + for member in archive.infolist() + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") + return 1 + + native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name + native_path.write_bytes(archive.read(native_members[0])) + result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False) + + if result.returncode != 0: + sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n") + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py new file mode 100644 index 00000000000..899e2a211c0 --- /dev/null +++ b/.github/scripts/verify_linux_native_wheel.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import importlib.util +import os +import re +import subprocess +import sys +import zipfile +from collections.abc import Callable, Mapping, Sequence +from itertools import product +from pathlib import Path, PurePosixPath +from types import MappingProxyType, ModuleType +from typing import Final, Protocol + +EXPECTED_PYTHON_TAG: Final = "cp310" +EXPECTED_ABI_TAG: Final = "abi3" +EXPECTED_PLATFORM_TAG: Final = "linux_x86_64" + + +class CommandRunner(Protocol): + def __call__( + self, + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +def _run_command( + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, check=check, capture_output=capture_output, text=text) + + +def _dist_info_directory(member: zipfile.ZipInfo) -> str | None: + parts: Final = PurePosixPath(member.filename).parts + if not parts or not parts[0].endswith(".dist-info"): + return None + return parts[0] + + +def _wheel_metadata_tags(archive: zipfile.ZipFile, members: tuple[zipfile.ZipInfo, ...]) -> tuple[str, ...]: + if len(members) != 1: + return () + lines: Final = archive.read(members[0]).splitlines() + return tuple(line.removeprefix(b"Tag:").strip().decode("ascii") for line in lines if line.startswith(b"Tag:")) + + +def _load_native_module(native_path: Path) -> ModuleType | None: + module_spec: Final = importlib.util.spec_from_file_location("litellm.rust_bridge._native", native_path) + if module_spec is None or module_spec.loader is None: + return None + try: + native_module: Final = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(native_module) + except Exception as error: # noqa: BLE001 # native module initialization can raise arbitrary exceptions + sys.stderr.write(f"native module load failed: {error}\n") + return None + return native_module + + +def main( + argv: Sequence[str] | None = None, + environment: Mapping[str, str] | None = None, + load_native_module: Callable[[Path], ModuleType | None] = _load_native_module, + run_command: CommandRunner = _run_command, +) -> int: + arguments: Final = tuple(sys.argv if argv is None else argv) + resolved_environment: Final = os.environ if environment is None else environment + if len(arguments) != 2: + sys.stderr.write(f"usage: {Path(arguments[0]).name} WHEEL\n") + return 2 + + wheel: Final = Path(arguments[1]) + wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) + if len(wheel_tags) != 4: + sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") + return 1 + + wheel_identity: Final = wheel_tags[0].split("-") + if len(wheel_identity) != 2 or wheel_identity[0] != "litellm" or not wheel_identity[1]: + sys.stderr.write(f"unexpected wheel identity: {wheel_tags[0]}\n") + return 1 + + expected_dist_info_directory: Final = f"{wheel_tags[0]}.dist-info" + expected_dist_info_directories: Final = frozenset((expected_dist_info_directory,)) + python_tag: Final = wheel_tags[1] + abi_tag: Final = wheel_tags[2] + platform_tag: Final = wheel_tags[3] + expanded_filename_tags: Final = frozenset( + "-".join(tag) for tag in product(python_tag.split("."), abi_tag.split("."), platform_tag.split(".")) + ) + + with zipfile.ZipFile(wheel) as archive: + wheel_members: Final = archive.infolist() + dist_info_directories: Final = frozenset( + directory for member in wheel_members if (directory := _dist_info_directory(member)) is not None + ) + required_dist_info_files: Final = ("METADATA", "RECORD", "WHEEL") + dist_info_file_counts: Final = MappingProxyType( + { + filename: sum( + member.filename == f"{expected_dist_info_directory}/{filename}" for member in wheel_members + ) + for filename in required_dist_info_files + } + ) + wheel_metadata_members: Final = tuple( + member for member in wheel_members if member.filename == f"{expected_dist_info_directory}/WHEEL" + ) + wheel_metadata_tags: Final = _wheel_metadata_tags(archive, wheel_metadata_members) + native_members: Final = tuple( + member + for member in wheel_members + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") + return 1 + + unexpected_members: Final = tuple( + member.filename + for member in wheel_members + if member.filename.endswith((".pdb", ".dwp", ".rlib", ".rmeta", "Cargo.toml", "Cargo.lock")) + or any(part.endswith(".dSYM") for part in PurePosixPath(member.filename).parts) + ) + native_member: Final = native_members[0] + uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members) + native_path: Final = wheel.parent / "native" / Path(native_member.filename).name + native_path.parent.mkdir(parents=True, exist_ok=True) + native_path.write_bytes(archive.read(native_member)) + + wheel_metadata_tags_match: Final = ( + len(wheel_metadata_tags) == len(expanded_filename_tags) + and frozenset(wheel_metadata_tags) == expanded_filename_tags + ) + commit_sha: Final = resolved_environment.get( + "RELEASE_WHEEL_COMMIT_SHA", resolved_environment.get("GITHUB_SHA", "unknown") + ) + rustc_version: Final = run_command( + ("rustc", "--version"), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text() + maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject) + if maturin_match is None: + sys.stderr.write("build-system does not pin an exact Maturin version\n") + return 1 + + maturin_version: Final = maturin_match.group(1) + native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100 + size_report: Final = "\n".join( + ( + "## Native wheel build report", + "", + "| Build | Value |", + "| --- | --- |", + f"| Commit | `{commit_sha}` |", + f"| Platform | `{platform_tag}` |", + f"| Python ABI | `{python_tag}-{abi_tag}` |", + f"| Rust compiler | `{rustc_version}` |", + f"| Maturin | `{maturin_version}` |", + "| Cargo profile | `release` |", + "", + "| Artifact | Size |", + "| --- | ---: |", + f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |", + f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |", + f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |", + f"| Native share | {native_percentage:.2f}% |", + "", + ) + ) + summary_path: Final = resolved_environment.get("GITHUB_STEP_SUMMARY") + if summary_path is None: + sys.stdout.write(size_report) + else: + Path(summary_path).write_text(size_report) + + sections: Final = run_command( + ("readelf", "--sections", "--wide", str(native_path)), + check=True, + capture_output=True, + text=True, + ).stdout + debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections) + debug_sections_absent: Final = not debug_sections + static_symbol_table_absent: Final = ".symtab" not in sections + + dynamic_symbols: Final = run_command( + ("readelf", "--dyn-syms", "--wide", str(native_path)), + check=True, + capture_output=True, + text=True, + ).stdout + extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols + native_module: Final = load_native_module(native_path) + native_module_loads: Final = native_module is not None + panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") + native_size_limit: Final = 20_000_000 + native_size_within_limit: Final = native_member.file_size <= native_size_limit + validations: Final = ( + (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), + (f"ABI tag is {EXPECTED_ABI_TAG}", abi_tag == EXPECTED_ABI_TAG), + (f"Platform tag is {EXPECTED_PLATFORM_TAG}", platform_tag == EXPECTED_PLATFORM_TAG), + ("Wheel dist-info directory matches the filename", dist_info_directories == expected_dist_info_directories), + ( + "Required dist-info files are present exactly once", + all(count == 1 for count in dist_info_file_counts.values()), + ), + ("Wheel metadata tags match the filename", wheel_metadata_tags_match), + ("Debug sections are absent", debug_sections_absent), + ("Static symbol table is absent", static_symbol_table_absent), + ("Python extension entry point is present", extension_entry_point_present), + ("Native module loads", native_module_loads), + ("Production module omits the panic test hook", panic_test_hook_absent), + ("Native extension does not exceed 20 MB", native_size_within_limit), + ("Wheel contents are valid", not unexpected_members), + ) + + verified_report: Final = size_report + "\n".join( + ("", "| Validation | Expected | Result |", "| --- | --- | :---: |") + + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations) + + ("",) + ) + if summary_path is not None: + Path(summary_path).write_text(verified_report) + + invalid_dist_info_files: Final = any(count != 1 for count in dist_info_file_counts.values()) + validation_errors: Final = tuple( + message + for failed, message in ( + (bool(debug_sections), f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}"), + (not static_symbol_table_absent, f"{native_member.filename} contains a static symbol table"), + (not extension_entry_point_present, "native extension does not export PyInit__native"), + ( + python_tag != EXPECTED_PYTHON_TAG, + f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}", + ), + (abi_tag != EXPECTED_ABI_TAG, f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}"), + ( + platform_tag != EXPECTED_PLATFORM_TAG, + f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}", + ), + ( + dist_info_directories != expected_dist_info_directories, + f"unexpected dist-info directories: expected {expected_dist_info_directory}, " + f"found {', '.join(sorted(dist_info_directories))}", + ), + (invalid_dist_info_files, f"required dist-info file counts are invalid: {dist_info_file_counts}"), + ( + not invalid_dist_info_files and not wheel_metadata_tags_match, + f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, " + f"found {', '.join(sorted(wheel_metadata_tags))}", + ), + ( + native_module is not None and not panic_test_hook_absent, + "production native module exposes _panic_for_test", + ), + ( + not native_size_within_limit, + f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB", + ), + (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"), + ) + if failed + ) + sys.stderr.write("".join(f"{message}\n" for message in validation_errors)) + + return 0 if all(passed for _, passed in validations) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/report-rust-release-wheel.yml b/.github/workflows/report-rust-release-wheel.yml new file mode 100644 index 00000000000..1d93b56f77f --- /dev/null +++ b/.github/workflows/report-rust-release-wheel.yml @@ -0,0 +1,130 @@ +name: Report LiteLLM Rust release wheel + +on: # zizmor: ignore[dangerous-triggers] reporter executes no PR code and consumes no PR artifacts or outputs + workflow_run: + workflows: + - LiteLLM Rust + types: + - completed + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || github.event.workflow_run.id }} + cancel-in-progress: false + +jobs: + report-release-wheel: + name: report release wheel + if: >- + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.path == '.github/workflows/test-rust.yml' && + github.event.workflow_run.head_repository.full_name == github.repository && + github.event.workflow_run.pull_requests[0].number != null + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write # PR comments use the issues API + pull-requests: read # Current-head validation rejects stale workflow runs + + steps: + - name: Link release wheel report on PR + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + COMMENT_MARKER: "" + with: + script: | + const marker = process.env.COMMENT_MARKER; + const workflowRun = context.payload.workflow_run; + const allowedConclusions = new Set([ + "action_required", + "cancelled", + "failure", + "neutral", + "skipped", + "stale", + "startup_failure", + "success", + "timed_out", + ]); + if ( + !allowedConclusions.has(workflowRun.conclusion) || + workflowRun.event !== "pull_request" || + workflowRun.path !== ".github/workflows/test-rust.yml" || + workflowRun.head_repository?.full_name !== + `${context.repo.owner}/${context.repo.repo}` || + workflowRun.pull_requests?.length !== 1 + ) { + throw new Error("unexpected source workflow"); + } + const pullRequest = workflowRun.pull_requests[0]; + const pullRequestNumber = pullRequest.number; + const headSha = workflowRun.head_sha; + const runId = workflowRun.id; + if ( + !Number.isSafeInteger(pullRequestNumber) || + pullRequestNumber <= 0 || + !Number.isSafeInteger(runId) || + runId <= 0 || + !/^[0-9a-f]{40}$/.test(headSha) || + pullRequest.head?.sha !== headSha + ) { + throw new Error("invalid source workflow metadata"); + } + const runUrl = + `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + + `/actions/runs/${runId}`; + const result = + workflowRun.conclusion === "success" + ? "successfully" + : `with \`${workflowRun.conclusion}\``; + const body = [ + marker, + "## LiteLLM Rust workflow", + "", + `Workflow completed ${result} for \`${headSha}\``, + "", + `[View workflow run](${runUrl})`, + ].join("\n"); + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + per_page: 100, + }); + const existing = comments.find( + (comment) => + comment.user?.login === "github-actions[bot]" && + comment.body?.startsWith(marker), + ); + const currentPullRequest = ( + await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullRequestNumber, + }) + ).data; + if ( + currentPullRequest.state !== "open" || + currentPullRequest.head.repo?.full_name !== + `${context.repo.owner}/${context.repo.repo}` || + currentPullRequest.head.sha !== headSha + ) { + core.info("source workflow no longer matches the current pull request head"); + return; + } + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body, + }); + } diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index c112bf2bb22..d02f5878396 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -128,6 +128,9 @@ jobs: - name: check_fastuuid_usage run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py + - name: check_py310_typing_imports + run: uv run --no-sync python ./tests/code_coverage_tests/check_py310_typing_imports.py + - name: check_e2e_no_raw_requests run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py @@ -145,3 +148,33 @@ jobs: - name: documentation_test_api_docs run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py + + python-310-import-smoke: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.10" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Install dependencies + run: uv sync --frozen --extra proxy --python 3.10 + + - run: uv run --no-sync python --version + + - name: Import litellm + run: uv run --no-sync python -c "import litellm" + + - name: Check litellm CLI + run: uv run --no-sync litellm --version diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 21e1bcb90c6..1b71232bc2e 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -4,6 +4,12 @@ on: push: paths: - "litellm-rust/**" + - ".cargo/**" + - "pyproject.toml" + - "rust-toolchain.toml" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" + - "tests/test_litellm/rust_bridge/native_route_wheel_test.py" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -13,6 +19,12 @@ on: - "litellm_**" paths: - "litellm-rust/**" + - ".cargo/**" + - "pyproject.toml" + - "rust-toolchain.toml" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" + - "tests/test_litellm/rust_bridge/native_route_wheel_test.py" - ".github/workflows/test-rust.yml" permissions: @@ -40,9 +52,7 @@ jobs: persist-credentials: false - name: Set up Rust - run: | - rustup toolchain install stable --profile minimal --component clippy,rustfmt - rustup default stable + run: rustup toolchain install - name: Cache Cargo registry and target uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 @@ -51,7 +61,7 @@ jobs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-cargo-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-cargo-${{ hashFiles('rust-toolchain.toml', 'litellm-rust/Cargo.lock') }} restore-keys: | ${{ runner.os }}-cargo- @@ -69,3 +79,50 @@ jobs: - name: Run core tests with Bedrock auth run: cargo test -p litellm-core --features bedrock-auth --locked + + release-wheel: + name: release wheel + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + env: + CARGO_TERM_COLOR: always + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Set up Rust + run: rustup toolchain install + + - name: Build release wheel + run: uv build --wheel --out-dir dist + + - name: Build panic contract wheel + run: >- + uv build --wheel --out-dir panic-dist + --config-setting "maturin.build-args=--features panic-test,extension-module" + + - name: Smoke-test native panic unwinding + run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl + + - name: Verify stripped native extension + env: + RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl + + - name: Test native route wheel + run: python tests/test_litellm/rust_bridge/native_route_wheel_test.py dist/*.whl diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index c2dff805772..6da5fc07e80 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -96,6 +96,7 @@ jobs: - shard: misc artifact-name: misc test-path: >- + tests/sdk_function_trace tests/test_litellm/batches tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol diff --git a/backend/Dockerfile b/backend/Dockerfile index aa01b9fba8b..622fedcd70d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -46,6 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ + --extra saml \ --python python3.13 # Stage 2 — copy source and install the project + workspace members. @@ -57,6 +58,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ + --extra saml \ --python python3.13 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index da788bf1ce3..3f96531cf6f 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,9 +1,9 @@ { "reportAny": { - "limit": 14076 + "limit": 14074 }, "reportArgumentType": { - "limit": 2216 + "limit": 2215 }, "reportAssignmentType": { "limit": 319 @@ -24,7 +24,7 @@ "limit": 19 }, "reportExplicitAny": { - "limit": 4128 + "limit": 4124 }, "reportFunctionMemberAccess": { "limit": 7 @@ -57,7 +57,7 @@ "limit": 5601 }, "reportMissingTypeArgument": { - "limit": 15306 + "limit": 15290 }, "reportMissingTypeStubs": { "limit": 40 @@ -99,31 +99,31 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 44364 + "limit": 44362 }, "reportUnknownLambdaType": { "limit": 109 }, "reportUnknownMemberType": { - "limit": 38350 + "limit": 38332 }, "reportUnknownParameterType": { - "limit": 19626 + "limit": 19625 }, "reportUnknownVariableType": { - "limit": 29890 + "limit": 29861 }, "reportUnnecessaryCast": { "limit": 111 }, "reportUnnecessaryComparison": { - "limit": 692 + "limit": 687 }, "reportUnnecessaryContains": { - "limit": 5 + "limit": 4 }, "reportUnnecessaryIsInstance": { - "limit": 826 + "limit": 823 }, "reportUntypedBaseClass": { "limit": 0 @@ -138,7 +138,7 @@ "limit": 138 }, "reportUnusedImport": { - "limit": 543 + "limit": 542 }, "reportUnusedVariable": { "limit": 137 diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py index 1d3268da9a0..190141470df 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py @@ -39,9 +39,9 @@ async def available_enterprise_users( if not premium_user: # check if SSO is enabled - show 5 user limit - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso - if _has_user_setup_sso(): + if has_user_setup_sso(): premium_user_data = EnterpriseLicenseData( max_users=5, ) diff --git a/helm/litellm-helm/Chart.yaml b/helm/litellm-helm/Chart.yaml index 3959d85edf3..a3cb388ffc6 100644 --- a/helm/litellm-helm/Chart.yaml +++ b/helm/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.1.2 +version: 1.1.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/litellm-helm/tests/hpa_tests.yaml b/helm/litellm-helm/tests/hpa_tests.yaml index ec18c3591d3..cd062dd5971 100644 --- a/helm/litellm-helm/tests/hpa_tests.yaml +++ b/helm/litellm-helm/tests/hpa_tests.yaml @@ -1,4 +1,4 @@ -suite: "hpa with behavior" +suite: "hpa" templates: - hpa.yaml tests: @@ -23,14 +23,44 @@ tests: - equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 } - equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 } ---- -suite: "hpa without behavior" -templates: - - hpa.yaml -tests: - it: "does not render behavior when not set" set: autoscaling.enabled: true asserts: - isKind: { of: HorizontalPodAutoscaler } - isNull: { path: spec.behavior } + + - it: "scales on cpu at the documented 60 percent by default" + set: + autoscaling.enabled: true + asserts: + - isKind: { of: HorizontalPodAutoscaler } + - equal: { path: "spec.metrics[0].resource.name", value: cpu } + - equal: { path: "spec.metrics[0].resource.target.type", value: Utilization } + - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 } + + - it: "does not scale on memory by default" + set: + autoscaling.enabled: true + asserts: + - lengthEqual: { path: spec.metrics, count: 1 } + + - it: "honours an explicit cpu target override" + set: + autoscaling.enabled: true + autoscaling.targetCPUUtilizationPercentage: 75 + asserts: + - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 } + + - it: "renders a memory metric only when a memory target is set" + set: + autoscaling.enabled: true + autoscaling.targetMemoryUtilizationPercentage: 80 + asserts: + - lengthEqual: { path: spec.metrics, count: 2 } + - equal: { path: "spec.metrics[1].resource.name", value: memory } + - equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 } + + - it: "renders no hpa when autoscaling is disabled" + asserts: + - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index f8df98de102..637be2322e3 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -200,7 +200,16 @@ autoscaling: enabled: false minReplicas: 1 maxReplicas: 100 - targetCPUUtilizationPercentage: 80 + # 60 is the documented recommendation. See "Recommended Machine Specifications" + # in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe + # above only after up to failureThreshold x periodSeconds = 300 seconds, so a target + # high enough to trip near saturation adds capacity minutes after it was needed. + targetCPUUtilizationPercentage: 60 + # Deliberately left unset rather than given a value. The prisma query engine's + # resident memory is a high-water mark that ratchets to the pod's worst-ever write + # and is never returned, so a memory target reads the largest write a pod ever did + # rather than what it is doing now, and replicas ratchet up without scaling back in. + # Memory is a floor to provision under 'resources', not a signal to scale on. # targetMemoryUtilizationPercentage: 80 # behavior: {} diff --git a/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py b/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py index f3b55fd4d96..2283814ab35 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py +++ b/litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py @@ -14,9 +14,22 @@ then fails on a Node binary that was never written. Deleting a cache directory that exists without a Node binary is what turns a killed bootstrap back into a recoverable one. -Both budgets are overridable so an operator can widen them without a release: -``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install and -``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every individual Prisma command. +``prisma migrate deploy`` is the other command whose runtime is not a +constant: it grows with the number of pending migrations, so a fresh database +that has to replay every migration this package ships overruns a per-command +budget sized for the short bookkeeping commands, on a laptop as much as on a +slow CI runner. The Python ``prisma`` wrapper spawns Node and the schema engine +as separate children, so killing the wrapper on timeout leaves them running: +the retry then contends with that orphan for Prisma's advisory lock and cannot +finish any sooner. Migrate deploy therefore runs under its own budget. + +All three budgets are overridable so an operator can widen them without a +release: ``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install, +``LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT`` for ``prisma migrate deploy`` and +``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every other Prisma command. The +per-command budget used to bound migrate deploy as well, so a deployment that +raised it above the deploy default keeps that larger budget for deploy unless +the deploy override says otherwise. """ import math @@ -36,10 +49,12 @@ except ImportError: PRISMA_COMMAND_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_COMMAND_TIMEOUT" PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_BOOTSTRAP_TIMEOUT" +PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT" NODEENV_CACHE_DIR_ENV_VAR = "PRISMA_NODEENV_CACHE_DIR" DEFAULT_PRISMA_COMMAND_TIMEOUT = 60.0 DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT = 600.0 +DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT = 600.0 BOOTSTRAP_ARG = "--version" @@ -88,6 +103,15 @@ def prisma_bootstrap_timeout() -> float: ) +def prisma_migrate_deploy_timeout() -> float: + """Seconds one ``prisma migrate deploy`` may run for, however many migrations are pending.""" + if os.getenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR) is not None: + return _timeout_from_env( + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT + ) + return max(DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT, prisma_command_timeout()) + + def nodeenv_cache_dir() -> Optional[Path]: """Where Prisma installs its private Node runtime, or None if unknowable.""" override = os.getenv(NODEENV_CACHE_DIR_ENV_VAR) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index b8032dd0d28..d22484bc0e8 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -15,8 +15,11 @@ from litellm_proxy_extras.replica_identity import ( apply_replica_identity_full, ) from litellm_proxy_extras.prisma_toolchain import ( + PRISMA_COMMAND_TIMEOUT_ENV_VAR, + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, ensure_prisma_toolchain, prisma_command_timeout, + prisma_migrate_deploy_timeout, ) @@ -40,6 +43,8 @@ def _get_prisma_env() -> dict: _MIGRATION_TS_RE = re.compile(r"^(\d{14})_") +_MIGRATION_DEADLOCK_MARKER = "deadlock detected" + _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE) _SPEND_LOGS_ARTIFACT_DROP_RE = re.compile( r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE @@ -262,6 +267,50 @@ class ProxyExtrasDBManager: env=prisma_env, ) + @staticmethod + def _roll_back_migration_best_effort(migration_name: str) -> None: + """Mark a migration rolled back, tolerating a concurrent resolver + having already done it.""" + try: + ProxyExtrasDBManager._roll_back_migration(migration_name) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + pass + + @staticmethod + def _failed_migration_logs(migration_name: str) -> Optional[str]: + """Return failed migration logs, or None if the ledger is unavailable.""" + database_url = os.getenv("DATABASE_URL") + if not database_url: + return None + + try: + import psycopg + except ImportError: + return None + + cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url) + ledger_table = psycopg.sql.SQL("{}.{}").format( + psycopg.sql.Identifier( + ProxyExtrasDBManager._prisma_schema_param(database_url) or "public" + ), + psycopg.sql.Identifier("_prisma_migrations"), + ) + try: + with psycopg.connect( + cleaned_url, connect_timeout=10, autocommit=True + ) as conn: + row = conn.execute( + psycopg.sql.SQL( + "SELECT logs FROM {} " + "WHERE migration_name = %s AND finished_at IS NULL " + "AND rolled_back_at IS NULL" + ).format(ledger_table), + (migration_name,), + ).fetchone() + except (psycopg.OperationalError, psycopg.DatabaseError): + return None + return (row[0] or "") if row else "" + @staticmethod def _resolve_specific_migration(migration_name: str): """Mark a specific migration as applied""" @@ -658,7 +707,8 @@ class ProxyExtrasDBManager: v2 migration resolver (opt-in via --use_v2_migration_resolver). Runs `prisma migrate deploy` and handles standard recovery paths - (P3005 baseline, P3009/P3018 idempotent errors). Critically, it does + (P3005 baseline, P3009/P3018 idempotent errors, deadlocks against a + concurrent migrate deploy). Critically, it does NOT call `_resolve_all_migrations` — the diff-and-force recovery that caused schema thrashing when two LiteLLM versions contended for the same DB during rolling deploys. @@ -698,12 +748,13 @@ class ProxyExtrasDBManager: original_dir = os.getcwd() os.chdir(migrations_dir) + deploy_timeout = prisma_migrate_deploy_timeout() try: for attempt in range(4): try: result = subprocess.run( [_get_prisma_command(), "migrate", "deploy"], - timeout=prisma_command_timeout(), + timeout=deploy_timeout, check=True, capture_output=True, text=True, @@ -713,8 +764,12 @@ class ProxyExtrasDBManager: return True except subprocess.TimeoutExpired: - logger.info( - f"prisma migrate deploy attempt {attempt + 1} timed out, retrying" + logger.warning( + "prisma migrate deploy attempt %s timed out after %ss, retrying. " + "Raise %s if this database needs longer to apply its pending migrations.", + attempt + 1, + deploy_timeout, + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, ) time.sleep(random.randrange(5, 15)) continue @@ -764,6 +819,20 @@ class ProxyExtrasDBManager: f"Detail: {resolve_err}" ) from resolve_err continue + if migration_match: + migration_name = migration_match.group(1) + ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name) + if ledger_logs is not None and ( + ledger_logs == "" or _MIGRATION_DEADLOCK_MARKER in ledger_logs + ): + logger.info( + "Migration %s failed in a concurrent migrate deploy " + "deadlock race, rolling its ledger row back and retrying", + migration_name, + ) + ProxyExtrasDBManager._roll_back_migration_best_effort(migration_name) + time.sleep(random.randrange(5, 15)) + continue raise RuntimeError( "Database migration failed and cannot be auto-recovered. " f"Manual intervention required.\n\nPrisma error:\n{stderr}" @@ -809,11 +878,42 @@ class ProxyExtrasDBManager: ) from resolve_err continue + if migration_match and _MIGRATION_DEADLOCK_MARKER in stderr: + logger.info( + "Migration %s deadlocked against a concurrent " + "migrate deploy, rolling its ledger row back " + "and retrying", + migration_match.group(1), + ) + ProxyExtrasDBManager._roll_back_migration_best_effort( + migration_match.group(1) + ) + time.sleep(random.randrange(5, 15)) + continue + raise RuntimeError( "Database migration failed and cannot be auto-recovered. " f"Manual intervention required.\n\nPrisma error:\n{stderr}" ) from e + if _MIGRATION_DEADLOCK_MARKER in stderr: + logger.info( + "prisma migrate deploy attempt %s deadlocked against " + "a concurrent migrate deploy, retrying", + attempt + 1, + ) + time.sleep(random.randrange(5, 15)) + continue + + if "P1002" in stderr and "advisory lock" in stderr: + logger.info( + "prisma migrate deploy attempt %s timed out waiting for " + "the advisory lock a concurrent migrate deploy holds, retrying", + attempt + 1, + ) + time.sleep(random.randrange(5, 15)) + continue + raise RuntimeError( "Database migration failed and cannot be auto-recovered. " f"Manual intervention required.\n\nPrisma error:\n{stderr}" @@ -821,9 +921,10 @@ class ProxyExtrasDBManager: raise RuntimeError( "Database migration failed after 4 attempts (retry loop " - "exhausted by timeouts or repeated idempotent-recovery " - "continues). Check database connectivity, load, and " - "_prisma_migrations ledger state." + "exhausted by timeouts, deadlock retries, or repeated " + "idempotent-recovery continues). Check database connectivity, " + "load, and _prisma_migrations ledger state, and raise " + f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out." ) finally: os.chdir(original_dir) @@ -908,7 +1009,7 @@ class ProxyExtrasDBManager: # Set migrations directory for Prisma result = subprocess.run( [_get_prisma_command(), "migrate", "deploy"], - timeout=prisma_command_timeout(), + timeout=prisma_migrate_deploy_timeout(), check=True, capture_output=True, text=True, @@ -1126,7 +1227,11 @@ class ProxyExtrasDBManager: ) return True except subprocess.TimeoutExpired: - logger.info(f"Attempt {attempt + 1} timed out") + logger.warning( + "Attempt %s timed out. Raise %s if this database needs longer to apply its schema.", + attempt + 1, + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR if use_migrate else PRISMA_COMMAND_TIMEOUT_ENV_VAR, + ) time.sleep(random.randrange(5, 15)) except subprocess.CalledProcessError as e: attempts_left = 3 - attempt diff --git a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py index 8d66bf872de..406f07eb792 100644 --- a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py +++ b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py @@ -240,3 +240,223 @@ def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path): ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) assert ok is True assert resolve_called["n"] == 0, "v2 must not invoke the diff-and-force recovery" + + +_DEADLOCK_P3018_STDERR = ( + "Error: P3018\n" + "Migration name: 20260415120000_health_check_latest_per_model_index\n" + "Database error code: 40P01\n" + "deadlock detected" +) + + +def _stub_v2_env(monkeypatch, tmp_path): + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setattr( + ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None + ) + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) + (tmp_path / "schema.prisma").write_text("// stub") + monkeypatch.setattr("time.sleep", lambda _: None) + + +def _succeed_after(failures: int, stderr: str): + calls = {"n": 0} + + class _OkResult: + stdout = "Applied migration.\n" + stderr = "" + + def _run(*args, **kwargs): + if "deploy" not in args[0]: + return _OkResult() + calls["n"] += 1 + if calls["n"] <= failures: + raise subprocess.CalledProcessError( + returncode=1, cmd=args[0], stderr=stderr, output="" + ) + return _OkResult() + + return _run + + +def test_v2_p3018_deadlock_rolls_back_and_retries(monkeypatch, tmp_path): + """v2: losing the migrate deploy deadlock race against a concurrent + instance rolls the ledger row back and retries instead of dying.""" + _stub_v2_env(monkeypatch, tmp_path) + + rolled_back = [] + monkeypatch.setattr( + ProxyExtrasDBManager, + "_roll_back_migration", + lambda name: rolled_back.append(name), + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_resolve_specific_migration", + lambda name: pytest.fail("a deadlocked migration must never be marked applied"), + ) + monkeypatch.setattr("subprocess.run", _succeed_after(1, _DEADLOCK_P3018_STDERR)) + + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert ok is True + assert rolled_back == ["20260415120000_health_check_latest_per_model_index"] + + +def test_v2_p3018_persistent_deadlock_exhausts_attempts(monkeypatch, tmp_path): + """v2: a deadlock on every attempt still fails after the retry budget.""" + _stub_v2_env(monkeypatch, tmp_path) + monkeypatch.setattr(ProxyExtrasDBManager, "_roll_back_migration", lambda name: None) + + with patch( + "subprocess.run", + side_effect=_fake_migrate_deploy_failure(1, _DEADLOCK_P3018_STDERR), + ): + with pytest.raises(RuntimeError, match="after 4 attempts"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + +def test_v2_p3009_deadlocked_ledger_row_rolls_back_and_retries(monkeypatch, tmp_path): + """v2: the surviving instance sees the victim's failed ledger row as P3009. + When that row's logs show a deadlock, roll it back and retry.""" + _stub_v2_env(monkeypatch, tmp_path) + + stderr = ( + "Error: P3009\n" + "migrate found failed migrations in the target database\n" + "The `20260415120000_health_check_latest_per_model_index` migration " + "started at 2026-09-01 18:46:13 UTC failed" + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_failed_migration_logs", + lambda name: "ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock", + ) + rolled_back = [] + monkeypatch.setattr( + ProxyExtrasDBManager, + "_roll_back_migration", + lambda name: rolled_back.append(name), + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_resolve_specific_migration", + lambda name: pytest.fail("a deadlocked migration must never be marked applied"), + ) + monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr)) + + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert ok is True + assert rolled_back == ["20260415120000_health_check_latest_per_model_index"] + + +def test_v2_p3009_empty_ledger_logs_rolls_back_and_retries(monkeypatch, tmp_path): + """v2: empty failed ledger logs mean a concurrent deploy moved it on.""" + _stub_v2_env(monkeypatch, tmp_path) + + stderr = ( + "Error: P3009\n" + "migrate found failed migrations in the target database\n" + "The `20260415120000_health_check_latest_per_model_index` migration " + "started at 2026-09-01 18:46:13 UTC failed" + ) + monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name: "") + rolled_back = [] + monkeypatch.setattr( + ProxyExtrasDBManager, + "_roll_back_migration", + lambda name: rolled_back.append(name), + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_resolve_specific_migration", + lambda name: pytest.fail("a deadlocked migration must never be marked applied"), + ) + monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr)) + + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert ok is True + assert rolled_back == ["20260415120000_health_check_latest_per_model_index"] + + +def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path): + """v2: an unreadable ledger cannot establish that P3009 was a deadlock.""" + _stub_v2_env(monkeypatch, tmp_path) + + stderr = ( + "Error: P3009\n" + "migrate found failed migrations in the target database\n" + "The `20260415120000_health_check_latest_per_model_index` migration " + "started at 2026-09-01 18:46:13 UTC failed" + ) + monkeypatch.setattr(ProxyExtrasDBManager, "_failed_migration_logs", lambda name: None) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_roll_back_migration", + lambda name: pytest.fail("an unreadable ledger must not trigger a retry"), + ) + monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr)) + + with pytest.raises(RuntimeError, match="cannot be auto-recovered"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + +def test_v2_p3009_non_deadlock_ledger_row_still_raises(monkeypatch, tmp_path): + """v2: a failed ledger row whose logs show a real SQL error stays fatal.""" + _stub_v2_env(monkeypatch, tmp_path) + + stderr = ( + "Error: P3009\n" + "migrate found failed migrations in the target database\n" + "The `20260101000000_genuinely_broken` migration started at " + "2026-09-01 18:46:13 UTC failed" + ) + monkeypatch.setattr( + ProxyExtrasDBManager, + "_failed_migration_logs", + lambda name: 'ERROR: syntax error at or near "BRKN"', + ) + + with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): + with pytest.raises(RuntimeError, match="cannot be auto-recovered"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + +def test_v2_bare_deadlock_stderr_retries(monkeypatch, tmp_path): + """v2: a deadlock reported without a Prisma error code (the advisory-lock + waiter as victim) is retried, not fatal.""" + _stub_v2_env(monkeypatch, tmp_path) + monkeypatch.setattr( + "subprocess.run", _succeed_after(1, "Database error: deadlock detected") + ) + + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert ok is True + + +_P1002_ADVISORY_LOCK_STDERR = ( + "Error: P1002\n\n" + "The database server at `127.0.0.1`:`45743` was reached but timed out.\n\n" + "Context: Timed out trying to acquire a postgres advisory lock " + "(SELECT pg_advisory_lock(72707369)). Elapsed: 10000ms." +) + + +def test_v2_advisory_lock_timeout_retries(monkeypatch, tmp_path): + """v2: the advisory-lock waiter that times out while a peer's retry holds + the lock retries instead of dying.""" + _stub_v2_env(monkeypatch, tmp_path) + monkeypatch.setattr("subprocess.run", _succeed_after(2, _P1002_ADVISORY_LOCK_STDERR)) + + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert ok is True + + +def test_v2_p1002_without_advisory_lock_context_still_raises(monkeypatch, tmp_path): + """v2: a plain P1002 (database unreachable) stays fatal.""" + _stub_v2_env(monkeypatch, tmp_path) + stderr = "Error: P1002\n\nThe database server at `db`:`5432` was reached but timed out." + monkeypatch.setattr("subprocess.run", _succeed_after(1, stderr)) + + with pytest.raises(RuntimeError, match="cannot be auto-recovered"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index 36a5ad5a8f4..b8b6291283d 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -1,6 +1,6 @@ # AGENTS.md -litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are MODULES inside the layers. +litellm-rust has four crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers. ## Crates @@ -8,9 +8,10 @@ litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes ( |-------|------| | litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. | | litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. ## Where a route lives @@ -28,7 +29,7 @@ core/src/messages/ Handlers never live in `ai-gateway`. `ocr`, `audio_transcription`, and `realtime` are still hosted there from before this rule; they move to `core` as they are touched. -Adding a crate: default to a MODULE. New crate ONLY on a real trigger — separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these. +Adding a crate: default to a module. A new crate requires a real trigger: separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these. Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional. diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md index fe6ceedbb86..3dcf1853efc 100644 --- a/litellm-rust/CLAUDE.md +++ b/litellm-rust/CLAUDE.md @@ -21,12 +21,13 @@ variants of it. The test for a good abstraction is that adding the next provider is a few declarative lines, not a new file of duplicated flow. Only diverge from the base when behavior is genuinely different, and say so explicitly in the PR. -## Crates (exactly three — see AGENTS.md) +## Crates (see AGENTS.md) `litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call. `litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and -`litellm-python-bridge` exposes it to the Python SDK. A crate is a **layer**, not -a route — add modules, not crates. +`litellm-python-bridge` exposes it to the Python SDK. `litellm-python-interop` +holds domain-neutral PyO3 primitives shared by Python-facing Rust code. A crate +is a layer or shared foundation, not a route; add modules, not crates. ## Core Boundary @@ -175,7 +176,7 @@ cd litellm-rust cargo fmt --check # the ai-gateway binary + server code is behind the `server` feature cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings -cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings +cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings cargo test --workspace ``` diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 4388e561026..b3dac5ca935 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -919,6 +919,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.33" @@ -972,6 +978,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "h2" version = "0.3.27" @@ -1380,6 +1392,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.186" @@ -1404,6 +1422,7 @@ dependencies = [ "tokio", "tokio-tungstenite", "tower", + "tracing", ] [[package]] @@ -1423,6 +1442,7 @@ dependencies = [ "sha2 0.10.9", "thiserror 2.0.19", "tokio", + "tracing", ] [[package]] @@ -1430,14 +1450,29 @@ name = "litellm-python-bridge" version = "0.1.0" dependencies = [ "criterion", + "futures-util", "litellm-ai-gateway", "litellm-core", + "litellm-python-interop", "pyo3", "pyo3-async-runtimes", - "pythonize", "serde", "serde_json", "tokio", + "tokio-tungstenite", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "litellm-python-interop" +version = "0.1.0" +dependencies = [ + "pyo3", + "pythonize", + "rstest", + "serde", + "serde_json", ] [[package]] @@ -1627,6 +1662,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -1638,9 +1682,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "libc", "once_cell", @@ -1666,18 +1710,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -1685,9 +1729,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1697,9 +1741,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", @@ -1899,6 +1943,12 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.28" @@ -1956,6 +2006,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.119", + "unicode-ident", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -2210,6 +2289,15 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "2.0.1" @@ -2348,6 +2436,15 @@ dependencies = [ "syn 3.0.0", ] +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + [[package]] name = "time" version = "0.3.53" @@ -2488,6 +2585,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + [[package]] name = "tower" version = "0.5.3" @@ -2566,6 +2693,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -2903,6 +3041,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index c17a0605fc7..a13dd4c04b0 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/core", "crates/ai-gateway", + "crates/python-interop", "crates/python-bridge", ] resolver = "2" @@ -13,14 +14,18 @@ license = "MIT" repository = "https://github.com/BerriAI/litellm" [workspace.dependencies] +tracing = "0.1" +tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] } litellm-core = { path = "crates/core" } litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false } +litellm-python-interop = { path = "crates/python-interop" } axum = "0.7" -pyo3 = "0.29.0" +pyo3 = "0.29.2" pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } pythonize = "0.29.0" rand = "0.8" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] } +rstest = "0.26.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10" diff --git a/litellm-rust/README.md b/litellm-rust/README.md index bcccf93300b..a0d79c6f0a5 100644 --- a/litellm-rust/README.md +++ b/litellm-rust/README.md @@ -26,9 +26,10 @@ coverage and production evidence. |-------|------| | litellm-core | The SDK. Per-route entrypoints (`messages::messages()`), types, provider transforms (modules under `providers/`), provider resolution, auth, the provider HTTP call, and the router. | | litellm-ai-gateway | The axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. ## Layout @@ -38,7 +39,8 @@ crates/ src/messages/ mod.rs (entrypoint), types, transformation, prepare, handler, client src/providers/anthropic/messages/transformation.rs ai-gateway/ Axum server + WebSocket hosts; calls core entrypoints. - python-bridge/ PyO3 bridge for Python LiteLLM. + python-interop/ Domain-neutral PyO3 conversion and GIL primitives. + python-bridge/ PyO3 API adapter for Python LiteLLM. ``` The folder shape follows the Python provider tree: diff --git a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md index a1860d8a9c9..4a689cb9579 100644 --- a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md +++ b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md @@ -54,6 +54,6 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages` cd litellm-rust cargo fmt --check cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings - cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings + cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings cargo test --workspace ``` diff --git a/litellm-rust/crates/ai-gateway/Cargo.toml b/litellm-rust/crates/ai-gateway/Cargo.toml index 541beabe170..e3dbdf24ce6 100644 --- a/litellm-rust/crates/ai-gateway/Cargo.toml +++ b/litellm-rust/crates/ai-gateway/Cargo.toml @@ -14,6 +14,7 @@ path = "src/main.rs" required-features = ["server"] [dependencies] +tracing.workspace = true litellm-core = { workspace = true, features = ["bedrock-auth"] } # reqwest (rustls + json) is used by io/ocr and ships realtime logs to the # Python proxy callbacks API. diff --git a/litellm-rust/crates/ai-gateway/README.md b/litellm-rust/crates/ai-gateway/README.md index 7a6c620ee84..5cbb47220be 100644 --- a/litellm-rust/crates/ai-gateway/README.md +++ b/litellm-rust/crates/ai-gateway/README.md @@ -6,15 +6,16 @@ dials OpenAI upstream, and splices the two sockets frame-by-frame. ## Crates -`litellm-rust` is exactly three crates (a crate is a **layer**, not a route): +`litellm-rust` has four crates. A crate is a layer or shared foundation, not a route: | Crate | Role | |-------|------| | litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. | | litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. - **Client endpoint:** `wss:///v1/realtime?model=` (WebSocket) - **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset) diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs deleted file mode 100644 index 270d5c2d97a..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs +++ /dev/null @@ -1,48 +0,0 @@ -use std::collections::BTreeMap; - -use litellm_core::CoreResult; -use litellm_core::audio_transcription::transformation::AudioTranscriptionProviderConfig; -use litellm_core::error::CoreError; -use litellm_core::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG; -use serde_json::{Map, Value}; - -pub(super) fn audio_transcription_provider_config( - provider: &str, -) -> Option<&'static dyn AudioTranscriptionProviderConfig> { - match provider { - "bedrock" => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG), - _ => None, - } -} - -pub(super) fn string_headers( - headers: Option>, -) -> CoreResult> { - headers - .unwrap_or_default() - .into_iter() - .map(|(key, value)| { - value - .as_str() - .map(|value| (key.clone(), value.to_string())) - .ok_or_else(|| { - CoreError::InvalidRequest(format!( - "audio transcription extra_headers.{key} must be a string" - )) - }) - }) - .collect() -} - -pub(super) fn has_header(headers: &BTreeMap, name: &str) -> bool { - headers.keys().any(|key| key.eq_ignore_ascii_case(name)) -} - -pub(super) fn truncate_error_body(body: &str) -> String { - let truncated: String = body.chars().take(256).collect(); - if truncated.chars().count() == body.chars().count() { - truncated - } else { - format!("{truncated}... (truncated)") - } -} diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs deleted file mode 100644 index 33c13550f58..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs +++ /dev/null @@ -1,89 +0,0 @@ -use std::time::SystemTime; - -use litellm_core::CoreResult; -use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth; -use litellm_core::error::CoreError; -use litellm_core::providers::bedrock::audio_transcription::aws_auth_config; -use litellm_core::providers::bedrock::aws_base::{resolve_credentials, sign_bedrock_post}; -use serde_json::Value; - -use super::common_utils::truncate_error_body; -use super::types::ProviderAudioTranscriptionRequest; -use crate::client::http_client; - -pub(crate) async fn execute_audio_transcription_provider_call( - request: ProviderAudioTranscriptionRequest, -) -> CoreResult { - let body = serde_json::to_vec(&request.body).map_err(|error| { - CoreError::InvalidRequest(format!("invalid audio request body: {error}")) - })?; - let mut request_builder = http_client().post(&request.url).body(body.clone()); - for (key, value) in &request.upstream_headers { - request_builder = request_builder.header(key, value); - } - if let Some(duration) = request.timeout { - request_builder = request_builder.timeout(duration); - } - let response = request_builder - .send() - .await - .map_err(|error| CoreError::Network(error.to_string()))?; - let status = response.status(); - let text = response - .text() - .await - .map_err(|error| CoreError::Network(error.to_string()))?; - if !status.is_success() { - return Err(CoreError::Http { - status: status.as_u16(), - body: truncate_error_body(&text), - }); - } - let response_json: Value = serde_json::from_str(&text).map_err(|error| { - CoreError::InvalidResponse(format!("invalid audio response JSON: {error}")) - })?; - Ok(request - .config - .transform_transcription_response(&request.model, response_json)? - .into_json()) -} - -pub(crate) async fn sign_request( - request: &ProviderAudioTranscriptionRequest, - optional_params: &serde_json::Map, -) -> CoreResult { - let env_lookup = environment_lookup; - let auth = request - .config - .auth_strategy(&request.model, optional_params, &env_lookup)?; - let body = serde_json::to_vec(&request.body).map_err(|error| { - CoreError::InvalidRequest(format!("invalid audio request body: {error}")) - })?; - let mut headers = super::common_utils::string_headers(None)?; - headers.insert("Content-Type".to_string(), "application/json".to_string()); - headers.extend(request.upstream_headers.iter().cloned()); - match auth { - AudioTranscriptionAuth::Bearer => {} - AudioTranscriptionAuth::AwsSigV4 { region, .. } => { - let credentials = - resolve_credentials(aws_auth_config(optional_params, &env_lookup), &env_lookup) - .await?; - headers.extend(sign_bedrock_post( - &request.url, - &body, - &headers, - ®ion, - &credentials, - SystemTime::now(), - )?); - } - } - Ok(ProviderAudioTranscriptionRequest { - upstream_headers: headers.into_iter().collect(), - ..request.clone() - }) -} - -pub(super) fn environment_lookup(key: &str) -> Option { - std::env::var(key).ok() -} diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs index 0c9faeda6e7..dbe2d3a325b 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs @@ -1,15 +1,14 @@ +use litellm_core::audio_transcription::{ + AudioTranscriptionRequest as CoreAudioTranscriptionRequest, ProviderAudioTranscriptionRequest, + prepare_audio_transcription_provider_call, +}; +use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; +use litellm_core::error::Error; +use serde_json::{Map, Value, json}; use std::future::Future; use std::pin::Pin; -use litellm_core::CoreResult; -use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth; -use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::CoreError; -use serde_json::{Map, Value, json}; - -use super::common_utils::{audio_transcription_provider_config, has_header, string_headers}; -use super::handler::sign_request; -use super::types::{PreparedAudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; +use super::types::PreparedAudioTranscriptionRequest; use crate::integrations::custom_guardrail::{ CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest, }; @@ -26,7 +25,7 @@ pub(crate) struct AudioTranscriptionLifecycleHooks { request_metadata: RequestMetadata, } -type AudioFuture<'a, T> = Pin> + Send + 'a>>; +type AudioFuture<'a, T> = Pin> + Send + 'a>>; type AudioLogFuture<'a> = Pin + Send + 'a>>; impl AudioTranscriptionLifecycleHooks { @@ -45,7 +44,7 @@ impl AudioTranscriptionLifecycleHooks { async fn run_pre_call_guardrails( &self, request: PreparedAudioTranscriptionRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -63,17 +62,17 @@ impl AudioTranscriptionLifecycleHooks { .await .map_err(guardrail_error_to_core_error)?; let Value::Object(mut data) = guardrail_request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription pre_call guardrail must return an object".to_string(), )); }; let audio = data.remove("audio").ok_or_else(|| { - CoreError::InvalidRequest("audio transcription guardrail removed audio".to_string()) + Error::InvalidRequest("audio transcription guardrail removed audio".to_string()) })?; let optional_params = match data.remove("optional_params") { Some(Value::Object(value)) => value, Some(_) => { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription optional_params must be an object".to_string(), )); } @@ -89,53 +88,36 @@ impl AudioTranscriptionLifecycleHooks { async fn prepare_provider_request( &self, request: PreparedAudioTranscriptionRequest, - ) -> CoreResult { - let config = audio_transcription_provider_config(&request.custom_llm_provider) - .ok_or_else(|| CoreError::InvalidProvider(request.custom_llm_provider.clone()))?; - let env_lookup = super::handler::environment_lookup; - let headers = string_headers(request.extra_headers)?; - let url = config.complete_url( - request.api_base.as_deref(), - &request.model, - &request.optional_params, - &env_lookup, - )?; - let filtered_params = config.map_transcription_params(&request.optional_params); - let body = config.transform_transcription_request( - &request.model, - request.audio, - filtered_params, - )?; - let auth = config.auth_strategy(&request.model, &request.optional_params, &env_lookup)?; - let mut upstream_headers = headers.into_iter().collect::>(); - if matches!(auth, AudioTranscriptionAuth::Bearer) - && !has_header( - &upstream_headers - .iter() - .cloned() - .collect::>(), - "authorization", - ) - && let Some(api_key) = request.api_key.as_deref() - { - upstream_headers.push(("Authorization".to_string(), format!("Bearer {api_key}"))); - } - let provider_request = ProviderAudioTranscriptionRequest { - model: request.model, - config, - url, - body: body.body, - upstream_headers, - timeout: request.timeout, - }; - let provider_request = self.run_during_call_guardrails(provider_request).await?; - sign_request(&provider_request, &request.optional_params).await + ) -> Result { + let PreparedAudioTranscriptionRequest { + model, + custom_llm_provider, + audio, + api_key, + api_base, + extra_headers, + optional_params, + timeout, + .. + } = request; + let provider_request = + prepare_audio_transcription_provider_call(CoreAudioTranscriptionRequest { + model: &model, + audio, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: Some(&custom_llm_provider), + extra_headers, + optional_params, + timeout, + })?; + self.run_during_call_guardrails(provider_request).await } async fn run_during_call_guardrails( &self, request: ProviderAudioTranscriptionRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -144,23 +126,23 @@ impl AudioTranscriptionLifecycleHooks { .run_during_call( &guardrail_context(&self.request_metadata), GuardrailRequest::new(json!({ - "model": request.model, - "custom_llm_provider": "bedrock", - "url": request.url, - "body": request.body, + "model": request.model(), + "custom_llm_provider": request.custom_llm_provider(), + "url": request.url(), + "body": request.body(), })), ) .await .map_err(guardrail_error_to_core_error)?; let Value::Object(mut data) = guardrail_request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription during_call guardrail must return an object".to_string(), )); }; let body = data.remove("body").ok_or_else(|| { - CoreError::InvalidRequest("audio transcription guardrail removed body".to_string()) + Error::InvalidRequest("audio transcription guardrail removed body".to_string()) })?; - Ok(ProviderAudioTranscriptionRequest { body, ..request }) + Ok(request.with_body(body)) } fn logging_payload( @@ -241,7 +223,7 @@ impl CallLifecycleHooks( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -281,22 +263,22 @@ fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { } } -fn guardrail_error_to_core_error(error: GuardrailError) -> CoreError { - CoreError::InvalidRequest(format!("{}: {}", error.kind, error.message)) +fn guardrail_error_to_core_error(error: GuardrailError) -> Error { + Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) } -fn core_error_kind(error: &CoreError) -> &'static str { +fn core_error_kind(error: &Error) -> &'static str { match error { - CoreError::Auth(_) => "AuthError", - CoreError::InvalidProvider(_) => "InvalidProvider", - CoreError::InvalidRequest(_) => "InvalidRequest", - CoreError::InvalidType { .. } => "InvalidType", - CoreError::MissingField(_) => "MissingField", - CoreError::Http { .. } => "HttpError", - CoreError::InvalidResponse(_) => "InvalidResponse", - CoreError::Network(_) => "NetworkError", - CoreError::Connect(_) => "ConnectError", - CoreError::Routing(_) => "RoutingError", - CoreError::Unsupported(_) => "UnsupportedRequest", + Error::Auth(_) => "AuthError", + Error::InvalidProvider(_) => "InvalidProvider", + Error::InvalidRequest(_) => "InvalidRequest", + Error::InvalidType { .. } => "InvalidType", + Error::MissingField(_) => "MissingField", + Error::Http { .. } => "HttpError", + Error::InvalidResponse(_) => "InvalidResponse", + Error::Network(_) => "NetworkError", + Error::Connect(_) => "ConnectError", + Error::Routing(_) => "RoutingError", + Error::Unsupported(_) => "UnsupportedRequest", } } diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs index 5d33d912c40..03d621b8414 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs @@ -1,19 +1,17 @@ -use litellm_core::CoreResult; +use litellm_core::Error; +use litellm_core::audio_transcription::execute_audio_transcription_provider_call; use litellm_core::call_lifecycle::CallLifecycle; use serde_json::Value; -mod common_utils; -mod handler; mod hooks; mod prepare; mod types; pub use types::AudioTranscriptionRequest; -use handler::execute_audio_transcription_provider_call; use prepare::{PreparedAudioTranscriptionCall, prepare_audio_transcription_call}; -pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> CoreResult { +pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { let PreparedAudioTranscriptionCall { request, hooks } = prepare_audio_transcription_call(request); CallLifecycle::default() diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs index 9697aa98b0a..b470638264e 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs @@ -1,7 +1,6 @@ use std::sync::Arc; use std::time::Duration; -use litellm_core::audio_transcription::transformation::AudioTranscriptionProviderConfig; use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleRequest}; use serde_json::{Map, Value}; @@ -46,13 +45,3 @@ impl CallLifecycleRequest for PreparedAudioTranscriptionRequest { ) } } - -#[derive(Clone)] -pub(crate) struct ProviderAudioTranscriptionRequest { - pub(crate) model: String, - pub(crate) config: &'static dyn AudioTranscriptionProviderConfig, - pub(crate) url: String, - pub(crate) body: Value, - pub(crate) upstream_headers: Vec<(String, String)>, - pub(crate) timeout: Option, -} diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime.rs b/litellm-rust/crates/ai-gateway/src/io/realtime.rs index 845e7bf9527..662f7328982 100644 --- a/litellm-rust/crates/ai-gateway/src/io/realtime.rs +++ b/litellm-rust/crates/ai-gateway/src/io/realtime.rs @@ -15,8 +15,7 @@ use std::time::Duration; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{Sink, SinkExt, Stream, StreamExt}; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::realtime::transformation::RealtimeProviderConfig; use litellm_core::realtime::types::RealtimeEvent; use tokio::net::TcpStream; @@ -48,7 +47,7 @@ pub(crate) type UpstreamRx = SplitStream; /// Resolve the OpenAI API key from the explicit param or the environment. /// /// Blank/whitespace values are treated as absent (guard at resolution time). -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { +pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) @@ -58,7 +57,7 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { .ok() .filter(|key| !key.trim().is_empty()) }) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } /// Open the upstream WebSocket to OpenAI for `(model, api_key, api_base)`. @@ -70,24 +69,24 @@ pub(crate) async fn dial_upstream( model: &str, api_key: &str, api_base: Option<&str>, -) -> CoreResult { +) -> Result { let url = OPENAI_REALTIME_CONFIG.complete_url(api_base, model); let mut request = url .as_str() .into_client_request() - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; // GA realtime: only Authorization. The legacy OpenAI-Beta header triggers // beta_api_shape_disabled, so we do not send it. request.headers_mut().insert( AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|err| CoreError::Auth(err.to_string()))?, + .map_err(|err| Error::Auth(err.to_string()))?, ); let (upstream, _response) = connect_async(request) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; Ok(upstream) } @@ -96,22 +95,22 @@ pub(crate) async fn dial_upstream( /// Used by the pool to pre-read OpenAI's unprompted `session.created`. Returns an /// error on a non-text frame, a closed socket, or undecodable JSON so the pool can /// discard a misbehaving socket rather than warm it. -pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> CoreResult { +pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> Result { loop { let message = upstream_rx .next() .await - .ok_or_else(|| CoreError::Network("upstream closed before first event".to_string()))? - .map_err(|err| CoreError::Network(err.to_string()))?; + .ok_or_else(|| Error::Network("upstream closed before first event".to_string()))? + .map_err(|err| Error::Network(err.to_string()))?; match message { Message::Text(text) => { return serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(err.to_string())); + .map_err(|err| Error::InvalidResponse(err.to_string())); } // Ignore protocol frames (ping/pong) while waiting for the first event. Message::Ping(_) | Message::Pong(_) => continue, Message::Close(_) => { - return Err(CoreError::Network( + return Err(Error::Network( "upstream closed before first event".to_string(), )); } @@ -139,7 +138,7 @@ pub(crate) async fn splice( mut observe: impl FnMut(&RealtimeEvent) + Send, mut client_in: In, mut client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -154,7 +153,7 @@ where client_out .send(outbound) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } @@ -175,26 +174,26 @@ where // inflate its own spend log. Logging observes upstream events only. for outbound in config.transform_realtime_request(&event, model)?.events { let payload = serde_json::to_string(&outbound) - .map_err(|err| CoreError::InvalidResponse(err.to_string()))?; + .map_err(|err| Error::InvalidResponse(err.to_string()))?; upstream_tx .send(Message::Text(payload)) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } // upstream -> client upstream_message = upstream_rx.next() => { let Some(message) = upstream_message else { break }; // upstream closed - match message.map_err(|err| CoreError::Network(err.to_string()))? { + match message.map_err(|err| Error::Network(err.to_string()))? { Message::Text(text) => { let event: RealtimeEvent = serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(err.to_string()))?; + .map_err(|err| Error::InvalidResponse(err.to_string()))?; observe(&event); for outbound in config.transform_realtime_response(&event, model)?.events { client_out .send(outbound) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } Message::Close(_) => break, @@ -225,7 +224,7 @@ pub async fn realtime( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -258,7 +257,7 @@ pub async fn realtime_warm( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs b/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs index 4a1a3cd1166..49e9c459a88 100644 --- a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs +++ b/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs @@ -28,7 +28,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use futures_util::StreamExt; -use litellm_core::CoreResult; +use litellm_core::Error; use litellm_core::realtime::types::RealtimeEvent; use crate::io::realtime::{ @@ -438,7 +438,7 @@ impl RealtimePool { /// /// `key.api_key` is already resolved (non-blank). The first frame OpenAI sends /// unprompted is `session.created`; we buffer exactly that and read nothing more. -async fn warm_one(key: &UpstreamKey) -> CoreResult { +async fn warm_one(key: &UpstreamKey) -> Result { let upstream: UpstreamWs = dial_upstream(&key.model, &key.api_key, key.api_base.as_deref()).await?; let (tx, mut rx) = upstream.split(); diff --git a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs index 9b51019f4bc..0b01747b1a5 100644 --- a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs +++ b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs @@ -4,10 +4,10 @@ use std::time::Duration; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{Sink, SinkExt, Stream, StreamExt}; +use litellm_core::Error; use litellm_core::providers::openai::responses::transformation::OPENAI_RESPONSES_WS_CONFIG; use litellm_core::responses::types::ResponsesWsEvent; use litellm_core::responses::websocket::ResponsesWebSocketProviderConfig; -use litellm_core::{CoreError, CoreResult}; use tokio::net::TcpStream; use tokio::sync::Mutex; use tokio_tungstenite::tungstenite::Message; @@ -37,51 +37,49 @@ impl ResponsesWebSocketConnection { url: &str, headers: &HashMap, timeout: Option, - ) -> CoreResult { + ) -> Result { let mut request = url .into_client_request() - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; for (name, value) in headers { let header_name = name .parse::() - .map_err(|error| CoreError::InvalidRequest(error.to_string()))?; + .map_err(|error| Error::InvalidRequest(error.to_string()))?; let header_value = HeaderValue::from_str(value) - .map_err(|error| CoreError::InvalidRequest(error.to_string()))?; + .map_err(|error| Error::InvalidRequest(error.to_string()))?; request.headers_mut().insert(header_name, header_value); } let connect = connect_async(request); let result = match timeout { Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| { - CoreError::Network("Responses WebSocket connection timed out".to_string()) + Error::Network("Responses WebSocket connection timed out".to_string()) })?, None => connect.await, }; let (socket, _) = result.map_err(|error| match error { - tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http { + tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http { status: response.status().as_u16(), body: String::new(), }, - other => CoreError::Network(other.to_string()), + other => Error::Network(other.to_string()), })?; Ok(Self { socket: Arc::new(Mutex::new(Some(socket))), }) } - pub async fn send_text(&self, text: String) -> CoreResult<()> { + pub async fn send_text(&self, text: String) -> Result<(), Error> { let mut socket = self.socket.lock().await; let Some(socket) = socket.as_mut() else { - return Err(CoreError::Network( - "Responses WebSocket is closed".to_string(), - )); + return Err(Error::Network("Responses WebSocket is closed".to_string())); }; socket .send(Message::Text(text)) .await - .map_err(|error| CoreError::Network(error.to_string())) + .map_err(|error| Error::Network(error.to_string())) } - pub async fn recv_text(&self) -> CoreResult> { + pub async fn recv_text(&self) -> Result, Error> { let mut socket_guard = self.socket.lock().await; let Some(socket) = socket_guard.as_mut() else { return Ok(None); @@ -90,27 +88,27 @@ impl ResponsesWebSocketConnection { Some(Ok(Message::Text(text))) => Ok(Some(text)), Some(Ok(Message::Binary(bytes))) => String::from_utf8(bytes.to_vec()) .map(Some) - .map_err(|error| CoreError::InvalidResponse(error.to_string())), + .map_err(|error| Error::InvalidResponse(error.to_string())), Some(Ok(Message::Close(_))) | None => Ok(None), Some(Ok(_)) => Ok(None), - Some(Err(error)) => Err(CoreError::Network(error.to_string())), + Some(Err(error)) => Err(Error::Network(error.to_string())), } } - pub async fn close(&self) -> CoreResult<()> { + pub async fn close(&self) -> Result<(), Error> { let mut socket = self.socket.lock().await; if let Some(socket) = socket.as_mut() { socket .close(None) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } *socket = None; Ok(()) } } -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { +pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { api_key .map(str::trim) .filter(|value| !value.is_empty()) @@ -120,38 +118,38 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { .ok() .filter(|value| !value.trim().is_empty()) }) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } async fn dial_upstream( model: &str, api_key: &str, api_base: Option<&str>, -) -> CoreResult { +) -> Result { let url = OPENAI_RESPONSES_WS_CONFIG.complete_websocket_url(api_base, model); let mut request = url .as_str() .into_client_request() - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; request.headers_mut().insert( AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|error| CoreError::Auth(error.to_string()))?, + .map_err(|error| Error::Auth(error.to_string()))?, ); let result = tokio::time::timeout( Duration::from_secs(DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS), connect_async(request), ) .await - .map_err(|_| CoreError::Network("Responses WebSocket connection timed out".to_string()))?; + .map_err(|_| Error::Network("Responses WebSocket connection timed out".to_string()))?; result .map(|(socket, _)| socket) .map_err(|error| match error { - tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http { + tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http { status: response.status().as_u16(), body: String::new(), }, - other => CoreError::Network(other.to_string()), + other => Error::Network(other.to_string()), }) } @@ -166,7 +164,7 @@ impl ResponsesWebSocketStreaming { observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, - ) -> CoreResult<()> + ) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -193,7 +191,7 @@ pub(crate) async fn splice( mut observe: impl FnMut(&ResponsesWsEvent) + Send, mut client_in: In, mut client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -210,18 +208,18 @@ where .events { let payload = serde_json::to_string(&outbound) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; upstream_tx.send(Message::Text(payload)) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } message = upstream_rx.next() => { let Some(message) = message else { break }; - match message.map_err(|error| CoreError::Network(error.to_string()))? { + match message.map_err(|error| Error::Network(error.to_string()))? { Message::Text(text) => { let event = serde_json::from_str::(&text) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; observe(&event); for outbound in OPENAI_RESPONSES_WS_CONFIG .transform_ws_response(&event, model)? @@ -229,7 +227,7 @@ where { client_out.send(outbound) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } Message::Close(_) => break, @@ -252,7 +250,7 @@ pub async fn async_responses_websocket( mut observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -267,11 +265,11 @@ where .events { let payload = serde_json::to_string(&outbound) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; upstream_tx .send(Message::Text(payload)) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } ResponsesWebSocketStreaming::bidirectional_forward( @@ -296,7 +294,7 @@ pub async fn responses_ws( observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -514,7 +512,7 @@ mod tests { ) .await .expect_err("status error"); - assert!(matches!(error, CoreError::Http { status: 401, .. })); + assert!(matches!(error, Error::Http { status: 401, .. })); server.await.expect("server task"); } @@ -543,7 +541,7 @@ mod tests { ) .await .expect_err("status error"); - assert!(matches!(error, CoreError::Http { status: 500, .. })); + assert!(matches!(error, Error::Http { status: 500, .. })); server.await.expect("server task"); } } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs b/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs index 9bc2818b6e7..c1fb328893b 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs @@ -3,8 +3,7 @@ use std::time::{Duration, Instant}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrProviderConfig; use reqwest::Url; use serde_json::{Map, Value}; @@ -33,6 +32,7 @@ pub(super) fn truncate_error_body(body: &str) -> String { format!("{truncated}... (truncated)") } +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) fn ocr_provider_config( provider: &str, model: &str, @@ -56,7 +56,7 @@ fn is_azure_document_intelligence_model(model: &str) -> bool { pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { extra_headers .unwrap_or_default() .into_iter() @@ -65,7 +65,7 @@ pub(super) fn string_headers( .as_str() .map(|value| (key.clone(), value.to_string())) .ok_or_else(|| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "OCR extra_headers.{key} must be a string, got {}", litellm_core::error::json_type_name(&value) )) @@ -74,13 +74,7 @@ pub(super) fn string_headers( .collect() } -pub(super) fn has_header(headers: &[(String, String)], name: &str) -> bool { - headers - .iter() - .any(|(key, _)| key.eq_ignore_ascii_case(name)) -} - -fn document_url_field(document: &Value) -> CoreResult> { +fn document_url_field(document: &Value) -> Result, Error> { let Some(object) = document.as_object() else { return Ok(None); }; @@ -138,13 +132,13 @@ fn is_blocked_ip(ip: IpAddr) -> bool { } } -fn blocked_url_error(url: &Url) -> CoreError { - CoreError::InvalidRequest(format!( +fn blocked_url_error(url: &Url) -> Error { + Error::InvalidRequest(format!( "OCR document URL rejected by SSRF protection: {url}" )) } -async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { +async fn validate_safe_fetch_url(url: &Url) -> Result<(), Error> { if !matches!(url.scheme(), "http" | "https") { return Err(blocked_url_error(url)); } @@ -162,7 +156,7 @@ async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { .ok_or_else(|| blocked_url_error(url))?; let addresses = tokio::net::lookup_host((host, port)) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let mut saw_address = false; for address in addresses { saw_address = true; @@ -176,25 +170,25 @@ async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { Ok(()) } -fn redirect_location(response: &reqwest::Response, url: &Url) -> CoreResult { +fn redirect_location(response: &reqwest::Response, url: &Url) -> Result { let location = response .headers() .get(reqwest::header::LOCATION) .and_then(|value| value.to_str().ok()) .ok_or_else(|| { - CoreError::InvalidResponse("OCR document redirect missing Location header".to_string()) + Error::InvalidResponse("OCR document redirect missing Location header".to_string()) })?; url.join(location) - .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR document redirect: {err}"))) + .map_err(|err| Error::InvalidResponse(format!("invalid OCR document redirect: {err}"))) } -async fn safe_get_document_url(url: &str) -> CoreResult<(Url, reqwest::Response)> { +async fn safe_get_document_url(url: &str) -> Result<(Url, reqwest::Response), Error> { let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .build() - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let mut current_url = Url::parse(url) - .map_err(|err| CoreError::InvalidRequest(format!("invalid OCR document URL: {err}")))?; + .map_err(|err| Error::InvalidRequest(format!("invalid OCR document URL: {err}")))?; for _ in 0..MAX_SAFE_FETCH_REDIRECTS { validate_safe_fetch_url(¤t_url).await?; @@ -202,28 +196,28 @@ async fn safe_get_document_url(url: &str) -> CoreResult<(Url, reqwest::Response) .get(current_url.clone()) .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !response.status().is_redirection() { return Ok((current_url, response)); } current_url = redirect_location(&response, ¤t_url)?; } - Err(CoreError::InvalidRequest( + Err(Error::InvalidRequest( "Too many redirects while fetching OCR document URL".to_string(), )) } -fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> CoreResult<()> { +fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Result<(), Error> { if max_bytes == 0 { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "OCR document URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}" ))); } if content_length > max_bytes { let size_mb = content_length as f64 / (1024.0 * 1024.0); let max_size_mb = max_bytes as f64 / (1024.0 * 1024.0); - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "OCR document size ({size_mb:.2}MB) exceeds maximum allowed size ({max_size_mb:.2}MB). url={url}" ))); } @@ -233,7 +227,7 @@ fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Core async fn read_response_with_limit( mut response: reqwest::Response, url: &Url, -) -> CoreResult> { +) -> Result, Error> { let max_bytes = max_document_download_bytes(); if let Some(content_length) = response.content_length() { enforce_download_size(content_length, max_bytes, url)?; @@ -246,7 +240,7 @@ async fn read_response_with_limit( while let Some(chunk) = response .chunk() .await - .map_err(|err| CoreError::Network(err.to_string()))? + .map_err(|err| Error::Network(err.to_string()))? { bytes_downloaded += chunk.len() as u64; enforce_download_size(bytes_downloaded, max_bytes, url)?; @@ -255,7 +249,7 @@ async fn read_response_with_limit( Ok(bytes) } -pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreResult { +pub(super) async fn convert_document_url_to_data_uri(document: Value) -> Result { let Some((field, url)) = document_url_field(&document)? else { return Ok(document); }; @@ -267,7 +261,7 @@ pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreRes let status = response.status(); if !status.is_success() { let body = response.text().await.unwrap_or_default(); - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&body), }); @@ -290,7 +284,7 @@ pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreRes let mut transformed = document .as_object() .cloned() - .ok_or_else(|| CoreError::InvalidRequest("OCR document must be an object".to_string()))?; + .ok_or_else(|| Error::InvalidRequest("OCR document must be an object".to_string()))?; transformed.insert(field.to_string(), Value::String(data_uri)); Ok(Value::Object(transformed)) } @@ -316,11 +310,11 @@ fn retry_after_secs(response: &reqwest::Response) -> u64 { .unwrap_or(2) } -fn operation_status(response_json: &Value) -> CoreResult<&str> { +fn operation_status(response_json: &Value) -> Result<&str, Error> { let status = response_json .get("status") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("status"))?; + .ok_or(Error::MissingField("status"))?; match status { "succeeded" => Ok("succeeded"), "running" | "notStarted" => Ok("running"), @@ -330,11 +324,11 @@ fn operation_status(response_json: &Value) -> CoreResult<&str> { .and_then(|error| error.get("message")) .and_then(Value::as_str) .unwrap_or("Unknown error"); - Err(CoreError::InvalidResponse(format!( + Err(Error::InvalidResponse(format!( "Azure Document Intelligence analysis failed: {message}" ))) } - other => Err(CoreError::InvalidResponse(format!( + other => Err(Error::InvalidResponse(format!( "Unknown operation status: {other}" ))), } @@ -345,9 +339,9 @@ pub(super) async fn poll_document_intelligence( original_url: &str, headers: &[(String, String)], timeout: Option, -) -> CoreResult { +) -> Result { if !same_origin(operation_url, original_url) { - return Err(CoreError::InvalidResponse( + return Err(Error::InvalidResponse( "Azure Document Intelligence: rejected cross-origin polling URL".to_string(), )); } @@ -358,7 +352,7 @@ pub(super) async fn poll_document_intelligence( )); loop { if start.elapsed() > timeout { - return Err(CoreError::Network(format!( + return Err(Error::Network(format!( "Azure Document Intelligence operation polling timed out after {} seconds", timeout.as_secs() ))); @@ -373,21 +367,21 @@ pub(super) async fn poll_document_intelligence( let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let retry_after = retry_after_secs(&response); let status = response.status(); let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let response_json: Value = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}")) + Error::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}")) })?; if operation_status(&response_json)? == "succeeded" { return Ok(response_json); @@ -426,7 +420,7 @@ mod tests { assert!(matches!( error, - CoreError::InvalidRequest(message) + Error::InvalidRequest(message) if message.contains("SSRF protection") )); } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs b/litellm-rust/crates/ai-gateway/src/ocr/handler.rs index 1de34eb400e..856d9571201 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/handler.rs @@ -1,13 +1,19 @@ -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; +use litellm_core::http_utils::http_request; use litellm_core::ocr::transformation::OcrResponseHandling; use serde_json::Value; use super::common_utils::{poll_document_intelligence, truncate_error_body}; -use super::types::ProviderOcrRequest; +use super::hooks::OcrLifecycleHooks; +use super::types::PreparedOcrRequest; use crate::client::http_client; -pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> CoreResult { +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub(crate) async fn execute_ocr_provider_call( + request: PreparedOcrRequest, + hooks: &OcrLifecycleHooks, +) -> Result { + let request = hooks.prepare_provider_request(request).await?; let mut request_builder = http_client().post(&request.url).json(&request.body); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -16,10 +22,9 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co request_builder = request_builder.timeout(duration); } - let response = request_builder - .send() + let response = http_request(request_builder) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); if request.config.response_handling() == OcrResponseHandling::AzureDocumentIntelligencePoll @@ -31,7 +36,7 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co .and_then(|value| value.to_str().ok()) .map(str::to_string) .ok_or_else(|| { - CoreError::InvalidResponse( + Error::InvalidResponse( "Azure Document Intelligence returned 202 but no Operation-Location header found" .to_string(), ) @@ -52,17 +57,17 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let response_json: Value = serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; + .map_err(|err| Error::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; Ok(request .config diff --git a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs index 95df566dc53..f8c4f8fe8c5 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs @@ -1,15 +1,10 @@ +use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; +use litellm_core::error::Error; +use serde_json::{Map, Value, json}; use std::future::Future; use std::pin::Pin; -use litellm_core::CoreResult; -use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::CoreError; -use litellm_core::ocr::transformation::OcrAuthStrategy; -use serde_json::{Map, Value, json}; - -use super::common_utils::{ - convert_document_url_to_data_uri, has_header, ocr_provider_config, string_headers, -}; +use super::common_utils::{convert_document_url_to_data_uri, string_headers}; use super::types::{PreparedOcrRequest, ProviderOcrRequest}; use crate::integrations::custom_guardrail::{ CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest, @@ -27,7 +22,7 @@ pub(crate) struct OcrLifecycleHooks { request_metadata: RequestMetadata, } -type OcrFuture<'a, T> = Pin> + Send + 'a>>; +type OcrFuture<'a, T> = Pin> + Send + 'a>>; type OcrLogFuture<'a> = Pin + Send + 'a>>; impl OcrLifecycleHooks { @@ -46,7 +41,7 @@ impl OcrLifecycleHooks { async fn run_pre_call_guardrails( &self, request: PreparedOcrRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -64,6 +59,10 @@ impl OcrLifecycleHooks { .await .map_err(guardrail_error_to_core_error)?; let (document, optional_params) = parse_ocr_pre_call_guardrail_request(guardrail_request)?; + let optional_params = match &request.config { + Ok(config) => config.map_ocr_params(&optional_params), + Err(_) => optional_params, + }; Ok(PreparedOcrRequest { document, optional_params, @@ -71,25 +70,23 @@ impl OcrLifecycleHooks { }) } - async fn prepare_provider_request( + pub(crate) async fn prepare_provider_request( &self, request: PreparedOcrRequest, - ) -> CoreResult { - let config = ocr_provider_config(&request.custom_llm_provider, &request.model) - .ok_or_else(|| CoreError::InvalidProvider(request.custom_llm_provider.clone()))?; + ) -> Result { + let config = request.config?; let env_lookup = |key: &str| std::env::var(key).ok(); - let headers = string_headers(request.extra_headers)?; - let auth_strategy = config.auth_strategy(); - let api_key = (!has_header(&headers, auth_strategy.header_name())) - .then(|| config.resolve_api_key(request.api_key.as_deref(), &env_lookup)) - .transpose()?; + let upstream_headers = config.validate_environment( + string_headers(request.extra_headers)?, + request.api_key.as_deref(), + &env_lookup, + )?; let url = config.complete_url( request.api_base.as_deref(), &request.model, &request.optional_params, &env_lookup, )?; - let filtered_params = config.map_ocr_params(&request.optional_params); let model = request.model.clone(); let custom_llm_provider = request.custom_llm_provider.clone(); let document = if config.requires_data_uri_document() { @@ -98,9 +95,8 @@ impl OcrLifecycleHooks { request.document }; let body = config - .transform_ocr_request(&request.model, document, filtered_params)? + .transform_ocr_request(&request.model, document, request.optional_params)? .data; - let upstream_headers = upstream_headers(&headers, auth_strategy, api_key.as_deref()); let body = self .run_during_call_guardrails(&model, &custom_llm_provider, &url, body) .await?; @@ -120,7 +116,7 @@ impl OcrLifecycleHooks { custom_llm_provider: &str, url: &str, body: Value, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(body); } @@ -169,9 +165,9 @@ impl OcrLifecycleHooks { } } -impl CallLifecycleHooks for OcrLifecycleHooks { +impl CallLifecycleHooks for OcrLifecycleHooks { type PreCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>; - type DuringCallFuture<'a> = OcrFuture<'a, ProviderOcrRequest>; + type DuringCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>; type SuccessFuture<'a> = OcrLogFuture<'a>; type FailureFuture<'a> = OcrLogFuture<'a>; @@ -188,7 +184,7 @@ impl CallLifecycleHooks for OcrLi _context: &'a CallLifecycleContext, request: PreparedOcrRequest, ) -> Self::DuringCallFuture<'a> { - Box::pin(async move { self.prepare_provider_request(request).await }) + Box::pin(async move { Ok(request) }) } fn async_log_success_event<'a>( @@ -217,7 +213,7 @@ impl CallLifecycleHooks for OcrLi fn async_log_failure_event<'a>( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -249,21 +245,6 @@ impl CallLifecycleHooks for OcrLi } } -fn upstream_headers( - headers: &[(String, String)], - auth_strategy: OcrAuthStrategy, - api_key: Option<&str>, -) -> Vec<(String, String)> { - api_key - .map(|api_key| match auth_strategy { - OcrAuthStrategy::Bearer => ("Authorization".to_string(), format!("Bearer {api_key}")), - OcrAuthStrategy::Header(header_name) => (header_name.to_string(), api_key.to_string()), - }) - .into_iter() - .chain(headers.iter().cloned()) - .collect() -} - fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { GuardrailContext { call_type: CallType::Ocr, @@ -278,19 +259,19 @@ fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { fn parse_ocr_pre_call_guardrail_request( request: GuardrailRequest, -) -> CoreResult<(Value, Map)> { +) -> Result<(Value, Map), Error> { let Value::Object(mut data) = request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR pre_call guardrail must return an object".to_string(), )); }; let document = data.remove("document").ok_or_else(|| { - CoreError::InvalidRequest("OCR pre_call guardrail removed document".to_string()) + Error::InvalidRequest("OCR pre_call guardrail removed document".to_string()) })?; let optional_params = match data.remove("optional_params") { Some(Value::Object(params)) => params, Some(_) => { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR pre_call guardrail optional_params must be an object".to_string(), )); } @@ -299,33 +280,32 @@ fn parse_ocr_pre_call_guardrail_request( Ok((document, optional_params)) } -fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> CoreResult { +fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> Result { let Value::Object(mut data) = request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR during_call guardrail must return an object".to_string(), )); }; - data.remove("body").ok_or_else(|| { - CoreError::InvalidRequest("OCR during_call guardrail removed body".to_string()) - }) + data.remove("body") + .ok_or_else(|| Error::InvalidRequest("OCR during_call guardrail removed body".to_string())) } -fn guardrail_error_to_core_error(error: GuardrailError) -> CoreError { - CoreError::InvalidRequest(format!("{}: {}", error.kind, error.message)) +fn guardrail_error_to_core_error(error: GuardrailError) -> Error { + Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) } -fn core_error_kind(error: &CoreError) -> &'static str { +fn core_error_kind(error: &Error) -> &'static str { match error { - CoreError::Auth(_) => "AuthError", - CoreError::InvalidProvider(_) => "InvalidProvider", - CoreError::InvalidRequest(_) => "InvalidRequest", - CoreError::InvalidType { .. } => "InvalidType", - CoreError::MissingField(_) => "MissingField", - CoreError::Http { .. } => "HttpError", - CoreError::InvalidResponse(_) => "InvalidResponse", - CoreError::Network(_) => "NetworkError", - CoreError::Connect(_) => "ConnectError", - CoreError::Routing(_) => "RoutingError", - CoreError::Unsupported(_) => "UnsupportedRequest", + Error::Auth(_) => "AuthError", + Error::InvalidProvider(_) => "InvalidProvider", + Error::InvalidRequest(_) => "InvalidRequest", + Error::InvalidType { .. } => "InvalidType", + Error::MissingField(_) => "MissingField", + Error::Http { .. } => "HttpError", + Error::InvalidResponse(_) => "InvalidResponse", + Error::Network(_) => "NetworkError", + Error::Connect(_) => "ConnectError", + Error::Routing(_) => "RoutingError", + Error::Unsupported(_) => "UnsupportedRequest", } } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs index c4c13e2300c..d9230af1c59 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs @@ -1,4 +1,4 @@ -use litellm_core::CoreResult; +use litellm_core::Error; use litellm_core::call_lifecycle::CallLifecycle; use serde_json::Value; @@ -13,10 +13,13 @@ pub use types::OcrRequest; use handler::execute_ocr_provider_call; use prepare::{PreparedOcrCall, prepare_ocr_call}; -pub async fn ocr(request: OcrRequest<'_>) -> CoreResult { +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub async fn ocr(request: OcrRequest<'_>) -> Result { let PreparedOcrCall { request, hooks } = prepare_ocr_call(request); CallLifecycle::default() - .run_request(request, &hooks, execute_ocr_provider_call) + .run_request(request, &hooks, |request| { + execute_ocr_provider_call(request, &hooks) + }) .await } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs b/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs index 6231393c889..fedacc62760 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs @@ -3,6 +3,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; +use super::common_utils::ocr_provider_config; use super::hooks::OcrLifecycleHooks; use super::types::{OcrRequest, PreparedOcrRequest}; use crate::integrations::custom_guardrail::CustomGuardrailRunner; @@ -13,6 +14,7 @@ pub(crate) struct PreparedOcrCall { pub(crate) hooks: OcrLifecycleHooks, } +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(crate) fn prepare_ocr_call(request: OcrRequest<'_>) -> PreparedOcrCall { let call_id = request .litellm_call_id @@ -25,9 +27,25 @@ pub(crate) fn prepare_ocr_call(request: OcrRequest<'_>) -> PreparedOcrCall { }); let model = provider_info.model.to_string(); let custom_llm_provider = provider_info.custom_llm_provider.to_string(); + let config = ocr_provider_config(&custom_llm_provider, &model) + .ok_or_else(|| litellm_core::Error::InvalidProvider(custom_llm_provider.clone())); + let optional_params = match &config { + Ok(config) => { + let supported = config.supported_ocr_params(); + config.map_ocr_params( + &request + .optional_params + .into_iter() + .filter(|(name, _)| supported.contains(&name.as_str())) + .collect(), + ) + } + Err(_) => request.optional_params, + }; PreparedOcrCall { request: PreparedOcrRequest { + config, model, custom_llm_provider, litellm_call_id: call_id, @@ -35,7 +53,7 @@ pub(crate) fn prepare_ocr_call(request: OcrRequest<'_>) -> PreparedOcrCall { api_key: request.api_key.map(str::to_string), api_base: request.api_base.map(str::to_string), extra_headers: request.extra_headers, - optional_params: request.optional_params, + optional_params, timeout: request.timeout, }, hooks: OcrLifecycleHooks::new( diff --git a/litellm-rust/crates/ai-gateway/src/ocr/tests.rs b/litellm-rust/crates/ai-gateway/src/ocr/tests.rs index bb2a6b06501..85e4c408045 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/tests.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/tests.rs @@ -1,13 +1,14 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; -use litellm_core::error::CoreError; +use litellm_core::error::Error; +use litellm_core::http_utils::has_header; use litellm_core::ocr::transformation::OcrResponseHandling; use serde_json::{Map, Value, json}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; -use super::common_utils::{has_header, ocr_provider_config, string_headers, truncate_error_body}; +use super::common_utils::{ocr_provider_config, string_headers, truncate_error_body}; use super::{OcrRequest, ocr}; use crate::integrations::custom_guardrail::{ CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailError, GuardrailEventHook, @@ -395,7 +396,7 @@ async fn ocr_lifecycle_runs_failure_hook_on_provider_error() { .await .expect_err("provider error propagates"); - assert!(matches!(err, CoreError::Http { status: 500, .. })); + assert!(matches!(err, Error::Http { status: 500, .. })); server.await.expect("server task completes"); assert_eq!( logger.events(), @@ -439,7 +440,7 @@ async fn ocr_lifecycle_pre_call_block_skips_provider_socket() { .await .expect_err("guardrail blocks request"); - assert!(matches!(err, CoreError::InvalidRequest(_))); + assert!(matches!(err, Error::InvalidRequest(_))); assert_eq!(guardrail.events(), vec!["async_pre_call_hook"]); assert_eq!( logger.events(), @@ -607,7 +608,7 @@ fn string_headers_rejects_non_string_values() { let err = string_headers(Some(headers)).expect_err("non-string header rejected"); assert_eq!( err, - CoreError::InvalidRequest( + Error::InvalidRequest( "OCR extra_headers.x-retry-count must be a string, got number".to_string() ) ); diff --git a/litellm-rust/crates/ai-gateway/src/ocr/types.rs b/litellm-rust/crates/ai-gateway/src/ocr/types.rs index bde734a4dd1..95e551d79ca 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/types.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/types.rs @@ -25,6 +25,7 @@ pub struct OcrRequest<'a> { } pub(crate) struct PreparedOcrRequest { + pub(crate) config: Result<&'static dyn OcrProviderConfig, litellm_core::Error>, pub(crate) model: String, pub(crate) custom_llm_provider: String, pub(crate) litellm_call_id: String, diff --git a/litellm-rust/crates/ai-gateway/src/python/config.rs b/litellm-rust/crates/ai-gateway/src/python/config.rs index c028d3d6b51..d5a4dd69c8d 100644 --- a/litellm-rust/crates/ai-gateway/src/python/config.rs +++ b/litellm-rust/crates/ai-gateway/src/python/config.rs @@ -6,33 +6,31 @@ //! (and recorded in [`crate::gil`]); the realtime hot path never touches Python. //! //! Compiled only under the `python-config` feature. - -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::router::{Deployment, Router}; use pyo3::prelude::*; use crate::gil; /// Load the router's `model_list` from `config_path` via the Python reader. -pub fn load_router_from_config(config_path: &str) -> CoreResult { +pub fn load_router_from_config(config_path: &str) -> Result { gil::record_acquisition(); Python::attach(|py| { let model_list = py .import("litellm.proxy.read_model_list") .and_then(|module| module.getattr("read_model_list")) .and_then(|reader| reader.call1((config_path,))) - .map_err(|err| CoreError::Routing(format!("read_model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("read_model_list failed: {err}")))?; let model_list_json: String = py .import("json") .and_then(|json| json.getattr("dumps")) .and_then(|dumps| dumps.call1((model_list,))) .and_then(|encoded| encoded.extract()) - .map_err(|err| CoreError::Routing(format!("serializing model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("serializing model_list failed: {err}")))?; let deployments: Vec = serde_json::from_str(&model_list_json) - .map_err(|err| CoreError::Routing(format!("parsing model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("parsing model_list failed: {err}")))?; Ok(Router::new(deployments)) }) diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs index 7e38d10c6ff..e9f8c477f36 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs @@ -9,7 +9,7 @@ use axum::http::StatusCode; use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE, HeaderMap, HeaderValue}; use axum::response::{IntoResponse, Response}; use axum::routing::post; -use litellm_core::CoreError; +use litellm_core::Error; use serde_json::{Map, Value}; use crate::auth::RequireMasterKey; @@ -46,7 +46,7 @@ fn stream_response(upstream: reqwest::Response) -> Result Result Result>, CoreError> { +fn forwarded_headers(headers: &HeaderMap) -> Result>, Error> { let forwarded = headers .iter() .filter(|(name, _)| { @@ -74,19 +74,19 @@ fn forwarded_headers(headers: &HeaderMap) -> Result>, }) .map(|(name, value)| { let value = value.to_str().map_err(|_| { - CoreError::InvalidRequest(format!("invalid value for header {}", name.as_str())) + Error::InvalidRequest(format!("invalid value for header {}", name.as_str())) })?; Ok((name.to_string(), Value::String(value.to_string()))) }) - .collect::, CoreError>>()?; + .collect::, Error>>()?; Ok((!forwarded.is_empty()).then_some(forwarded)) } #[derive(Debug)] -struct MessagesRouteError(CoreError); +struct MessagesRouteError(Error); -impl From for MessagesRouteError { - fn from(error: CoreError) -> Self { +impl From for MessagesRouteError { + fn from(error: Error) -> Self { Self(error) } } @@ -94,28 +94,28 @@ impl From for MessagesRouteError { impl IntoResponse for MessagesRouteError { fn into_response(self) -> Response { let (status, message) = match self.0 { - CoreError::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message), - CoreError::InvalidProvider(_) | CoreError::Routing(_) => ( + Error::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message), + Error::InvalidProvider(_) | Error::Routing(_) => ( StatusCode::NOT_FOUND, "no messages deployment is configured for this model".to_string(), ), - CoreError::Auth(_) => ( + Error::Auth(_) => ( StatusCode::BAD_GATEWAY, "messages provider authentication failed".to_string(), ), - CoreError::Http { .. } - | CoreError::Network(_) - | CoreError::Connect(_) - | CoreError::InvalidResponse(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) => ( + Error::Http { .. } + | Error::Network(_) + | Error::Connect(_) + | Error::InvalidResponse(_) + | Error::InvalidType { .. } + | Error::MissingField(_) => ( StatusCode::BAD_GATEWAY, "messages provider request failed".to_string(), ), // The gateway has no Python implementation to decline to, so a // request the core cannot serve is reported to the caller. The // reason is a fixed internal string, never provider content. - CoreError::Unsupported(reason) => ( + Error::Unsupported(reason) => ( StatusCode::BAD_REQUEST, format!("messages request is not supported: {reason}"), ), diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs index 5f4c5fe8de4..4fd29db05d6 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs @@ -1,10 +1,10 @@ use std::sync::Arc; +use litellm_core::Error; use litellm_core::constants::ANTHROPIC_MESSAGES_PROVIDER; use litellm_core::messages::types::MessagesRequest; use litellm_core::messages::{messages, messages_stream}; use litellm_core::router::Router; -use litellm_core::{CoreError, CoreResult}; use serde_json::{Map, Value}; pub(crate) enum MessagesResponse { @@ -16,16 +16,16 @@ pub async fn run( router: &Arc, body: Value, extra_headers: Option>, -) -> CoreResult { +) -> Result { let model = body .get("model") .and_then(Value::as_str) .map(str::trim) .filter(|model| !model.is_empty()) - .ok_or_else(|| CoreError::InvalidRequest("messages body requires a model".to_string()))?; - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + .ok_or_else(|| Error::InvalidRequest("messages body requires a model".to_string()))?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let provider_model = deployment.litellm_params.model.as_str(); let upstream_model = provider_model .split_once('/') @@ -37,7 +37,7 @@ pub async fn run( }; let mut body = body; body.as_object_mut() - .ok_or_else(|| CoreError::InvalidRequest("messages body must be an object".to_string()))? + .ok_or_else(|| Error::InvalidRequest("messages body must be an object".to_string()))? .insert( "model".to_string(), Value::String(upstream_model.to_string()), @@ -60,6 +60,6 @@ pub async fn run( serde_json::to_value(response) .map(MessagesResponse::Json) .map_err(|err| { - CoreError::InvalidResponse(format!("failed to serialize messages response: {err}")) + Error::InvalidResponse(format!("failed to serialize messages response: {err}")) }) } diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs index 4ae8cfe7379..b8ee77c4269 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs @@ -11,8 +11,7 @@ use std::time::Duration; use crate::io::realtime_pool::{RealtimePool, upstream_key}; use futures_util::{Sink, Stream}; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::realtime::types::RealtimeEvent; use litellm_core::router::Router; @@ -29,15 +28,15 @@ pub async fn run( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, >::Error: std::fmt::Display, { - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let params = &deployment.litellm_params; // Strip a leading `openai/` so the OpenAI-only realtime fn gets the bare model. let provider_model = params diff --git a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs b/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs index 165c95695d3..e8f840c0c8e 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs @@ -2,13 +2,13 @@ use std::sync::Arc; use std::time::Duration; use futures_util::{Sink, Stream}; +use litellm_core::Error; use litellm_core::call_lifecycle::{CallLifecycle, CallLifecycleContext}; use litellm_core::responses::instrumentation::{ ResponsesWsCallbackPayload, ResponsesWsInstrumentation, ResponsesWsLogOutcome, ResponsesWsMetadata, }; use litellm_core::responses::types::ResponsesWsEvent; -use litellm_core::{CoreError, CoreResult}; use crate::integrations::custom_logger::{ CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, ModelCallDetails, @@ -26,22 +26,22 @@ pub async fn run( metadata: RequestMetadata, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, Out::Error: std::fmt::Display, { - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let params = &deployment.litellm_params; let provider_model = params .model .strip_prefix("openai/") .unwrap_or(¶ms.model); if params.model.contains('/') && !params.model.starts_with("openai/") { - return Err(CoreError::InvalidProvider( + return Err(Error::InvalidProvider( "Responses WebSocket route supports OpenAI deployments only".to_string(), )); } diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index ab8050734f2..389dbd49505 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -11,6 +11,7 @@ reqwest.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true +tracing.workspace = true sha2.workspace = true aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true } diff --git a/litellm-rust/crates/core/src/audio_transcription/client.rs b/litellm-rust/crates/core/src/audio_transcription/client.rs new file mode 100644 index 00000000000..0e612628dc6 --- /dev/null +++ b/litellm-rust/crates/core/src/audio_transcription/client.rs @@ -0,0 +1,14 @@ +use std::sync::OnceLock; +use std::time::Duration; + +use crate::constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS; + +pub(super) fn http_client() -> &'static reqwest::Client { + static CLIENT: OnceLock = OnceLock::new(); + CLIENT.get_or_init(|| { + reqwest::Client::builder() + .timeout(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()) + }) +} diff --git a/litellm-rust/crates/core/src/audio_transcription/handler.rs b/litellm-rust/crates/core/src/audio_transcription/handler.rs new file mode 100644 index 00000000000..9a96b9d1140 --- /dev/null +++ b/litellm-rust/crates/core/src/audio_transcription/handler.rs @@ -0,0 +1,91 @@ +use serde_json::Value; + +use crate::error::Error; +use crate::http_utils::{http_request, truncate_error_body}; + +use super::client::http_client; +use super::types::ProviderAudioTranscriptionRequest; + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub async fn execute_audio_transcription_provider_call( + request: ProviderAudioTranscriptionRequest, +) -> Result { + let body = serde_json::to_vec(&request.body) + .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?; + let headers = signed_headers(&request, &body).await?; + let mut request_builder = http_client().post(&request.url).body(body); + for (key, value) in headers { + request_builder = request_builder.header(key, value); + } + if let Some(duration) = request.timeout { + request_builder = request_builder.timeout(duration); + } + let response = http_request(request_builder) + .await + .map_err(|error| Error::Network(error.to_string()))?; + let status = response.status(); + let text = response + .text() + .await + .map_err(|error| Error::Network(error.to_string()))?; + if !status.is_success() { + return Err(Error::Http { + status: status.as_u16(), + body: truncate_error_body(&text), + }); + } + let response_json = serde_json::from_str(&text) + .map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?; + Ok(request + .config + .transform_transcription_response(&request.model, response_json)? + .into_json()) +} + +#[cfg(feature = "bedrock-auth")] +async fn signed_headers( + request: &ProviderAudioTranscriptionRequest, + body: &[u8], +) -> Result, Error> { + use std::collections::BTreeMap; + use std::time::SystemTime; + + use crate::audio_transcription::transformation::AudioTranscriptionAuth; + use crate::providers::bedrock::audio_transcription::aws_auth_config; + use crate::providers::bedrock::aws_base::{resolve_credentials, sign_bedrock_post}; + + let AudioTranscriptionAuth::AwsSigV4 { region, .. } = &request.auth else { + return Ok(request.upstream_headers.clone()); + }; + let env_lookup = |key: &str| std::env::var(key).ok(); + let credentials = resolve_credentials( + aws_auth_config(&request.optional_params, &env_lookup), + &env_lookup, + ) + .await?; + let unsigned: BTreeMap = request.upstream_headers.iter().cloned().collect(); + let signature = sign_bedrock_post( + &request.url, + body, + &unsigned, + region, + &credentials, + SystemTime::now(), + )?; + Ok(unsigned.into_iter().chain(signature).collect()) +} + +#[cfg(not(feature = "bedrock-auth"))] +async fn signed_headers( + request: &ProviderAudioTranscriptionRequest, + _body: &[u8], +) -> Result, Error> { + use crate::audio_transcription::transformation::AudioTranscriptionAuth; + + match request.auth { + AudioTranscriptionAuth::AwsSigV4 { .. } => Err(Error::Unsupported( + "AWS SigV4 requires the bedrock-auth feature", + )), + AudioTranscriptionAuth::Bearer => Ok(request.upstream_headers.clone()), + } +} diff --git a/litellm-rust/crates/core/src/audio_transcription/mod.rs b/litellm-rust/crates/core/src/audio_transcription/mod.rs index ec2fbb969a6..31b6de4b3e4 100644 --- a/litellm-rust/crates/core/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/core/src/audio_transcription/mod.rs @@ -1,2 +1,21 @@ +use crate::Error; +mod client; +mod handler; +mod prepare; pub mod transformation; pub mod types; + +use serde_json::Value; + +pub use handler::execute_audio_transcription_provider_call; +pub use prepare::prepare_audio_transcription_provider_call; +pub use types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { + execute_audio_transcription_provider_call(prepare_audio_transcription_provider_call(request)?) + .await +} + +#[cfg(test)] +mod tests; diff --git a/litellm-rust/crates/core/src/audio_transcription/prepare.rs b/litellm-rust/crates/core/src/audio_transcription/prepare.rs new file mode 100644 index 00000000000..bbef97341a9 --- /dev/null +++ b/litellm-rust/crates/core/src/audio_transcription/prepare.rs @@ -0,0 +1,74 @@ +use crate::error::Error; +use crate::http_utils::{has_header, string_headers}; +#[cfg(feature = "bedrock-auth")] +use crate::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG; +use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; + +use super::transformation::{AudioTranscriptionAuth, AudioTranscriptionProviderConfig}; +use super::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +fn provider_config(provider: &str) -> Option<&'static dyn AudioTranscriptionProviderConfig> { + #[cfg(feature = "bedrock-auth")] + if provider == "bedrock" { + return Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG); + } + let _ = provider; + None +} + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub fn prepare_audio_transcription_provider_call( + request: AudioTranscriptionRequest<'_>, +) -> Result { + let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider) + .or_else(|| { + request + .custom_llm_provider + .map(|provider| CustomLlmProvider { + model: request.model, + custom_llm_provider: provider, + }) + }) + .ok_or_else(|| { + Error::InvalidProvider( + "unable to resolve custom_llm_provider for audio transcription request".to_string(), + ) + })?; + let model = provider_info.model.to_string(); + let config = provider_config(provider_info.custom_llm_provider) + .ok_or_else(|| Error::InvalidProvider(provider_info.custom_llm_provider.to_string()))?; + let env_lookup = |key: &str| std::env::var(key).ok(); + let mut headers = string_headers("audio transcription", request.extra_headers)?; + let auth = config.auth_strategy(&model, &request.optional_params, &env_lookup)?; + if matches!(auth, AudioTranscriptionAuth::Bearer) + && !has_header(&headers, "authorization") + && let Some(api_key) = request.api_key + { + headers.push(("Authorization".to_string(), format!("Bearer {api_key}"))); + } + if !has_header(&headers, "content-type") { + headers.push(("Content-Type".to_string(), "application/json".to_string())); + } + let url = config.complete_url( + request.api_base, + &model, + &request.optional_params, + &env_lookup, + )?; + let filtered_params = config.map_transcription_params(&request.optional_params); + let transformed = + config.transform_transcription_request(&model, request.audio, filtered_params)?; + Ok(ProviderAudioTranscriptionRequest { + model, + custom_llm_provider: provider_info.custom_llm_provider.to_string(), + config, + url, + body: transformed.body, + upstream_headers: headers, + auth, + #[cfg(feature = "bedrock-auth")] + optional_params: request.optional_params, + timeout: request.timeout, + }) +} diff --git a/litellm-rust/crates/core/src/audio_transcription/tests.rs b/litellm-rust/crates/core/src/audio_transcription/tests.rs new file mode 100644 index 00000000000..263d63337b0 --- /dev/null +++ b/litellm-rust/crates/core/src/audio_transcription/tests.rs @@ -0,0 +1,50 @@ +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::thread; + +use serde_json::{Map, json}; + +use super::audio_transcription; +use super::types::AudioTranscriptionRequest; + +#[tokio::test] +async fn bedrock_request_is_signed_and_contains_audio() { + let listener = TcpListener::bind("127.0.0.1:0").expect("listener"); + let address = listener.local_addr().expect("address"); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("connection"); + let mut request = Vec::new(); + let mut buffer = [0_u8; 16_384]; + let count = stream.read(&mut buffer).expect("request"); + request.extend_from_slice(&buffer[..count]); + let request = String::from_utf8_lossy(&request); + assert!(request.contains("POST /model/mistral.voxtral-mini-3b-2507/converse")); + assert!(request.contains("authorization: AWS4-HMAC-SHA256")); + assert!(request.contains("x-amz-date:")); + assert!(request.contains("\"bytes\":\"AQI=\"")); + assert!(request.contains("Transcribe the audio. Respond with only the transcript.")); + let response = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 53\r\nConnection: close\r\n\r\n{\"output\":{\"message\":{\"content\":[{\"text\":\"hello\"}]}}}"; + stream.write_all(response).expect("response"); + }); + + let optional_params = Map::from_iter([ + ("aws_access_key_id".to_string(), json!("access-key")), + ("aws_secret_access_key".to_string(), json!("secret-key")), + ("aws_region_name".to_string(), json!("us-east-1")), + ]); + let api_base = format!("http://{address}"); + let response = audio_transcription(AudioTranscriptionRequest { + model: "mistral.voxtral-mini-3b-2507", + audio: json!({"data": "AQI=", "format": "wav", "filename": "audio.wav"}), + api_key: None, + api_base: Some(&api_base), + custom_llm_provider: Some("bedrock"), + extra_headers: None, + optional_params, + timeout: None, + }) + .await + .expect("transcription"); + assert_eq!(response, json!({"text": "hello"})); + server.join().expect("server"); +} diff --git a/litellm-rust/crates/core/src/audio_transcription/transformation.rs b/litellm-rust/crates/core/src/audio_transcription/transformation.rs index eab34c13843..aa9846427dc 100644 --- a/litellm-rust/crates/core/src/audio_transcription/transformation.rs +++ b/litellm-rust/crates/core/src/audio_transcription/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::CoreResult; - use super::types::{AudioTranscriptionRequestData, AudioTranscriptionResponseData}; #[derive(Clone, Debug, PartialEq, Eq)] @@ -16,6 +15,7 @@ pub enum AudioTranscriptionAuth { pub trait AudioTranscriptionProviderConfig: Sync { fn supported_transcription_params(&self) -> &'static [&'static str]; + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn map_transcription_params(&self, params: &Map) -> Map { params .iter() @@ -32,13 +32,13 @@ pub trait AudioTranscriptionProviderConfig: Sync { model: &str, audio: Value, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_transcription_response( &self, model: &str, response_json: Value, - ) -> CoreResult; + ) -> Result; fn complete_url( &self, @@ -46,12 +46,12 @@ pub trait AudioTranscriptionProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth_strategy( &self, model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; } diff --git a/litellm-rust/crates/core/src/audio_transcription/types.rs b/litellm-rust/crates/core/src/audio_transcription/types.rs index 3a9e1ecd88c..559d7837027 100644 --- a/litellm-rust/crates/core/src/audio_transcription/types.rs +++ b/litellm-rust/crates/core/src/audio_transcription/types.rs @@ -1,5 +1,56 @@ +use std::time::Duration; + use serde::{Deserialize, Serialize}; -use serde_json::Value; +use serde_json::{Map, Value}; + +use super::transformation::{AudioTranscriptionAuth, AudioTranscriptionProviderConfig}; + +pub struct AudioTranscriptionRequest<'a> { + pub model: &'a str, + pub audio: Value, + pub api_key: Option<&'a str>, + pub api_base: Option<&'a str>, + pub custom_llm_provider: Option<&'a str>, + pub extra_headers: Option>, + pub optional_params: Map, + pub timeout: Option, +} + +#[derive(Clone)] +pub struct ProviderAudioTranscriptionRequest { + pub(super) model: String, + pub(super) custom_llm_provider: String, + pub(super) config: &'static dyn AudioTranscriptionProviderConfig, + pub(super) url: String, + pub(super) body: Value, + pub(super) upstream_headers: Vec<(String, String)>, + pub(super) auth: AudioTranscriptionAuth, + #[cfg(feature = "bedrock-auth")] + pub(super) optional_params: Map, + pub(super) timeout: Option, +} + +impl ProviderAudioTranscriptionRequest { + pub fn model(&self) -> &str { + &self.model + } + + pub fn custom_llm_provider(&self) -> &str { + &self.custom_llm_provider + } + + pub fn url(&self) -> &str { + &self.url + } + + pub fn body(&self) -> &Value { + &self.body + } + + pub fn with_body(self, body: Value) -> Self { + Self { body, ..self } + } +} #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct AudioTranscriptionRequestData { diff --git a/litellm-rust/crates/core/src/call_lifecycle/mod.rs b/litellm-rust/crates/core/src/call_lifecycle/mod.rs index d9b68a1b726..637c156e192 100644 --- a/litellm-rust/crates/core/src/call_lifecycle/mod.rs +++ b/litellm-rust/crates/core/src/call_lifecycle/mod.rs @@ -1,7 +1,7 @@ use std::future::Future; use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use crate::{CoreError, CoreResult}; +use crate::Error; pub mod types; @@ -11,14 +11,14 @@ pub use types::{ }; pub trait CallLifecycleHooks: Send + Sync { - type PreCallFuture<'a>: Future> + Send + 'a + type PreCallFuture<'a>: Future> + Send + 'a where Self: 'a, InitialReq: 'a, ProviderReq: 'a, Resp: 'a; - type DuringCallFuture<'a>: Future> + Send + 'a + type DuringCallFuture<'a>: Future> + Send + 'a where Self: 'a, InitialReq: 'a, @@ -56,7 +56,7 @@ pub trait CallLifecycleHooks: Send + Sync { fn async_log_failure_event<'a>( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a>; } @@ -86,12 +86,12 @@ impl<'a> CallLifecycle<'a> { request: InitialReq, hooks: &Hooks, provider_call: ProviderCall, - ) -> CoreResult + ) -> Result where InitialReq: CallLifecycleRequest, Hooks: CallLifecycleHooks, ProviderCall: FnOnce(ProviderReq) -> ProviderFuture, - ProviderFuture: Future>, + ProviderFuture: Future>, { let context = request.lifecycle_context(); self.run(context, request, hooks, provider_call).await @@ -103,11 +103,11 @@ impl<'a> CallLifecycle<'a> { request: InitialReq, hooks: &Hooks, provider_call: ProviderCall, - ) -> CoreResult + ) -> Result where Hooks: CallLifecycleHooks, ProviderCall: FnOnce(ProviderReq) -> ProviderFuture, - ProviderFuture: Future>, + ProviderFuture: Future>, { let call_start = epoch_seconds(); let mut phases = Vec::new(); @@ -166,7 +166,7 @@ impl<'a> CallLifecycle<'a> { &self, context: &CallLifecycleContext, hooks: &Hooks, - error: &CoreError, + error: &Error, call_start: f64, phases: &mut Vec, ) where @@ -251,8 +251,8 @@ mod tests { } impl CallLifecycleHooks for RecordingHooks { - type PreCallFuture<'a> = BoxFuture<'a, CoreResult>; - type DuringCallFuture<'a> = BoxFuture<'a, CoreResult>; + type PreCallFuture<'a> = BoxFuture<'a, Result>; + type DuringCallFuture<'a> = BoxFuture<'a, Result>; type SuccessFuture<'a> = BoxFuture<'a, ()>; type FailureFuture<'a> = BoxFuture<'a, ()>; @@ -294,7 +294,7 @@ mod tests { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -304,8 +304,8 @@ mod tests { } impl CallLifecycleHooks for RecordingHooks { - type PreCallFuture<'a> = BoxFuture<'a, CoreResult>; - type DuringCallFuture<'a> = BoxFuture<'a, CoreResult>; + type PreCallFuture<'a> = BoxFuture<'a, Result>; + type DuringCallFuture<'a> = BoxFuture<'a, Result>; type SuccessFuture<'a> = BoxFuture<'a, ()>; type FailureFuture<'a> = BoxFuture<'a, ()>; @@ -345,7 +345,7 @@ mod tests { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -383,13 +383,13 @@ mod tests { "request".to_string(), &hooks, |_request| async move { - Err::(CoreError::Network("provider down".to_string())) + Err::(Error::Network("provider down".to_string())) }, ) .await .expect_err("call fails"); - assert_eq!(error, CoreError::Network("provider down".to_string())); + assert_eq!(error, Error::Network("provider down".to_string())); assert_eq!(hooks.events(), vec!["pre_call", "during_call", "failure"]); } diff --git a/litellm-rust/crates/core/src/chat_completions/common_utils.rs b/litellm-rust/crates/core/src/chat_completions/common_utils.rs index 36eaf242a5a..69e5f175ad5 100644 --- a/litellm-rust/crates/core/src/chat_completions/common_utils.rs +++ b/litellm-rust/crates/core/src/chat_completions/common_utils.rs @@ -1,13 +1,13 @@ -use serde_json::{Map, Value}; - -use crate::error::CoreResult; +use crate::Error; use crate::http_utils::string_headers as shared_string_headers; use crate::providers::anthropic::chat_completions::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG; +use serde_json::{Map, Value}; use super::transformation::ChatCompletionsProviderConfig; const HEADER_CONTEXT: &str = "chat completions"; +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) fn chat_completions_provider_config( provider: &str, ) -> Option<&'static dyn ChatCompletionsProviderConfig> { @@ -23,6 +23,6 @@ pub(super) fn chat_completions_provider_config( pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { shared_string_headers(HEADER_CONTEXT, extra_headers) } diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index afc4529fd26..96d001e2892 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,19 +1,23 @@ use serde_json::Value; -use crate::error::{CoreError, CoreResult}; -use crate::http_utils::truncate_error_body; +use crate::error::Error; +use crate::http_utils::{http_request, truncate_error_body}; use super::client::http_client; +use super::prepare::prepare_provider_request; use super::transformation::ChatCompletionsAuth; use super::types::{ ChatCompletionsResponse, ProviderChatCompletionsRequest, ProviderChatResponseData, + ResolvedChatCompletionsRequest, }; +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) async fn execute_chat_completions_provider_call( - request: ProviderChatCompletionsRequest, -) -> CoreResult { + request: ResolvedChatCompletionsRequest<'_>, +) -> Result { + let request = prepare_provider_request(request)?; let body = serde_json::to_vec(&request.body).map_err(|err| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "failed to serialize chat completions request: {err}" )) })?; @@ -27,14 +31,14 @@ pub(super) async fn execute_chat_completions_provider_call( request_builder = request_builder.timeout(duration); } - let response = request_builder.send().await.map_err(|err| { + let response = http_request(request_builder).await.map_err(|err| { // Failing to establish the connection means the request never went out, // so the host can still serve it. Everything else here, a timeout // above all, may have reached the provider and been answered. if err.is_connect() || err.is_builder() { - CoreError::Connect(err.to_string()) + Error::Connect(err.to_string()) } else { - CoreError::Network(err.to_string()) + Error::Network(err.to_string()) } })?; @@ -42,17 +46,17 @@ pub(super) async fn execute_chat_completions_provider_call( let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let body: Value = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid chat completions response JSON: {err}")) + Error::InvalidResponse(format!("invalid chat completions response JSON: {err}")) })?; request .config @@ -69,10 +73,10 @@ pub(super) async fn execute_chat_completions_provider_call( /// second kind has already been billed, and a host that keeps a reference /// implementation must not retry those, so collapse them to one variant that /// can only mean the provider was already called. -pub(super) fn as_response_error(err: CoreError) -> CoreError { +pub(super) fn as_response_error(err: Error) -> Error { match err { - already @ (CoreError::InvalidResponse(_) | CoreError::Http { .. }) => already, - other => CoreError::InvalidResponse(other.to_string()), + already @ (Error::InvalidResponse(_) | Error::Http { .. }) => already, + other => Error::InvalidResponse(other.to_string()), } } @@ -80,7 +84,7 @@ pub(super) fn as_response_error(err: CoreError) -> CoreError { pub(super) async fn signed_headers( request: &ProviderChatCompletionsRequest, body: &[u8], -) -> CoreResult> { +) -> Result, Error> { use std::collections::BTreeMap; use std::time::SystemTime; @@ -101,7 +105,7 @@ pub(super) async fn signed_headers( .iter() .any(|(name, _)| is_sigv4_computed_header(name)) { - return Err(CoreError::Unsupported( + return Err(Error::Unsupported( "request forwards a header AWS SigV4 computes", )); } @@ -137,9 +141,9 @@ pub(super) async fn signed_headers( pub(super) async fn signed_headers( request: &ProviderChatCompletionsRequest, _body: &[u8], -) -> CoreResult> { +) -> Result, Error> { match &request.auth { - ChatCompletionsAuth::AwsSigV4 { .. } => Err(CoreError::Unsupported( + ChatCompletionsAuth::AwsSigV4 { .. } => Err(Error::Unsupported( "AWS SigV4 requires the bedrock-auth feature", )), _ => Ok(request.upstream_headers.clone()), diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index f30ac1a24bf..32dea17d202 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -6,6 +6,7 @@ //! credentials, and it resolves the provider, translates the conversation, //! calls the provider, and returns a typed OpenAI-shaped response. +use crate::Error; mod client; mod common_utils; pub mod conversation; @@ -17,16 +18,15 @@ pub mod types; use serde_json::{Map, Value}; -use crate::error::CoreResult; - use handler::execute_chat_completions_provider_call; -use prepare::{parse_messages, prepare_chat_completions_call, resolve_provider_config}; +use prepare::{parse_messages, resolve_provider_config, resolve_request}; use types::{ChatCompletionsRequest, ChatCompletionsResponse}; +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn chat_completions( request: ChatCompletionsRequest<'_>, -) -> CoreResult { - execute_chat_completions_provider_call(prepare_chat_completions_call(request)?).await +) -> Result { + execute_chat_completions_provider_call(resolve_request(request)?).await } /// Whether the core would accept this request, without resolving credentials or diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index 1e1c8d1bafd..3be2ba21de4 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -1,17 +1,20 @@ use serde_json::Value; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::http_utils::has_header; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{chat_completions_provider_config, string_headers}; use super::transformation::{ChatCompletionsAuth, ChatCompletionsProviderConfig}; -use super::types::{ChatCompletionsRequest, ChatMessage, ProviderChatCompletionsRequest}; +use super::types::{ + ChatCompletionsRequest, ChatMessage, ProviderChatCompletionsRequest, + ResolvedChatCompletionsRequest, +}; pub(super) fn resolve_provider_config<'a>( model: &'a str, custom_llm_provider: Option<&'a str>, -) -> CoreResult<(String, &'static dyn ChatCompletionsProviderConfig)> { +) -> Result<(String, &'static dyn ChatCompletionsProviderConfig), Error> { let provider_info = get_custom_llm_provider(model, custom_llm_provider) .or_else(|| { custom_llm_provider.map(|provider| CustomLlmProvider { @@ -20,41 +23,56 @@ pub(super) fn resolve_provider_config<'a>( }) }) .ok_or_else(|| { - CoreError::InvalidProvider( + Error::InvalidProvider( "unable to resolve custom_llm_provider for chat completions request".to_string(), ) })?; let config = chat_completions_provider_config(provider_info.custom_llm_provider) - .ok_or_else(|| CoreError::InvalidProvider(provider_info.custom_llm_provider.to_string()))?; + .ok_or_else(|| Error::InvalidProvider(provider_info.custom_llm_provider.to_string()))?; Ok((provider_info.model.to_string(), config)) } -pub(super) fn parse_messages(messages: Value) -> CoreResult> { - serde_json::from_value(messages).map_err(|err| { - CoreError::InvalidRequest(format!("invalid chat completions messages: {err}")) - }) +pub(super) fn parse_messages(messages: Value) -> Result, Error> { + serde_json::from_value(messages) + .map_err(|err| Error::InvalidRequest(format!("invalid chat completions messages: {err}"))) } -pub(super) fn prepare_chat_completions_call( +pub(super) fn resolve_request( request: ChatCompletionsRequest<'_>, -) -> CoreResult { +) -> Result, Error> { let (model, config) = resolve_provider_config(request.model, request.custom_llm_provider)?; - let env_lookup = |key: &str| std::env::var(key).ok(); - let messages = parse_messages(request.messages)?; if messages.is_empty() { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "chat completions requires at least one message".to_string(), )); } if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) { - return Err(CoreError::Unsupported(reason.0)); + return Err(Error::Unsupported(reason.0)); } + Ok(ResolvedChatCompletionsRequest { + model, + config, + messages, + optional_params: request.optional_params, + api_key: request.api_key, + api_base: request.api_base, + extra_headers: request.extra_headers, + timeout: request.timeout, + }) +} - let mut headers = string_headers(request.extra_headers)?; +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +fn validate_environment( + request: &ResolvedChatCompletionsRequest<'_>, + model: &str, + config: &dyn ChatCompletionsProviderConfig, +) -> Result<(Vec<(String, String)>, ChatCompletionsAuth), Error> { + let env_lookup = |key: &str| std::env::var(key).ok(); + let mut headers = string_headers(request.extra_headers.clone())?; let auth = config.auth( request.api_key, - &model, + model, &request.optional_params, &env_lookup, )?; @@ -95,7 +113,16 @@ pub(super) fn prepare_chat_completions_call( headers.push(((*name).to_string(), (*value).to_string())); } } + Ok((headers, auth)) +} +pub(super) fn prepare_provider_request( + request: ResolvedChatCompletionsRequest<'_>, +) -> Result { + let (headers, auth) = validate_environment(&request, &request.model, request.config)?; + let model = request.model; + let config = request.config; + let env_lookup = |key: &str| std::env::var(key).ok(); let url = config.complete_url( request.api_base, &model, @@ -103,7 +130,7 @@ pub(super) fn prepare_chat_completions_call( &env_lookup, )?; let transformed = - config.transform_request(&model, messages, request.optional_params.clone())?; + config.transform_request(&model, request.messages, request.optional_params.clone())?; Ok(ProviderChatCompletionsRequest { model, diff --git a/litellm-rust/crates/core/src/chat_completions/tests.rs b/litellm-rust/crates/core/src/chat_completions/tests.rs index e2383723cb0..f8594dee447 100644 --- a/litellm-rust/crates/core/src/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/chat_completions/tests.rs @@ -1,10 +1,16 @@ use serde_json::{Map, Value, json}; -use crate::error::CoreError; +use crate::error::Error; -use super::prepare::prepare_chat_completions_call; +use super::prepare::{prepare_provider_request, resolve_request}; use super::transformation::ChatCompletionsAuth; -use super::types::ChatCompletionsRequest; +use super::types::{ChatCompletionsRequest, ProviderChatCompletionsRequest}; + +fn prepare_chat_completions_call( + request: ChatCompletionsRequest<'_>, +) -> Result { + prepare_provider_request(resolve_request(request)?) +} fn request<'a>( model: &'a str, @@ -29,7 +35,7 @@ fn request<'a>( /// `ProviderChatCompletionsRequest` deliberately has no `Debug` (its headers /// carry resolved credentials), so unwrap the failure case by hand. -fn decline(request: ChatCompletionsRequest<'_>) -> CoreError { +fn decline(request: ChatCompletionsRequest<'_>) -> Error { match prepare_chat_completions_call(request) { Err(error) => error, Ok(prepared) => panic!("expected a decline, prepared a call to {}", prepared.url), @@ -196,7 +202,7 @@ fn declines_an_unsupported_request_before_resolving_credentials() { call.api_key = None; // No api_key is set and no env is consulted: the gate must run first, so the // error is the decline rather than a missing-credential error. - assert_eq!(decline(call), CoreError::Unsupported("streaming")); + assert_eq!(decline(call), Error::Unsupported("streaming")); } #[test] @@ -208,7 +214,7 @@ fn rejects_an_unknown_provider() { json!([{"role": "user", "content": "hi"}]), json!({}), )), - CoreError::InvalidProvider("openai".to_string()) + Error::InvalidProvider("openai".to_string()) ); } @@ -221,7 +227,7 @@ fn rejects_a_model_with_no_resolvable_provider() { json!([{"role": "user", "content": "hi"}]), json!({}), )), - CoreError::InvalidProvider(_) + Error::InvalidProvider(_) )); } @@ -234,7 +240,7 @@ fn rejects_an_empty_or_malformed_message_list() { json!([]), json!({}), )), - CoreError::InvalidRequest("chat completions requires at least one message".to_string()) + Error::InvalidRequest("chat completions requires at least one message".to_string()) ); assert!(matches!( decline(request( @@ -243,7 +249,7 @@ fn rejects_an_empty_or_malformed_message_list() { json!("not a list"), json!({}), )), - CoreError::InvalidRequest(_) + Error::InvalidRequest(_) )); } @@ -258,7 +264,7 @@ fn rejects_non_string_extra_headers() { call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))])); assert_eq!( decline(call), - CoreError::InvalidRequest( + Error::InvalidRequest( "chat completions extra_headers.x-trace must be a string, got number".to_string() ) ); @@ -374,7 +380,7 @@ async fn a_forwarded_header_the_signer_computes_declines_to_python() { .await .expect_err("{forwarded} should decline instead of being signed"); assert!( - matches!(error, CoreError::Unsupported(_)), + matches!(error, Error::Unsupported(_)), "{forwarded} declined as {error:?}, which the host would not fall back on" ); } @@ -727,7 +733,7 @@ mod round_trip { .expect_err("response cannot be normalized"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::InvalidResponse(_)), + matches!(err, Error::InvalidResponse(_)), "expected a post-send error, got {err:?}" ); } @@ -745,7 +751,7 @@ mod round_trip { .expect_err("response cannot be normalized"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::InvalidResponse(_)), + matches!(err, Error::InvalidResponse(_)), "expected a post-send error, got {err:?}" ); } @@ -763,7 +769,7 @@ mod round_trip { .expect_err("upstream rejects"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::Http { status: 429, .. }), + matches!(err, Error::Http { status: 429, .. }), "expected a 429, got {err:?}" ); } @@ -787,7 +793,7 @@ mod round_trip { .await .expect_err("nothing is listening"); assert!( - matches!(err, CoreError::Connect(_)), + matches!(err, Error::Connect(_)), "expected a pre-send connect failure, got {err:?}" ); } @@ -797,24 +803,24 @@ mod round_trip { use crate::chat_completions::handler::as_response_error; for original in [ - CoreError::MissingField("usage"), - CoreError::Unsupported("non-text response content block"), - CoreError::InvalidRequest("whatever".to_string()), - CoreError::Auth("whatever".to_string()), + Error::MissingField("usage"), + Error::Unsupported("non-text response content block"), + Error::InvalidRequest("whatever".to_string()), + Error::Auth("whatever".to_string()), ] { let label = format!("{original:?}"); assert!( - matches!(as_response_error(original), CoreError::InvalidResponse(_)), + matches!(as_response_error(original), Error::InvalidResponse(_)), "{label} must not stay retryable once the provider has answered" ); } // An upstream status is already unambiguous, so it survives intact. assert!(matches!( - as_response_error(CoreError::Http { + as_response_error(Error::Http { status: 500, body: "boom".to_string() }), - CoreError::Http { status: 500, .. } + Error::Http { status: 500, .. } )); } } diff --git a/litellm-rust/crates/core/src/chat_completions/transformation.rs b/litellm-rust/crates/core/src/chat_completions/transformation.rs index a30ce9dc77c..d7b9704c46c 100644 --- a/litellm-rust/crates/core/src/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/chat_completions/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::error::CoreResult; - use super::types::{ ChatCompletionsResponse, ChatMessage, ChatMessageContent, ProviderChatRequestData, ProviderChatResponseData, @@ -39,7 +38,7 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth( &self, @@ -47,7 +46,7 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn default_headers(&self) -> &'static [(&'static str, &'static str)] { &[("content-type", "application/json")] @@ -63,9 +62,8 @@ pub trait ChatCompletionsProviderConfig: Sync { false } - /// Provider parameter names (post-mapping) the Rust path knows how to place - /// in the upstream body. Anything outside this set declines the request. - fn supported_params(&self) -> &'static [&'static str]; + /// Supported OpenAI parameter names paired with their provider names. + fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)]; /// Parameters consumed as call configuration (credentials, endpoints) /// rather than placed in the body. Accepted, never serialized. @@ -79,7 +77,7 @@ pub trait ChatCompletionsProviderConfig: Sync { optional_params: &Map, ) -> Option { unsupported_param( - self.supported_params(), + self.supported_openai_params(), self.config_params(), optional_params, ) @@ -91,17 +89,17 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_response( &self, model: &str, response: ProviderChatResponseData, - ) -> CoreResult; + ) -> Result; } pub fn unsupported_param( - supported: &'static [&'static str], + supported: &'static [(&'static str, &'static str)], config: &'static [&'static str], optional_params: &Map, ) -> Option { @@ -116,7 +114,9 @@ pub fn unsupported_param( .keys() .any(|key| { key != STREAM_PARAM - && !supported.contains(&key.as_str()) + && !supported + .iter() + .any(|(_, provider_name)| *provider_name == key) && !config.contains(&key.as_str()) }) .then_some(Unsupported("unrecognized request parameter")) diff --git a/litellm-rust/crates/core/src/chat_completions/types.rs b/litellm-rust/crates/core/src/chat_completions/types.rs index 35dd543a986..3238d09b6b5 100644 --- a/litellm-rust/crates/core/src/chat_completions/types.rs +++ b/litellm-rust/crates/core/src/chat_completions/types.rs @@ -22,6 +22,17 @@ pub struct ChatCompletionsRequest<'a> { pub timeout: Option, } +pub(super) struct ResolvedChatCompletionsRequest<'a> { + pub(super) model: String, + pub(super) config: &'static dyn ChatCompletionsProviderConfig, + pub(super) messages: Vec, + pub(super) optional_params: Map, + pub(super) api_key: Option<&'a str>, + pub(super) api_base: Option<&'a str>, + pub(super) extra_headers: Option>, + pub(super) timeout: Option, +} + pub(super) struct ProviderChatCompletionsRequest { pub(super) model: String, pub(super) config: &'static dyn ChatCompletionsProviderConfig, diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index e1ac0a4fc8f..a73961060eb 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -30,6 +30,8 @@ pub(crate) const CHAT_COMPLETIONS_TIMEOUT_SECS: u64 = 600; /// Connect timeout for chat completions provider calls, in seconds. pub(crate) const CHAT_COMPLETIONS_CONNECT_TIMEOUT_SECS: u64 = 10; +pub(crate) const AUDIO_TRANSCRIPTION_TIMEOUT_SECS: u64 = 600; + /// `object` field every non-streaming chat completion response carries. pub const CHAT_COMPLETION_OBJECT: &str = "chat.completion"; diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index 739532f8cb5..db3fa2ec704 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -1,9 +1,7 @@ -use thiserror::Error; +use thiserror::Error as ThisError; -pub type CoreResult = Result; - -#[derive(Debug, Error, PartialEq, Eq)] -pub enum CoreError { +#[derive(Debug, ThisError, PartialEq, Eq)] +pub enum Error { #[error("expected {expected}, got {actual}")] InvalidType { expected: &'static str, diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index c541f50275b..3633130528d 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -3,7 +3,14 @@ use serde_json::{Map, Value}; use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub async fn http_request( + request: reqwest::RequestBuilder, +) -> Result { + request.send().await +} /// Bound an upstream error body before it crosses a host boundary, so provider /// bodies stay data-minimized. @@ -18,7 +25,7 @@ pub fn truncate_error_body(body: &str) -> String { pub fn string_headers( context: &'static str, extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { extra_headers .unwrap_or_default() .into_iter() @@ -27,7 +34,7 @@ pub fn string_headers( .as_str() .map(|value| (key.clone(), value.to_string())) .ok_or_else(|| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "{context} extra_headers.{key} must be a string, got {}", json_type_name(&value) )) @@ -81,7 +88,7 @@ mod tests { let err = string_headers("chat completions", Some(headers)).expect_err("non-string value"); assert_eq!( err, - CoreError::InvalidRequest( + Error::InvalidRequest( "chat completions extra_headers.x-trace must be a string, got number".to_string() ) ); diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index dce4a425ea0..0e18d24e5d8 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -13,4 +13,4 @@ pub mod responses; pub mod router; pub mod routing_utils; -pub use error::{CoreError, CoreResult}; +pub use error::Error; diff --git a/litellm-rust/crates/core/src/messages/common_utils.rs b/litellm-rust/crates/core/src/messages/common_utils.rs index a14dffbc1fe..8f0f6652fa4 100644 --- a/litellm-rust/crates/core/src/messages/common_utils.rs +++ b/litellm-rust/crates/core/src/messages/common_utils.rs @@ -1,9 +1,8 @@ -use serde_json::{Map, Value}; - -use crate::error::CoreResult; +use crate::Error; use crate::http_utils::string_headers as shared_string_headers; use crate::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG; use crate::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG; +use serde_json::{Map, Value}; use super::transformation::AnthropicMessagesProviderConfig; @@ -11,6 +10,7 @@ pub(super) use crate::http_utils::{has_bearer_auth, has_header, truncate_error_b const HEADER_CONTEXT: &str = "messages"; +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) fn messages_provider_config( provider: &str, ) -> Option<&'static dyn AnthropicMessagesProviderConfig> { @@ -23,6 +23,6 @@ pub(super) fn messages_provider_config( pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { shared_string_headers(HEADER_CONTEXT, extra_headers) } diff --git a/litellm-rust/crates/core/src/messages/handler.rs b/litellm-rust/crates/core/src/messages/handler.rs index 1c895f66eba..61ff81bcdc8 100644 --- a/litellm-rust/crates/core/src/messages/handler.rs +++ b/litellm-rust/crates/core/src/messages/handler.rs @@ -1,13 +1,17 @@ use crate::constants::ANTHROPIC_MESSAGES_PROVIDER; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; +use crate::http_utils::http_request; use super::client::http_client; use super::common_utils::truncate_error_body; -use super::types::{AnthropicMessagesResponse, ProviderMessagesRequest}; +use super::prepare::prepare_provider_request; +use super::types::{AnthropicMessagesResponse, MessagesRequest}; +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) async fn execute_messages_provider_call( - request: ProviderMessagesRequest, -) -> CoreResult { + request: MessagesRequest<'_>, +) -> Result { + let request = prepare_provider_request(request)?; let mut request_builder = http_client().post(&request.url).json(&request.body); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -16,35 +20,34 @@ pub(super) async fn execute_messages_provider_call( request_builder = request_builder.timeout(duration); } - let response = request_builder - .send() + let response = http_request(request_builder) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } - let response = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid messages response JSON: {err}")) - })?; + let response = serde_json::from_str(&text) + .map_err(|err| Error::InvalidResponse(format!("invalid messages response JSON: {err}")))?; request.config.transform_response(&request.model, response) } pub(super) async fn execute_messages_provider_stream( - request: ProviderMessagesRequest, -) -> CoreResult { + request: MessagesRequest<'_>, +) -> Result { + let request = prepare_provider_request(request)?; if request.provider != ANTHROPIC_MESSAGES_PROVIDER { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "streaming messages is not supported for this provider".to_string(), )); } @@ -57,17 +60,16 @@ pub(super) async fn execute_messages_provider_stream( request_builder = request_builder.timeout(duration); } - let response = request_builder - .send() + let response = http_request(request_builder) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); if !status.is_success() { let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; - return Err(CoreError::Http { + .map_err(|err| Error::Network(err.to_string()))?; + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index acb36d89daf..cfa8bda1104 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -7,6 +7,7 @@ //! is the streaming variant; it hands the raw upstream response back so a host //! can splice the event stream to its own caller. +use crate::Error; mod client; mod common_utils; mod handler; @@ -14,18 +15,16 @@ mod prepare; pub mod transformation; pub mod types; -use crate::error::CoreResult; - use handler::{execute_messages_provider_call, execute_messages_provider_stream}; -use prepare::prepare_messages_call; use types::{AnthropicMessagesResponse, MessagesRequest}; -pub async fn messages(request: MessagesRequest<'_>) -> CoreResult { - execute_messages_provider_call(prepare_messages_call(request)?).await +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub async fn messages(request: MessagesRequest<'_>) -> Result { + execute_messages_provider_call(request).await } -pub async fn messages_stream(request: MessagesRequest<'_>) -> CoreResult { - execute_messages_provider_stream(prepare_messages_call(request)?).await +pub async fn messages_stream(request: MessagesRequest<'_>) -> Result { + execute_messages_provider_stream(request).await } #[cfg(test)] diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index 94b5b1eaed7..ec83d03f535 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,13 +1,14 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}; -use super::transformation::MessagesAuthStrategy; +use super::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; use super::types::{MessagesRequest, ProviderMessagesRequest}; +use serde_json::{Map, Value}; -pub(super) fn prepare_messages_call( +pub(super) fn prepare_provider_request( request: MessagesRequest<'_>, -) -> CoreResult { +) -> Result { let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider) .or_else(|| { request @@ -18,7 +19,7 @@ pub(super) fn prepare_messages_call( }) }) .ok_or_else(|| { - CoreError::InvalidProvider( + Error::InvalidProvider( "unable to resolve custom_llm_provider for messages request".to_string(), ) })?; @@ -26,16 +27,49 @@ pub(super) fn prepare_messages_call( let provider = provider_info.custom_llm_provider; let config = messages_provider_config(provider) - .ok_or_else(|| CoreError::InvalidProvider(provider.to_string()))?; + .ok_or_else(|| Error::InvalidProvider(provider.to_string()))?; let env_lookup = |key: &str| std::env::var(key).ok(); - let mut headers = string_headers(request.extra_headers)?; + let headers = + validate_environment(config, request.extra_headers, request.api_key, &env_lookup)?; + + let typed_request = serde_json::from_value(request.body).map_err(|err| { + Error::InvalidRequest(format!("invalid Anthropic messages request: {err}")) + })?; + let transformed = config.transform_request(typed_request)?; + let body = serde_json::to_value(transformed).map_err(|err| { + Error::InvalidRequest(format!( + "failed to serialize Anthropic messages request: {err}" + )) + })?; + + let url = config.complete_url(request.api_base, &model, &env_lookup)?; + + Ok(ProviderMessagesRequest { + provider: provider.to_string(), + model, + config, + url, + body, + upstream_headers: headers, + timeout: request.timeout, + }) +} + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +fn validate_environment( + config: &dyn AnthropicMessagesProviderConfig, + extra_headers: Option>, + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result, Error> { + let mut headers = string_headers(extra_headers)?; let auth_strategy = config.auth_strategy(); let already_authorized = has_header(&headers, auth_strategy.header_name()) || (config.accepts_bearer_auth() && has_bearer_auth(&headers)); if !already_authorized { - let api_key = config.resolve_api_key(request.api_key, &env_lookup)?; + let api_key = config.resolve_api_key(api_key, env_lookup)?; let auth_header = match auth_strategy { MessagesAuthStrategy::Bearer => { ("authorization".to_string(), format!("Bearer {api_key}")) @@ -51,24 +85,5 @@ pub(super) fn prepare_messages_call( } } - let url = config.complete_url(request.api_base, &model, &env_lookup)?; - let typed_request = serde_json::from_value(request.body).map_err(|err| { - CoreError::InvalidRequest(format!("invalid Anthropic messages request: {err}")) - })?; - let transformed = config.transform_request(typed_request)?; - let body = serde_json::to_value(transformed).map_err(|err| { - CoreError::InvalidRequest(format!( - "failed to serialize Anthropic messages request: {err}" - )) - })?; - - Ok(ProviderMessagesRequest { - provider: provider.to_string(), - model, - config, - url, - body, - upstream_headers: headers, - timeout: request.timeout, - }) + Ok(headers) } diff --git a/litellm-rust/crates/core/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs index 9fc1763683b..df9f7051011 100644 --- a/litellm-rust/crates/core/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -4,7 +4,7 @@ use serde_json::{Map, Value, json}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; -use crate::error::CoreError; +use crate::error::Error; use super::common_utils::{ has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body, @@ -77,7 +77,7 @@ fn truncate_error_body_caps_long_payloads() { fn string_headers_rejects_non_string_values() { let headers = json!({"x-count": 3}).as_object().unwrap().clone(); let err = string_headers(Some(headers)).expect_err("non-string header rejected"); - assert!(matches!(err, CoreError::InvalidRequest(_))); + assert!(matches!(err, Error::InvalidRequest(_))); } #[test] @@ -341,7 +341,7 @@ async fn messages_requires_auth_when_no_key_and_no_header() { .await .expect_err("missing auth errors"); - assert!(matches!(err, CoreError::Auth(_))); + assert!(matches!(err, Error::Auth(_))); } #[tokio::test] @@ -420,7 +420,7 @@ async fn messages_maps_provider_error_status_to_http_error() { .await .expect_err("provider error propagates"); - assert!(matches!(err, CoreError::Http { status: 401, .. })); + assert!(matches!(err, Error::Http { status: 401, .. })); } #[tokio::test] @@ -437,5 +437,5 @@ async fn messages_rejects_unsupported_provider() { .await .expect_err("unsupported provider errors"); - assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "openai")); + assert!(matches!(err, Error::InvalidProvider(provider) if provider == "openai")); } diff --git a/litellm-rust/crates/core/src/messages/transformation.rs b/litellm-rust/crates/core/src/messages/transformation.rs index b478e20d24b..a5904c085a0 100644 --- a/litellm-rust/crates/core/src/messages/transformation.rs +++ b/litellm-rust/crates/core/src/messages/transformation.rs @@ -1,6 +1,5 @@ -use crate::error::CoreResult; - use super::types::{AnthropicMessagesRequest, AnthropicMessagesResponse}; +use crate::Error; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MessagesAuthStrategy { @@ -23,13 +22,13 @@ pub trait AnthropicMessagesProviderConfig: Sync { api_base: Option<&str>, model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn resolve_api_key( &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth_strategy(&self) -> MessagesAuthStrategy { MessagesAuthStrategy::Header("x-api-key") @@ -46,18 +45,20 @@ pub trait AnthropicMessagesProviderConfig: Sync { ] } + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_request( &self, request: AnthropicMessagesRequest, - ) -> CoreResult { + ) -> Result { Ok(request) } + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_response( &self, _model: &str, response: AnthropicMessagesResponse, - ) -> CoreResult { + ) -> Result { Ok(response) } } diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs index cb3e735e533..ad484c8f968 100644 --- a/litellm-rust/crates/core/src/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::CoreResult; - use super::types::{OcrRequestData, OcrResponseData}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -28,6 +27,7 @@ pub enum OcrResponseHandling { pub trait OcrProviderConfig: Sync { fn supported_ocr_params(&self) -> &'static [&'static str]; + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn map_ocr_params(&self, non_default_params: &Map) -> Map { let mut mapped_params = Map::new(); for (param, value) in non_default_params { @@ -43,13 +43,13 @@ pub trait OcrProviderConfig: Sync { model: &str, document: Value, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_ocr_response( &self, model: &str, response_json: Value, - ) -> CoreResult; + ) -> Result; fn complete_url( &self, @@ -57,13 +57,32 @@ pub trait OcrProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn resolve_api_key( &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; + + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] + fn validate_environment( + &self, + headers: Vec<(String, String)>, + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result, Error> { + let strategy = self.auth_strategy(); + if crate::http_utils::has_header(&headers, strategy.header_name()) { + return Ok(headers); + } + let api_key = self.resolve_api_key(api_key, env_lookup)?; + let auth_header = match strategy { + OcrAuthStrategy::Bearer => ("Authorization".to_string(), format!("Bearer {api_key}")), + OcrAuthStrategy::Header(name) => (name.to_string(), api_key), + }; + Ok(std::iter::once(auth_header).chain(headers).collect()) + } fn auth_strategy(&self) -> OcrAuthStrategy { OcrAuthStrategy::Bearer diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs index 4534ac0182c..b22de6c47de 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::Error; use serde_json::json; fn messages(value: Value) -> Vec { @@ -19,7 +20,7 @@ fn transform(model: &str, msgs: Value, opts: Value) -> Value { .body } -fn transform_response(body: Value) -> CoreResult { +fn transform_response(body: Value) -> Result { ANTHROPIC_CHAT_COMPLETIONS_CONFIG .transform_response("claude-sonnet-4-5", ProviderChatResponseData { body }) } @@ -390,29 +391,26 @@ fn declines_a_response_carrying_a_non_text_block() { "usage": {"input_tokens": 1, "output_tokens": 1} })) .expect_err("non-text block"); - assert_eq!( - err, - CoreError::Unsupported("non-text response content block") - ); + assert_eq!(err, Error::Unsupported("non-text response content block")); } #[test] fn errors_on_a_response_missing_required_fields() { assert_eq!( transform_response(json!("nope")).expect_err("not an object"), - CoreError::InvalidResponse("messages response is not an object".to_string()) + Error::InvalidResponse("messages response is not an object".to_string()) ); assert_eq!( transform_response(json!({"model": "m", "usage": {}})).expect_err("no content"), - CoreError::MissingField("content") + Error::MissingField("content") ); assert_eq!( transform_response(json!({"model": "m", "content": []})).expect_err("no usage"), - CoreError::MissingField("usage") + Error::MissingField("usage") ); assert_eq!( transform_response(json!({"content": [], "usage": {}})).expect_err("no model"), - CoreError::MissingField("model") + Error::MissingField("model") ); } diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs index 3658642b539..a7d5a8ad0cf 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs @@ -10,7 +10,7 @@ use crate::chat_completions::types::{ ProviderChatRequestData, ProviderChatResponseData, }; use crate::constants::ANTHROPIC_OAUTH_TOKEN_PREFIX; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::providers::anthropic::messages::transformation::{ complete_anthropic_url, resolve_anthropic_api_key, }; @@ -27,7 +27,12 @@ use crate::chat_completions::response_utils::{finish_reason_for, unix_now, usage /// per-model gate inside `transform_request`, the function this route replaces. /// Forwarding it would send `top_k` to a model that removed sampling params and /// take a 400 after the call, where Python drops it and succeeds. -const SUPPORTED_PARAMS: &[&str] = &["max_tokens", "temperature", "top_p", "stop_sequences"]; +const SUPPORTED_PARAMS: &[(&str, &str)] = &[ + ("max_tokens", "max_tokens"), + ("temperature", "temperature"), + ("top_p", "top_p"), + ("stop", "stop_sequences"), +]; pub struct AnthropicChatCompletionsConfig; @@ -74,7 +79,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_anthropic_url(api_base, env_lookup)) } @@ -84,7 +89,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(ChatCompletionsAuth::Header { name: "x-api-key", value: resolve_anthropic_api_key(api_key, env_lookup)?, @@ -112,7 +117,8 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { }) } - fn supported_params(&self) -> &'static [&'static str] { + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] + fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)] { SUPPORTED_PARAMS } @@ -121,7 +127,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { messages: &[ChatMessage], optional_params: &Map, ) -> Option { - unsupported_param(SUPPORTED_PARAMS, &[], optional_params) + unsupported_param(self.supported_openai_params(), &[], optional_params) .or_else(|| messages.iter().find_map(unsupported_message)) // Anthropic rejects a request whose first turn is not a user turn. // Python only repairs that under `litellm.modify_params`, which the @@ -132,30 +138,33 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { }) } + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_request( &self, model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult { + ) -> Result { Ok(ProviderChatRequestData { body: anthropic_body(model, &build_conversation(&messages), optional_params), }) } + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_response( &self, _model: &str, response: ProviderChatResponseData, - ) -> CoreResult { - let body = response.body.as_object().ok_or_else(|| { - CoreError::InvalidResponse("messages response is not an object".into()) - })?; + ) -> Result { + let body = response + .body + .as_object() + .ok_or_else(|| Error::InvalidResponse("messages response is not an object".into()))?; let content = body .get("content") .and_then(Value::as_array) - .ok_or(CoreError::MissingField("content"))?; + .ok_or(Error::MissingField("content"))?; // The route declines tool and thinking requests, so a non-text block // means the response carries something this path never asked for. // Decline rather than silently dropping it; the host falls back. @@ -163,7 +172,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { .iter() .any(|block| block.get("type").and_then(Value::as_str) != Some("text")) { - return Err(CoreError::Unsupported("non-text response content block")); + return Err(Error::Unsupported("non-text response content block")); } let text: String = content .iter() @@ -173,7 +182,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { let usage = body .get("usage") .and_then(Value::as_object) - .ok_or(CoreError::MissingField("usage"))?; + .ok_or(Error::MissingField("usage"))?; let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0); Ok(ChatCompletionsResponse { @@ -181,7 +190,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { model: body .get("model") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("model"))? + .ok_or(Error::MissingField("model"))? .to_string(), choices: vec![ChatCompletionsChoice { index: 0, diff --git a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs index 829f2260d3c..f31b961e78a 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; @@ -17,12 +17,12 @@ pub fn non_empty(value: Option<&str>) -> Option<&str> { pub fn resolve_anthropic_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(ANTHROPIC_API_KEY_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY \ environment variable" .to_string(), @@ -47,12 +47,13 @@ pub fn complete_anthropic_url( } impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn complete_url( &self, api_base: Option<&str>, _model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_anthropic_url(api_base, env_lookup)) } @@ -60,7 +61,7 @@ impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_anthropic_api_key(api_key, env_lookup) } @@ -121,7 +122,7 @@ mod tests { ); assert!(matches!( resolve_anthropic_api_key(None, &|_| None).expect_err("missing key"), - CoreError::Auth(_) + Error::Auth(_) )); } diff --git a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs index 7b958c77ba3..b8ca10461fb 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; use crate::messages::types::{ AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock, @@ -28,12 +28,12 @@ pub const AZURE_ANTHROPIC_MESSAGES_CONFIG: AzureAnthropicMessagesConfig = pub fn resolve_azure_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_KEY_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Azure API Key - Set `api_key` or the AZURE_API_KEY environment variable" .to_string(), ) @@ -43,12 +43,12 @@ pub fn resolve_azure_api_key( pub fn complete_azure_anthropic_url( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let api_base = non_empty(api_base) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_BASE_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Azure API Base - Set `api_base` or the AZURE_API_BASE environment variable. \ Expected format: https://.services.ai.azure.com/anthropic" .to_string(), @@ -142,12 +142,13 @@ fn fold_system_role_messages(request: AnthropicMessagesRequest) -> AnthropicMess } impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn complete_url( &self, api_base: Option<&str>, _model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_azure_anthropic_url(api_base, env_lookup) } @@ -155,7 +156,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_azure_api_key(api_key, env_lookup) } @@ -174,7 +175,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { fn transform_request( &self, request: AnthropicMessagesRequest, - ) -> CoreResult { + ) -> Result { let mut request = fold_system_role_messages(request); if let Some(system) = request.system.as_mut() { strip_scope_from_system(system); @@ -190,7 +191,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { &self, model: &str, response: AnthropicMessagesResponse, - ) -> CoreResult { + ) -> Result { self.anthropic.transform_response(model, response) } } @@ -268,7 +269,7 @@ mod tests { "https://env.services.ai.azure.com/anthropic/v1/messages" ); let err = complete_azure_anthropic_url(Some(" "), &|_| None).expect_err("missing base"); - assert!(matches!(err, CoreError::Auth(_))); + assert!(matches!(err, Error::Auth(_))); } #[test] @@ -284,7 +285,7 @@ mod tests { ); assert!(matches!( resolve_azure_api_key(None, &|_| None).expect_err("missing key"), - CoreError::Auth(_) + Error::Auth(_) )); } diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs index eabd15677cc..b26a7925e8a 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs @@ -1,6 +1,6 @@ use std::collections::BTreeSet; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::{OcrAuthStrategy, OcrProviderConfig, OcrResponseHandling}; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value, json}; @@ -32,17 +32,17 @@ fn resolve_value( env_name: &str, env_lookup: &dyn Fn(&str) -> Option, missing_message: &str, -) -> CoreResult { +) -> Result { non_empty(explicit) .map(str::to_string) .or_else(|| env_lookup(env_name).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| CoreError::Auth(missing_message.to_string())) + .ok_or_else(|| Error::Auth(missing_message.to_string())) } pub fn resolve_azure_ai_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_key, AZURE_AI_API_KEY_ENV, @@ -54,7 +54,7 @@ pub fn resolve_azure_ai_api_key( pub fn resolve_azure_ai_api_base( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_base, AZURE_AI_API_BASE_ENV, @@ -66,7 +66,7 @@ pub fn resolve_azure_ai_api_base( pub fn complete_azure_ai_url( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let base = resolve_azure_ai_api_base(api_base, env_lookup)?; Ok(format!( "{}/providers/mistral/azure/ocr", @@ -77,7 +77,7 @@ pub fn complete_azure_ai_url( pub fn resolve_document_intelligence_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_key, AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV, @@ -89,7 +89,7 @@ pub fn resolve_document_intelligence_api_key( pub fn resolve_document_intelligence_endpoint( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_base, AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV, @@ -127,7 +127,7 @@ fn pages_token_is_valid(token: &str) -> bool { } } -fn normalize_pages_param(pages: &Value) -> CoreResult> { +fn normalize_pages_param(pages: &Value) -> Result, Error> { match pages { Value::String(value) => { let normalized = value @@ -138,7 +138,7 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { if normalized.split(',').all(pages_token_is_valid) { Ok(Some(normalized)) } else { - Err(CoreError::InvalidRequest(format!( + Err(Error::InvalidRequest(format!( "Invalid `pages` string for Azure Document Intelligence: {value:?}. Expected format like '1-3,5,7-9'." ))) } @@ -152,7 +152,7 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { for value in values { let page = value.as_i64().expect("checked is_i64"); if page < 0 { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "`pages` integers must be >= 0 (Mistral 0-based indices)".to_string(), )); } @@ -176,16 +176,16 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { if normalized.split(',').all(pages_token_is_valid) { return Ok(Some(normalized)); } - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Invalid `pages` list for Azure Document Intelligence: {values:?}. Expected tokens like '1' or '3-5'." ))); } - Err(CoreError::InvalidRequest( + Err(Error::InvalidRequest( "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." .to_string(), )) } - _ => Err(CoreError::InvalidRequest( + _ => Err(Error::InvalidRequest( "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." .to_string(), )), @@ -197,7 +197,7 @@ pub fn complete_document_intelligence_url( model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let endpoint = resolve_document_intelligence_endpoint(api_base, env_lookup)?; let mut url = format!( "{}/documentintelligence/documentModels/{}:analyze?api-version={}", @@ -216,20 +216,20 @@ pub fn complete_document_intelligence_url( Ok(url) } -fn document_url_from_mistral_document(document: &Value) -> CoreResult<&str> { - let object = document.as_object().ok_or_else(|| CoreError::InvalidType { +fn document_url_from_mistral_document(document: &Value) -> Result<&str, Error> { + let object = document.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(document), })?; let doc_type = object .get("type") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("document.type"))?; + .ok_or(Error::MissingField("document.type"))?; let field_name = match doc_type { "document_url" => "document_url", "image_url" => "image_url", other => { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Invalid document type: {other}. Must be 'document_url' or 'image_url'" ))); } @@ -238,7 +238,7 @@ fn document_url_from_mistral_document(document: &Value) -> CoreResult<&str> { .get(field_name) .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField(field_name)) + .ok_or(Error::MissingField(field_name)) } fn extract_base64_from_data_uri(data_uri: &str) -> &str { @@ -290,7 +290,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } @@ -298,7 +298,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -308,7 +308,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_azure_ai_url(api_base, env_lookup) } @@ -316,7 +316,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_azure_ai_api_key(api_key, env_lookup) } @@ -335,7 +335,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { _model: &str, document: Value, _optional_params: Map, - ) -> CoreResult { + ) -> Result { let document_url = document_url_from_mistral_document(&document)?; let mut data = Map::new(); if document_url.starts_with("data:") { @@ -359,19 +359,19 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; let status = response .get("status") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("status"))?; + .ok_or(Error::MissingField("status"))?; if status != "succeeded" { - return Err(CoreError::InvalidResponse(format!( + return Err(Error::InvalidResponse(format!( "Azure Document Intelligence analysis failed with status: {status}" ))); } @@ -414,7 +414,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_document_intelligence_url(api_base, model, optional_params, env_lookup) } @@ -422,7 +422,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_document_intelligence_api_key(api_key, env_lookup) } diff --git a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs index 5e885734182..9bf1f73a74d 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs @@ -6,7 +6,7 @@ use crate::audio_transcription::transformation::{ use crate::audio_transcription::types::{ AudioTranscriptionRequestData, AudioTranscriptionResponseData, }; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; pub use super::aws_base::{aws_auth_config, bedrock_model_id_and_region, resolve_bedrock_region}; use super::constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}; @@ -18,8 +18,8 @@ pub static BEDROCK_AUDIO_TRANSCRIPTION_CONFIG: BedrockAudioTranscriptionConfig = pub struct BedrockAudioTranscriptionConfig; -fn audio_fields(audio: Value) -> CoreResult<(String, String)> { - let object = audio.as_object().ok_or_else(|| CoreError::InvalidType { +fn audio_fields(audio: Value) -> Result<(String, String), Error> { + let object = audio.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&audio), })?; @@ -27,13 +27,13 @@ fn audio_fields(audio: Value) -> CoreResult<(String, String)> { .get("data") .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField("audio.data"))?; + .ok_or(Error::MissingField("audio.data"))?; let format = object .get("format") .and_then(Value::as_str) .filter(|value| matches!(*value, "wav" | "mp3" | "flac" | "ogg")) .ok_or_else(|| { - CoreError::InvalidRequest("audio.format must be wav, mp3, flac, or ogg".to_string()) + Error::InvalidRequest("audio.format must be wav, mp3, flac, or ogg".to_string()) })?; Ok((data.to_string(), format.to_string())) } @@ -46,16 +46,18 @@ fn optional_string<'a>(params: &'a Map, key: &str) -> Option<&'a } impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn supported_transcription_params(&self) -> &'static [&'static str] { SUPPORTED_PARAMS } + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_transcription_request( &self, _model: &str, audio: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { let (data, format) = audio_fields(audio)?; let mut instruction = "Transcribe the audio. Respond with only the transcript.".to_string(); if let Some(language) = optional_string(&optional_params, "language") { @@ -83,18 +85,19 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { }) } + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_transcription_response( &self, _model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let content = response_json .get("output") .and_then(|value| value.get("message")) .and_then(|value| value.get("content")) .and_then(Value::as_array) .ok_or_else(|| { - CoreError::InvalidResponse("Bedrock response has no output content".to_string()) + Error::InvalidResponse("Bedrock response has no output content".to_string()) })?; let mut text = String::new(); for block in content { @@ -111,7 +114,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (model_id, model_region) = bedrock_model_id_and_region(model); let region = resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup); let endpoint = optional_params @@ -133,7 +136,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (_, model_region) = bedrock_model_id_and_region(model); Ok(AudioTranscriptionAuth::AwsSigV4 { region: resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup), diff --git a/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs b/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs index b11639aa09b..e5e52bfce95 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs @@ -4,7 +4,7 @@ use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use crate::caching::in_memory_cache::InMemoryCache; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use aws_credential_types::Credentials; use aws_credential_types::provider::ProvideCredentials; use aws_sigv4::http_request::{ @@ -197,7 +197,7 @@ pub fn classify_auth( pub async fn resolve_credentials( config: AwsAuthConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), -) -> CoreResult { +) -> Result { let resolved = config.clone().with_environment(env_lookup); let flow = classify_auth(config, env_lookup); match flow { @@ -244,9 +244,10 @@ pub async fn resolve_credentials( let provider = aws_config::profile::ProfileFileCredentialsProvider::builder() .profile_name(name) .build(); - provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS profile credentials failed: {error}")) - }) + provider + .provide_credentials() + .await + .map_err(|error| Error::Auth(format!("AWS profile credentials failed: {error}"))) } AwsAuthFlow::AssumeRole { role, session_name } => { if is_already_running_as_role(&role, &resolved).await? { @@ -260,7 +261,7 @@ pub async fn resolve_credentials( .build() .await; let credentials = provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS default credentials failed: {error}")) + Error::Auth(format!("AWS default credentials failed: {error}")) })?; set_cached_credentials( key, @@ -301,7 +302,7 @@ pub async fn resolve_credentials( provider .provide_credentials() .await - .map_err(|error| CoreError::Auth(format!("AWS role credentials failed: {error}"))) + .map_err(|error| Error::Auth(format!("AWS role credentials failed: {error}"))) } AwsAuthFlow::WebIdentity { token, @@ -325,13 +326,13 @@ pub async fn resolve_credentials( .send() .await .map_err(|error| { - CoreError::Auth(format!("AWS web identity credentials failed: {error}")) + Error::Auth(format!("AWS web identity credentials failed: {error}")) })?; let credentials = response.credentials().ok_or_else(|| { - CoreError::Auth("AWS web identity response had no credentials".to_string()) + Error::Auth("AWS web identity response had no credentials".to_string()) })?; let expiration = SystemTime::try_from(*credentials.expiration()).map_err(|error| { - CoreError::Auth(format!("AWS web identity expiration was invalid: {error}")) + Error::Auth(format!("AWS web identity expiration was invalid: {error}")) })?; Ok(Credentials::new( credentials.access_key_id(), @@ -350,9 +351,10 @@ pub async fn resolve_credentials( aws_config::default_provider::credentials::DefaultCredentialsChain::builder() .build() .await; - let credentials = provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS default credentials failed: {error}")) - })?; + let credentials = provider + .provide_credentials() + .await + .map_err(|error| Error::Auth(format!("AWS default credentials failed: {error}")))?; set_cached_credentials( key, credentials.clone(), @@ -363,7 +365,7 @@ pub async fn resolve_credentials( } } -async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> CoreResult { +async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> Result { if role_identity(role).is_none() { return Ok(false); } @@ -437,7 +439,7 @@ pub fn sign_bedrock_post( region: &str, credentials: &Credentials, signing_time: SystemTime, -) -> CoreResult> { +) -> Result, Error> { let identity: Identity = credentials.clone().into(); let params = v4::SigningParams::builder() .identity(&identity) @@ -447,14 +449,14 @@ pub fn sign_bedrock_post( .settings(SigningSettings::default()) .build() .map(SigningParams::from) - .map_err(|error| CoreError::Auth(format!("AWS signing parameters failed: {error}")))?; + .map_err(|error| Error::Auth(format!("AWS signing parameters failed: {error}")))?; let header_refs = headers .iter() .map(|(name, value)| (name.as_str(), value.as_str())); let request = SignableRequest::new("POST", url, header_refs, SignableBody::Bytes(body)) - .map_err(|error| CoreError::Auth(format!("AWS signable request failed: {error}")))?; + .map_err(|error| Error::Auth(format!("AWS signable request failed: {error}")))?; let (instructions, _) = sign(request, ¶ms) - .map_err(|error| CoreError::Auth(format!("AWS request signing failed: {error}")))? + .map_err(|error| Error::Auth(format!("AWS request signing failed: {error}")))? .into_parts(); Ok(instructions .headers() diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs index 4b75dcb8e9d..c86f061b9ca 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::Error; use serde_json::json; fn messages(value: Value) -> Vec { @@ -23,7 +24,7 @@ fn transform(msgs: Value, opts: Value) -> Value { .body } -fn transform_response(body: Value) -> CoreResult { +fn transform_response(body: Value) -> Result { BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response( "anthropic.claude-sonnet-4-5-v1:0", ProviderChatResponseData { body }, @@ -478,25 +479,22 @@ fn declines_a_response_carrying_a_tool_use_block() { "usage": {"inputTokens": 1, "outputTokens": 1} })) .expect_err("tool use block"); - assert_eq!( - err, - CoreError::Unsupported("non-text response content block") - ); + assert_eq!(err, Error::Unsupported("non-text response content block")); } #[test] fn errors_on_a_response_missing_required_fields() { assert_eq!( transform_response(json!("nope")).expect_err("not an object"), - CoreError::InvalidResponse("converse response is not an object".to_string()) + Error::InvalidResponse("converse response is not an object".to_string()) ); assert_eq!( transform_response(json!({"usage": {}})).expect_err("no output"), - CoreError::MissingField("output.message.content") + Error::MissingField("output.message.content") ); assert_eq!( transform_response(json!({"output": {"message": {"content": []}}})).expect_err("no usage"), - CoreError::MissingField("usage") + Error::MissingField("usage") ); } diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs index b107950748e..7be3d108d44 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs @@ -11,7 +11,7 @@ use crate::chat_completions::types::{ ChatCompletionsUsage, ChatMessage, ChatMessageContent, ProviderChatRequestData, ProviderChatResponseData, }; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use super::super::aws_base::{bedrock_model_id_and_region, resolve_bedrock_region}; use super::super::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE}; @@ -23,11 +23,12 @@ use super::super::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT /// `additionalModelRequestFields` for Anthropic base models and to /// `inferenceConfig` otherwise, and that branch reads the model catalog the /// core cannot see. -const SUPPORTED_PARAMS: &[&str] = &["maxTokens", "temperature", "topP", "stopSequences"]; - -/// Params that belong in `inferenceConfig`, in the order Python's -/// `AmazonConverseConfig` declares them, so bodies compare cleanly. -const INFERENCE_CONFIG_PARAMS: &[&str] = SUPPORTED_PARAMS; +const SUPPORTED_PARAMS: &[(&str, &str)] = &[ + ("max_tokens", "maxTokens"), + ("temperature", "temperature"), + ("top_p", "topP"), + ("stop", "stopSequences"), +]; const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = "aws_bedrock_runtime_endpoint"; @@ -66,7 +67,7 @@ fn converse_body(conversation: &Conversation, params: &Map) -> Va }) .collect(); - let inference_config = Map::from_iter(INFERENCE_CONFIG_PARAMS.iter().filter_map(|name| { + let inference_config = Map::from_iter(SUPPORTED_PARAMS.iter().filter_map(|(_, name)| { params .get(*name) .map(|value| ((*name).to_string(), value.clone())) @@ -110,7 +111,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (model_id, model_region) = bedrock_model_id_and_region(model); let region = resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup); let endpoint = optional_params @@ -137,7 +138,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { // Python reads `api_key` as the Bedrock bearer token and consults the // env only when the caller passed none, so a caller-supplied empty key // falls through to SigV4 without reaching for the environment. An @@ -162,7 +163,8 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { &[("Content-Type", "application/json")] } - fn supported_params(&self) -> &'static [&'static str] { + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] + fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)] { SUPPORTED_PARAMS } @@ -175,32 +177,36 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { messages: &[ChatMessage], optional_params: &Map, ) -> Option { - unsupported_param(SUPPORTED_PARAMS, CONFIG_PARAMS, optional_params) - .or_else(|| messages.iter().find_map(unsupported_message)) - // Python's Converse translation drops blank text blocks instead of - // substituting the placeholder the shared conversation builder - // applies, so decline blank text rather than diverge. - .or_else(|| { - messages - .iter() - .any(has_blank_text) - .then_some(Unsupported("blank message text")) - }) - // Converse has no assistant prefill: Python inserts a continue turn - // when a conversation opens or closes on an assistant message, and - // only under `litellm.modify_params`, which the core cannot see. - // Declining both ends also keeps the shared builder's final - // assistant right-strip (an Anthropic rule) unreachable here. - .or_else(|| { - let conversation = build_conversation(messages); - let ends_on_assistant = conversation - .turns - .last() - .is_some_and(|turn| turn.role == TurnRole::Assistant); - (!conversation.opens_on_user_turn() || ends_on_assistant).then_some(Unsupported( - "conversation does not run user turn to user turn", - )) - }) + unsupported_param( + self.supported_openai_params(), + CONFIG_PARAMS, + optional_params, + ) + .or_else(|| messages.iter().find_map(unsupported_message)) + // Python's Converse translation drops blank text blocks instead of + // substituting the placeholder the shared conversation builder + // applies, so decline blank text rather than diverge. + .or_else(|| { + messages + .iter() + .any(has_blank_text) + .then_some(Unsupported("blank message text")) + }) + // Converse has no assistant prefill: Python inserts a continue turn + // when a conversation opens or closes on an assistant message, and + // only under `litellm.modify_params`, which the core cannot see. + // Declining both ends also keeps the shared builder's final + // assistant right-strip (an Anthropic rule) unreachable here. + .or_else(|| { + let conversation = build_conversation(messages); + let ends_on_assistant = conversation + .turns + .last() + .is_some_and(|turn| turn.role == TurnRole::Assistant); + (!conversation.opens_on_user_turn() || ends_on_assistant).then_some(Unsupported( + "conversation does not run user turn to user turn", + )) + }) } fn transform_request( @@ -208,7 +214,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { _model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult { + ) -> Result { Ok(ProviderChatRequestData { body: converse_body(&build_conversation(&messages), &optional_params), }) @@ -218,17 +224,18 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { &self, model: &str, response: ProviderChatResponseData, - ) -> CoreResult { - let body = response.body.as_object().ok_or_else(|| { - CoreError::InvalidResponse("converse response is not an object".into()) - })?; + ) -> Result { + let body = response + .body + .as_object() + .ok_or_else(|| Error::InvalidResponse("converse response is not an object".into()))?; let content = body .get("output") .and_then(|output| output.get("message")) .and_then(|message| message.get("content")) .and_then(Value::as_array) - .ok_or(CoreError::MissingField("output.message.content"))?; + .ok_or(Error::MissingField("output.message.content"))?; // The route declines tool requests, so anything other than a text block // is something this path never asked for. Decline; the host falls back. if content.iter().any(|block| { @@ -236,7 +243,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { .as_object() .is_none_or(|block| block.len() != 1 || !block.contains_key("text")) }) { - return Err(CoreError::Unsupported("non-text response content block")); + return Err(Error::Unsupported("non-text response content block")); } let text: String = content .iter() @@ -246,7 +253,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { let usage = body .get("usage") .and_then(Value::as_object) - .ok_or(CoreError::MissingField("usage"))?; + .ok_or(Error::MissingField("usage"))?; let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0); let computed = usage_from_parts( field("inputTokens"), diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs index dc720cc4244..9648321d7ff 100644 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value}; @@ -47,7 +47,7 @@ pub fn complete_url(api_base: Option<&str>) -> String { /// Resolve the Mistral API key from the explicit param or the environment. /// -/// Blank/whitespace values are treated as absent. Returns `CoreError::Auth` +/// Blank/whitespace values are treated as absent. Returns `Error::Auth` /// when no usable key is available. /// /// Note: the env fallback only reads the process environment. Secret-manager @@ -56,13 +56,13 @@ pub fn complete_url(api_base: Option<&str>) -> String { pub fn resolve_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) .map(str::to_string) .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty())) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } pub struct MistralOcrConfig; @@ -70,18 +70,20 @@ pub struct MistralOcrConfig; pub const MISTRAL_OCR_CONFIG: MistralOcrConfig = MistralOcrConfig; impl OcrProviderConfig for MistralOcrConfig { + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn supported_ocr_params(&self) -> &'static [&'static str] { SUPPORTED_OCR_PARAMS } + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_ocr_request( &self, model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { if !document.is_object() { - return Err(CoreError::InvalidType { + return Err(Error::InvalidType { expected: "object", actual: json_type_name(&document), }); @@ -100,14 +102,15 @@ impl OcrProviderConfig for MistralOcrConfig { }) } + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_ocr_response( &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response_object = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; @@ -134,13 +137,14 @@ impl OcrProviderConfig for MistralOcrConfig { }) } + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn complete_url( &self, api_base: Option<&str>, _model: &str, _optional_params: &Map, _env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_url(api_base)) } @@ -148,11 +152,12 @@ impl OcrProviderConfig for MistralOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_api_key(api_key, env_lookup) } } +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub fn supported_ocr_params() -> &'static [&'static str] { MISTRAL_OCR_CONFIG.supported_ocr_params() } @@ -161,15 +166,17 @@ pub fn map_ocr_params(non_default_params: &Map) -> Map, -) -> CoreResult { +) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } -pub fn transform_ocr_response(model: &str, response_json: Value) -> CoreResult { +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub fn transform_ocr_response(model: &str, response_json: Value) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -250,7 +257,7 @@ mod tests { assert_eq!( err, - CoreError::InvalidType { + Error::InvalidType { expected: "object", actual: "string", } @@ -307,6 +314,6 @@ mod tests { #[test] fn resolve_api_key_errors_when_absent() { let err = resolve_api_key(None, &|_| None).expect_err("missing key should error"); - assert_eq!(err, CoreError::Auth(MISSING_KEY_MESSAGE.to_string())); + assert_eq!(err, Error::Auth(MISSING_KEY_MESSAGE.to_string())); } } diff --git a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs index b3f6b03b28a..f1985f81b7d 100644 --- a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs +++ b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::realtime::transformation::RealtimeProviderConfig; use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; @@ -72,7 +72,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { &self, event: &RealtimeEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(RealtimeTransformResult::passthrough(event.clone())) } @@ -80,7 +80,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { &self, event: &RealtimeEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(RealtimeTransformResult::passthrough(event.clone())) } } @@ -88,14 +88,14 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { pub fn transform_realtime_request( event: &RealtimeEvent, model: &str, -) -> CoreResult { +) -> Result { OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model) } pub fn transform_realtime_response( event: &RealtimeEvent, model: &str, -) -> CoreResult { +) -> Result { OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model) } diff --git a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs index e15197c468c..be86bb90311 100644 --- a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs +++ b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::responses::types::{ResponsesWsEvent, ResponsesWsTransformResult}; use crate::responses::websocket::{ResponsesWebSocketProviderConfig, enforce_model}; @@ -15,7 +15,7 @@ impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig { &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult { + ) -> Result { Ok(ResponsesWsTransformResult::passthrough(enforce_model( event, model, ))) @@ -25,7 +25,7 @@ impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig { &self, event: &ResponsesWsEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(ResponsesWsTransformResult::passthrough(event.clone())) } } diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs index 6300149c237..ee095447028 100644 --- a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value, json}; @@ -43,7 +43,7 @@ pub fn is_deepseek_model(model: &str) -> bool { pub fn resolve_vertex_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) @@ -51,7 +51,7 @@ pub fn resolve_vertex_api_key( .or_else(|| env_lookup(VERTEX_AI_API_KEY_ENV).filter(|key| !key.trim().is_empty())) .or_else(|| env_lookup(VERTEXAI_API_KEY_ENV).filter(|key| !key.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Vertex AI access token - pass api_key or provide Authorization via extra_headers" .to_string(), ) @@ -61,12 +61,12 @@ pub fn resolve_vertex_api_key( fn vertex_project( params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { string_param(params, &["vertex_project", "vertex_ai_project"]) .map(str::to_string) .or_else(|| env_lookup(VERTEXAI_PROJECT_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::InvalidRequest( + Error::InvalidRequest( "Missing vertex_project - Set VERTEXAI_PROJECT environment variable or pass vertex_project parameter" .to_string(), ) @@ -99,7 +99,7 @@ pub fn complete_vertex_mistral_url( model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let project = vertex_project(optional_params, env_lookup)?; let location = vertex_location(optional_params, env_lookup); let base = vertex_mistral_api_base(api_base, &location); @@ -112,7 +112,7 @@ pub fn complete_vertex_deepseek_url( api_base: Option<&str>, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let project = vertex_project(optional_params, env_lookup)?; let location = vertex_location(optional_params, env_lookup); let base = api_base @@ -125,20 +125,20 @@ pub fn complete_vertex_deepseek_url( )) } -fn document_content_item(document: &Value) -> CoreResult { - let object = document.as_object().ok_or_else(|| CoreError::InvalidType { +fn document_content_item(document: &Value) -> Result { + let object = document.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(document), })?; let doc_type = object .get("type") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("document.type"))?; + .ok_or(Error::MissingField("document.type"))?; let url_field = match doc_type { "image_url" => "image_url", "document_url" => "document_url", other => { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Unsupported document type: {other}. Expected 'image_url' or 'document_url'" ))); } @@ -147,7 +147,7 @@ fn document_content_item(document: &Value) -> CoreResult { .get(url_field) .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField(url_field))?; + .ok_or(Error::MissingField(url_field))?; Ok(json!({ "type": "image_url", @@ -163,7 +163,7 @@ fn deepseek_model_name(model: &str) -> String { } } -fn first_choice_content(response: &Value) -> CoreResult { +fn first_choice_content(response: &Value) -> Result { response .get("choices") .and_then(Value::as_array) @@ -176,9 +176,7 @@ fn first_choice_content(response: &Value) -> CoreResult { Value::Object(_) => true, _ => false, }) - .ok_or_else(|| { - CoreError::InvalidResponse("No content in DeepSeek OCR response".to_string()) - }) + .ok_or_else(|| Error::InvalidResponse("No content in DeepSeek OCR response".to_string())) } fn ocr_data_from_content(content: Value, usage: Option, model: &str) -> Value { @@ -219,7 +217,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } @@ -227,7 +225,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -237,7 +235,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_vertex_mistral_url(api_base, model, optional_params, env_lookup) } @@ -245,7 +243,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_vertex_api_key(api_key, env_lookup) } @@ -264,7 +262,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { let mut data = Map::new(); data.insert( "model".to_string(), @@ -289,10 +287,10 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; @@ -314,7 +312,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { }); } - let object = ocr_data.as_object().ok_or_else(|| CoreError::InvalidType { + let object = ocr_data.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&ocr_data), })?; @@ -346,7 +344,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { _model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_vertex_deepseek_url(api_base, optional_params, env_lookup) } @@ -354,7 +352,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_vertex_api_key(api_key, env_lookup) } } diff --git a/litellm-rust/crates/core/src/realtime/transformation.rs b/litellm-rust/crates/core/src/realtime/transformation.rs index 69b88687000..b08084514ef 100644 --- a/litellm-rust/crates/core/src/realtime/transformation.rs +++ b/litellm-rust/crates/core/src/realtime/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; pub trait RealtimeProviderConfig { @@ -11,12 +11,12 @@ pub trait RealtimeProviderConfig { &self, event: &RealtimeEvent, model: &str, - ) -> CoreResult; + ) -> Result; /// Transform a backend → client event before it is forwarded downstream. fn transform_realtime_response( &self, event: &RealtimeEvent, model: &str, - ) -> CoreResult; + ) -> Result; } diff --git a/litellm-rust/crates/core/src/responses/instrumentation.rs b/litellm-rust/crates/core/src/responses/instrumentation.rs index ec04571da14..b1098f4d386 100644 --- a/litellm-rust/crates/core/src/responses/instrumentation.rs +++ b/litellm-rust/crates/core/src/responses/instrumentation.rs @@ -5,9 +5,9 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde_json::Value; +use crate::Error; use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType}; -use crate::{CoreError, CoreResult}; #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ResponsesWsUsage { @@ -205,7 +205,7 @@ impl ResponsesWsInstrumentation { } } -type LifecycleFuture<'a, T> = Pin> + Send + 'a>>; +type LifecycleFuture<'a, T> = Pin> + Send + 'a>>; impl CallLifecycleHooks<(), (), ()> for ResponsesWsInstrumentation { type PreCallFuture<'a> = LifecycleFuture<'a, ()>; @@ -246,7 +246,7 @@ impl CallLifecycleHooks<(), (), ()> for ResponsesWsInstrumentation { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -342,7 +342,7 @@ mod tests { ), (), &instrumentation, - |_| async { Ok::<(), CoreError>(()) }, + |_| async { Ok::<(), Error>(()) }, ) .await; diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/core/src/responses/websocket.rs index 92dc19627a0..5d037e9cf1b 100644 --- a/litellm-rust/crates/core/src/responses/websocket.rs +++ b/litellm-rust/crates/core/src/responses/websocket.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}; use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult}; @@ -19,13 +19,13 @@ pub trait ResponsesWebSocketProviderConfig: Sync { &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult; + ) -> Result; fn transform_ws_response( &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult; + ) -> Result; } pub fn complete_websocket_url( diff --git a/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs b/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs index 656ba033b62..8a8a5ea263a 100644 --- a/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs +++ b/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs @@ -1,7 +1,8 @@ -//! Enforcement: the litellm-rust workspace has exactly three crates. +//! Enforcement: the litellm-rust workspace has exactly four crates. //! -//! `core` (pure translation), `ai-gateway` (routes + all network I/O), and -//! `python-bridge` (the PyO3 cdylib). Adding or removing a crate must be a +//! `core` (the Rust SDK), `ai-gateway` (the HTTP/WebSocket host), +//! `python-interop` (domain-neutral PyO3 primitives), and `python-bridge` (the +//! PyO3 cdylib). Adding or removing a crate must be a //! deliberate act: this test fails until the allowlist here is updated, forcing //! whoever changes the crate set to justify the new crate per the rule that a //! crate is a layer needing independent compilation / its own deps / a separate @@ -16,10 +17,15 @@ use std::path::{Path, PathBuf}; /// The one true crate set. Update BOTH this and `litellm-rust/AGENTS.md` when the /// workspace legitimately gains or loses a crate. -const EXPECTED_MEMBERS: &[&str] = &["crates/core", "crates/ai-gateway", "crates/python-bridge"]; +const EXPECTED_MEMBERS: &[&str] = &[ + "crates/core", + "crates/ai-gateway", + "crates/python-interop", + "crates/python-bridge", +]; /// The crate subdirectory names that must exist under `crates/`. -const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-bridge"]; +const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-interop", "python-bridge"]; const MISMATCH: &str = "litellm-rust crate set changed — update this allowlist AND litellm-rust/AGENTS.md, and justify the crate per the rule (crate = layer needing independent compilation / its own deps / a separate artifact)."; diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index ad3cddfa5fd..42282ca4da4 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -1,3 +1,3 @@ -litellm-python-bridge is the PyO3 cdylib that exposes Rust to the litellm Python SDK — a thin adapter (Python objects → Rust calls → Python results) over the litellm-core route entrypoints (e.g. `litellm_core::messages::messages`). +litellm-python-bridge is the PyO3 cdylib that exposes LiteLLM Rust APIs to the Python SDK. Keep API registration, domain dependency wiring, request assembly, and Python exception mapping here. Put domain-neutral Python/Serde conversion and GIL primitives in litellm-python-interop. Keep it thin: no business logic, no transforms, no I/O orchestration — just marshal in/out and call the core entrypoint. diff --git a/litellm-rust/crates/python-bridge/CLAUDE.md b/litellm-rust/crates/python-bridge/CLAUDE.md index 3ce8b8c639a..d25ae5a8130 100644 --- a/litellm-rust/crates/python-bridge/CLAUDE.md +++ b/litellm-rust/crates/python-bridge/CLAUDE.md @@ -5,8 +5,9 @@ Rules for `litellm-rust/crates/python-bridge`. ## Responsibility `python-bridge` is the PyO3 boundary between Python LiteLLM and Rust transforms. -Keep this crate thin. It adapts Python objects to Rust payloads and returns -Python-compatible dictionaries. +Keep this crate thin. It exposes LiteLLM Rust APIs, assembles domain requests, +maps domain errors to Python exceptions, and delegates generic conversion and +GIL handling to `litellm-python-interop`. ## Bridge Shape diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index d461a483ae0..637e5580170 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -13,19 +13,24 @@ crate-type = ["cdylib"] default = ["abi3"] abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] +panic-test = [] [dependencies] +futures-util.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true litellm-core = { workspace = true, features = ["bedrock-auth"] } litellm-ai-gateway = { workspace = true, default-features = false } +litellm-python-interop.workspace = true pyo3.workspace = true pyo3-async-runtimes.workspace = true -pythonize.workspace = true serde.workspace = true serde_json.workspace = true tokio.workspace = true [dev-dependencies] criterion = "0.8.2" +tokio-tungstenite.workspace = true [[bench]] name = "serialization" diff --git a/litellm-rust/crates/python-bridge/benches/serialization.rs b/litellm-rust/crates/python-bridge/benches/serialization.rs index 8a90cf667d0..0b9436d0cb7 100644 --- a/litellm-rust/crates/python-bridge/benches/serialization.rs +++ b/litellm-rust/crates/python-bridge/benches/serialization.rs @@ -2,6 +2,7 @@ use std::hint::black_box; use std::time::Duration; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use litellm_python_interop::{from_py, to_py}; use pyo3::prelude::*; use pyo3::types::PyDict; use serde_json::{Value, json}; @@ -25,7 +26,7 @@ fn former_json_roundtrip_from_py(py: Python<'_>, value: &Bound<'_, PyAny>) -> Va } fn pythonize_from_py(value: &Bound<'_, PyAny>) -> Value { - pythonize::depythonize(value).expect("payload should depythonize") + from_py(value).expect("payload should depythonize") } fn former_json_roundtrip_to_py(py: Python<'_>, value: &Value) -> Py { @@ -37,12 +38,10 @@ fn former_json_roundtrip_to_py(py: Python<'_>, value: &Value) -> Py { } fn pythonize_to_py(py: Python<'_>, value: &Value) -> Py { - pythonize::pythonize(py, value) - .expect("response should pythonize") - .unbind() + to_py(py, value).expect("response should pythonize") } -fn serialization(c: &mut Criterion) { +fn bridge_serialization(c: &mut Criterion) { Python::initialize(); Python::attach(|py| { for &(label, payload_bytes) in PAYLOAD_SIZES { @@ -98,6 +97,6 @@ criterion_group! { .sample_size(20) .warm_up_time(Duration::from_secs(1)) .measurement_time(Duration::from_secs(4)); - targets = serialization + targets = bridge_serialization } criterion_main!(benches); diff --git a/litellm-rust/crates/python-bridge/src/constants.rs b/litellm-rust/crates/python-bridge/src/constants.rs new file mode 100644 index 00000000000..07b2836b838 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/constants.rs @@ -0,0 +1 @@ +pub(crate) const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace"; diff --git a/litellm-rust/crates/python-bridge/src/diagnostics.rs b/litellm-rust/crates/python-bridge/src/diagnostics.rs new file mode 100644 index 00000000000..cc153a89b8f --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/diagnostics.rs @@ -0,0 +1,23 @@ +use litellm_python_interop::release_count; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +#[pyfunction] +fn gil_stats(py: Python<'_>) -> PyResult> { + let stats = PyDict::new(py); + stats.set_item("releases", release_count())?; + Ok(stats.into_any().unbind()) +} + +#[cfg(feature = "panic-test")] +#[pyfunction] +fn _panic_for_test() { + panic!("intentional PyO3 panic smoke test"); +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(gil_stats, module)?)?; + #[cfg(feature = "panic-test")] + module.add_function(wrap_pyfunction!(_panic_for_test, module)?)?; + Ok(()) +} diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs new file mode 100644 index 00000000000..914e2e1e033 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -0,0 +1,61 @@ +use litellm_core::error::Error; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::*; + +pyo3::create_exception!( + _native, + RustBridgeDeclined, + pyo3::exceptions::PyException, + "The route declined before calling the provider, so the host may retry on its own path." +); + +pyo3::create_exception!( + _native, + RustUpstreamError, + pyo3::exceptions::PyException, + "The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response." +); + +pub(crate) fn core_error_to_pyerr(err: Error) -> PyErr { + match err { + Error::Auth(message) => PyValueError::new_err(message), + Error::InvalidProvider(_) + | Error::InvalidRequest(_) + | Error::InvalidType { .. } + | Error::MissingField(_) => PyValueError::new_err(err.to_string()), + other => PyRuntimeError::new_err(other.to_string()), + } +} + +/// Map a core error for a route whose host keeps a Python implementation. +/// +/// The distinction the host needs is whether the provider was already called. +/// Everything raised before the request goes out is safe for the host to retry +/// on its own path; anything after it is not, because the provider has already +/// done the work and billed for it. +pub(crate) fn chat_completions_error_to_pyerr(err: Error) -> PyErr { + match err { + Error::Unsupported(_) + | Error::Auth(_) + | Error::InvalidProvider(_) + | Error::InvalidRequest(_) + | Error::InvalidType { .. } + | Error::MissingField(_) + | Error::Routing(_) + // Nothing reached the provider, so serving it on Python cannot double + // bill and is the only way the caller gets an answer at all. + | Error::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), + Error::Http { status, body } => { + RustUpstreamError::new_err((status, format!("{status}: {body}"))) + } + Error::Network(message) | Error::InvalidResponse(message) => { + RustUpstreamError::new_err((0u16, message)) + } + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + module.add("RustBridgeDeclined", py.get_type::())?; + module.add("RustUpstreamError", py.get_type::()) +} diff --git a/litellm-rust/crates/python-bridge/src/execution.rs b/litellm-rust/crates/python-bridge/src/execution.rs new file mode 100644 index 00000000000..f3648158cf6 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/execution.rs @@ -0,0 +1,423 @@ +use std::future::Future; +use std::panic::AssertUnwindSafe; +use std::time::Duration; + +use futures_util::FutureExt; +use litellm_core::error::Error; +use litellm_python_interop::{Pythonized, panic_to_pyerr, release_gil}; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::*; +use serde::Serialize; +use tokio::runtime::{Handle, Runtime}; +use tokio::time::{self, MissedTickBehavior}; + +pub(crate) fn run_sync( + py: Python<'_>, + future: F, + map_error: fn(Error) -> PyErr, +) -> PyResult> +where + T: Serialize + Send + 'static, + F: Future> + Send + 'static, +{ + run_sync_on( + py, + pyo3_async_runtimes::tokio::get_runtime(), + future, + map_error, + ) +} + +fn run_sync_on( + py: Python<'_>, + runtime: &Runtime, + future: F, + map_error: fn(Error) -> PyErr, +) -> PyResult> +where + T: Serialize + Send + 'static, + F: Future> + Send + 'static, +{ + if Handle::try_current().is_ok() { + return Err(PyRuntimeError::new_err( + "synchronous native routes cannot run from a Tokio context; use the async route", + )); + } + + let result = release_gil(py, move || runtime.block_on(wait_for_sync_result(future)))?; + let result = map_core_result(result, map_error)?; + Pythonized(result).into_pyobject(py).map(Bound::unbind) +} + +pub(crate) fn run_async( + py: Python<'_>, + future: F, + map_error: fn(Error) -> PyErr, +) -> PyResult> +where + T: Serialize + Send + 'static, + F: Future> + Send + 'static, +{ + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let result = catch_future_panic(future).await?; + let result = map_core_result(result, map_error)?; + Ok(Pythonized(result)) + }) +} + +fn map_core_result(result: Result, map_error: fn(Error) -> PyErr) -> PyResult { + match result { + Ok(value) => Ok(value), + Err(error) => Err( + std::panic::catch_unwind(AssertUnwindSafe(|| map_error(error))) + .map_err(panic_to_pyerr)?, + ), + } +} + +async fn catch_future_panic(future: F) -> PyResult> +where + F: Future>, +{ + AssertUnwindSafe(future) + .catch_unwind() + .await + .map_err(panic_to_pyerr) +} + +async fn wait_for_sync_result(future: F) -> PyResult> +where + F: Future>, +{ + let future = catch_future_panic(future); + tokio::pin!(future); + + let signal_interval = Duration::from_millis(50); + let mut signal_checks = + time::interval_at(time::Instant::now() + signal_interval, signal_interval); + signal_checks.set_missed_tick_behavior(MissedTickBehavior::Delay); + loop { + tokio::select! { + result = &mut future => return result, + _ = signal_checks.tick() => Python::attach(|py| py.check_signals())?, + } + } +} + +#[cfg(test)] +mod tests { + use std::ffi::CString; + use std::future::poll_fn; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, mpsc}; + use std::task::Poll; + use std::thread; + use std::time::Instant; + + use pyo3::panic::PanicException; + use pyo3::types::{PyDict, PyModule}; + use serde::Serializer; + use tokio::runtime::Builder; + + use super::*; + + fn runtime_error(error: Error) -> PyErr { + PyRuntimeError::new_err(error.to_string()) + } + + fn panicking_error_mapper(_error: Error) -> PyErr { + panic!("error mapper panicked") + } + + struct PanickingOutput; + + static ASYNC_PROBE_COMPLETED: AtomicUsize = AtomicUsize::new(0); + + impl Serialize for PanickingOutput { + fn serialize(&self, _serializer: S) -> Result + where + S: Serializer, + { + panic!("serializer panicked") + } + } + + #[pyfunction] + fn async_serialization_panic(py: Python<'_>) -> PyResult> { + run_async(py, async { Ok(PanickingOutput) }, runtime_error) + } + + #[pyfunction] + fn async_runtime_probe(py: Python<'_>) -> PyResult> { + run_async( + py, + async { + ASYNC_PROBE_COMPLETED.fetch_add(1, Ordering::SeqCst); + Ok(true) + }, + runtime_error, + ) + } + + #[pyfunction] + fn runtime_worker_count() -> usize { + pyo3_async_runtimes::tokio::get_runtime() + .metrics() + .num_workers() + } + + #[pyfunction] + fn runtime_is_responsive(_py: Python<'_>, expected_completions: usize) -> bool { + let completion_deadline = Instant::now() + Duration::from_secs(2); + while ASYNC_PROBE_COMPLETED.load(Ordering::SeqCst) < expected_completions { + if Instant::now() >= completion_deadline { + return false; + } + thread::sleep(Duration::from_millis(1)); + } + + let (heartbeat_tx, heartbeat_rx) = mpsc::sync_channel(1); + pyo3_async_runtimes::tokio::get_runtime().spawn(async move { + let _ = heartbeat_tx.send(()); + }); + heartbeat_rx.recv_timeout(Duration::from_secs(2)).is_ok() + } + + fn extract_bool(py: Python<'_>, result: PyResult>) -> bool { + result + .expect("route should complete") + .bind(py) + .extract() + .expect("result should convert") + } + + #[test] + fn sync_runner_polls_future_on_the_caller_thread() { + Python::initialize(); + Python::attach(|py| { + let caller_thread = std::thread::current().id(); + let result = run_sync( + py, + async move { Ok(std::thread::current().id() == caller_thread) }, + runtime_error, + ); + + assert!(extract_bool(py, result)); + }); + } + + #[test] + fn sync_runner_releases_gil_while_waiting() { + Python::initialize(); + Python::attach(|py| { + let result = run_sync( + py, + async { + let gil_acquired = tokio::time::timeout( + Duration::from_secs(2), + tokio::task::spawn_blocking(|| Python::attach(|_| true)), + ) + .await; + Ok(matches!(gil_acquired, Ok(Ok(true)))) + }, + runtime_error, + ); + + assert!(extract_bool(py, result)); + }); + } + + #[test] + fn sync_runner_rejects_calls_from_a_tokio_context() { + Python::initialize(); + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + + let error = runtime.block_on(async { + Python::attach(|py| { + run_sync::(py, async { Ok(true) }, runtime_error) + .expect_err("sync route should reject a nested Tokio runtime") + }) + }); + + assert_eq!( + error.to_string(), + "RuntimeError: synchronous native routes cannot run from a Tokio context; use the async route" + ); + } + + #[test] + fn sync_runner_can_drive_a_current_thread_runtime() { + Python::initialize(); + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + Python::attach(|py| { + let result = run_sync_on( + py, + &runtime, + async { + tokio::task::yield_now().await; + Ok(true) + }, + runtime_error, + ); + assert!(extract_bool(py, result)); + }); + } + + #[test] + fn sync_runner_maps_a_panicked_future() { + Python::initialize(); + Python::attach(|py| { + let error = run_sync::( + py, + poll_fn(|_| -> Poll> { panic!("route future panicked") }), + runtime_error, + ) + .expect_err("panicked route should become a Python exception"); + + assert!(error.is_instance_of::(py)); + assert_eq!(error.to_string(), "PanicException: route future panicked"); + }); + } + + #[test] + fn sync_runner_maps_a_panicked_error_mapper() { + Python::initialize(); + Python::attach(|py| { + let error = run_sync::( + py, + async { Err(Error::InvalidRequest("invalid".to_string())) }, + panicking_error_mapper, + ) + .expect_err("panicked mapper should become a Python exception"); + + assert!(error.is_instance_of::(py)); + assert_eq!(error.to_string(), "PanicException: error mapper panicked"); + }); + } + + #[test] + fn sync_runner_surfaces_serializer_panics() { + Python::initialize(); + Python::attach(|py| { + let error = run_sync(py, async { Ok(PanickingOutput) }, runtime_error) + .expect_err("serializer panic should become a Python exception"); + + assert!(error.is_instance_of::(py)); + assert_eq!(error.to_string(), "PanicException: serializer panicked"); + }); + } + + #[test] + fn sync_runner_supports_concurrent_callers_on_the_shared_runtime() { + Python::initialize(); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let callers: Vec<_> = (0..2) + .map(|_| { + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + Python::attach(|py| { + extract_bool( + py, + run_sync( + py, + async move { + Ok(tokio::time::timeout(Duration::from_secs(2), barrier.wait()) + .await + .is_ok()) + }, + runtime_error, + ), + ) + }) + }) + }) + .collect(); + let results: Vec<_> = callers + .into_iter() + .map(|caller| caller.join().expect("caller should not panic")) + .collect(); + + assert_eq!(results, vec![true, true]); + } + + #[test] + fn async_runner_surfaces_serializer_panics() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "runtime").expect("module should be created"); + module + .add_function( + wrap_pyfunction!(async_serialization_panic, &module) + .expect("function should wrap"), + ) + .expect("function should register"); + let locals = PyDict::new(py); + locals + .set_item("runtime", &module) + .expect("module should enter Python locals"); + let code = CString::new( + r#" +import asyncio + +async def exercise(): + try: + await runtime.async_serialization_panic() + except BaseException as error: + assert type(error).__name__ == "PanicException" + assert str(error) == "serializer panicked" + else: + raise AssertionError("serializer panic was not raised") + +asyncio.run(exercise()) +"#, + ) + .expect("Python source should not contain null bytes"); + py.run(&code, Some(&locals), Some(&locals)) + .expect("serializer panic should reach the Python awaiter"); + }); + } + + #[test] + fn async_result_delivery_does_not_stall_tokio_workers() { + Python::initialize(); + ASYNC_PROBE_COMPLETED.store(0, Ordering::SeqCst); + Python::attach(|py| { + let module = PyModule::new(py, "runtime").expect("module should be created"); + for function in [ + wrap_pyfunction!(async_runtime_probe, &module).expect("function should wrap"), + wrap_pyfunction!(runtime_worker_count, &module).expect("function should wrap"), + wrap_pyfunction!(runtime_is_responsive, &module).expect("function should wrap"), + ] { + module + .add_function(function) + .expect("function should register"); + } + let locals = PyDict::new(py); + locals + .set_item("runtime", &module) + .expect("module should enter Python locals"); + let code = CString::new( + r#" +import asyncio + +async def exercise(): + worker_count = runtime.runtime_worker_count() + awaitables = [runtime.async_runtime_probe() for _ in range(worker_count)] + assert runtime.runtime_is_responsive(worker_count) + assert await asyncio.gather(*awaitables) == [True] * worker_count + +asyncio.run(exercise()) +"#, + ) + .expect("Python source should not contain null bytes"); + py.run(&code, Some(&locals), Some(&locals)) + .expect("result delivery should leave Tokio workers responsive"); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/function_trace.rs b/litellm-rust/crates/python-bridge/src/function_trace.rs new file mode 100644 index 00000000000..420d237c79d --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/function_trace.rs @@ -0,0 +1,216 @@ +use std::future::Future; +use std::sync::{Arc, Mutex}; + +use serde::Serialize; +use tracing::instrument::WithSubscriber; +use tracing::span::{Attributes, Id}; +use tracing::{Dispatch, Level, Subscriber}; +use tracing_subscriber::filter::{LevelFilter, filter_fn}; +use tracing_subscriber::layer::Context; +use tracing_subscriber::prelude::*; +use tracing_subscriber::registry::LookupSpan; +use tracing_subscriber::{Layer, Registry}; + +use crate::constants::FUNCTION_TRACE_TARGET; + +#[derive(Serialize)] +#[serde(untagged)] +pub(crate) enum TraceResponse { + Plain(T), + Traced { + response: T, + trace: Vec, + }, +} + +pub(crate) async fn trace_call( + future: impl Future>, + enabled: bool, +) -> Result, E> { + if !enabled { + return future.await.map(TraceResponse::Plain); + } + let trace = FunctionTrace::default(); + let response = future.with_subscriber(trace.dispatcher()).await?; + Ok(TraceResponse::Traced { + response, + trace: trace.events(), + }) +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct FunctionTraceEvent { + pub function: &'static str, + pub depth: usize, +} + +#[derive(Clone, Default)] +pub struct FunctionTrace { + events: Arc>>, +} + +impl FunctionTrace { + pub fn dispatcher(&self) -> Dispatch { + let filter = filter_fn(|metadata| { + metadata.is_span() + && metadata.target() == FUNCTION_TRACE_TARGET + && *metadata.level() == Level::TRACE + }) + .with_max_level_hint(LevelFilter::TRACE); + Dispatch::new( + Registry::default().with( + FunctionTraceLayer { + trace: self.clone(), + } + .with_filter(filter), + ), + ) + } + + pub fn events(&self) -> Vec { + self.events + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone() + } +} + +struct FunctionTraceLayer { + trace: FunctionTrace, +} + +impl Layer for FunctionTraceLayer +where + S: Subscriber + for<'lookup> LookupSpan<'lookup>, +{ + fn on_new_span(&self, attributes: &Attributes<'_>, id: &Id, context: Context<'_, S>) { + let depth = context + .span(id) + .map(|span| span.scope().skip(1).count()) + .unwrap_or_default(); + self.trace + .events + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(FunctionTraceEvent { + function: attributes.metadata().name(), + depth, + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] + async fn outer() { + tokio::task::yield_now().await; + inner().await; + } + + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] + async fn inner() { + tokio::task::yield_now().await; + } + + #[tokio::test] + async fn concurrent_futures_keep_separate_traces_across_yields() { + use tracing::instrument::WithSubscriber; + + let first = FunctionTrace::default(); + let second = FunctionTrace::default(); + let outside = FunctionTrace::default(); + + async { + tokio::join!( + outer().with_subscriber(first.dispatcher()), + inner().with_subscriber(second.dispatcher()), + ); + inner().await; + } + .with_subscriber(outside.dispatcher()) + .await; + + assert_eq!( + first.events(), + vec![ + FunctionTraceEvent { + function: "outer", + depth: 0 + }, + FunctionTraceEvent { + function: "inner", + depth: 1 + }, + ], + ); + assert_eq!( + second.events(), + vec![FunctionTraceEvent { + function: "inner", + depth: 0 + }], + ); + assert_eq!( + outside.events(), + vec![FunctionTraceEvent { + function: "inner", + depth: 0 + }], + ); + } + + #[test] + fn records_matching_spans_in_creation_order() { + let trace = FunctionTrace::default(); + let dispatch = trace.dispatcher(); + + tracing::dispatcher::with_default(&dispatch, || { + let _ignored = tracing::trace_span!(target: "other", "ignored"); + let _first = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name"); + let _wrong_level = tracing::debug_span!(target: FUNCTION_TRACE_TARGET, "wrong_level"); + let _second = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name"); + }); + + assert_eq!( + trace.events(), + vec![ + FunctionTraceEvent { + function: "same_name", + depth: 0, + }, + FunctionTraceEvent { + function: "same_name", + depth: 0, + }, + ] + ); + } + + #[test] + fn records_matching_span_nesting_depth() { + let trace = FunctionTrace::default(); + let dispatch = trace.dispatcher(); + + tracing::dispatcher::with_default(&dispatch, || { + let outer = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "outer"); + let _outer_guard = outer.enter(); + let _inner = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "inner"); + }); + + assert_eq!( + trace.events(), + vec![ + FunctionTraceEvent { + function: "outer", + depth: 0, + }, + FunctionTraceEvent { + function: "inner", + depth: 1, + }, + ] + ); + } +} diff --git a/litellm-rust/crates/python-bridge/src/gil.rs b/litellm-rust/crates/python-bridge/src/gil.rs deleted file mode 100644 index e887c8ec1e3..00000000000 --- a/litellm-rust/crates/python-bridge/src/gil.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! GIL accounting. -//! -//! A single chokepoint for releasing the GIL around blocking work. Every -//! blocking call in the bridge goes through [`release_gil`] instead of calling -//! `Python::detach` directly, so the release count stays accurate and we -//! have one place to extend later (timing histograms, per-call labels, etc.). - -use std::sync::atomic::{AtomicU64, Ordering}; - -use pyo3::prelude::*; - -/// Number of times the bridge has released the GIL since process start. -static GIL_RELEASES: AtomicU64 = AtomicU64::new(0); - -/// Release the GIL around `f`, recording the release. -/// -/// `f` must not touch any Python state — that is what makes releasing the GIL -/// safe. Returning the value back to Python re-acquires the GIL at the call -/// site, after `f` has finished. -pub fn release_gil(py: Python<'_>, f: F) -> T -where - F: FnOnce() -> T + Send, - T: Send, -{ - GIL_RELEASES.fetch_add(1, Ordering::Relaxed); - py.detach(f) -} - -/// Total GIL releases performed by the bridge so far. -pub fn release_count() -> u64 { - GIL_RELEASES.load(Ordering::Relaxed) -} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f9e75f45f75..5f36a22370a 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -1,146 +1,18 @@ -use std::collections::HashMap; -use std::time::Duration; - -use litellm_ai_gateway::io::audio_transcription::{ - AudioTranscriptionRequest, audio_transcription as run_audio_transcription, -}; -use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr}; -use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; -use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse}; -use litellm_core::chat_completions::{ - chat_completions as run_chat_completions, chat_completions_decline_reason, -}; -use litellm_core::error::CoreError; -use litellm_core::messages::messages as run_messages; -use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::*; -use pyo3::types::{PyAny, PyDict}; -use serde_json::{Map, Value}; - -mod gil; +mod constants; +mod diagnostics; +mod errors; +mod execution; +pub mod function_trace; mod marshal; +mod routes; -use marshal::{from_py, to_py}; +use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; +use pyo3::prelude::*; +use pyo3::types::PyAny; +use serde_json::Value; -pyo3::create_exception!( - _native, - RustBridgeDeclined, - pyo3::exceptions::PyException, - "The route declined before calling the provider, so the host may retry on its own path." -); - -pyo3::create_exception!( - _native, - RustUpstreamError, - pyo3::exceptions::PyException, - "The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response." -); - -type MarshaledOcrInputs = ( - Value, - Option>, - Map, - Option, -); - -fn messages_response_to_py( - py: Python<'_>, - response: AnthropicMessagesResponse, -) -> PyResult> { - to_py(py, &response) -} - -fn chat_completions_response_to_py( - py: Python<'_>, - response: ChatCompletionsResponse, -) -> PyResult> { - to_py(py, &response) -} - -fn core_error_to_pyerr(err: CoreError) -> PyErr { - match err { - CoreError::Auth(message) => PyValueError::new_err(message), - CoreError::InvalidProvider(_) - | CoreError::InvalidRequest(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) => PyValueError::new_err(err.to_string()), - other => PyRuntimeError::new_err(other.to_string()), - } -} - -/// Map a core error for a route whose host keeps a Python implementation. -/// -/// The distinction the host needs is whether the provider was already called. -/// Everything raised before the request goes out is safe for the host to retry -/// on its own path; anything after it is not, because the provider has already -/// done the work and billed for it. -fn chat_completions_error_to_pyerr(err: CoreError) -> PyErr { - match err { - CoreError::Unsupported(_) - | CoreError::Auth(_) - | CoreError::InvalidProvider(_) - | CoreError::InvalidRequest(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) - | CoreError::Routing(_) - // Nothing reached the provider, so serving it on Python cannot double - // bill and is the only way the caller gets an answer at all. - | CoreError::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), - CoreError::Http { status, body } => { - RustUpstreamError::new_err((status, format!("{status}: {body}"))) - } - CoreError::Network(message) | CoreError::InvalidResponse(message) => { - RustUpstreamError::new_err((0u16, message)) - } - } -} - -fn optional_object_to_map( - py: Python<'_>, - name: &'static str, - value: Option>, -) -> PyResult> { - match value { - Some(value) => match from_py(value.bind(py))? { - Value::Object(map) => Ok(map), - _ => Err(PyValueError::new_err(format!("{name} must be a dict"))), - }, - None => Ok(Map::new()), - } -} - -fn optional_timeout(timeout_seconds: Option) -> Option { - timeout_seconds.and_then(|secs| { - if secs.is_finite() && secs > 0.0 { - Some(Duration::from_secs_f64(secs)) - } else { - None - } - }) -} - -fn marshal_headers( - py: Python<'_>, - headers: Option>, -) -> PyResult> { - let value = match headers { - Some(headers) => from_py(headers.bind(py))?, - None => Value::Object(Map::new()), - }; - let Value::Object(headers) = value else { - return Err(PyValueError::new_err("headers must be a dict")); - }; - headers - .into_iter() - .map(|(name, value)| { - value - .as_str() - .map(|value| (name, value.to_string())) - .ok_or_else(|| PyValueError::new_err("header values must be strings")) - }) - .collect() -} +use crate::errors::core_error_to_pyerr; +use crate::marshal::{marshal_headers, optional_timeout}; #[pyclass] struct ResponsesWebSocketConnection { @@ -155,16 +27,16 @@ impl ResponsesWebSocketConnection { _cls: &Bound<'py, pyo3::types::PyType>, py: Python<'py>, url: String, - headers: Option>, + #[pyo3(from_py_with = litellm_python_interop::from_py)] headers: Option, timeout_seconds: Option, ) -> PyResult> { - let headers = marshal_headers(py, headers)?; + let headers = marshal_headers(headers)?; let timeout = optional_timeout(timeout_seconds); pyo3_async_runtimes::tokio::future_into_py(py, async move { let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout) .await .map_err(core_error_to_pyerr)?; - Python::attach(|py| Py::new(py, ResponsesWebSocketConnection { inner })) + Ok(ResponsesWebSocketConnection { inner }) }) } @@ -190,445 +62,126 @@ impl ResponsesWebSocketConnection { } } -fn marshal_inputs( - py: Python<'_>, - document: Py, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult { - let document = from_py(document.bind(py))?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let timeout = optional_timeout(timeout_seconds); +#[pymodule(gil_used = false)] +mod _native { + use pyo3::prelude::*; - Ok((document, extra_headers, optional_params, timeout)) -} - -#[pyfunction] -#[pyo3(signature = (model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn ocr( - py: Python<'_>, - model: String, - document: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (document, extra_headers, optional_params, timeout) = marshal_inputs( - py, - document, - extra_headers, - optional_params, - timeout_seconds, - )?; - - let result = gil::release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_ocr(OcrRequest { - model: &model, - document, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - })) - }); - - match result { - Ok(value) => to_py(py, &value), - Err(err) => Err(core_error_to_pyerr(err)), + #[pymodule_init] + fn init(module: &Bound<'_, PyModule>) -> PyResult<()> { + super::errors::register(module)?; + super::routes::register(module)?; + module.add_class::()?; + super::diagnostics::register(module) } } -#[pyfunction] -#[pyo3(signature = (model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn aocr( - py: Python<'_>, - model: String, - document: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (document, extra_headers, optional_params, timeout) = marshal_inputs( - py, - document, - extra_headers, - optional_params, - timeout_seconds, - )?; +#[cfg(test)] +mod tests { + use std::ffi::CString; + use std::time::Duration; - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let value = run_ocr(OcrRequest { - model: &model, - document, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - }) - .await - .map_err(core_error_to_pyerr)?; + use futures_util::{SinkExt, StreamExt}; + use pyo3::types::PyDict; + use tokio::net::TcpListener; + use tokio_tungstenite::{accept_async, tungstenite::Message}; - Python::attach(|py| to_py(py, &value)) - }) -} + use super::*; -#[pyfunction] -#[pyo3(signature = (model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn transcription( - py: Python<'_>, - model: String, - audio: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let audio = from_py(audio.bind(py))?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let timeout = optional_timeout(timeout_seconds); - let result = gil::release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_audio_transcription( - AudioTranscriptionRequest { - model: &model, - audio, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - }, - )) - }); - match result { - Ok(value) => to_py(py, &value), - Err(err) => Err(core_error_to_pyerr(err)), + #[test] + fn module_registration_preserves_the_public_surface() { + Python::initialize(); + Python::attach(|py| { + let module = pyo3::wrap_pymodule!(_native)(py).into_bound(py); + + let expected = [ + "RustBridgeDeclined", + "RustUpstreamError", + "ocr", + "aocr", + "transcription", + "atranscription", + "messages", + "amessages", + "chat_completions_decline", + "chat_completions", + "achat_completions", + "ResponsesWebSocketConnection", + "gil_stats", + ]; + + let public_names: Vec = module + .dict() + .keys() + .extract::>() + .expect("module names should be strings") + .into_iter() + .filter(|name| !name.starts_with("__")) + .collect(); + assert_eq!(public_names, expected); + }); + } + + #[test] + fn responses_websocket_connection_round_trips_through_python() { + Python::initialize(); + let runtime = pyo3_async_runtimes::tokio::get_runtime(); + let listener = runtime + .block_on(TcpListener::bind("127.0.0.1:0")) + .expect("listener should bind"); + let address = listener + .local_addr() + .expect("listener should have an address"); + let server = runtime.spawn(async move { + let (stream, _) = listener.accept().await.expect("server should accept"); + let mut socket = accept_async(stream) + .await + .expect("handshake should succeed"); + + let message = socket + .next() + .await + .expect("client should send a frame") + .expect("client frame should be valid"); + assert_eq!(message, Message::Text("from-python".into())); + socket + .send(Message::Text("from-server".into())) + .await + .expect("server should reply"); + assert!(matches!(socket.next().await, Some(Ok(Message::Close(_))))); + }); + + Python::attach(|py| { + let module = pyo3::wrap_pymodule!(_native)(py).into_bound(py); + let locals = PyDict::new(py); + locals + .set_item("native", &module) + .expect("module should enter Python locals"); + locals + .set_item("url", format!("ws://{address}")) + .expect("URL should enter Python locals"); + let code = CString::new( + r#" +import asyncio + +async def exercise(): + connection = await native.ResponsesWebSocketConnection.connect(url) + assert type(connection) is native.ResponsesWebSocketConnection + await connection.send_text("from-python") + assert await connection.recv_text() == "from-server" + await connection.close() + assert await connection.recv_text() is None + +asyncio.run(asyncio.wait_for(exercise(), timeout=5)) +"#, + ) + .expect("Python source should not contain null bytes"); + py.run(&code, Some(&locals), Some(&locals)) + .expect("Python WebSocket methods should round trip"); + }); + + runtime + .block_on(async { tokio::time::timeout(Duration::from_secs(5), server).await }) + .expect("server should finish") + .expect("server task should not panic"); } } - -#[pyfunction] -#[pyo3(signature = (model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn atranscription( - py: Python<'_>, - model: String, - audio: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - optional_params: Option>, - timeout_seconds: Option, -) -> PyResult> { - let audio = from_py(audio.bind(py))?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let timeout = optional_timeout(timeout_seconds); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let value = run_audio_transcription(AudioTranscriptionRequest { - model: &model, - audio, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - }) - .await - .map_err(core_error_to_pyerr)?; - Python::attach(|py| to_py(py, &value)) - }) -} - -type MarshaledMessagesInputs = (Value, Option>, Option); - -fn marshal_messages_inputs( - py: Python<'_>, - body: Py, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult { - let body: Value = from_py(body.bind(py))?; - if !body.is_object() { - return Err(PyValueError::new_err("body must be a dict")); - } - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - Ok((body, extra_headers, optional_timeout(timeout_seconds))) -} - -#[pyfunction] -#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn messages( - py: Python<'_>, - model: String, - body: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (body, extra_headers, timeout) = - marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; - - let result = gil::release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_messages(MessagesRequest { - model: &model, - body, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - })) - }); - - match result { - Ok(response) => messages_response_to_py(py, response), - Err(err) => Err(core_error_to_pyerr(err)), - } -} - -#[pyfunction] -#[pyo3(signature = (model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn amessages( - py: Python<'_>, - model: String, - body: Py, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (body, extra_headers, timeout) = - marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let response = run_messages(MessagesRequest { - model: &model, - body, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }) - .await - .map_err(core_error_to_pyerr)?; - - Python::attach(|py| messages_response_to_py(py, response)) - }) -} - -type MarshaledChatCompletionsInputs = ( - Value, - Map, - Option>, - Option, -); - -fn marshal_chat_completions_inputs( - py: Python<'_>, - messages: Py, - optional_params: Option>, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult { - let messages: Value = from_py(messages.bind(py))?; - if !messages.is_array() { - return Err(PyValueError::new_err("messages must be a list")); - } - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - let extra_headers = match extra_headers { - Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), - None => None, - }; - Ok(( - messages, - optional_params, - extra_headers, - optional_timeout(timeout_seconds), - )) -} - -/// The decline reason for this request, or `None` when the Rust path accepts -/// it. Resolves no credentials and performs no I/O, so a host can ask before -/// committing to either path. -#[pyfunction] -#[pyo3(signature = (model, messages, optional_params=None, custom_llm_provider=None))] -fn chat_completions_decline( - py: Python<'_>, - model: String, - messages: Py, - optional_params: Option>, - custom_llm_provider: Option, -) -> PyResult> { - let messages = from_py(messages.bind(py))?; - let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; - Ok(chat_completions_decline_reason( - &model, - custom_llm_provider.as_deref(), - messages, - &optional_params, - ) - .map(str::to_string)) -} - -#[pyfunction] -#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn chat_completions( - py: Python<'_>, - model: String, - messages: Py, - optional_params: Option>, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (messages, optional_params, extra_headers, timeout) = marshal_chat_completions_inputs( - py, - messages, - optional_params, - extra_headers, - timeout_seconds, - )?; - - let result = gil::release_gil(py, || { - pyo3_async_runtimes::tokio::get_runtime().block_on(run_chat_completions( - ChatCompletionsRequest { - model: &model, - messages, - optional_params, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }, - )) - }); - - match result { - Ok(response) => chat_completions_response_to_py(py, response), - Err(err) => Err(chat_completions_error_to_pyerr(err)), - } -} - -#[pyfunction] -#[pyo3(signature = (model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None))] -#[allow(clippy::too_many_arguments)] -fn achat_completions( - py: Python<'_>, - model: String, - messages: Py, - optional_params: Option>, - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - extra_headers: Option>, - timeout_seconds: Option, -) -> PyResult> { - let (messages, optional_params, extra_headers, timeout) = marshal_chat_completions_inputs( - py, - messages, - optional_params, - extra_headers, - timeout_seconds, - )?; - - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let response = run_chat_completions(ChatCompletionsRequest { - model: &model, - messages, - optional_params, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }) - .await - .map_err(chat_completions_error_to_pyerr)?; - - Python::attach(|py| chat_completions_response_to_py(py, response)) - }) -} - -#[pyfunction] -fn gil_stats(py: Python<'_>) -> PyResult> { - let stats = PyDict::new(py); - stats.set_item("releases", gil::release_count())?; - Ok(stats.into_any().unbind()) -} - -#[pymodule] -fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py = module.py(); - module.add_function(wrap_pyfunction!(ocr, module)?)?; - module.add_function(wrap_pyfunction!(aocr, module)?)?; - module.add_function(wrap_pyfunction!(transcription, module)?)?; - module.add_function(wrap_pyfunction!(atranscription, module)?)?; - module.add_function(wrap_pyfunction!(messages, module)?)?; - module.add_function(wrap_pyfunction!(amessages, module)?)?; - module.add("RustBridgeDeclined", py.get_type::())?; - module.add("RustUpstreamError", py.get_type::())?; - module.add_function(wrap_pyfunction!(chat_completions_decline, module)?)?; - module.add_function(wrap_pyfunction!(chat_completions, module)?)?; - module.add_function(wrap_pyfunction!(achat_completions, module)?)?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(gil_stats, module)?)?; - Ok(()) -} diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index c3d0638427c..a14e4b55d82 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -1,20 +1,104 @@ +use std::collections::HashMap; +use std::time::Duration; + use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use serde::Serialize; -use serde::de::DeserializeOwned; +use serde_json::{Map, Value}; -pub fn from_py(value: &Bound<'_, PyAny>) -> PyResult -where - T: DeserializeOwned, -{ - pythonize::depythonize(value).map_err(|error| PyValueError::new_err(error.to_string())) +pub(crate) struct RouteOptions { + pub(crate) model: String, + pub(crate) api_key: Option, + pub(crate) api_base: Option, + pub(crate) custom_llm_provider: Option, + pub(crate) extra_headers: Option>, + pub(crate) timeout: Option, } -pub fn to_py(py: Python<'_>, value: &T) -> PyResult> -where - T: Serialize + ?Sized, -{ - pythonize::pythonize(py, value) - .map(Bound::unbind) - .map_err(|error| PyValueError::new_err(error.to_string())) +pub(crate) struct RouteOptionsInputs { + pub(crate) model: String, + pub(crate) api_key: Option, + pub(crate) api_base: Option, + pub(crate) custom_llm_provider: Option, + pub(crate) extra_headers: Option, + pub(crate) timeout_seconds: Option, +} + +impl RouteOptions { + pub(crate) fn from_python(inputs: RouteOptionsInputs) -> PyResult { + Ok(Self { + model: inputs.model, + api_key: inputs.api_key, + api_base: inputs.api_base, + custom_llm_provider: inputs.custom_llm_provider, + extra_headers: optional_object("extra_headers", inputs.extra_headers)?, + timeout: optional_timeout(inputs.timeout_seconds), + }) + } +} + +pub(crate) fn required_value( + name: &'static str, + value: Value, + expected: fn(&Value) -> bool, + expected_name: &'static str, +) -> PyResult { + if expected(&value) { + return Ok(value); + } + Err(PyValueError::new_err(format!( + "{name} must be a {expected_name}" + ))) +} + +pub(crate) fn object_or_empty( + name: &'static str, + value: Option, +) -> PyResult> { + match value { + Some(value) => object(name, value), + None => Ok(Map::new()), + } +} + +fn optional_object( + name: &'static str, + value: Option, +) -> PyResult>> { + value.map(|value| object(name, value)).transpose() +} + +fn object(name: &'static str, value: Value) -> PyResult> { + match value { + Value::Object(map) => Ok(map), + _ => Err(PyValueError::new_err(format!("{name} must be a dict"))), + } +} + +pub(crate) fn optional_timeout(timeout_seconds: Option) -> Option { + timeout_seconds.and_then(|secs| { + if secs.is_finite() && secs > 0.0 { + Some(Duration::from_secs_f64(secs)) + } else { + None + } + }) +} + +pub(crate) fn marshal_headers(headers: Option) -> PyResult> { + let value = match headers { + Some(headers) => headers, + None => Value::Object(Map::new()), + }; + let Value::Object(headers) = value else { + return Err(PyValueError::new_err("headers must be a dict")); + }; + headers + .into_iter() + .map(|(name, value)| { + value + .as_str() + .map(|value| (name, value.to_string())) + .ok_or_else(|| PyValueError::new_err("header values must be strings")) + }) + .collect() } diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs new file mode 100644 index 00000000000..10b86132be7 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs @@ -0,0 +1,71 @@ +use litellm_core::Error; +use std::future::Future; + +use litellm_core::audio_transcription::{ + AudioTranscriptionRequest, audio_transcription as run_audio_transcription, +}; +use pyo3::prelude::*; +use serde_json::Value; + +use crate::errors::core_error_to_pyerr; +use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty}; + +fn prepare_transcription( + inputs: AudioTranscriptionInputs, +) -> PyResult> + Send + 'static> { + let audio = inputs.audio; + let options = RouteOptions::from_python(RouteOptionsInputs { + model: inputs.model, + api_key: inputs.api_key, + api_base: inputs.api_base, + custom_llm_provider: inputs.custom_llm_provider, + extra_headers: inputs.extra_headers, + timeout_seconds: inputs.timeout_seconds, + })?; + let optional_params = object_or_empty("optional_params", inputs.optional_params)?; + + Ok(async move { + let RouteOptions { + model, + api_key, + api_base, + custom_llm_provider, + extra_headers, + timeout, + } = options; + run_audio_transcription(AudioTranscriptionRequest { + model: &model, + audio, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + }) + .await + }) +} + +bridge_route! { + sync = transcription, + asynchronous = atranscription, + inputs = AudioTranscriptionInputs, + required = { + model: String, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + audio: Value, + }, + optional = { + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + extra_headers: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + optional_params: Option, + timeout_seconds: Option, + }, + prepare = prepare_transcription, + errors = core_error_to_pyerr, +} diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs new file mode 100644 index 00000000000..68b7762cb10 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs @@ -0,0 +1,91 @@ +use litellm_core::Error; +use std::future::Future; + +use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse}; +use litellm_core::chat_completions::{ + chat_completions as run_chat_completions, chat_completions_decline_reason, +}; +use pyo3::prelude::*; +use serde_json::Value; + +use crate::errors::chat_completions_error_to_pyerr; +use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty, required_value}; + +fn prepare_chat_completions( + inputs: ChatCompletionsInputs, +) -> PyResult> + Send + 'static> { + let messages = required_value("messages", inputs.messages, Value::is_array, "list")?; + let optional_params = object_or_empty("optional_params", inputs.optional_params)?; + let options = RouteOptions::from_python(RouteOptionsInputs { + model: inputs.model, + api_key: inputs.api_key, + api_base: inputs.api_base, + custom_llm_provider: inputs.custom_llm_provider, + extra_headers: inputs.extra_headers, + timeout_seconds: inputs.timeout_seconds, + })?; + + Ok(async move { + let RouteOptions { + model, + api_key, + api_base, + custom_llm_provider, + extra_headers, + timeout, + } = options; + run_chat_completions(ChatCompletionsRequest { + model: &model, + messages, + optional_params, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + }) + .await + }) +} + +#[pyfunction] +#[pyo3(signature = (model, messages, optional_params=None, custom_llm_provider=None))] +fn chat_completions_decline( + model: String, + #[pyo3(from_py_with = litellm_python_interop::from_py)] messages: Value, + #[pyo3(from_py_with = litellm_python_interop::from_py)] optional_params: Option, + custom_llm_provider: Option, +) -> PyResult> { + let optional_params = object_or_empty("optional_params", optional_params)?; + Ok(chat_completions_decline_reason( + &model, + custom_llm_provider.as_deref(), + messages, + &optional_params, + ) + .map(str::to_string)) +} + +bridge_route! { + sync = chat_completions, + asynchronous = achat_completions, + inputs = ChatCompletionsInputs, + required = { + model: String, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + messages: Value, + }, + optional = { + #[pyo3(from_py_with = litellm_python_interop::from_py)] + optional_params: Option, + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + extra_headers: Option, + timeout_seconds: Option, + }, + prepare = prepare_chat_completions, + errors = chat_completions_error_to_pyerr, + extra = [chat_completions_decline], +} diff --git a/litellm-rust/crates/python-bridge/src/routes/definition.rs b/litellm-rust/crates/python-bridge/src/routes/definition.rs new file mode 100644 index 00000000000..21a7fd5a766 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/definition.rs @@ -0,0 +1,429 @@ +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::*; +use pyo3::types::PyCFunction; + +macro_rules! bridge_route { + ( + sync = $sync_name:ident, + asynchronous = $async_name:ident, + inputs = $inputs:ident, + required = { $($(#[$required_attr:meta])* $required_name:ident: $required_type:ty),+ $(,)? }, + optional = { $($(#[$optional_attr:meta])* $optional_name:ident: $optional_type:ty),* $(,)? }, + prepare = $prepare:path, + errors = $map_error:path + $(, extra = [$($extra:ident),* $(,)?])? + $(,)? + ) => { + struct $inputs { + $($required_name: $required_type,)* + $($optional_name: $optional_type),* + } + + #[pyfunction] + #[pyo3(signature = ($($required_name),*, $($optional_name=None,)* trace=false))] + #[allow(clippy::too_many_arguments)] + fn $sync_name( + py: pyo3::Python<'_>, + $($(#[$required_attr])* $required_name: $required_type,)* + $($(#[$optional_attr])* $optional_name: $optional_type,)* + trace: bool, + ) -> pyo3::PyResult> { + let future = $prepare($inputs { + $($required_name,)* + $($optional_name),* + })?; + $crate::execution::run_sync( + py, + $crate::function_trace::trace_call(future, trace), + $map_error, + ) + } + + #[pyfunction] + #[pyo3(signature = ($($required_name),*, $($optional_name=None,)* trace=false))] + #[allow(clippy::too_many_arguments)] + fn $async_name( + py: pyo3::Python<'_>, + $($(#[$required_attr])* $required_name: $required_type,)* + $($(#[$optional_attr])* $optional_name: $optional_type,)* + trace: bool, + ) -> pyo3::PyResult> { + let future = $prepare($inputs { + $($required_name,)* + $($optional_name),* + })?; + $crate::execution::run_async( + py, + $crate::function_trace::trace_call(future, trace), + $map_error, + ) + } + + pub(super) fn register( + module: &pyo3::Bound<'_, pyo3::types::PyModule>, + ) -> pyo3::PyResult<()> { + $($($crate::routes::definition::add_function(module, pyo3::wrap_pyfunction!($extra, module)?)?;)*)? + $crate::routes::definition::add_function(module, pyo3::wrap_pyfunction!($sync_name, module)?)?; + $crate::routes::definition::add_function(module, pyo3::wrap_pyfunction!($async_name, module)?)?; + Ok(()) + } + }; +} + +pub(super) fn add_function( + module: &Bound<'_, PyModule>, + function: Bound<'_, PyCFunction>, +) -> PyResult<()> { + let name: String = function.getattr("__name__")?.extract()?; + if module.hasattr(&name)? { + return Err(PyRuntimeError::new_err(format!( + "duplicate native route: {name}" + ))); + } + module.add_function(function) +} + +#[cfg(test)] +mod tests { + use std::ffi::CString; + use std::sync::atomic::{AtomicBool, Ordering}; + + use litellm_core::error::Error; + use pyo3::exceptions::PyLookupError; + use pyo3::types::{PyDict, PyList}; + + use super::*; + + mod synthetic { + use std::future::{Future, pending}; + + use super::*; + + static FUTURE_DROPPED: AtomicBool = AtomicBool::new(false); + + struct DropGuard; + + impl Drop for DropGuard { + fn drop(&mut self) { + FUTURE_DROPPED.store(true, Ordering::SeqCst); + } + } + + #[pyfunction] + fn future_dropped() -> bool { + FUTURE_DROPPED.load(Ordering::SeqCst) + } + + bridge_route! { + sync = echo, + asynchronous = aecho, + inputs = EchoInputs, + required = { value: String }, + optional = {}, + prepare = prepare_echo, + errors = map_error, + extra = [future_dropped], + } + + fn prepare_echo( + inputs: EchoInputs, + ) -> PyResult> + Send + 'static> { + FUTURE_DROPPED.store(false, Ordering::SeqCst); + let drop_guard = (inputs.value == "pending").then_some(DropGuard); + Ok(async move { + let _drop_guard = drop_guard; + tokio::task::yield_now().await; + match inputs.value.as_str() { + "error" => Err(Error::InvalidRequest("synthetic error".to_string())), + "map_panic" => Err(Error::InvalidRequest("panic in mapper".to_string())), + "panic" => panic!("synthetic panic"), + "pending" => { + pending::<()>().await; + unreachable!() + } + _ => Ok(inputs.value), + } + }) + } + + fn map_error(error: Error) -> PyErr { + if matches!(&error, Error::InvalidRequest(message) if message == "panic in mapper") { + panic!("synthetic mapper panic") + } + PyLookupError::new_err(error.to_string()) + } + } + + #[test] + fn sync_and_async_route_signatures_match_the_python_contract() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "routes").expect("module should be created"); + crate::routes::register(&module).expect("routes should register"); + let routes = [ + ( + "ocr", + "aocr", + "(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None, trace=False)", + ), + ( + "transcription", + "atranscription", + "(model, audio, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None, trace=False)", + ), + ( + "messages", + "amessages", + "(model, body, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None, trace=False)", + ), + ( + "chat_completions", + "achat_completions", + "(model, messages, optional_params=None, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, timeout_seconds=None, trace=False)", + ), + ]; + + for (sync_name, async_name, expected) in routes { + let sync_signature: String = module + .getattr(sync_name) + .and_then(|function| function.getattr("__text_signature__")) + .and_then(|signature| signature.extract()) + .expect("sync signature should be available"); + let async_signature: String = module + .getattr(async_name) + .and_then(|function| function.getattr("__text_signature__")) + .and_then(|signature| signature.extract()) + .expect("async signature should be available"); + + assert_eq!(sync_signature, expected); + assert_eq!(async_signature, expected); + } + }); + } + + #[test] + fn sync_and_async_routes_apply_the_same_input_validation() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "routes").expect("module should be created"); + crate::routes::register(&module).expect("routes should register"); + + let invalid_messages = PyDict::new(py); + let sync_chat_error = module + .getattr("chat_completions") + .and_then(|function| function.call1(("model", &invalid_messages))) + .expect_err("sync chat should reject a non-list messages value"); + let async_chat_error = module + .getattr("achat_completions") + .and_then(|function| function.call1(("model", &invalid_messages))) + .expect_err("async chat should reject a non-list messages value"); + + assert_eq!( + sync_chat_error.to_string(), + "ValueError: messages must be a list" + ); + assert_eq!(async_chat_error.to_string(), sync_chat_error.to_string()); + + let invalid_body = PyList::empty(py); + let sync_messages_error = module + .getattr("messages") + .and_then(|function| function.call1(("model", &invalid_body))) + .expect_err("sync Messages should reject a non-dict body"); + let async_messages_error = module + .getattr("amessages") + .and_then(|function| function.call1(("model", &invalid_body))) + .expect_err("async Messages should reject a non-dict body"); + + assert_eq!( + sync_messages_error.to_string(), + "ValueError: body must be a dict" + ); + assert_eq!( + async_messages_error.to_string(), + sync_messages_error.to_string() + ); + + let invalid_headers = PyList::empty(py); + let kwargs = PyDict::new(py); + kwargs + .set_item("extra_headers", &invalid_headers) + .expect("kwargs should accept extra_headers"); + let document = PyDict::new(py); + + for (sync_name, async_name) in [("ocr", "aocr"), ("transcription", "atranscription")] { + let sync_error = module + .getattr(sync_name) + .and_then(|function| function.call(("model", &document), Some(&kwargs))) + .expect_err("sync route should reject non-dict extra_headers"); + let async_error = module + .getattr(async_name) + .and_then(|function| function.call(("model", &document), Some(&kwargs))) + .expect_err("async route should reject non-dict extra_headers"); + + assert_eq!( + sync_error.to_string(), + "ValueError: extra_headers must be a dict" + ); + assert_eq!(async_error.to_string(), sync_error.to_string()); + } + }); + } + + #[test] + fn route_input_validation_preserves_left_to_right_order() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "routes").expect("module should be created"); + crate::routes::register(&module).expect("routes should register"); + let invalid = PyList::empty(py); + + let chat_kwargs = PyDict::new(py); + chat_kwargs + .set_item("optional_params", &invalid) + .expect("kwargs should accept optional_params"); + chat_kwargs + .set_item("extra_headers", &invalid) + .expect("kwargs should accept extra_headers"); + let invalid_messages = PyDict::new(py); + let error = module + .getattr("chat_completions") + .and_then(|function| { + function.call(("model", &invalid_messages), Some(&chat_kwargs)) + }) + .expect_err("messages should be validated first"); + assert_eq!(error.to_string(), "ValueError: messages must be a list"); + + let valid_messages = PyList::empty(py); + let error = module + .getattr("chat_completions") + .and_then(|function| function.call(("model", &valid_messages), Some(&chat_kwargs))) + .expect_err("optional_params should be validated before headers"); + assert_eq!( + error.to_string(), + "ValueError: optional_params must be a dict" + ); + + let headers_kwargs = PyDict::new(py); + headers_kwargs + .set_item("extra_headers", &invalid) + .expect("kwargs should accept extra_headers"); + let invalid_body = PyList::empty(py); + let error = module + .getattr("messages") + .and_then(|function| function.call(("model", &invalid_body), Some(&headers_kwargs))) + .expect_err("body should be validated before headers"); + assert_eq!(error.to_string(), "ValueError: body must be a dict"); + + let invalid_payload = + PyModule::new(py, "invalid_payload").expect("invalid payload should be created"); + for name in ["ocr", "transcription"] { + let error = module + .getattr(name) + .and_then(|function| { + function.call(("model", &invalid_payload), Some(&headers_kwargs)) + }) + .expect_err("payload should be validated before headers"); + assert!(!error.to_string().contains("extra_headers")); + } + }); + } + + #[test] + fn generated_routes_execute_sync_and_async_contracts() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "synthetic").expect("module should be created"); + synthetic::register(&module).expect("routes should register"); + + let sync_value: String = module + .getattr("echo") + .and_then(|function| function.call1(("sync",))) + .and_then(|value| value.extract()) + .expect("sync route should return its value"); + assert_eq!(sync_value, "sync"); + + let sync_error = module + .getattr("echo") + .and_then(|function| function.call1(("error",))) + .expect_err("sync route should map its error"); + assert!(sync_error.is_instance_of::(py)); + assert_eq!( + sync_error.to_string(), + "LookupError: invalid request: synthetic error" + ); + + let locals = PyDict::new(py); + locals + .set_item("routes", &module) + .expect("module should enter Python locals"); + let code = CString::new( + r#" +import asyncio + +async def exercise(): + assert await routes.aecho("async") == "async" + + try: + await routes.aecho("error") + except LookupError as error: + assert str(error) == "invalid request: synthetic error" + else: + raise AssertionError("mapped error was not raised") + + try: + await routes.aecho("panic") + except BaseException as error: + assert type(error).__name__ == "PanicException" + assert str(error) == "synthetic panic" + else: + raise AssertionError("panic was not raised") + + try: + await routes.aecho("map_panic") + except BaseException as error: + assert type(error).__name__ == "PanicException" + assert str(error) == "synthetic mapper panic" + else: + raise AssertionError("mapper panic was not raised") + + task = asyncio.ensure_future(routes.aecho("pending")) + await asyncio.sleep(0) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + else: + raise AssertionError("cancelled route completed") + + for _ in range(100): + if routes.future_dropped(): + break + await asyncio.sleep(0.001) + assert routes.future_dropped() + +asyncio.run(exercise()) +"#, + ) + .expect("Python source should not contain null bytes"); + py.run(&code, Some(&locals), Some(&locals)) + .expect("async route contract should hold"); + }); + } + + #[test] + fn route_registration_rejects_duplicate_python_names() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "synthetic").expect("module should be created"); + synthetic::register(&module).expect("first registration should succeed"); + let error = synthetic::register(&module) + .expect_err("duplicate registration should be rejected"); + + assert_eq!( + error.to_string(), + "RuntimeError: duplicate native route: future_dropped" + ); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages.rs b/litellm-rust/crates/python-bridge/src/routes/messages.rs new file mode 100644 index 00000000000..2bb64a7a763 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages.rs @@ -0,0 +1,65 @@ +use litellm_core::Error; +use litellm_core::messages::messages as run_messages; +use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; +use pyo3::prelude::*; +use serde_json::Value; +use std::future::Future; + +use crate::errors::core_error_to_pyerr; +use crate::marshal::{RouteOptions, RouteOptionsInputs, required_value}; + +fn prepare_messages( + inputs: MessagesInputs, +) -> PyResult> + Send + 'static> { + let body = required_value("body", inputs.body, Value::is_object, "dict")?; + let options = RouteOptions::from_python(RouteOptionsInputs { + model: inputs.model, + api_key: inputs.api_key, + api_base: inputs.api_base, + custom_llm_provider: inputs.custom_llm_provider, + extra_headers: inputs.extra_headers, + timeout_seconds: inputs.timeout_seconds, + })?; + + Ok(async move { + let RouteOptions { + model, + api_key, + api_base, + custom_llm_provider, + extra_headers, + timeout, + } = options; + run_messages(MessagesRequest { + model: &model, + body, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + }) + .await + }) +} + +bridge_route! { + sync = messages, + asynchronous = amessages, + inputs = MessagesInputs, + required = { + model: String, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + body: Value, + }, + optional = { + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + extra_headers: Option, + timeout_seconds: Option, + }, + prepare = prepare_messages, + errors = core_error_to_pyerr, +} diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs new file mode 100644 index 00000000000..bf611c26d44 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -0,0 +1,16 @@ +use pyo3::prelude::*; + +#[macro_use] +mod definition; + +mod audio_transcription; +mod chat_completions; +mod messages; +mod ocr; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + ocr::register(module)?; + audio_transcription::register(module)?; + messages::register(module)?; + chat_completions::register(module) +} diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr.rs b/litellm-rust/crates/python-bridge/src/routes/ocr.rs new file mode 100644 index 00000000000..5588c400972 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/ocr.rs @@ -0,0 +1,73 @@ +use litellm_core::Error; +use std::future::Future; + +use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr}; +use pyo3::prelude::*; +use serde_json::Value; + +use crate::errors::core_error_to_pyerr; +use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty}; + +fn prepare_ocr( + inputs: OcrInputs, +) -> PyResult> + Send + 'static> { + let document = inputs.document; + let options = RouteOptions::from_python(RouteOptionsInputs { + model: inputs.model, + api_key: inputs.api_key, + api_base: inputs.api_base, + custom_llm_provider: inputs.custom_llm_provider, + extra_headers: inputs.extra_headers, + timeout_seconds: inputs.timeout_seconds, + })?; + let optional_params = object_or_empty("optional_params", inputs.optional_params)?; + + Ok(async move { + let RouteOptions { + model, + api_key, + api_base, + custom_llm_provider, + extra_headers, + timeout, + } = options; + run_ocr(OcrRequest { + model: &model, + document, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + callbacks: Vec::new(), + guardrails: Vec::new(), + request_metadata: Default::default(), + litellm_call_id: None, + }) + .await + }) +} + +bridge_route! { + sync = ocr, + asynchronous = aocr, + inputs = OcrInputs, + required = { + model: String, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + document: Value, + }, + optional = { + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + extra_headers: Option, + #[pyo3(from_py_with = litellm_python_interop::from_py)] + optional_params: Option, + timeout_seconds: Option, + }, + prepare = prepare_ocr, + errors = core_error_to_pyerr, +} diff --git a/litellm-rust/crates/python-bridge/src/routes/runtime.rs b/litellm-rust/crates/python-bridge/src/routes/runtime.rs new file mode 100644 index 00000000000..87a0c3e0104 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/runtime.rs @@ -0,0 +1,423 @@ +use std::future::Future; +use std::panic::AssertUnwindSafe; +use std::time::Duration; + +use futures_util::FutureExt; +use litellm_core::error::Error; +use litellm_python_interop::{Pythonized, panic_to_pyerr, release_gil}; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::*; +use serde::Serialize; +use tokio::runtime::{Handle, Runtime}; +use tokio::time::{self, MissedTickBehavior}; + +pub(super) fn run_sync( + py: Python<'_>, + future: F, + map_error: fn(Error) -> PyErr, +) -> PyResult> +where + T: Serialize + Send + 'static, + F: Future> + Send + 'static, +{ + run_sync_on( + py, + pyo3_async_runtimes::tokio::get_runtime(), + future, + map_error, + ) +} + +fn run_sync_on( + py: Python<'_>, + runtime: &Runtime, + future: F, + map_error: fn(Error) -> PyErr, +) -> PyResult> +where + T: Serialize + Send + 'static, + F: Future> + Send + 'static, +{ + if Handle::try_current().is_ok() { + return Err(PyRuntimeError::new_err( + "synchronous native routes cannot run from a Tokio context; use the async route", + )); + } + + let result = release_gil(py, move || runtime.block_on(wait_for_sync_result(future)))?; + let result = map_core_result(result, map_error)?; + Pythonized(result).into_pyobject(py).map(Bound::unbind) +} + +pub(super) fn run_async( + py: Python<'_>, + future: F, + map_error: fn(Error) -> PyErr, +) -> PyResult> +where + T: Serialize + Send + 'static, + F: Future> + Send + 'static, +{ + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let result = catch_route_panic(future).await?; + let result = map_core_result(result, map_error)?; + Ok(Pythonized(result)) + }) +} + +fn map_core_result(result: Result, map_error: fn(Error) -> PyErr) -> PyResult { + match result { + Ok(value) => Ok(value), + Err(error) => Err( + std::panic::catch_unwind(AssertUnwindSafe(|| map_error(error))) + .map_err(panic_to_pyerr)?, + ), + } +} + +async fn catch_route_panic(future: F) -> PyResult> +where + F: Future>, +{ + AssertUnwindSafe(future) + .catch_unwind() + .await + .map_err(panic_to_pyerr) +} + +async fn wait_for_sync_result(future: F) -> PyResult> +where + F: Future>, +{ + let future = catch_route_panic(future); + tokio::pin!(future); + + let signal_interval = Duration::from_millis(50); + let mut signal_checks = + time::interval_at(time::Instant::now() + signal_interval, signal_interval); + signal_checks.set_missed_tick_behavior(MissedTickBehavior::Delay); + loop { + tokio::select! { + result = &mut future => return result, + _ = signal_checks.tick() => Python::attach(|py| py.check_signals())?, + } + } +} + +#[cfg(test)] +mod tests { + use std::ffi::CString; + use std::future::poll_fn; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, mpsc}; + use std::task::Poll; + use std::thread; + use std::time::Instant; + + use pyo3::panic::PanicException; + use pyo3::types::{PyDict, PyModule}; + use serde::Serializer; + use tokio::runtime::Builder; + + use super::*; + + fn runtime_error(error: Error) -> PyErr { + PyRuntimeError::new_err(error.to_string()) + } + + fn panicking_error_mapper(_error: Error) -> PyErr { + panic!("error mapper panicked") + } + + struct PanickingOutput; + + static ASYNC_PROBE_COMPLETED: AtomicUsize = AtomicUsize::new(0); + + impl Serialize for PanickingOutput { + fn serialize(&self, _serializer: S) -> Result + where + S: Serializer, + { + panic!("serializer panicked") + } + } + + #[pyfunction] + fn async_serialization_panic(py: Python<'_>) -> PyResult> { + run_async(py, async { Ok(PanickingOutput) }, runtime_error) + } + + #[pyfunction] + fn async_runtime_probe(py: Python<'_>) -> PyResult> { + run_async( + py, + async { + ASYNC_PROBE_COMPLETED.fetch_add(1, Ordering::SeqCst); + Ok(true) + }, + runtime_error, + ) + } + + #[pyfunction] + fn runtime_worker_count() -> usize { + pyo3_async_runtimes::tokio::get_runtime() + .metrics() + .num_workers() + } + + #[pyfunction] + fn runtime_is_responsive(_py: Python<'_>, expected_completions: usize) -> bool { + let completion_deadline = Instant::now() + Duration::from_secs(2); + while ASYNC_PROBE_COMPLETED.load(Ordering::SeqCst) < expected_completions { + if Instant::now() >= completion_deadline { + return false; + } + thread::sleep(Duration::from_millis(1)); + } + + let (heartbeat_tx, heartbeat_rx) = mpsc::sync_channel(1); + pyo3_async_runtimes::tokio::get_runtime().spawn(async move { + let _ = heartbeat_tx.send(()); + }); + heartbeat_rx.recv_timeout(Duration::from_secs(2)).is_ok() + } + + fn extract_bool(py: Python<'_>, result: PyResult>) -> bool { + result + .expect("route should complete") + .bind(py) + .extract() + .expect("result should convert") + } + + #[test] + fn sync_runner_polls_future_on_the_caller_thread() { + Python::initialize(); + Python::attach(|py| { + let caller_thread = std::thread::current().id(); + let result = run_sync( + py, + async move { Ok(std::thread::current().id() == caller_thread) }, + runtime_error, + ); + + assert!(extract_bool(py, result)); + }); + } + + #[test] + fn sync_runner_releases_gil_while_waiting() { + Python::initialize(); + Python::attach(|py| { + let result = run_sync( + py, + async { + let gil_acquired = tokio::time::timeout( + Duration::from_secs(2), + tokio::task::spawn_blocking(|| Python::attach(|_| true)), + ) + .await; + Ok(matches!(gil_acquired, Ok(Ok(true)))) + }, + runtime_error, + ); + + assert!(extract_bool(py, result)); + }); + } + + #[test] + fn sync_runner_rejects_calls_from_a_tokio_context() { + Python::initialize(); + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + + let error = runtime.block_on(async { + Python::attach(|py| { + run_sync::(py, async { Ok(true) }, runtime_error) + .expect_err("sync route should reject a nested Tokio runtime") + }) + }); + + assert_eq!( + error.to_string(), + "RuntimeError: synchronous native routes cannot run from a Tokio context; use the async route" + ); + } + + #[test] + fn sync_runner_can_drive_a_current_thread_runtime() { + Python::initialize(); + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + Python::attach(|py| { + let result = run_sync_on( + py, + &runtime, + async { + tokio::task::yield_now().await; + Ok(true) + }, + runtime_error, + ); + assert!(extract_bool(py, result)); + }); + } + + #[test] + fn sync_runner_maps_a_panicked_future() { + Python::initialize(); + Python::attach(|py| { + let error = run_sync::( + py, + poll_fn(|_| -> Poll> { panic!("route future panicked") }), + runtime_error, + ) + .expect_err("panicked route should become a Python exception"); + + assert!(error.is_instance_of::(py)); + assert_eq!(error.to_string(), "PanicException: route future panicked"); + }); + } + + #[test] + fn sync_runner_maps_a_panicked_error_mapper() { + Python::initialize(); + Python::attach(|py| { + let error = run_sync::( + py, + async { Err(Error::InvalidRequest("invalid".to_string())) }, + panicking_error_mapper, + ) + .expect_err("panicked mapper should become a Python exception"); + + assert!(error.is_instance_of::(py)); + assert_eq!(error.to_string(), "PanicException: error mapper panicked"); + }); + } + + #[test] + fn sync_runner_surfaces_serializer_panics() { + Python::initialize(); + Python::attach(|py| { + let error = run_sync(py, async { Ok(PanickingOutput) }, runtime_error) + .expect_err("serializer panic should become a Python exception"); + + assert!(error.is_instance_of::(py)); + assert_eq!(error.to_string(), "PanicException: serializer panicked"); + }); + } + + #[test] + fn sync_runner_supports_concurrent_callers_on_the_shared_runtime() { + Python::initialize(); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let callers: Vec<_> = (0..2) + .map(|_| { + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + Python::attach(|py| { + extract_bool( + py, + run_sync( + py, + async move { + Ok(tokio::time::timeout(Duration::from_secs(2), barrier.wait()) + .await + .is_ok()) + }, + runtime_error, + ), + ) + }) + }) + }) + .collect(); + let results: Vec<_> = callers + .into_iter() + .map(|caller| caller.join().expect("caller should not panic")) + .collect(); + + assert_eq!(results, vec![true, true]); + } + + #[test] + fn async_runner_surfaces_serializer_panics() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::new(py, "runtime").expect("module should be created"); + module + .add_function( + wrap_pyfunction!(async_serialization_panic, &module) + .expect("function should wrap"), + ) + .expect("function should register"); + let locals = PyDict::new(py); + locals + .set_item("runtime", &module) + .expect("module should enter Python locals"); + let code = CString::new( + r#" +import asyncio + +async def exercise(): + try: + await runtime.async_serialization_panic() + except BaseException as error: + assert type(error).__name__ == "PanicException" + assert str(error) == "serializer panicked" + else: + raise AssertionError("serializer panic was not raised") + +asyncio.run(exercise()) +"#, + ) + .expect("Python source should not contain null bytes"); + py.run(&code, Some(&locals), Some(&locals)) + .expect("serializer panic should reach the Python awaiter"); + }); + } + + #[test] + fn async_result_delivery_does_not_stall_tokio_workers() { + Python::initialize(); + ASYNC_PROBE_COMPLETED.store(0, Ordering::SeqCst); + Python::attach(|py| { + let module = PyModule::new(py, "runtime").expect("module should be created"); + for function in [ + wrap_pyfunction!(async_runtime_probe, &module).expect("function should wrap"), + wrap_pyfunction!(runtime_worker_count, &module).expect("function should wrap"), + wrap_pyfunction!(runtime_is_responsive, &module).expect("function should wrap"), + ] { + module + .add_function(function) + .expect("function should register"); + } + let locals = PyDict::new(py); + locals + .set_item("runtime", &module) + .expect("module should enter Python locals"); + let code = CString::new( + r#" +import asyncio + +async def exercise(): + worker_count = runtime.runtime_worker_count() + awaitables = [runtime.async_runtime_probe() for _ in range(worker_count)] + assert runtime.runtime_is_responsive(worker_count) + assert await asyncio.gather(*awaitables) == [True] * worker_count + +asyncio.run(exercise()) +"#, + ) + .expect("Python source should not contain null bytes"); + py.run(&code, Some(&locals), Some(&locals)) + .expect("result delivery should leave Tokio workers responsive"); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs index 6a6ede22e85..d397d20b9fd 100644 --- a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs +++ b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs @@ -1,7 +1,7 @@ use std::fs; use std::path::{Path, PathBuf}; -const DISALLOWED_OUTSIDE_MARSHAL: &[&str] = &[ +const DISALLOWED_OUTSIDE_INTEROP: &[&str] = &[ "py.import(\"json\")", "pythonize::", "serde_json::to_string", @@ -33,18 +33,15 @@ fn rust_sources(directory: &Path) -> Vec { } #[test] -fn serialization_is_centralized_in_marshal_module() { +fn serialization_uses_the_interop_boundary() { let root = source_root(); for path in rust_sources(&root) { - if path == root.join("marshal.rs") { - continue; - } let source = fs::read_to_string(&path).expect("bridge source should be readable"); - for disallowed in DISALLOWED_OUTSIDE_MARSHAL { + for disallowed in DISALLOWED_OUTSIDE_INTEROP { assert!( !source.contains(disallowed), - "{} bypasses the typed marshal module with `{disallowed}`", + "{} bypasses litellm-python-interop with `{disallowed}`", path.display() ); } diff --git a/litellm-rust/crates/python-interop/AGENTS.md b/litellm-rust/crates/python-interop/AGENTS.md new file mode 100644 index 00000000000..d1d61e5dfa0 --- /dev/null +++ b/litellm-rust/crates/python-interop/AGENTS.md @@ -0,0 +1 @@ +litellm-python-interop is the domain-neutral PyO3 foundation. Keep generic Python/Serde conversion and interpreter primitives here. Do not add LiteLLM domain crates, route types, API registration, or cdylib build features. diff --git a/litellm-rust/crates/python-interop/Cargo.toml b/litellm-rust/crates/python-interop/Cargo.toml new file mode 100644 index 00000000000..9da6af6e2e2 --- /dev/null +++ b/litellm-rust/crates/python-interop/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "litellm-python-interop" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +pyo3.workspace = true +pythonize.workspace = true +serde.workspace = true + +[dev-dependencies] +rstest.workspace = true +serde_json.workspace = true diff --git a/litellm-rust/crates/python-interop/src/gil.rs b/litellm-rust/crates/python-interop/src/gil.rs new file mode 100644 index 00000000000..04b966a6002 --- /dev/null +++ b/litellm-rust/crates/python-interop/src/gil.rs @@ -0,0 +1,21 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use pyo3::prelude::*; + +static GIL_RELEASES: AtomicU64 = AtomicU64::new(0); + +/// Runs work detached from the interpreter and records the release. +/// +/// `f` must not access Python state while the interpreter is detached. +pub fn release_gil(py: Python<'_>, f: F) -> T +where + F: FnOnce() -> T + Send, + T: Send, +{ + GIL_RELEASES.fetch_add(1, Ordering::Relaxed); + py.detach(f) +} + +pub fn release_count() -> u64 { + GIL_RELEASES.load(Ordering::Relaxed) +} diff --git a/litellm-rust/crates/python-interop/src/lib.rs b/litellm-rust/crates/python-interop/src/lib.rs new file mode 100644 index 00000000000..2e562bdae70 --- /dev/null +++ b/litellm-rust/crates/python-interop/src/lib.rs @@ -0,0 +1,5 @@ +mod gil; +mod marshal; + +pub use gil::{release_count, release_gil}; +pub use marshal::{Pythonized, from_py, panic_to_pyerr, to_py}; diff --git a/litellm-rust/crates/python-interop/src/marshal.rs b/litellm-rust/crates/python-interop/src/marshal.rs new file mode 100644 index 00000000000..a16d1e0ae13 --- /dev/null +++ b/litellm-rust/crates/python-interop/src/marshal.rs @@ -0,0 +1,92 @@ +use std::any::Any; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use pyo3::exceptions::PyValueError; +use pyo3::panic::PanicException; +use pyo3::prelude::*; +use serde::Serialize; +use serde::de::DeserializeOwned; + +pub fn from_py(value: &Bound<'_, PyAny>) -> PyResult +where + T: DeserializeOwned, +{ + pythonize::depythonize(value).map_err(|error| PyValueError::new_err(error.to_string())) +} + +pub fn to_py(py: Python<'_>, value: &T) -> PyResult> +where + T: Serialize + ?Sized, +{ + pythonize::pythonize(py, value) + .map(Bound::unbind) + .map_err(|error| PyValueError::new_err(error.to_string())) +} + +pub struct Pythonized(pub T); + +impl<'py, T> IntoPyObject<'py> for Pythonized +where + T: Serialize, +{ + type Target = PyAny; + type Output = Bound<'py, PyAny>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> PyResult { + catch_unwind(AssertUnwindSafe(|| pythonize::pythonize(py, &self.0))) + .map_err(panic_to_pyerr)? + .map_err(|error| PyValueError::new_err(error.to_string())) + } +} + +pub fn panic_to_pyerr(payload: Box) -> PyErr { + let message = payload + .downcast_ref::() + .map(String::as_str) + .or_else(|| payload.downcast_ref::<&str>().copied()) + .unwrap_or("panic from Rust code"); + PanicException::new_err(message.to_string()) +} + +#[cfg(test)] +mod tests { + use serde::Serializer; + + use super::*; + + struct PanickingSerializer; + + impl Serialize for PanickingSerializer { + fn serialize(&self, _serializer: S) -> Result + where + S: Serializer, + { + panic!("serializer panicked") + } + } + + #[test] + fn pythonized_converts_on_the_attached_thread() { + Python::initialize(); + Python::attach(|py| { + let value: Vec = Pythonized(vec![1, 2, 3]) + .into_pyobject(py) + .and_then(|value| value.extract()) + .expect("value should convert"); + assert_eq!(value, vec![1, 2, 3]); + }); + } + + #[test] + fn pythonized_maps_serializer_panics_to_a_base_exception() { + Python::initialize(); + Python::attach(|py| { + let error = Pythonized(PanickingSerializer) + .into_pyobject(py) + .expect_err("serializer panic should become a Python exception"); + assert!(error.is_instance_of::(py)); + assert_eq!(error.to_string(), "PanicException: serializer panicked"); + }); + } +} diff --git a/litellm-rust/crates/python-interop/tests/interop.rs b/litellm-rust/crates/python-interop/tests/interop.rs new file mode 100644 index 00000000000..9c456dcb938 --- /dev/null +++ b/litellm-rust/crates/python-interop/tests/interop.rs @@ -0,0 +1,44 @@ +use pyo3::Python; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; + +use litellm_python_interop::{from_py, release_count, release_gil, to_py}; + +struct InitializedPython; + +impl InitializedPython { + fn attach(&self, f: F) -> R + where + F: for<'py> FnOnce(Python<'py>) -> R, + { + Python::attach(f) + } +} + +#[fixture] +#[once] +fn initialized_python() -> InitializedPython { + Python::initialize(); + InitializedPython +} + +#[rstest] +fn serde_values_round_trip_through_python(#[from(initialized_python)] python: &InitializedPython) { + python.attach(|py| { + let expected = json!({"model": "test", "items": [1, true, null]}); + let python_value = to_py(py, &expected).expect("value should convert to Python"); + let actual: Value = + from_py(python_value.bind(py)).expect("Python value should convert to serde"); + + assert_eq!(actual, expected); + }); +} + +#[rstest] +fn release_gil_runs_work_and_records_it(#[from(initialized_python)] python: &InitializedPython) { + let before = release_count(); + let result = python.attach(|py| release_gil(py, || 42)); + + assert_eq!(result, 42); + assert_eq!(release_count(), before + 1); +} diff --git a/litellm/__init__.py b/litellm/__init__.py index 4eeececdb7e..41a3789ab0d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -29,7 +29,7 @@ def _dev_env_hot_reload_enabled() -> bool: if os.getenv("LITELLM_MODE", "DEV") == "DEV": _dotenv.load_dotenv(override=_dev_env_hot_reload_enabled()) -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import ( Any, Callable, @@ -424,6 +424,10 @@ anthropic_beta_headers_url: str = os.getenv( "LITELLM_ANTHROPIC_BETA_HEADERS_URL", "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json", ) +autorouter_presets_url: str = os.getenv( + "LITELLM_AUTOROUTER_PRESETS_URL", + "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/proxy/public_endpoints/autorouter_presets.json", +) suppress_debug_info: bool = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None @@ -490,6 +494,7 @@ public_mcp_hub_strict_whitelist: bool = True public_model_groups: Optional[List[str]] = None public_agent_groups: Optional[List[str]] = None agent_search_embedding_model: Optional[str] = None +mcp_tool_search: Optional[Mapping[str, object]] = None # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } # Old format: { "displayName": "url" } (for backward compatibility) @@ -1416,7 +1421,7 @@ from .skills.main import ( ) from .containers.main import * from .ocr.main import * -from .rust_bridge.ocr import use_litellm_rust +from .rust_bridge import use_litellm_rust from .rag.main import * from .sandbox.main import * from .search.main import * diff --git a/litellm/_logging.py b/litellm/_logging.py index 9435562f890..c73b5175a31 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -6,6 +6,7 @@ import sys from datetime import datetime from logging import Formatter from typing import Any, Final, TextIO +from urllib.parse import unquote import litellm from litellm.constants import ( @@ -16,7 +17,11 @@ from litellm.constants import ( from litellm.litellm_core_utils.env_utils import get_env_int from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.litellm_core_utils.secret_redaction import redact_string, redact_structured_value +from litellm.litellm_core_utils.secret_redaction import ( + redact_internal_details, + redact_string, + redact_structured_value, +) set_verbose = False @@ -88,6 +93,14 @@ def redact_secrets(value: str) -> str: return _redact_string(value) +def redact_internal_details_from_client_message(value: str) -> str: + """Public API: redact_secrets() plus filesystem paths, internal hostnames, and an + embedded traceback, for a string about to leave the process in an HTTP response.""" + if not _ENABLE_SECRET_REDACTION: + return value + return redact_internal_details(value) + + def _substituted_color_message(record: logging.LogRecord) -> str | None: """Render a record's ``color_message`` against its args, or None if absent. @@ -146,6 +159,72 @@ class SecretRedactionFilter(logging.Filter): _secret_filter: Final = SecretRedactionFilter() +_MAX_SCRUBBED_ACCESS_ARG: Final = 512 + +_REDACTION_PLACEHOLDER: Final = "REDACTED" + + +def _hides_a_credential(value: str) -> bool: + """Whether *value* only looks clean until it is percent-decoded.""" + decoded: Final = unquote(value) + return _redact_string(decoded) != decoded + + +def _drop_encoded_credential(scrubbed: str) -> str: + """Drop the part of a request target that only decoding shows to be a secret. + + The request parser decodes query names and values, so `?k%65y=sk%2D...` is a + working credential that the patterns, which match literal text, do not see. + The decoded text is never logged back: it can carry a newline, and forging + log lines is not a trade worth making for a readable request target. + """ + path, separator, _query = scrubbed.partition("?") + if _hides_a_credential(path): + return _REDACTION_PLACEHOLDER + if separator and _hides_a_credential(scrubbed): + return f"{path}?{_REDACTION_PLACEHOLDER}" + return scrubbed + + +def _scrub_access_arg(value: str) -> str: + """Redact one access-log positional arg, bounding the scanned length. + + The request target is the only input to the secret regex an unauthenticated + caller controls end to end, so it is cut back to a whole query parameter + before it is scanned; a half-parameter would be too short to match its + pattern and would then be logged raw. + """ + if len(value) <= _MAX_SCRUBBED_ACCESS_ARG: + return _drop_encoded_credential(_redact_string(value)) + head: Final = value[:_MAX_SCRUBBED_ACCESS_ARG] + kept: Final = head[: max(head.rfind("?"), head.rfind("&"))] if "?" in head else head + scrubbed: Final = _drop_encoded_credential(_redact_string(kept)) + return f"{scrubbed}... ({len(value) - len(kept)} more chars truncated) ..." + + +class AccessLogRedactionFilter(logging.Filter): + """Scrubs known secret/credential patterns from HTTP access-log records. + + uvicorn's AccessFormatter unpacks ``record.args`` as a five-element tuple at + emit time, so SecretRedactionFilter cannot be reused here: it collapses the + record into ``record.msg`` and clears the args, and the formatter then raises. + """ + + def filter(self, record: logging.LogRecord) -> bool: + if not _ENABLE_SECRET_REDACTION: + return True + if isinstance(record.args, tuple) and record.args: + record.args = tuple( # rebind-ok: a Filter scrubs records in place + _scrub_access_arg(arg) if isinstance(arg, str) else arg for arg in record.args + ) + return True + # No positional args means everything is in msg, where collapsing is correct. + return _secret_filter.filter(record) + + +_access_log_filter: Final = AccessLogRedactionFilter() + + def _get_max_string_length_stdout_log() -> int: """Read the limit per record so a value loaded later via proxy config environment_variables is honored.""" @@ -553,6 +632,14 @@ _REDACTED_THIRD_PARTY_LOGGERS: Final[tuple[str, ...]] = ( "uvicorn.error", ) +# Access loggers, which emit the full request target, so a credential passed as a +# query parameter (e.g. `/key/info?key=`) lands on stdout verbatim. uvicorn.access +# covers uvicorn.run, --run_gunicorn (its worker_class is UvicornWorker, so the +# access line is still uvicorn's) and an embedding host app. --run_hypercorn and +# --run_granian log through their own loggers in their own record shapes, and +# both ship with access logging off. +_REDACTED_ACCESS_LOGGERS: Final[tuple[str, ...]] = ("uvicorn.access",) + def _redact_third_party_loggers() -> None: """Extend secret redaction to records litellm does not emit directly. @@ -575,6 +662,8 @@ def _redact_third_party_loggers() -> None: """ for name in _REDACTED_THIRD_PARTY_LOGGERS: logging.getLogger(name).addFilter(_secret_filter) + for name in _REDACTED_ACCESS_LOGGERS: + logging.getLogger(name).addFilter(_access_log_filter) # Call the suppression function diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py index 32252711997..1e8cc4ff90e 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py @@ -10,8 +10,19 @@ from collections.abc import AsyncIterator, Mapping from typing import Any, Final from litellm._logging import verbose_logger +from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, +) +from litellm.a2a_protocol.utils import ( + get_session_id_from_a2a_params, + scope_session_to_principal, +) +from litellm.exceptions import BadRequestError from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig +RUNTIME_SESSION_ID_MIN_LENGTH: Final = 33 +RUNTIME_SESSION_ID_MAX_LENGTH: Final = 256 + # Reserved outbound header names that must never be sourced from per-request # ``agent_extra_headers`` for AgentCore requests. ``agent_extra_headers`` carries # values rewritten from the client-controlled ``x-a2a-{agent}-*`` convention, so @@ -19,8 +30,9 @@ from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreCo # request identity / SigV4 metadata by overwriting headers the proxy sets from # trusted server-side config. # -# The runtime headers (session / user id) are derived server-side from -# ``runtimeSessionId`` / ``runtimeUserId`` in the agent's ``litellm_params``; +# The runtime headers (session / user id) are derived server-side from the A2A +# ``message.contextId`` and ``runtimeSessionId`` / ``runtimeUserId`` in the +# agent's ``litellm_params``; # ``authorization`` is set by the AgentCore signer (JWT or SigV4); ``host`` and # the ``x-amz-*`` family are owned by SigV4 itself. _RESERVED_EXACT_HEADERS: Final = frozenset( @@ -66,6 +78,31 @@ def _filter_reserved_headers( return filtered or None +def _request_scoped_runtime_session_id( + params: Mapping[str, Any], + litellm_params: Mapping[str, Any], +) -> str | None: + context_id: Final = get_session_id_from_a2a_params(params) + if not isinstance(context_id, str) or not context_id: + return None + return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)) + + +def _validate_runtime_session_id(session_id: str, model: str) -> str: + if RUNTIME_SESSION_ID_MIN_LENGTH <= len(session_id) <= RUNTIME_SESSION_ID_MAX_LENGTH: + return session_id + raise BadRequestError( + message=( + f"Invalid AgentCore runtime session id {session_id!r}: AWS requires " + f"{RUNTIME_SESSION_ID_MIN_LENGTH}-{RUNTIME_SESSION_ID_MAX_LENGTH} characters. It is built from the A2A " + "message.contextId (prefixed with a 16-hex-char hash of the calling key and '-') when set, " + "otherwise from the agent's configured runtimeSessionId." + ), + model=model, + llm_provider="bedrock", + ) + + class BedrockAgentCoreA2ATransformation: """ Request/response transformation for Bedrock AgentCore A2A agents. @@ -100,7 +137,9 @@ class BedrockAgentCoreA2ATransformation: here to prevent a caller-controlled ``x-a2a-{agent}-*`` header from spoofing the AgentCore runtime user id or other SigV4 metadata. Use ``api_key`` / ``runtimeUserId`` / ``runtimeSessionId`` in litellm_params - (not ``agent_extra_headers``) to override those values. + (not ``agent_extra_headers``) to override those values. The runtime + session id is taken from ``params["message"]["contextId"]`` (scoped to + the calling key) when present, then ``runtimeSessionId``, else generated. Returns: Tuple of (url, signed_headers, signed_body_bytes) @@ -139,7 +178,11 @@ class BedrockAgentCoreA2ATransformation: # Set required AgentCore session headers (normally set by transform_request, # which we skip because it also builds {"prompt": "..."}) headers: Final[dict] = {} - session_id: Final = agentcore_config._get_runtime_session_id(optional_params) + session_id: Final = _validate_runtime_session_id( + _request_scoped_runtime_session_id(params, litellm_params) + or agentcore_config._get_runtime_session_id(optional_params), + model=model, + ) headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = session_id runtime_user_id: Final = agentcore_config._get_runtime_user_id(optional_params) if runtime_user_id: diff --git a/litellm/a2a_protocol/utils.py b/litellm/a2a_protocol/utils.py index f2e61f66105..7c459daf720 100644 --- a/litellm/a2a_protocol/utils.py +++ b/litellm/a2a_protocol/utils.py @@ -2,6 +2,8 @@ Utility functions for A2A protocol. """ +import hashlib +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final import litellm @@ -140,6 +142,29 @@ class A2ARequestUtils: return prompt_tokens, completion_tokens, total_tokens +def get_session_id_from_a2a_params(params: Mapping[str, Any]) -> str | None: + message: Final = params.get("message", {}) + if isinstance(message, dict): + return message.get("contextId") + return getattr(message, "contextId", None) + + +def scope_session_to_principal(session_id: str, principal: str | None) -> str: + """ + Bind a client-supplied A2A contextId to the authenticated principal. + + Without this, two distinct keys authorized for the same agent could set the + same contextId and read/append to each other's backend memory. The + principal is hashed (it is already a hashed token) so the raw value is never + sent to the agent backend, while the original contextId is kept as a suffix + for operator-side correlation. + """ + if not principal: + return session_id + principal_prefix: Final = hashlib.sha256(principal.encode("utf-8")).hexdigest()[:16] + return f"{principal_prefix}-{session_id}" + + # Backwards compatibility aliases def extract_text_from_a2a_message(message: Any) -> str: return A2ARequestUtils.extract_text_from_message(message) diff --git a/litellm/constants.py b/litellm/constants.py index 1bd977dd9a9..ef9329b9dfc 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -9,6 +9,38 @@ DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT" AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5)) ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000 +RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( + { + "routing_strategy_args", + "routing_strategy", + "routing_groups", + "allowed_fails", + "cooldown_time", + "num_retries", + "timeout", + "max_retries", + "retry_after", + "fallbacks", + "context_window_fallbacks", + "retry_policy", + "model_group_retry_policy", + "model_group_alias", + "enable_weighted_failover", + "enable_tag_filtering", + "tag_routing_prefix", + "optional_pre_call_checks", + } +) +ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset( + { + "model_list", + "search_tools", + "assistants_config", + "router_general_settings", + "ignore_invalid_deployments", + "fallback_access_check", + } +) DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) @@ -1417,6 +1449,7 @@ RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl" CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" +SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated" LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = ( "Truncation is a DB storage safeguard. " @@ -1710,6 +1743,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "anthropic_prompt_caching_ttl", "max_ui_session_budget", "budget_rollover", + "mcp_tool_search", ] SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"] DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) diff --git a/litellm/containers/main.py b/litellm/containers/main.py index 97ca11872c1..90d59af009f 100644 --- a/litellm/containers/main.py +++ b/litellm/containers/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars import json -from collections.abc import Coroutine, Mapping +from collections.abc import Callable, Coroutine, Mapping from functools import partial from typing import Final, Literal, overload @@ -47,6 +47,13 @@ __all__ = [ ##### Container Create ####################### +async def _encode_created_container_id( + pending: Coroutine[object, object, ContainerObject], + encode: Callable[[ContainerObject], ContainerObject], +) -> ContainerObject: + return encode(await pending) + + @client async def acreate_container( name: str, @@ -256,16 +263,16 @@ def create_container( _is_async=_is_async, ) - # Encode container_id with provider/model metadata for routing + encode: Final = partial( + ContainerRequestUtils.encode_container_id_in_response, + custom_llm_provider=custom_llm_provider, + litellm_metadata=kwargs.get("litellm_metadata"), + extra_body=extra_body, + ) if isinstance(container_obj, ContainerObject): - container_obj = ContainerRequestUtils.encode_container_id_in_response( - response_obj=container_obj, - custom_llm_provider=custom_llm_provider, - litellm_metadata=kwargs.get("litellm_metadata"), - extra_body=extra_body, - ) + return encode(container_obj) - return container_obj + return _encode_created_container_id(pending=container_obj, encode=encode) except Exception as e: raise litellm.exception_type( diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 748ef938cea..dc41c7dadc8 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1955,11 +1955,10 @@ Model Info: if not thresholds_enabled and not anomalies_enabled: return - if prisma_client is None: - from litellm.proxy.proxy_server import prisma_client as global_prisma_client + from litellm.proxy.proxy_server import prisma_client as global_prisma_client - prisma_client = global_prisma_client # rebind-ok: fall back to the proxy's global client - if prisma_client is None: + client: Final = prisma_client if prisma_client is not None else global_prisma_client + if client is None: return from litellm.integrations.SlackAlerting.user_spend_alerts import ( @@ -1970,7 +1969,7 @@ Model Info: try: today: Final = datetime.datetime.now(datetime.timezone.utc).date() rows: Final = await fetch_user_spend_rows( - prisma_client=prisma_client, + prisma_client=client, today=today, baseline_days=self.alerting_args.spend_anomaly_baseline_days, ) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 545b0f40018..3519240dda9 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -755,6 +755,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): def record_gateway_injection( request_kwargs: Mapping[str, object], added: int, + injected_for_every_deployment: bool = False, ) -> None: """Name the deployment whose payload the gateway, not the client, put breakpoints on. @@ -771,7 +772,16 @@ class AnthropicCacheControlHook(CustomPromptManagement): A pass that runs before a deployment is chosen, which is what the proxy does for prompt templates, injects into the payload every leg goes on to send, so it marks - the request for all of them rather than for one. + the request for all of them rather than for one. Such a pass says so with + ``injected_for_every_deployment`` instead of relying on the shape of + ``request_kwargs``: the router's prompt-management factory stamps a provisional + deployment's ``model_info`` into kwargs before the prompt pass runs, and billing + the request through any other deployment would silently drop the credit. An + every-deployment mark, once written, also never narrows: a later per-leg stamp + (the Bedrock converse tool_config one included) describes one leg of a payload + every leg sends, so narrowing to it would uncredit whichever leg gets billed + after a failover. Both losses are fail-closed under-crediting, which is why the + guard only protects the sentinel and per-leg marks still overwrite each other. Only what this pass actually placed counts. A ``tool_config`` point is placed by the Bedrock converse transform, and only when the request carries tools, so the @@ -801,13 +811,19 @@ class AnthropicCacheControlHook(CustomPromptManagement): ), None, ) - if bucket is not None: - model_info: Final = request_kwargs.get("model_info") - bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = ( - model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT) - if isinstance(model_info, dict) - else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT - ) + if bucket is None: + return + if bucket.get(GATEWAY_INJECTED_CACHE_METADATA_KEY) == GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT: + return + if injected_for_every_deployment: + bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT + return + model_info: Final = request_kwargs.get("model_info") + bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = ( + model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT) + if isinstance(model_info, dict) + else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT + ) @staticmethod def maybe_inject_cache_control( diff --git a/litellm/integrations/azure_storage/azure_storage.py b/litellm/integrations/azure_storage/azure_storage.py index cb7175691df..16ef6920114 100644 --- a/litellm/integrations/azure_storage/azure_storage.py +++ b/litellm/integrations/azure_storage/azure_storage.py @@ -1,7 +1,9 @@ import asyncio import os import time +from collections.abc import Callable from datetime import datetime, timedelta +from functools import cache from typing import Final from litellm._logging import verbose_logger @@ -19,21 +21,40 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) +from litellm.secret_managers.get_azure_ad_token_provider import ( + get_azure_ad_token_provider, +) +from litellm.types.secret_managers.get_azure_ad_token_provider import ( + AzureCredentialType, +) from litellm.types.utils import StandardLoggingPayload +AZURE_STORAGE_TOKEN_SCOPE: Final = "https://storage.azure.com/.default" + + +@cache +def _cached_credential_chain_token_provider() -> Callable[[], str]: + return get_azure_ad_token_provider( + azure_scope=AZURE_STORAGE_TOKEN_SCOPE, + azure_credential=AzureCredentialType.DefaultAzureCredential, + ) + class AzureBlobStorageLogger(CustomBatchLogger): def __init__( self, + build_credential_chain_token_provider: Callable[ + [], Callable[[], str] + ] = _cached_credential_chain_token_provider, **kwargs, ): try: verbose_logger.debug("AzureBlobStorageLogger: in init azure blob storage logger") # Env Variables used for Azure Storage Authentication - self.tenant_id = os.getenv("AZURE_STORAGE_TENANT_ID") - self.client_id = os.getenv("AZURE_STORAGE_CLIENT_ID") - self.client_secret = os.getenv("AZURE_STORAGE_CLIENT_SECRET") + self.tenant_id = os.getenv("AZURE_STORAGE_TENANT_ID") or None + self.client_id = os.getenv("AZURE_STORAGE_CLIENT_ID") or None + self.client_secret = os.getenv("AZURE_STORAGE_CLIENT_SECRET") or None self.azure_storage_account_key: str | None = os.getenv("AZURE_STORAGE_ACCOUNT_KEY") # Required Env Variables for Azure Storage @@ -55,6 +76,9 @@ class AzureBlobStorageLogger(CustomBatchLogger): # Internal variables used for Token based authentication self.azure_auth_token: str | None = None # the Azure AD token to use for Azure Storage API requests self.token_expiry: datetime | None = None # the expiry time of the currentAzure AD token + self._build_credential_chain_token_provider: Callable[[], Callable[[], str]] = ( + build_credential_chain_token_provider + ) asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() @@ -231,10 +255,15 @@ class AzureBlobStorageLogger(CustomBatchLogger): """ Wrapper to set self.azure_auth_token to a valid Azure AD token, refreshing if necessary - Refreshes the token when: - - Token is expired - - Token is not set + Without a service principal configured, the credential chain provider is read every + time; it caches internally and refreshes against the token's real expiry. The read runs + in a worker thread because the chain walk (IMDS probe, CLI subprocess) is blocking """ + if self.tenant_id is None and self.client_id is None and self.client_secret is None: + token_provider: Final = self._build_credential_chain_token_provider() + self.azure_auth_token = await asyncio.to_thread(token_provider) + return + # Check if token needs refresh if self._azure_ad_token_is_expired() or self.azure_auth_token is None: verbose_logger.debug("Azure AD token needs refresh") @@ -273,13 +302,9 @@ class AzureBlobStorageLogger(CustomBatchLogger): tenant_id=tenant_id, client_id=client_id, client_secret=client_secret, - scope="https://storage.azure.com/.default", + scope=AZURE_STORAGE_TOKEN_SCOPE, ) - token: Final = token_provider() - - verbose_logger.debug("azure auth token %s", token) - - return token + return token_provider() def _azure_ad_token_is_expired(self): """ diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index e87ac9521ae..372c9bf6b91 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1,4 +1,5 @@ import contextvars +import copy import hashlib import os import secrets @@ -39,6 +40,7 @@ except ImportError: if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation dc: Final = DualCache() @@ -852,6 +854,69 @@ class CustomGuardrail(CustomLogger): return result + async def async_logging_hook( + self, + kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract + result: object, + call_type: str, + ) -> tuple[dict, object]: # mutable-ok: CustomLogger.async_logging_hook contract + """logging_only: run apply_guardrail on copies of the logged request/response and record the verdict.""" + from litellm.llms import get_guardrail_translation_mapping + + if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks: + return kwargs, result + try: + translation: Final = get_guardrail_translation_mapping(CallTypes(call_type))() + except ValueError: + verbose_logger.debug( + "Guardrail %s: no guardrail translation for call_type=%s, skipping logging_only scan", + self.guardrail_name, + call_type, + ) + return kwargs, result + litellm_params: Final = kwargs.get("litellm_params") or {} + scratch_metadata: Final = { + key: value + for key, value in (litellm_params.get("metadata") or {}).items() + if key != "standard_logging_guardrail_information" + } + try: + await self._scan_logged_call(kwargs, result, translation, scratch_metadata) + except Exception as e: + verbose_logger.warning("Guardrail %s: logging_only scan raised: %s", self.guardrail_name, e) + recorded: Final = scratch_metadata.get("standard_logging_guardrail_information") + standard_logging_object: Final = kwargs.get("standard_logging_object") + if not recorded or not isinstance(standard_logging_object, dict): + return kwargs, result + entries: Final = recorded if isinstance(recorded, list) else [recorded] + existing: Final = standard_logging_object.get("guardrail_information") or [] + return { + **kwargs, + "standard_logging_object": {**standard_logging_object, "guardrail_information": [*existing, *entries]}, + }, result + + async def _scan_logged_call( + self, + kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract + result: object, + translation: "BaseTranslation", + scratch_metadata: dict, # mutable-ok: apply_guardrail records its verdict into request metadata + ) -> None: + optional_params: Final = kwargs.get("optional_params") or {} + scratch_input: Final = copy.deepcopy(kwargs.get("messages") or kwargs.get("input")) + scratch_request: Final = { + "model": kwargs.get("model"), + "messages": scratch_input, + "input": scratch_input, + "tools": copy.deepcopy(optional_params.get("tools")), + "litellm_call_id": kwargs.get("litellm_call_id"), + "metadata": scratch_metadata, + } + await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self) + await translation.process_output_response( + response=copy.deepcopy(result), guardrail_to_apply=self, request_data=scratch_request + ) + def supports_scan_only_tool_results(self) -> bool: """Whether this guardrail can scan tool-result content. diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py new file mode 100644 index 00000000000..ed47533e700 --- /dev/null +++ b/litellm/integrations/otel/langfuse_logger.py @@ -0,0 +1,60 @@ +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_logger +from litellm.integrations.otel.logger import OpenTelemetryV2 +from litellm.integrations.otel.mappers.langfuse import LANGFUSE_OBSERVATION_INPUT, LANGFUSE_OBSERVATION_OUTPUT +from litellm.integrations.otel.model.request_io import request_input, response_output, stream_output +from litellm.integrations.otel.plumbing.context import request_root_span + +if TYPE_CHECKING: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.utils import ModelResponseStream + + +class LangfuseOpenTelemetryV2(OpenTelemetryV2): + """Stamps the request's input and output on the root observation while it is still recording. + + Langfuse shows a trace's input and output from its root observation. The proxy's root span ends + when the response is sent, before the success callback runs, so both stamps come from the + post-call hooks in the request task: the request as it stands after the pre-call chain and the + response as it is returned, for the call types whose response renders as a message. + """ + + async def async_post_call_success_hook( + self, + data: Mapping[str, object], + user_api_key_dict: "UserAPIKeyAuth", + response: object, + ) -> None: + self._stamp_root_io(data, lambda: response_output(response)) + + async def async_post_call_streaming_iterator_hook( + self, + user_api_key_dict: "UserAPIKeyAuth", + response: "AsyncIterator[ModelResponseStream]", + request_data: Mapping[str, object], + ) -> "AsyncGenerator[ModelResponseStream, None]": + relayed: Final[list[ModelResponseStream]] = [] # mutable-ok: relayed as they arrive, assembled at end of stream + async for chunk in response: + relayed.append(chunk) + yield chunk + self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data)) + + def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: + root: Final = request_root_span() + if root is None or not root.is_recording(): + return + try: + output: Final = render_output() + if output is None: + return + root.set_attribute(LANGFUSE_OBSERVATION_OUTPUT, output) + rendered_input: Final = request_input(data) + except Exception: # noqa: BLE001 # telemetry must never fail the request it describes + verbose_logger.debug( + "otel v2 langfuse: could not render the root observation input or output", exc_info=True + ) + return + if rendered_input is not None: + root.set_attribute(LANGFUSE_OBSERVATION_INPUT, rendered_input) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index d2a32ef73b6..a550dca6cc8 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -4,6 +4,7 @@ from collections import OrderedDict from collections.abc import Callable, Iterator, Mapping, Sequence from contextlib import contextmanager from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast from opentelemetry.context import Context, attach, get_current @@ -909,3 +910,29 @@ def phase_span(name: str) -> "Iterator[Span | None]": return with logger.start_phase_span(name) as span: yield span + + +def build_otel_v2_logger( + config: OpenTelemetryV2Config, + callback_name: str | None = None, + tracer_provider: TracerProvider | None = None, + logger_provider: LoggerProvider | None = None, + meter_provider: "MeterProvider | None" = None, + settings: Mapping[str, object] = MappingProxyType({}), +) -> OpenTelemetryV2: + return _logger_class(config)( + config=config, + callback_name=callback_name, + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + **settings, + ) + + +def _logger_class(config: OpenTelemetryV2Config) -> type[OpenTelemetryV2]: + if "langfuse" not in config.mapper_names or not config.capture_span_content: + return OpenTelemetryV2 + from litellm.integrations.otel.langfuse_logger import LangfuseOpenTelemetryV2 + + return LangfuseOpenTelemetryV2 diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index 6d4f1b4fd0a..01063d85355 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -11,6 +11,7 @@ the JSON-serialized payloads. ``_llm_call`` just applies both tables. import json from collections.abc import Callable +from typing import Final from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData from litellm.integrations.otel.mappers.utils import ( @@ -25,6 +26,9 @@ from litellm.integrations.otel.model.payloads import ( LLMUsage, ) +LANGFUSE_OBSERVATION_INPUT: Final = "langfuse.observation.input" +LANGFUSE_OBSERVATION_OUTPUT: Final = "langfuse.observation.output" + class LangfuseMapper: _LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { @@ -56,8 +60,8 @@ class LangfuseMapper: "langfuse.observation.model.parameters": lambda d: json_if( collect(LangfuseMapper._MODEL_PARAMS, d.request_params) ), - "langfuse.observation.input": lambda d: serialize_messages(d.messages_in), - "langfuse.observation.output": lambda d: serialize_messages(output_messages(d)), + LANGFUSE_OBSERVATION_INPUT: lambda d: serialize_messages(d.messages_in), + LANGFUSE_OBSERVATION_OUTPUT: lambda d: serialize_messages(output_messages(d)), "langfuse.observation.usage_details": lambda d: json_if(collect(LangfuseMapper._USAGE_FIELDS, d.usage)), "langfuse.observation.cost_details": lambda d: ( json.dumps({"total": d.response_cost}) if d.response_cost is not None else None diff --git a/litellm/integrations/otel/model/request_io.py b/litellm/integrations/otel/model/request_io.py new file mode 100644 index 00000000000..4e80fb91993 --- /dev/null +++ b/litellm/integrations/otel/model/request_io.py @@ -0,0 +1,90 @@ +from collections.abc import Mapping, Sequence +from typing import Final, Literal + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError +from typing_extensions import ReadOnly, TypedDict + +import litellm +from litellm.integrations.otel.mappers.utils import json_or_none +from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_stream, is_raw_sse_stream +from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse +from litellm.types.utils import ModelResponse, ModelResponseStream + +_SYSTEM_KEYS: Final = ("system", "instructions") +_TURNS: Final = TypeAdapter(tuple[object, ...]) +_MESSAGES: Final = TypeAdapter(list[object] | None) + + +class _Turn(TypedDict): + role: ReadOnly[str] + content: ReadOnly[object] + + +class _AnthropicMessage(BaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["message"] = Field(exclude=True) + role: str = "assistant" + content: object = None + + +def request_input(data: Mapping[str, object]) -> str | None: + turns: Final = data.get("messages", data.get("input")) + if turns is None: + return None + return json_or_none((*_system_turns(data), *_user_turns(turns))) + + +def _system_turns(data: Mapping[str, object]) -> tuple[_Turn, ...]: + return tuple(_Turn(role="system", content=data[key]) for key in _SYSTEM_KEYS if data.get(key) is not None) + + +def _user_turns(turns: object) -> tuple[object, ...]: + if isinstance(turns, str): + return (_Turn(role="user", content=turns),) + try: + return _TURNS.validate_python(turns) + except ValidationError: + return (_Turn(role="user", content=turns),) + + +def response_output(response: object) -> str | None: + match response: + case ModelResponse(): + return json_or_none(tuple(choice.message.model_dump(exclude_none=True) for choice in response.choices)) + case ResponsesAPIResponse(): + return json_or_none(response.model_dump(exclude_none=True).get("output")) + case _: + return _anthropic_message_output(response) + + +def _anthropic_message_output(message: object) -> str | None: + try: + parsed: Final = _AnthropicMessage.model_validate(message) + except ValidationError: + return None + return json_or_none((parsed.model_dump(),)) + + +def stream_output(chunks: Sequence[object], data: Mapping[str, object]) -> str | None: + if not chunks: + return None + if is_raw_sse_stream(chunks): + return response_output(assemble_anthropic_sse_stream(chunks)) + if all(isinstance(chunk, ModelResponseStream) for chunk in chunks): + return response_output(_assembled_chat_stream(chunks, data)) + return response_output(_completed_response(chunks)) + + +def _assembled_chat_stream(chunks: Sequence[object], data: Mapping[str, object]) -> object: + try: + return litellm.stream_chunk_builder( # pyright: ignore[reportUnknownMemberType] # upstream types chunks as a bare list + chunks=list(chunks), # mutable-ok: stream_chunk_builder takes a list + messages=_MESSAGES.validate_python(data.get("messages")), + ) + except (litellm.APIError, ValidationError): + return None + + +def _completed_response(chunks: Sequence[object]) -> ResponsesAPIResponse | None: + return next((chunk.response for chunk in reversed(chunks) if isinstance(chunk, ResponseCompletedEvent)), None) diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index 07d4f959489..e012d35b8f3 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -5,6 +5,7 @@ This hook is called before making an LLM request when a vector store is configur It searches the vector store for relevant context and appends it to the messages. """ +from collections.abc import Awaitable, Callable from typing import TYPE_CHECKING, Any, Final, cast import litellm @@ -80,10 +81,17 @@ class VectorStorePreCallHook(CustomLogger): # Get prisma_client for database fallback prisma_client = None + llm_router = None try: - from litellm.proxy.proxy_server import prisma_client as _prisma_client + from litellm.proxy.proxy_server import ( + llm_router as _llm_router, + ) + from litellm.proxy.proxy_server import ( + prisma_client as _prisma_client, + ) prisma_client = _prisma_client + llm_router = _llm_router except ImportError: pass @@ -114,12 +122,26 @@ class VectorStorePreCallHook(CustomLogger): vector_store_id = vector_store_to_run.get("vector_store_id", "") custom_llm_provider = vector_store_to_run.get("custom_llm_provider") litellm_params_for_vector_store = vector_store_to_run.get("litellm_params", {}) or {} - # Call litellm.vector_stores.search() with the required parameters - search_response = await litellm.vector_stores.asearch( + request_litellm_params = litellm_logging_obj.model_call_details.get("litellm_params", {}) + request_metadata = ( + request_litellm_params.get("metadata", {}) if isinstance(request_litellm_params, dict) else {} + ) + if llm_router is not None: + search_function = cast( # cast-ok: normalize router search callable + Callable[..., Awaitable[VectorStoreSearchResponse]], + llm_router.avector_store_search, + ) + else: + search_function = cast( # cast-ok: normalize SDK search callable + Callable[..., Awaitable[VectorStoreSearchResponse]], + litellm.vector_stores.asearch, + ) + search_response = await search_function( **{ "vector_store_id": vector_store_id, "query": query, "custom_llm_provider": custom_llm_provider, + "metadata": request_metadata, **litellm_params_for_vector_store, }, ) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index dc61ee38a8c..587da997f94 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -10,9 +10,9 @@ import asyncio import math import uuid from collections.abc import AsyncIterator, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, Never, TypedDict, TypeVar, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast -from typing_extensions import ReadOnly +from typing_extensions import Never, ReadOnly import litellm from litellm._logging import verbose_logger @@ -419,7 +419,6 @@ class WebSearchInterceptionLogger(CustomLogger): if call_type in (CallTypes.responses, CallTypes.aresponses): return self._convert_responses_tools(kwargs=kwargs, tools=tools) - # Check if any tool is a web search tool (native or already LiteLLM standard) has_websearch: Final = any(is_web_search_tool(t) for t in tools) if not has_websearch: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9a6fb11f978..f54eeca5178 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -901,6 +901,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_label: str | None = None, prompt_version: int | None = None, request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs + injected_for_every_deployment: bool = False, ) -> tuple[str, list[AllMessageValues], dict]: from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook @@ -933,6 +934,7 @@ class Logging(LiteLLMLoggingBaseClass): AnthropicCacheControlHook.record_gateway_injection( request_kwargs, AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before, + injected_for_every_deployment=injected_for_every_deployment, ) self.messages = messages return model, messages, non_default_params @@ -950,6 +952,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_label: str | None = None, prompt_version: int | None = None, request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs + injected_for_every_deployment: bool = False, ) -> tuple[str, list[AllMessageValues], dict]: from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook @@ -985,6 +988,7 @@ class Logging(LiteLLMLoggingBaseClass): AnthropicCacheControlHook.record_gateway_injection( request_kwargs, AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before, + injected_for_every_deployment=injected_for_every_deployment, ) self.messages = messages return model, messages, non_default_params @@ -4390,13 +4394,15 @@ def _init_custom_logger_compatible_class( from litellm.integrations.otel.model.config import is_otel_v2_enabled if is_otel_v2_enabled(): - from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger + from litellm.integrations.otel.model.config import OpenTelemetryV2Config for callback in _in_memory_loggers: - if type(callback) is OpenTelemetryV2: + if isinstance(callback, OpenTelemetryV2): return callback - otel_logger_v2: Final = OpenTelemetryV2( - **_get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) + otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) + otel_logger_v2: Final = build_otel_v2_logger( + config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings ) _in_memory_loggers.append(otel_logger_v2) _maybe_auto_initialize_arize_phoenix(_in_memory_loggers) @@ -4759,7 +4765,7 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom if not is_otel_v2_enabled(): return None - from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger from litellm.integrations.otel.presets import PRESET_BY_CALLBACK preset_fn: Final = PRESET_BY_CALLBACK.get(callback_name) @@ -4774,7 +4780,7 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None - v2_logger: Final = OpenTelemetryV2(config=config, callback_name=callback_name) + v2_logger: Final = build_otel_v2_logger(config=config, callback_name=callback_name) _in_memory_loggers.append(v2_logger) return v2_logger diff --git a/litellm/litellm_core_utils/secret_redaction.py b/litellm/litellm_core_utils/secret_redaction.py index 5d5bd547d22..b62226a6a19 100644 --- a/litellm/litellm_core_utils/secret_redaction.py +++ b/litellm/litellm_core_utils/secret_redaction.py @@ -30,6 +30,11 @@ def _build_secret_patterns() -> "re.Pattern[str]": r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}", # OpenAI / Anthropic sk- prefixed keys rf"sk-[A-Za-z0-9\-_]{{{MINIMUM_CUSTOM_KEY_LENGTH - len('sk-')},}}", + # Credentials passed as URL query params. Terminated by "&" like the key= + # and sig= patterns below, so the rest of the request line survives in an + # access log. Must precede the generic patterns to win at the same position. + r"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))" + r"=[^\s&'\"]+", # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr) r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}", # x-api-key / api-key header values (handles 'key': 'value' dict repr) @@ -45,8 +50,10 @@ def _build_secret_patterns() -> "re.Pattern[str]": # Word boundary prevents O(n^2) backtracking on long word-char runs. r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)" r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", - # Database connection string credentials (scheme://user:pass@host) - r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)", + # Database connection string credentials (scheme://user:pass@host). + # The user half stops at the ":" separator and both halves are length-capped, + # so a long attacker-supplied URL cannot backtrack quadratically. + r"(?<=://)[^\s'\":]{0,4096}:[^\s'\"]{1,4096}(?=@)", # Databricks personal access tokens r"dapi[0-9a-f]{32}", # Module-level provider keys logged as litellm._key= @@ -67,8 +74,10 @@ def _build_secret_patterns() -> "re.Pattern[str]": r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""", # Raw JWTs (without Bearer prefix) r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*", - # Azure SAS tokens in URLs - r"[?&]sig=[A-Za-z0-9%+/=]+", + # Azure SAS tokens in URLs. The delimiter is a lookbehind, like the + # `key=` pattern above, so the `?` or `&` survives and the redacted URL + # stays well formed (this string is often a request line in a log). + r"(?<=[?&])sig=[A-Za-z0-9%+/=]+", # Full JSON service-account blobs (single-line and multi-line) r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', ] @@ -83,6 +92,27 @@ def redact_string(value: str) -> str: return _SECRET_RE.sub(_REDACTED, value) +_UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+" +_WINDOWS_DRIVE_PATH: Final = r"[A-Za-z]:\\[^\s'\"\)\]}>,]+" +_PRIVATE_OR_LOOPBACK_IPV4: Final = ( + r"\b(?:10(?:\.\d{1,3}){3}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|192\.168(?:\.\d{1,3}){2}|127(?:\.\d{1,3}){3})\b" +) +_INTERNAL_SUFFIX_HOSTNAME: Final = r"\b[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.(?:internal|local|corp|lan|intra|private)\b" +_INTERNAL_DETAIL_RE: Final = re.compile( + "|".join((_UNIX_SYSTEM_PATH, _WINDOWS_DRIVE_PATH, _PRIVATE_OR_LOOPBACK_IPV4, _INTERNAL_SUFFIX_HOSTNAME)), + re.IGNORECASE, +) +_TRACEBACK_MARKER: Final = "Traceback (most recent call last):" + + +def redact_internal_details(value: str) -> str: + """Drop an embedded traceback and scrub filesystem paths and internal hostnames, + on top of redact_string(). For client-facing messages only: server logs keep this detail.""" + marker_index: Final = value.find(_TRACEBACK_MARKER) + without_traceback: Final = value[:marker_index].rstrip() if marker_index != -1 else value + return _INTERNAL_DETAIL_RE.sub(_REDACTED, redact_string(without_traceback)) + + def redact_structured_value(key: str | None, value: str) -> str: """Scrub *value* as it appeared under *key* inside a structured record. diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 3978a01a5db..0e01577b20e 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -36,6 +36,8 @@ from litellm.types.utils import ( from litellm.utils import print_verbose, token_counter if TYPE_CHECKING: + from openai.types.completion_usage import CompletionUsage + from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.litellm_core_utils.streaming_chunk_builder_utils import ( UsagePerChunk, @@ -794,7 +796,7 @@ class ChunkProcessor: @staticmethod def _extract_usage_chunk(chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream") -> Usage | None: - usage_chunk: Usage | None = None + usage_chunk: Usage | CompletionUsage | None = None if hasattr(chunk, "usage") and chunk.usage is not None: usage_chunk = chunk.usage elif "usage" in chunk: @@ -806,7 +808,9 @@ class ChunkProcessor: if isinstance(usage_chunk, dict): return Usage(**usage_chunk) - return usage_chunk + if usage_chunk is None or isinstance(usage_chunk, Usage): + return usage_chunk + return Usage(**usage_chunk.model_dump()) def _calculate_usage_per_chunk( self, diff --git a/litellm/llms/a2a/chat/guardrail_translation/handler.py b/litellm/llms/a2a/chat/guardrail_translation/handler.py index f1c7451796d..5c30ff4747a 100644 --- a/litellm/llms/a2a/chat/guardrail_translation/handler.py +++ b/litellm/llms/a2a/chat/guardrail_translation/handler.py @@ -17,7 +17,10 @@ from typing import TYPE_CHECKING, Any, Final, Optional from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger -from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.llms.base_llm.guardrail_translation.base_translation import ( + BaseTranslation, + StreamingScanKey, +) from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: @@ -313,9 +316,14 @@ class A2AGuardrailHandler(BaseTranslation): return responses_so_far + def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: + _, valid_parsed = self._parse_streaming_responses(responses_so_far) + combined_text, _ = self._collect_text_from_parsed_chunks(valid_parsed) + return StreamingScanKey(texts=(combined_text,)) + def _parse_streaming_responses( self, - responses_so_far: list[object], + responses_so_far: Sequence[object], ) -> tuple[list[dict[str, object] | None], list[tuple[int, dict[str, object]]]]: """Parse JSON-RPC items, returning aligned parsed list and valid entries.""" parsed: Final[list[dict[str, object] | None]] = [None] * len(responses_so_far) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index c7d12e5cf3a..c23797f72af 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -26,7 +26,10 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im LiteLLMAnthropicMessagesAdapter, is_provider_native_tool_dict, ) -from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.llms.base_llm.guardrail_translation.base_translation import ( + BaseTranslation, + StreamingScanKey, +) from litellm.llms.base_llm.guardrail_translation.utils import ( anthropic_tool_name, anthropic_tool_names, @@ -36,6 +39,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( merge_guardrailed_scoped_messages, merge_returned_tools_into_request_tools, scoped_structured_message_indices, + stream_item_fingerprint, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, @@ -1176,6 +1180,25 @@ class AnthropicMessagesHandler(BaseTranslation): inputs["model"] = response_model return inputs + def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: + stream_ended: Final = self._check_streaming_has_ended(responses_so_far) + return StreamingScanKey( + texts=(self.get_streaming_string_so_far(responses_so_far),), + tool_calls=self._streamed_tool_use_fingerprints(responses_so_far) if stream_ended else (), + stream_ended=stream_ended, + ) + + @classmethod + def _streamed_tool_use_fingerprints(cls, responses_so_far: Sequence[object]) -> tuple[str, ...]: + return tuple( + stream_item_fingerprint(block) + for item in responses_so_far + for event in cls._iter_sse_events(item) + if event.get("type") == "content_block_start" + and isinstance(block := event.get("content_block"), Mapping) + and block.get("type") == "tool_use" + ) + def get_streaming_string_so_far(self, responses_so_far: Sequence[object]) -> str: """ Parse streaming responses and extract accumulated text content. diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index aa805ccea71..5f7ac73c919 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1565,6 +1565,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): optional_params.pop("thinking", None) else: optional_params["thinking"] = value + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model=model, optional_params=optional_params, custom_llm_provider=self._resolved_provider + ) elif param == "reasoning_effort": # Accept both string ("low") and dict ({"effort": "low", # "summary": "concise"}). The Responses->Chat parser keeps the diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index c60ebd844ba..6079b709bcc 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -13,7 +13,12 @@ import httpx from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm -from litellm.constants import DEFAULT_MODEL_CREATED_AT_TIME +from litellm.constants import ( + DEFAULT_MODEL_CREATED_AT_TIME, + DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, +) from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_file_ids_from_messages, ) @@ -534,6 +539,51 @@ class AnthropicModelInfo(BaseLLMModelInfo): ) optional_params.pop("thinking", None) + @staticmethod + def translate_legacy_thinking_for_adaptive_model( + model: str, + optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param like the sibling helpers + custom_llm_provider: str, + ) -> None: + """Translate legacy ``thinking.type=enabled`` to adaptive for the + adaptive-thinking models that reject it (4.7+ and the 5 families). + Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the + legacy shape natively, so it is forwarded verbatim and the caller's + ``budget_tokens`` cap keeps applying. Caller-provided + ``output_config.effort`` is never overridden. + """ + if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider): + return + if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider): + return + thinking: Final = optional_params.get("thinking") + if not isinstance(thinking, dict) or thinking.get("type") != "enabled": + return + + effort: Final = AnthropicModelInfo._legacy_budget_to_effort( + model=model, + budget_tokens=int(thinking.get("budget_tokens") or 0), + custom_llm_provider=custom_llm_provider, + ) + existing_output_config: Final = optional_params.get("output_config") + optional_params["thinking"] = {"type": "adaptive"} + optional_params["output_config"] = { + "effort": effort, + **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})), + } + + @staticmethod + def _legacy_budget_to_effort(model: str, budget_tokens: int, custom_llm_provider: str) -> str: + if budget_tokens >= DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET and ( + AnthropicModelInfo._supports_model_capability(model, "supports_xhigh_reasoning_effort", custom_llm_provider) + ): + return "xhigh" + if budget_tokens >= DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET: + return "high" + if budget_tokens >= DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET: + return "medium" + return "low" + def is_effort_used( self, optional_params: dict | None, @@ -1361,6 +1411,97 @@ def flatten_unencrypted_web_search_results_in_anthropic_messages( # mutable-ok: return [_flatten_web_search_results_in_message(m) for m in messages] # mutable-ok: JSON wire format +def _normalized_cache_control(cache_control: object) -> dict[str, str] | None: # mutable-ok: JSON wire format + if not isinstance(cache_control, Mapping): + return None + cache_type: Final = cache_control.get("type") + return {"type": cache_type if isinstance(cache_type, str) else "ephemeral"} # mutable-ok: JSON wire format + + +def _with_portable_cache_control(block: Mapping[str, object]) -> dict[str, object]: # mutable-ok: JSON wire format + if "cache_control" not in block: + return dict(block) # mutable-ok: JSON wire format + normalized: Final = _normalized_cache_control(block["cache_control"]) + rest: Final = {key: value for key, value in block.items() if key != "cache_control"} # mutable-ok: JSON wire format + return rest if normalized is None else {**rest, "cache_control": normalized} # mutable-ok: JSON wire format + + +def _with_portable_cache_control_in_blocks(blocks: object) -> object: + if isinstance(blocks, str) or not isinstance(blocks, Sequence): + return blocks + return [ # mutable-ok: JSON wire format + _with_portable_cache_control(block) if isinstance(block, Mapping) else block for block in blocks + ] + + +def _with_portable_cache_control_in_content_block(block: object) -> object: + if not isinstance(block, Mapping): + return block + portable: Final = _with_portable_cache_control(block) + if portable.get("type") != "tool_result" or "content" not in portable: + return portable + return { # mutable-ok: JSON wire format + **portable, + "content": _with_portable_cache_control_in_blocks(portable["content"]), + } + + +def _with_portable_cache_control_in_message(message: object) -> object: + if not isinstance(message, Mapping) or "content" not in message: + return message + content: Final = message["content"] + if isinstance(content, str) or not isinstance(content, Sequence): + return message + return { # mutable-ok: JSON wire format + **message, + "content": [ # mutable-ok: JSON wire format + _with_portable_cache_control_in_content_block(block) for block in content + ], + } + + +def _with_portable_cache_control_in_messages(messages: object) -> object: + if isinstance(messages, str) or not isinstance(messages, Sequence): + return messages + return [ # mutable-ok: JSON wire format + _with_portable_cache_control_in_message(message) for message in messages + ] + + +def _with_portable_cache_control_in_scoped_value(key: str, value: object) -> object: + match key: + case "system" | "tools": + return _with_portable_cache_control_in_blocks(value) + case "messages": + return _with_portable_cache_control_in_messages(value) + case _: + return value + + +def normalize_cache_control_in_anthropic_payload( + payload: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: JSON wire format + """ + Return a copy of an Anthropic /v1/messages payload with every + ``cache_control`` entry reduced to ``{"type": }`` + at the places the Messages API defines it: the request itself, system + blocks, tools, message content blocks, and ``tool_result`` content blocks. + Application data such as ``tool_use.input`` and tool ``input_schema`` is + never touched, even when it happens to contain a ``cache_control`` key. + + Anthropic itself accepts prompt-caching extensions such as ``ttl``, but + strict non-Anthropic implementations of the Messages API validate the field + literally and reject the whole request (``cache_control.ttl: 1h is not + supported``, ``cache_control.type is required``), which 400s clients like + Claude Code that send cache hints. Non-dict ``cache_control`` values are + dropped entirely. The caller's payload is never mutated. + """ + portable: Final = _with_portable_cache_control(payload) + return { # mutable-ok: JSON wire format + key: _with_portable_cache_control_in_scoped_value(key, value) for key, value in portable.items() + } + + def process_anthropic_headers(headers: httpx.Headers | dict) -> dict: openai_headers: Final = {} if "anthropic-ratelimit-requests-limit" in headers: @@ -1378,31 +1519,38 @@ def process_anthropic_headers(headers: httpx.Headers | dict) -> dict: return additional_headers -def _anthropic_model_entry(model: ModelInfoResponse, created_at: str) -> Mapping[str, object]: +def _anthropic_model_entry( + model: ModelInfoResponse, created_at: str, display_names: Mapping[str, str] +) -> Mapping[str, object]: return { # mutable-ok: JSON response body, serialized by the route and never mutated "type": "model", "id": model["id"], - "display_name": model["id"], + "display_name": display_names.get(model["id"], model["id"]), "created_at": created_at, "max_input_tokens": model.get("max_input_tokens"), "max_tokens": model.get("max_output_tokens"), } -def create_anthropic_model_list_response(models: Sequence[ModelInfoResponse]) -> Mapping[str, object]: +def create_anthropic_model_list_response( + models: Sequence[ModelInfoResponse], + display_names: Mapping[str, str] = MappingProxyType({}), +) -> Mapping[str, object]: """Build the Anthropic-native /v1/models envelope. Clients that send an anthropic-version header parse the Anthropic Models API shape (type/display_name/created_at plus has_more/first_id/last_id) and filter the list themselves, so every model is returned here. The token limits carry over from the OpenAI-shaped listing, named as the Messages API names them, and - are always present because the vendor shape declares them nullable, not optional + are always present because the vendor shape declares them nullable, not optional. + display_names maps a listed model id to a configured human-readable name; ids + without an entry fall back to the id itself, matching the vendor behavior """ created_at: Final = ( datetime.fromtimestamp(DEFAULT_MODEL_CREATED_AT_TIME, tz=timezone.utc).isoformat().replace("+00:00", "Z") ) data: Final = [ # mutable-ok: JSON response body, serialized by the route and never mutated - _anthropic_model_entry(model, created_at) for model in models + _anthropic_model_entry(model, created_at, display_names) for model in models ] return { # mutable-ok: JSON response body, serialized by the route and never mutated "data": data, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 69985bcdaa3..b82903d6f87 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -99,6 +99,10 @@ def _deployment_passes_through_anthropic_messages(model_info: object) -> bool: return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints +def _deployment_supports_cache_control_ttl(model_info: object) -> bool: + return isinstance(model_info, dict) and model_info.get("cache_control_ttl") is True + + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -568,7 +572,9 @@ def anthropic_messages_handler( OpenAILikeAnthropicMessagesConfig, ) - anthropic_messages_provider_config = OpenAILikeAnthropicMessagesConfig() + anthropic_messages_provider_config = OpenAILikeAnthropicMessagesConfig( + cache_control_ttl=_deployment_supports_cache_control_ttl(kwargs.get("model_info")), + ) if anthropic_messages_provider_config is None: # Route to Responses API for OpenAI / Azure, chat/completions for everything else. if _should_route_to_responses_api(custom_llm_provider, original_model, model): diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 3d62b8b4784..988f81c9eb4 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -3,11 +3,6 @@ from typing import Any, Final import httpx -from litellm.constants import ( - DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, - DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, -) from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import verbose_logger @@ -400,46 +395,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): existing_output_config.setdefault("effort", mapped_effort) optional_params["output_config"] = existing_output_config - @staticmethod - def _translate_legacy_thinking_for_adaptive_model( - model: str, optional_params: dict, custom_llm_provider: str - ) -> None: - """Translate legacy ``thinking.type=enabled`` to adaptive for the - adaptive-thinking models that reject it (4.7+ and the 5 families). - Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the - legacy shape natively, so it is forwarded verbatim and the caller's - ``budget_tokens`` cap keeps applying. Caller-provided - ``output_config.effort`` is never overridden. - """ - from litellm.llms.anthropic.chat.transformation import AnthropicConfig - - if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider): - return - if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider): - return - thinking: Final = optional_params.get("thinking") - if not isinstance(thinking, dict) or thinking.get("type") != "enabled": - return - - budget: Final = int(thinking.get("budget_tokens") or 0) - if budget >= DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET and ( - AnthropicConfig._supports_effort_level(model, "xhigh", custom_llm_provider) - ): - effort = "xhigh" - elif budget >= DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET: - effort = "high" - elif budget >= DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET: - effort = "medium" - else: - effort = "low" - - optional_params["thinking"] = {"type": "adaptive"} - existing_output_config = optional_params.get("output_config") - if not isinstance(existing_output_config, dict): - existing_output_config = {} - existing_output_config.setdefault("effort", effort) - optional_params["output_config"] = existing_output_config - @staticmethod def _translate_adaptive_effort_for_non_adaptive_model( model: str, optional_params: dict, max_tokens: int | None, custom_llm_provider: str @@ -606,7 +561,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): custom_llm_provider=self._resolved_provider, ) - self._translate_legacy_thinking_for_adaptive_model( + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( model=model, optional_params=anthropic_messages_optional_request_params, custom_llm_provider=self._resolved_provider, diff --git a/litellm/llms/azure_ai/anthropic/transformation.py b/litellm/llms/azure_ai/anthropic/transformation.py index c5053448627..864d2134a84 100644 --- a/litellm/llms/azure_ai/anthropic/transformation.py +++ b/litellm/llms/azure_ai/anthropic/transformation.py @@ -17,13 +17,14 @@ def _promote_extra_body_to_optional_params(optional_params: dict) -> None: ``output_config`` get auto-routed into ``extra_body`` by ``add_provider_specific_params_to_optional_params``. For the Azure→Anthropic route those keys must reach the request body and be validated, so promote - them. ``setdefault`` keeps explicit top-level values authoritative. + them. The caller's values overwrite mapped top-level duplicates, matching + the native ``anthropic`` provider, where the same passthrough lands on + top-level ``optional_params`` after mapping. """ extra_body: Final = optional_params.get("extra_body") if not isinstance(extra_body, dict) or not extra_body: return - for k, v in extra_body.items(): - optional_params.setdefault(k, v) + optional_params.update(extra_body) optional_params.pop("extra_body", None) diff --git a/litellm/llms/azure_ai/vector_stores/transformation.py b/litellm/llms/azure_ai/vector_stores/transformation.py index 5e61d0a1dd9..db1a0fc89a3 100644 --- a/litellm/llms/azure_ai/vector_stores/transformation.py +++ b/litellm/llms/azure_ai/vector_stores/transformation.py @@ -1,10 +1,15 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx -import litellm from litellm.llms.azure.common_utils import BaseAzureLLM -from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig +from litellm.llms.base_llm.vector_store.transformation import ( + BaseQueryEmbeddingVectorStoreConfig, + VectorStoreEmbeddingExecutor, +) from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( BaseVectorStoreAuthCredentials, @@ -19,13 +24,14 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: LiteLLMLoggingObj = Any -class AzureAIVectorStoreConfig(BaseVectorStoreConfig, BaseAzureLLM): +class AzureAIVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, BaseAzureLLM): """ Configuration for Azure AI Search Vector Store @@ -109,82 +115,73 @@ class AzureAIVectorStoreConfig(BaseVectorStoreConfig, BaseAzureLLM): def transform_search_vector_store_request( self, vector_store_id: str, - query: str | list[str], + query: str | Sequence[str], vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, api_base: str, litellm_logging_obj: LiteLLMLoggingObj, - litellm_params: dict, - extra_body: dict[str, Any] | None = None, - ) -> tuple[str, dict[str, Any]]: - """ - Transform search request for Azure AI Search API + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + router: Router | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: + query_text: Final = self.query_text(query) + query_vector: Final = self.embed_query(query_text, litellm_params, embedding_executor, router) + return self._search_request( + vector_store_id, + query_text, + query_vector, + vector_store_search_optional_params, + api_base, + litellm_logging_obj, + litellm_params, + ) - Generates embeddings using litellm.embeddings and constructs Azure AI Search request - """ - # Convert query to string if it's a list - if isinstance(query, list): - query = " ".join(query) + async def atransform_search_vector_store_request( + self, + vector_store_id: str, + query: str | Sequence[str], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: LiteLLMLoggingObj, + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + router: Router | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: + query_text: Final = self.query_text(query) + query_vector: Final = await self.aembed_query(query_text, litellm_params, embedding_executor, router) + return self._search_request( + vector_store_id, + query_text, + query_vector, + vector_store_search_optional_params, + api_base, + litellm_logging_obj, + litellm_params, + ) - # Get embedding model from litellm_params (required) - embedding_model: Final = litellm_params.get("litellm_embedding_model") - if not embedding_model: - raise ValueError( - "embedding_model is required in litellm_params for Azure AI Search. " - "Example: litellm_params['embedding_model'] = 'azure/text-embedding-3-large'" - ) - - embedding_config: Final = litellm_params.get("litellm_embedding_config", {}) - if not embedding_config: - raise ValueError( - "embedding_config is required in litellm_params for Azure AI Search. " - "Example: litellm_params['embedding_config'] = {'api_base': 'https://krris-mh44uf7y-eastus2.cognitiveservices.azure.com/', 'api_key': 'os.environ/AZURE_API_KEY', 'api_version': '2025-09-01'}" - ) - - # Get vector field name (defaults to contentVector) + @staticmethod + def _search_request( + vector_store_id: str, + query_text: str, + query_vector: Sequence[float], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: LiteLLMLoggingObj, + litellm_params: Mapping[str, object], + ) -> tuple[str, dict[str, object]]: vector_field: Final = litellm_params.get("azure_search_vector_field", "contentVector") - - # Get top_k (number of results to return) top_k: Final = vector_store_search_optional_params.get("top_k", 10) - - # Generate embedding for the query using litellm.embeddings - try: - embedding_response: Final = litellm.embedding( - model=embedding_model, - input=[query], - **embedding_config, - ) - query_vector: Final = embedding_response.data[0]["embedding"] - except Exception as e: - raise Exception(f"Failed to generate embedding for query: {e}") - - # Azure AI Search endpoint for search - index_name: Final = vector_store_id # vector_store_id is the index name - url: Final = f"{api_base}/indexes/{index_name}/docs/search?api-version=2024-07-01" - - # Build the request body for Azure AI Search with vector search - request_body: Final = { - "search": "*", # Get all documents (filtered by vector similarity) - "vectorQueries": [ - { - "vector": query_vector, - "fields": vector_field, - "kind": "vector", - "k": top_k, # Number of nearest neighbors to return - } - ], - "select": "id,content", # Fields to return (customize based on schema) + litellm_logging_obj.model_call_details["input"] = query_text + litellm_logging_obj.model_call_details["embedding_model"] = litellm_params.get("litellm_embedding_model") + litellm_logging_obj.model_call_details["top_k"] = top_k + return f"{api_base}/indexes/{vector_store_id}/docs/search?api-version=2024-07-01", { + "search": "*", + "vectorQueries": [{"vector": query_vector, "fields": vector_field, "kind": "vector", "k": top_k}], + "select": "id,content", "top": top_k, } - ######################################################### - # Update logging object with details of the request - ######################################################### - litellm_logging_obj.model_call_details["input"] = query - litellm_logging_obj.model_call_details["embedding_model"] = embedding_model - litellm_logging_obj.model_call_details["top_k"] = top_k - - return url, request_body - def transform_search_vector_store_response( self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj ) -> VectorStoreSearchResponse: diff --git a/litellm/llms/base_llm/files/azure_blob_storage_backend.py b/litellm/llms/base_llm/files/azure_blob_storage_backend.py index e22cf528856..a192226110c 100644 --- a/litellm/llms/base_llm/files/azure_blob_storage_backend.py +++ b/litellm/llms/base_llm/files/azure_blob_storage_backend.py @@ -7,16 +7,36 @@ to reuse all authentication and Azure Storage operations. """ import time +from pathlib import Path from typing import Final from urllib.parse import quote, urlparse from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger +from litellm.proxy.common_utils.path_utils import safe_filename from .storage_backend import BaseFileStorageBackend +def _safe_basename(original_filename: str) -> str: + try: + return safe_filename(original_filename) + except ValueError: + return "file" + + +def _safe_extension(original_filename: str) -> str: + """The extension off a basename, with no path separators or traversal sequences. + + original_filename.split(".")[-1] does not parse path structure, so a filename + like "a.jsonl/../../etc/cron.d/x" would put "../../etc/cron.d/x" straight into + the blob path built below. Path.suffix only ever looks at the last path + component, so routing through safe_filename() first closes that off. + """ + return Path(_safe_basename(original_filename)).suffix.lstrip(".") + + class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): """ Azure Blob Storage backend implementation. @@ -81,16 +101,15 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): def _generate_file_name(self, original_filename: str, file_naming_strategy: str) -> str: """Generate file name based on naming strategy.""" if file_naming_strategy == "original_filename": - # Use original filename, but sanitize it - return quote(original_filename, safe="") + return quote(_safe_basename(original_filename), safe="") elif file_naming_strategy == "timestamp": # Use timestamp - extension = original_filename.split(".")[-1] if "." in original_filename else "" + extension = _safe_extension(original_filename) timestamp: Final = int(time.time() * 1000) # milliseconds return f"{timestamp}.{extension}" if extension else str(timestamp) else: # default to "uuid" # Use UUID - extension = original_filename.split(".")[-1] if "." in original_filename else "" + extension = _safe_extension(original_filename) file_uuid: Final = str(uuid.uuid4()) return f"{file_uuid}.{extension}" if extension else file_uuid diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 220fcedb0f8..b28daf73bc4 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -35,6 +35,22 @@ class StreamTransformSink: holdback_per_choice: dict[int, int] = field(default_factory=dict) +@dataclass(frozen=True, slots=True) +class StreamingScanKey: + """What a streaming guardrail round would hand to ``apply_guardrail``. Two keys + compare equal when the round would scan the same content again; ``stream_ended`` + stays out of the comparison and only says whether the handler is on its + end-of-stream path, where an empty payload is still scanned today.""" + + texts: tuple[str, ...] + tool_calls: tuple[str, ...] = () + stream_ended: bool = field(default=False, compare=False) + + @property + def has_nothing_to_scan(self) -> bool: + return not self.stream_ended and not any(self.texts) and not self.tool_calls + + class BaseTranslation(ABC): @staticmethod def transform_user_api_key_dict_to_metadata( @@ -151,6 +167,9 @@ class BaseTranslation(ABC): """ return responses_so_far + def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: + return None + def build_block_sse_chunks( self, exc: "ModifyResponseException", diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index 9b6f9c47105..8dee262001d 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -4,6 +4,8 @@ import json from collections.abc import Callable, Iterator, Sequence from typing import Any, Final, TypeVar +from pydantic import BaseModel + from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage @@ -130,6 +132,16 @@ def stream_item_field(item: object, field: str) -> object | None: return getattr(item, field, None) +def stream_item_fingerprint(item: object) -> str: + plain: Final = item.model_dump() if isinstance(item, BaseModel) else item + return json.dumps(plain, sort_keys=True, default=str) + + +def stream_item_items(item: object, field: str) -> tuple[object, ...]: + value: Final = stream_item_field(item, field) + return tuple(value) if isinstance(value, (list, tuple)) else () + + def blocked_chat_stream_usage(original_response: object) -> tuple[int, int]: """ ``(prompt_tokens, completion_tokens)`` for a synthetic guardrail-blocked diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 02a51a8bace..9624a721870 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -1,10 +1,16 @@ +from __future__ import annotations + from abc import abstractmethod from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, NoReturn +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, runtime_checkable import httpx +from pydantic import TypeAdapter from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import EmbeddingResponse from litellm.types.vector_stores import ( VECTOR_STORE_OPENAI_PARAMS, BaseVectorStoreAuthCredentials, @@ -17,6 +23,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router from ..chat.transformation import BaseLLMException as _BaseLLMException @@ -27,6 +34,95 @@ else: BaseLLMException = Any +@runtime_checkable +class VectorStoreEmbeddingExecutor(Protocol): + def embed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: ... + + async def aembed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: ... + + +@dataclass(frozen=True, slots=True) +class LiteLLMVectorStoreEmbeddingExecutor: + def embed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: + import litellm + + return litellm.embedding( # pyright: ignore[reportCallIssue, reportUnknownMemberType, reportUnknownVariableType] # provider kwargs are intentionally dynamic + model=model, + input=[query], # mutable-ok: LiteLLM embedding requires a mutable input list + **dict(configuration), # pyright: ignore[reportArgumentType] # provider-specific embedding config is validated downstream # mutable-ok: kwargs require a concrete dict + ) + + async def aembed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: + import litellm + + return await litellm.aembedding( # pyright: ignore[reportUnknownMemberType] # provider kwargs are intentionally dynamic + model=model, + input=[query], # mutable-ok: LiteLLM embedding requires a mutable input list + **dict(configuration), # pyright: ignore[reportArgumentType] # provider-specific embedding config is validated downstream # mutable-ok: kwargs require a concrete dict + ) + + +_REQUEST_METADATA: Final = TypeAdapter(dict[str, object]) + + +def vector_store_request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: + litellm_metadata: Final = kwargs.get("litellm_metadata") + if isinstance(litellm_metadata, dict): + return _REQUEST_METADATA.validate_python(litellm_metadata) + metadata: Final = kwargs.get("metadata") + if isinstance(metadata, dict): + return _REQUEST_METADATA.validate_python(metadata) + return MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class RouterVectorStoreEmbeddingExecutor: + router: Router + metadata: Mapping[str, object] + + def _embedding_kwargs(self, configuration: Mapping[str, object]) -> Mapping[str, object]: + configured_metadata: Final = configuration.get("metadata") + metadata: Final = { + **(configured_metadata if isinstance(configured_metadata, Mapping) else {}), + **self.metadata, + } + return { + **{key: value for key, value in configuration.items() if key not in ("input", "metadata", "model")}, + "metadata": metadata, + } + + def _router_serves(self, model: str) -> bool: + team_id: Final = self.metadata.get("user_api_key_team_id") + resolved: Final = self.router.resolved_litellm_models(model, team_id if isinstance(team_id, str) else None) + deployment_models: Final = ( + deployment.get("litellm_params", {}).get("model") for deployment in self.router.get_model_list() or () + ) + return bool(resolved) or model in deployment_models + + def _embeds_through_sdk(self, model: str, configuration: Mapping[str, object]) -> bool: + return bool(configuration) and not self._router_serves(model) + + def embed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: + embedding_kwargs: Final = self._embedding_kwargs(configuration) + if self._embeds_through_sdk(model, configuration): + return LiteLLMVectorStoreEmbeddingExecutor().embed(model, query, embedding_kwargs) + return self.router.embedding( # pyright: ignore[reportUnknownMemberType] # Router embedding input retains a legacy untyped list + model=model, + input=[query], # mutable-ok: Router embedding requires a mutable input list + **embedding_kwargs, # pyright: ignore[reportArgumentType] # provider kwargs are intentionally dynamic + ) + + async def aembed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse: + embedding_kwargs: Final = self._embedding_kwargs(configuration) + if self._embeds_through_sdk(model, configuration): + return await LiteLLMVectorStoreEmbeddingExecutor().aembed(model, query, embedding_kwargs) + return await self.router.aembedding( # pyright: ignore[reportUnknownMemberType] # Router embedding input retains a legacy untyped list + model=model, + input=[query], # mutable-ok: Router embedding requires a mutable input list + **embedding_kwargs, # pyright: ignore[reportArgumentType] # provider kwargs are intentionally dynamic + ) + + class BaseVectorStoreConfig: def get_supported_openai_params(self, model: str) -> list[VECTOR_STORE_OPENAI_PARAMS]: return [] @@ -57,6 +153,7 @@ class BaseVectorStoreConfig: litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, + router: Router | None = None, ) -> tuple[str, dict]: pass @@ -69,6 +166,7 @@ class BaseVectorStoreConfig: litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, + router: Router | None = None, ) -> tuple[str, dict]: """ Optional async version of transform_search_vector_store_request. @@ -84,6 +182,7 @@ class BaseVectorStoreConfig: litellm_logging_obj=litellm_logging_obj, litellm_params=litellm_params, extra_body=extra_body, + router=router, ) @abstractmethod @@ -157,6 +256,116 @@ class BaseVectorStoreConfig: return 0.0, 0.0 +_EMPTY_EMBEDDING_CONFIGURATION: Final[Mapping[str, object]] = MappingProxyType({}) +_QUERY_VECTOR: Final = TypeAdapter(list[float]) + + +class BaseQueryEmbeddingVectorStoreConfig(BaseVectorStoreConfig): + @abstractmethod + def transform_search_vector_store_request( + self, + vector_store_id: str, + query: str | Sequence[str], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: LiteLLMLoggingObj, + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + router: Router | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: + pass + + async def atransform_search_vector_store_request( + self, + vector_store_id: str, + query: str | Sequence[str], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: LiteLLMLoggingObj, + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + router: Router | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: + return self.transform_search_vector_store_request( + vector_store_id=vector_store_id, + query=query, + vector_store_search_optional_params=vector_store_search_optional_params, + api_base=api_base, + litellm_logging_obj=litellm_logging_obj, + litellm_params=litellm_params, + extra_body=extra_body, + router=router, + embedding_executor=embedding_executor, + ) + + @staticmethod + def query_text(query: str | Sequence[str]) -> str: + return query if isinstance(query, str) else " ".join(query) + + @staticmethod + def query_embedding_model(litellm_params: Mapping[str, object]) -> str: + embedding_model: Final = litellm_params.get("litellm_embedding_model") + if isinstance(embedding_model, str) and embedding_model: + return embedding_model + raise ValueError( + "litellm_embedding_model is required in litellm_params for this vector store. " + "Example: litellm_params['litellm_embedding_model'] = 'openai/text-embedding-3-small'" + ) + + @staticmethod + def query_embedding_configuration(litellm_params: Mapping[str, object]) -> Mapping[str, object]: + configuration: Final = litellm_params.get("litellm_embedding_config") + if isinstance(configuration, Mapping): + return {str(key): value for key, value in configuration.items()} # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # litellm_params is an untyped dict, keys are re-validated as str here + return _EMPTY_EMBEDDING_CONFIGURATION + + @staticmethod + def query_embedding_executor( + embedding_executor: VectorStoreEmbeddingExecutor | None, + router: Router | None, + request_metadata: Mapping[str, object] = MappingProxyType({}), + ) -> VectorStoreEmbeddingExecutor: + if embedding_executor is not None: + return embedding_executor + if router is not None: + return RouterVectorStoreEmbeddingExecutor(router=router, metadata=request_metadata) + return LiteLLMVectorStoreEmbeddingExecutor() + + def embed_query( + self, + query_text: str, + litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None, + router: Router | None = None, + ) -> Sequence[float]: + model: Final = self.query_embedding_model(litellm_params) + configuration: Final = self.query_embedding_configuration(litellm_params) + executor: Final = self.query_embedding_executor(embedding_executor, router) + try: + response: Final = executor.embed(model, query_text, configuration) + except Exception as e: + raise Exception(f"Failed to generate embedding for query: {e}") + return _QUERY_VECTOR.validate_python(response.data[0]["embedding"]) # pyright: ignore[reportUnknownMemberType] # EmbeddingResponse.data is an untyped list, the vector is validated here + + async def aembed_query( + self, + query_text: str, + litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None, + router: Router | None = None, + ) -> Sequence[float]: + model: Final = self.query_embedding_model(litellm_params) + configuration: Final = self.query_embedding_configuration(litellm_params) + executor: Final = self.query_embedding_executor(embedding_executor, router) + try: + response: Final = await executor.aembed(model, query_text, configuration) + except Exception as e: + raise Exception(f"Failed to generate embedding for query: {e}") + return _QUERY_VECTOR.validate_python(response.data[0]["embedding"]) # pyright: ignore[reportUnknownMemberType] # EmbeddingResponse.data is an untyped list, the vector is validated here + + class BaseDirectVectorStoreConfig(BaseVectorStoreConfig): """ Base config for vector store providers whose datastore has no HTTP API @@ -172,6 +381,7 @@ class BaseDirectVectorStoreConfig(BaseVectorStoreConfig): vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None = None, timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: pass @@ -184,6 +394,7 @@ class BaseDirectVectorStoreConfig(BaseVectorStoreConfig): vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None = None, timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: pass @@ -197,6 +408,7 @@ class BaseDirectVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: Mapping[str, object], extra_body: Mapping[str, object] | None = None, + router: Router | None = None, ) -> NoReturn: raise NotImplementedError("Direct vector store providers execute the search themselves; no HTTP request shape") diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 5363c3c0366..e097805f54a 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -943,6 +943,9 @@ class AmazonConverseConfig(BaseConfig): litellm.verbose_logger.warning(DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, model) else: optional_params["thinking"] = value + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model=model, optional_params=optional_params, custom_llm_provider="bedrock" + ) elif param == "reasoning_effort" and isinstance(value, str): self._handle_reasoning_effort_parameter( model=model, reasoning_effort=value, optional_params=optional_params @@ -1334,6 +1337,7 @@ class AmazonConverseConfig(BaseConfig): ) additional_request_params.pop("parallel_tool_calls", None) + additional_request_params.pop("client_metadata", None) # Only set the topK value in for models that support it additional_request_params.update(self._handle_top_k_value(model, inference_params, drop_params)) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 2a4c38e71ea..67720451c00 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -107,6 +107,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): # Restore original model name model = original_model + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model=original_model, optional_params=optional_params, custom_llm_provider="bedrock" + ) + # The stub model hides the original model from the parent's forced-tool-use backstop response_format_tool_choice: Final = optional_params.get("tool_choice") if ( diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 66ee5f10679..1e5329c90dd 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -748,6 +748,15 @@ def strip_bedrock_throughput_suffix(model: str) -> str: MANTLE_MESSAGES_PATH: Final = "/anthropic/v1/messages" +_MANTLE_OPENAI_BASE_SUFFIXES: Final = ("/openai/v1", "/v1") + + +def _mantle_api_base_from_env() -> str | None: + env_base: Final = get_secret_str("BEDROCK_MANTLE_API_BASE") + if env_base is None: + return None + base: Final = env_base.rstrip("/") + return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base) def build_mantle_messages_url( @@ -758,12 +767,15 @@ def build_mantle_messages_url( """Build the bedrock-mantle Anthropic /messages URL. Honors an explicit endpoint override (``api_base``, then - ``aws_bedrock_runtime_endpoint``) so private VPC / VPCE / GovCloud Mantle - endpoints are reachable; otherwise falls back to the public regional host. + ``aws_bedrock_runtime_endpoint``, then ``BEDROCK_MANTLE_API_BASE``) so + private VPC / VPCE / GovCloud Mantle endpoints are reachable; otherwise + falls back to the public regional host. The mantle messages path is appended unless the override already carries it, - so callers can pass either the host or the full messages URL. + so callers can pass either the host or the full messages URL. The env var is + shared with the OpenAI-surface ``bedrock_mantle/*`` routes, which need it to + carry their ``/v1`` or ``/openai/v1`` base, so that suffix is dropped first. """ - override: Final = api_base or aws_bedrock_runtime_endpoint + override: Final = api_base or aws_bedrock_runtime_endpoint or _mantle_api_base_from_env() if override: base: Final = override.rstrip("/") if base.endswith(MANTLE_MESSAGES_PATH): diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index 2d72db0cdba..bad17a2181d 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -27,6 +27,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -196,6 +197,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, + router: "Router | None" = None, ) -> tuple[str, dict]: if isinstance(query, list): query = " ".join(query) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index b6e93f590ca..e1f0fc9e7d3 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -9,13 +9,15 @@ import threading import time from collections.abc import AsyncIterable, Callable, Iterable, Mapping from http.cookiejar import CookieJar, DefaultCookiePolicy -from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, Optional, TypeAlias, TypedDict +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, Optional, TypeAlias, TypedDict, TypeVar import certifi import httpx from aiohttp import ClientSession, DummyCookieJar, TCPConnector from httpx import USE_CLIENT_DEFAULT, AsyncHTTPTransport, HTTPTransport -from httpx._types import RequestFiles +from httpx._types import CertTypes, RequestFiles +from httpx._utils import get_environment_proxies import litellm from litellm._logging import verbose_logger @@ -66,6 +68,22 @@ _AddrInfo: TypeAlias = tuple[int | socket.AddressFamily, int | socket.SocketKind _RequestContent: TypeAlias = str | bytes | Iterable[bytes] | AsyncIterable[bytes] +_IPV4_LOCAL_ADDRESS: Final = "0.0.0.0" + +_HttpxTransportT = TypeVar("_HttpxTransportT", HTTPTransport, AsyncHTTPTransport) + + +def _environment_proxy_mounts( + build_proxy_transport: Callable[[str], _HttpxTransportT], +) -> Mapping[str, _HttpxTransportT | None]: + """httpx skips its own HTTP(S)_PROXY / NO_PROXY mounts whenever an explicit `transport=` is passed.""" + return MappingProxyType( + { + pattern: None if proxy_url is None else build_proxy_transport(proxy_url) + for pattern, proxy_url in get_environment_proxies().items() + } + ) + class _TCPConnectorKwargs(TypedDict, total=False): local_addr: tuple[str, int] | None @@ -607,6 +625,7 @@ class AsyncHTTPHandler: return httpx.AsyncClient( transport=transport, + mounts=AsyncHTTPHandler._create_httpx_proxy_mounts(transport, verify=ssl_config, cert=cert), event_hooks=event_hooks, timeout=timeout, verify=ssl_config, @@ -1191,10 +1210,22 @@ class AsyncHTTPHandler: - [Default] If force_ipv4 is False, it will return None """ if litellm.force_ipv4: - return AsyncHTTPTransport(local_address="0.0.0.0") + return AsyncHTTPTransport(local_address=_IPV4_LOCAL_ADDRESS) else: return None + @staticmethod + def _create_httpx_proxy_mounts( + transport: LiteLLMAiohttpTransport | AsyncHTTPTransport | None, + verify: VerifyTypes, + cert: CertTypes | None, + ) -> Mapping[str, AsyncHTTPTransport | None] | None: + if not isinstance(transport, AsyncHTTPTransport): + return None + return _environment_proxy_mounts( + lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert) + ) + class HTTPHandler: def __init__( @@ -1227,6 +1258,7 @@ class HTTPHandler: # Create a client with a connection pool return httpx.Client( transport=self._create_sync_transport(), + mounts=self._create_sync_proxy_mounts(verify=ssl_config, cert=cert), timeout=self.timeout if self.timeout is not None else _DEFAULT_TIMEOUT, verify=ssl_config, cert=cert, @@ -1507,10 +1539,19 @@ class HTTPHandler: Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them """ if litellm.force_ipv4: - return HTTPTransport(local_address="0.0.0.0") + return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS) else: return getattr(litellm, "sync_transport", None) + @staticmethod + def _create_sync_proxy_mounts( + verify: VerifyTypes, + cert: CertTypes | None, + ) -> Mapping[str, HTTPTransport | None] | None: + if not litellm.force_ipv4: + return None + return _environment_proxy_mounts(lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert)) + def get_async_httpx_client( llm_provider: LlmProviders | httpxSpecialProvider, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 834f7d564a2..0f6966b0ae2 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1,6 +1,5 @@ import asyncio import json -import os import ssl from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from contextlib import asynccontextmanager @@ -29,6 +28,7 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) +from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming @@ -69,7 +69,9 @@ from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeechConfig from litellm.llms.base_llm.vector_store.transformation import ( BaseDirectVectorStoreConfig, + BaseQueryEmbeddingVectorStoreConfig, BaseVectorStoreConfig, + VectorStoreEmbeddingExecutor, ) from litellm.llms.base_llm.vector_store_files.transformation import ( BaseVectorStoreFilesConfig, @@ -159,7 +161,11 @@ def _rust_responses_websocket_enabled( custom_llm_provider: str | None, litellm_params: GenericLiteLLMParams, ) -> bool: - return custom_llm_provider == "openai" and litellm_params.get("rust") is True + from litellm.rust_bridge.configuration import rust_enabled + + raw_request_override: Final = litellm_params.get("rust") + request_override: Final = raw_request_override if isinstance(raw_request_override, bool) else None + return custom_llm_provider == "openai" and rust_enabled(request_override=request_override) from .http_handler import get_shared_realtime_ssl_context @@ -178,6 +184,7 @@ if TYPE_CHECKING: AnthropicMessagesStreamingResponse, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig + from litellm.router import Router from litellm.types.llms.openai_evals import ( CancelEvalResponse, CancelRunResponse, @@ -270,6 +277,16 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool: return False +def _aws_signing_overrides(optional_params: Mapping[str, Any], litellm_params: Mapping[str, Any]) -> Mapping[str, Any]: + return MappingProxyType( + { + key: litellm_params[key] + for key in AWS_CREDENTIAL_KWARGS_KEYS + if optional_params.get(key) is None and litellm_params.get(key) is not None + } + ) + + def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]: """Duck-type discover proxy hooks exposing per-frame project ITPM/OTPM enforcement, so the Responses WebSocket loop can charge every @@ -534,7 +551,10 @@ class BaseLLMHTTPHandler: headers, signed_json_body = provider_config.sign_request( headers=headers, - optional_params=optional_params, + optional_params={ + **optional_params, + **_aws_signing_overrides(optional_params, litellm_params), + }, request_data=data, api_base=api_base, api_key=api_key, @@ -2363,10 +2383,6 @@ class BaseLLMHTTPHandler: "anthropic_messages", ) - @staticmethod - def _rust_env_enabled() -> bool: - return os.getenv("LITELLM_RUST", "").strip().lower() in {"1", "true", "yes", "on"} - @staticmethod async def _maybe_rust_anthropic_messages( *, @@ -2382,7 +2398,11 @@ class BaseLLMHTTPHandler: ) -> AnthropicMessagesResponse | None: if custom_llm_provider not in ("azure_ai", "anthropic"): return None - if litellm_params.get("rust") is not True and not BaseLLMHTTPHandler._rust_env_enabled(): + from litellm.rust_bridge.configuration import rust_enabled + + raw_request_override: Final = litellm_params.get("rust") + request_override: Final = raw_request_override if isinstance(raw_request_override, bool) else None + if not rust_enabled(request_override=request_override): return None if has_agentic_hook: return None @@ -2923,7 +2943,7 @@ class BaseLLMHTTPHandler: final_response: Final = await self._call_agentic_completion_hooks( response=initial_response, model=model, - messages=(input if isinstance(input, list) else [{"role": "user", "content": input}]), + messages=(input if isinstance(input, list) else [{"role": "user", "content": input}]), # pyright: ignore[reportArgumentType] # pre-existing mismatch surfaced by the Router import; the hook accepts response input items at runtime anthropic_messages_provider_config=responses_api_provider_config, anthropic_messages_optional_request_params=response_api_optional_request_params, logging_obj=logging_obj, @@ -5415,7 +5435,7 @@ class BaseLLMHTTPHandler: try: response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator = await litellm.aresponses( model=patch.model or model, - input=patch.messages, + input=patch.messages, # pyright: ignore[reportArgumentType] # pre-existing mismatch surfaced by the Router import; patch messages are valid response input at runtime **optional_params, **kwargs_for_followup, ) @@ -9683,11 +9703,13 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, extra_headers: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, timeout: float | httpx.Timeout | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, _is_async: bool = False, + router: "Router | None" = None, ) -> VectorStoreSearchResponse: if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig): self._pre_call_direct_vector_store_search( @@ -9702,6 +9724,7 @@ class BaseLLMHTTPHandler: vector_store_search_optional_params=vector_store_search_optional_params, litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), # mutable-ok: snapshot GenericLiteLLMParams into the Mapping shape + embedding_executor=embedding_executor, timeout=timeout, ) @@ -9725,8 +9748,7 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - # Check if provider has async transform method - if hasattr(vector_store_provider_config, "atransform_search_vector_store_request"): + if isinstance(vector_store_provider_config, BaseQueryEmbeddingVectorStoreConfig): ( url, request_body, @@ -9738,12 +9760,14 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, + router=router, + embedding_executor=embedding_executor, ) else: ( url, request_body, - ) = vector_store_provider_config.transform_search_vector_store_request( + ) = await vector_store_provider_config.atransform_search_vector_store_request( vector_store_id=vector_store_id, query=query, vector_store_search_optional_params=vector_store_search_optional_params, @@ -9751,6 +9775,7 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, + router=router, ) all_optional_params: Final[dict[str, object]] = dict(litellm_params) all_optional_params.update(vector_store_search_optional_params or {}) @@ -9797,11 +9822,13 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, litellm_params: GenericLiteLLMParams, logging_obj: LiteLLMLoggingObj, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, extra_headers: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, timeout: float | httpx.Timeout | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, _is_async: bool = False, + router: "Router | None" = None, ) -> VectorStoreSearchResponse | Coroutine[object, object, VectorStoreSearchResponse]: if _is_async: return self.async_vector_store_search_handler( @@ -9812,10 +9839,12 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + embedding_executor=embedding_executor, extra_headers=extra_headers, extra_body=extra_body, timeout=timeout, client=client, + router=router, ) if isinstance(vector_store_provider_config, BaseDirectVectorStoreConfig): @@ -9831,6 +9860,7 @@ class BaseLLMHTTPHandler: vector_store_search_optional_params=vector_store_search_optional_params, litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), # mutable-ok: snapshot GenericLiteLLMParams into the Mapping shape + embedding_executor=embedding_executor, timeout=timeout, ) @@ -9851,18 +9881,35 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) - ( - url, - request_body, - ) = vector_store_provider_config.transform_search_vector_store_request( - vector_store_id=vector_store_id, - query=query, - vector_store_search_optional_params=vector_store_search_optional_params, - api_base=api_base, - litellm_logging_obj=logging_obj, - litellm_params=dict(litellm_params), - extra_body=extra_body, - ) + if isinstance(vector_store_provider_config, BaseQueryEmbeddingVectorStoreConfig): + ( + url, + request_body, + ) = vector_store_provider_config.transform_search_vector_store_request( + vector_store_id=vector_store_id, + query=query, + vector_store_search_optional_params=vector_store_search_optional_params, + api_base=api_base, + litellm_logging_obj=logging_obj, + litellm_params=dict(litellm_params), + extra_body=extra_body, + router=router, + embedding_executor=embedding_executor, + ) + else: + ( + url, + request_body, + ) = vector_store_provider_config.transform_search_vector_store_request( + vector_store_id=vector_store_id, + query=query, + vector_store_search_optional_params=vector_store_search_optional_params, + api_base=api_base, + litellm_logging_obj=logging_obj, + litellm_params=dict(litellm_params), + extra_body=extra_body, + router=router, + ) all_optional_params: Final[dict[str, object]] = dict(litellm_params) all_optional_params.update(vector_store_search_optional_params or {}) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index c587146005f..65622d62af2 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -330,6 +330,10 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): ) -> dict: is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params) mapped_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params) + if "claude" in model: + AnthropicConfig.translate_legacy_thinking_for_adaptive_model( + model=model, optional_params=mapped_params, custom_llm_provider="databricks" + ) if "tools" in mapped_params: mapped_params["tools"] = self._map_openai_to_dbrx_tool(model=model, tools=mapped_params["tools"]) if "max_completion_tokens" in non_default_params and replace_max_completion_tokens_with_max_tokens: diff --git a/litellm/llms/fireworks_ai/common_utils.py b/litellm/llms/fireworks_ai/common_utils.py index 8c306faa036..ac934ad0cb5 100644 --- a/litellm/llms/fireworks_ai/common_utils.py +++ b/litellm/llms/fireworks_ai/common_utils.py @@ -2,6 +2,7 @@ from typing import Final from httpx import Headers +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -16,16 +17,18 @@ def get_fireworks_session_id(litellm_params: dict) -> str | None: """ Session id to send as `x-session-affinity`, or None when the caller gave none. - Deliberately does not fall back to `litellm_trace_id`: that is generated per - request (`str(uuid.uuid4())` when absent), so using it pins every request to a - different Fireworks node and prompt caching never hits. + Deliberately does not fall back to `litellm_trace_id`, and ignores session ids the + proxy generated for a request that had none: both are per request, so using them + pins every request to a different Fireworks node and prompt caching never hits. """ params: Final = litellm_params + metadata: Final = params.get("metadata") + if isinstance(metadata, dict) and metadata.get(SESSION_ID_GENERATED_METADATA_KEY): + return None for key in ("litellm_session_id", "session_id"): value = params.get(key) if value: return str(value) - metadata: Final = params.get("metadata") if isinstance(metadata, dict): value = metadata.get("session_id") if value: diff --git a/litellm/llms/gemini/vector_stores/transformation.py b/litellm/llms/gemini/vector_stores/transformation.py index f6525a449b6..82586b1f638 100644 --- a/litellm/llms/gemini/vector_stores/transformation.py +++ b/litellm/llms/gemini/vector_stores/transformation.py @@ -33,6 +33,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -168,6 +169,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Mapping[str, object] | None = None, + router: "Router | None" = None, ) -> tuple[str, dict]: """ Transform search request to Gemini's generateContent format. diff --git a/litellm/llms/gigachat/authenticator.py b/litellm/llms/gigachat/authenticator.py index d6b217d5746..73086ba395b 100644 --- a/litellm/llms/gigachat/authenticator.py +++ b/litellm/llms/gigachat/authenticator.py @@ -8,6 +8,7 @@ Based on official GigaChat SDK authentication flow. import time import uuid from collections.abc import Mapping +from types import MappingProxyType from typing import Final import httpx @@ -32,8 +33,8 @@ GIGACHAT_SCOPE: Final = "GIGACHAT_API_PERS" # Token expiry buffer in milliseconds (refresh token 60s before expiry) TOKEN_EXPIRY_BUFFER_MS: Final = 60000 -# Cache for access tokens _token_cache: Final = InMemoryCache() +_NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) class GigaChatAuthError(BaseLLMException): @@ -80,10 +81,9 @@ def get_access_token( Raises: GigaChatAuthError: If authentication fails """ - if not litellm_params: - litellm_params = {} # mutable-ok: empty dict default; rebind-ok: provide default + params: Final = litellm_params or _NO_LITELLM_PARAMS - access_token: Final = litellm_params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN") + access_token: Final = params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN") if access_token: return access_token @@ -94,24 +94,20 @@ def get_access_token( message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.", ) - effective_scope: Final = scope or litellm_params.get("gigachat_scope") or _get_scope() - effective_auth_url: Final = auth_url or litellm_params.get("gigachat_auth_url") or _get_auth_url() + effective_scope: Final = scope or params.get("gigachat_scope") or _get_scope() + effective_auth_url: Final = auth_url or params.get("gigachat_auth_url") or _get_auth_url() - # Check cache cache_key: Final = f"gigachat_token:{effective_credentials[:16]}" cached: Final = _token_cache.get_cache(cache_key) if cached: _token, _expires_at = cached - # Check if token is still valid (with buffer) if time.time() * 1000 < _expires_at - TOKEN_EXPIRY_BUFFER_MS: verbose_logger.debug("Using cached GigaChat access token") return _token - # Request new token new_token, new_expires_at = _request_token_sync(effective_credentials, effective_scope, effective_auth_url) # pyright: ignore[reportArgumentType] # credential keys may be broader than str if new_expires_at: - # Cache token ttl_seconds: Final = max(0, (new_expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) if ttl_seconds > 0: _token_cache.set_cache(cache_key, (new_token, new_expires_at), ttl=ttl_seconds) @@ -126,10 +122,9 @@ async def get_access_token_async( litellm_params: Mapping[str, object] | None = None, ) -> str: """Async version of get_access_token.""" - if not litellm_params: - litellm_params = {} # mutable-ok: empty dict default; rebind-ok: provide default + params: Final = litellm_params or _NO_LITELLM_PARAMS - access_token: Final = litellm_params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN") + access_token: Final = params.get("gigachat_access_token") or get_secret_str("GIGACHAT_ACCESS_TOKEN") if access_token: return access_token @@ -140,10 +135,9 @@ async def get_access_token_async( message="GigaChat credentials not provided. Set GIGACHAT_CREDENTIALS or GIGACHAT_API_KEY environment variable.", ) - effective_scope: Final = scope or litellm_params.get("gigachat_scope") or _get_scope() - effective_auth_url: Final = auth_url or litellm_params.get("gigachat_auth_url") or _get_auth_url() + effective_scope: Final = scope or params.get("gigachat_scope") or _get_scope() + effective_auth_url: Final = auth_url or params.get("gigachat_auth_url") or _get_auth_url() - # Check cache cache_key: Final = f"gigachat_token:{effective_credentials[:16]}" cached: Final = _token_cache.get_cache(cache_key) if cached: @@ -152,11 +146,9 @@ async def get_access_token_async( verbose_logger.debug("Using cached GigaChat access token") return _token - # Request new token new_token, new_expires_at = await _request_token_async(effective_credentials, effective_scope, effective_auth_url) # pyright: ignore[reportArgumentType] # credential keys may be broader than str if new_expires_at: - # Cache token ttl_seconds: Final = max(0, (new_expires_at - TOKEN_EXPIRY_BUFFER_MS - time.time() * 1000) / 1000) if ttl_seconds > 0: _token_cache.set_cache(cache_key, (new_token, new_expires_at), ttl=ttl_seconds) diff --git a/litellm/llms/gigachat/chat/streaming.py b/litellm/llms/gigachat/chat/streaming.py index 2875b30232e..0a4cbd8e520 100644 --- a/litellm/llms/gigachat/chat/streaming.py +++ b/litellm/llms/gigachat/chat/streaming.py @@ -52,7 +52,6 @@ class GigaChatModelResponseIterator: tool_use: ChatCompletionToolCallChunk | None = None # rebind-ok: conditionally assigned on function_call finish_reason: str | None = chunk_finish_reason - # Handle function_call in stream raw_function_call: Final = delta.get("function_call") if chunk_finish_reason == "function_call" and isinstance(raw_function_call, Mapping) and raw_function_call: func_call: Final[Mapping[str, object]] = raw_function_call diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 8f23c5175ec..89920ebd27b 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -10,6 +10,7 @@ import json import time import uuid from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx @@ -34,6 +35,9 @@ else: LiteLLMLoggingObj = Any +_EMPTY_FUNCTION: Final[Mapping[str, object]] = MappingProxyType({}) + + def is_valid_json(value: str) -> bool: """Checks whether the value passed is a valid serialized JSON string""" try: @@ -111,11 +115,9 @@ class GigaChatConfig(BaseConfig): """ Set up headers with OAuth token. """ - # Get access token credentials: Final = api_key or get_secret_str("GIGACHAT_CREDENTIALS") or get_secret_str("GIGACHAT_API_KEY") access_token: Final = get_access_token(credentials=credentials, litellm_params=litellm_params) - # Store credentials for image uploads self._current_credentials = credentials self._current_api_base = api_base @@ -208,18 +210,18 @@ class GigaChatConfig(BaseConfig): def _convert_tools_to_functions(self, tools: Sequence) -> Sequence[dict]: """Convert OpenAI tools format to GigaChat functions format.""" - functions: Final[list[dict]] = [] # mutable-ok: accumulator for building functions list - for tool in tools: - if isinstance(tool, dict) and tool.get("type") == "function": - func = tool.get("function", {}) - functions.append( - { - "name": func.get("name", ""), - "description": func.get("description", ""), - "parameters": func.get("parameters", {}), - } - ) - return functions + return [ + { + "name": function.get("name", ""), + "description": function.get("description", ""), + "parameters": function.get("parameters", {}), + } + for function in ( + tool.get("function", _EMPTY_FUNCTION) + for tool in tools + if isinstance(tool, dict) and tool.get("type") == "function" + ) + ] def _map_tool_choice(self, tool_choice: str | Mapping[str, object]) -> str | Mapping[str, object] | None: """ @@ -299,7 +301,6 @@ class GigaChatConfig(BaseConfig): if part.get("type") == "text": texts.append(part.get("text", "")) elif part.get("type") == "image_url": - # Extract image URL and upload to GigaChat image_url: object = part.get("image_url", {}) upload_url: str if isinstance(image_url, str): @@ -322,16 +323,13 @@ class GigaChatConfig(BaseConfig): headers: Mapping[str, object], ) -> dict: # mutable-ok: request payload sent to httpx """Transform OpenAI request to GigaChat format.""" - # Transform messages giga_messages: Final = self._transform_messages(messages) - # Build request request_data: Final[dict[str, object]] = { "model": model.replace("gigachat/", ""), "messages": giga_messages, } - # Add optional params for key in [ "temperature", "top_p", @@ -343,7 +341,6 @@ class GigaChatConfig(BaseConfig): if key in optional_params: request_data[key] = optional_params[key] - # Add functions if present if "functions" in optional_params: request_data["functions"] = optional_params["functions"] if "function_call" in optional_params: @@ -358,10 +355,8 @@ class GigaChatConfig(BaseConfig): for i, msg in enumerate(messages): message = dict(msg) - # Remove unsupported fields message.pop("name", None) - # Transform roles role = message.get("role", "user") if role == "developer": message["role"] = "system" @@ -374,18 +369,15 @@ class GigaChatConfig(BaseConfig): if not isinstance(content, str) or not is_valid_json(content): message["content"] = json.dumps(content, ensure_ascii=False) - # Handle None content if message.get("content") is None: message["content"] = "" - # Handle list content (multimodal) - extract text and images content = message.get("content") if isinstance(content, list): message["content"], attachments = self._transform_list_content(content) if attachments: message["attachments"] = attachments - # Transform tool_calls to function_call tool_calls = message.get("tool_calls") if tool_calls and isinstance(tool_calls, list) and len(tool_calls) > 0: tool_call = tool_calls[0] @@ -436,13 +428,11 @@ class GigaChatConfig(BaseConfig): message_data = choice.get("message", {}) finish_reason = choice.get("finish_reason", "stop") - # Transform function_call to tool_calls or content if finish_reason == "function_call" and message_data.get("function_call"): func_call = message_data["function_call"] args = func_call.get("arguments", {}) if is_structured_output: - # Convert to content for structured output if isinstance(args, dict): content = json.dumps(args, ensure_ascii=False) else: @@ -452,7 +442,6 @@ class GigaChatConfig(BaseConfig): message_data.pop("functions_state_id", None) finish_reason = "stop" else: - # Convert to tool_calls format if isinstance(args, dict): args = json.dumps(args, ensure_ascii=False) message_data["tool_calls"] = [ @@ -468,7 +457,6 @@ class GigaChatConfig(BaseConfig): message_data.pop("function_call", None) finish_reason = "tool_calls" - # Clean up GigaChat-specific fields message_data.pop("functions_state_id", None) choices.append( diff --git a/litellm/llms/gigachat/embedding/transformation.py b/litellm/llms/gigachat/embedding/transformation.py index 2ec8324e33c..0db4475be8f 100644 --- a/litellm/llms/gigachat/embedding/transformation.py +++ b/litellm/llms/gigachat/embedding/transformation.py @@ -112,18 +112,10 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig): "input": ["text1", "text2", ...] } """ - # Normalize input to list - if isinstance(input, str): - input_list: list = [input] # rebind-ok: locally scoped conversion - else: - input_list = input - - # Remove gigachat/ prefix from model if present - model = model.removeprefix("gigachat/") # rebind-ok: parameter reassignment for normalization - + normalized_input: Final = [input] if isinstance(input, str) else input # mutable-ok: preserve list API return { - "model": model, - "input": input_list, + "model": model.removeprefix("gigachat/"), + "input": normalized_input, } def transform_embedding_response( diff --git a/litellm/llms/gigachat/passthrough/transformation.py b/litellm/llms/gigachat/passthrough/transformation.py index a0edc6f5682..e1f73d04275 100644 --- a/litellm/llms/gigachat/passthrough/transformation.py +++ b/litellm/llms/gigachat/passthrough/transformation.py @@ -60,7 +60,6 @@ class GigaChatPassthroughConfig(BasePassthroughConfig): """ Set up headers with OAuth token. """ - # Get access token access_token: Final = get_access_token(credentials=api_key, litellm_params=litellm_params) headers["Authorization"] = f"Bearer {access_token}" # rebind-ok: mutating for OAuth setup @@ -82,7 +81,6 @@ class GigaChatPassthroughConfig(BasePassthroughConfig): from litellm.types.utils import LlmProviders, ModelResponse from litellm.utils import ProviderConfigManager - # cost tracking only for completions and embeddings if "completions" in endpoint: provider_chat_config: Final = ProviderConfigManager.get_provider_chat_config( provider=LlmProviders(custom_llm_provider), diff --git a/litellm/llms/gigachat/utils.py b/litellm/llms/gigachat/utils.py index cbb35cd1b57..ce7e848ed7f 100644 --- a/litellm/llms/gigachat/utils.py +++ b/litellm/llms/gigachat/utils.py @@ -4,7 +4,6 @@ from typing import Final from litellm.secret_managers.main import get_secret_str from litellm.types.utils import PromptTokensDetailsWrapper, Usage -# GigaChat API endpoint GIGACHAT_BASE_URL: Final = "https://gigachat.devices.sberbank.ru/api/v1" diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 0e8fa294f5d..764d80c6f82 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -3,16 +3,20 @@ Transformation logic for Hosted VLLM rerank """ from collections.abc import Mapping +from types import MappingProxyType from typing import Any, Final import httpx +from pydantic import ValidationError from litellm._uuid import uuid +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.secret_managers.main import get_secret_str from litellm.types.rerank import ( + HostedVLLMRerankTruncationParams, OptionalRerankParams, RerankBilledUnits, RerankRequest, @@ -34,6 +38,13 @@ class HostedVLLMRerankError(BaseLLMException): super().__init__(status_code=status_code, message=message, headers=headers) +def validated_truncation_params(non_default_params: Mapping[str, object] | None) -> HostedVLLMRerankTruncationParams: + try: + return HostedVLLMRerankTruncationParams.model_validate(non_default_params or MappingProxyType({})) + except ValidationError as error: + raise UnsupportedParamsError(status_code=400, message=f"hosted_vllm rerank: {error}") from error + + class HostedVLLMRerankConfig(BaseRerankConfig): def __init__(self) -> None: pass @@ -62,7 +73,11 @@ class HostedVLLMRerankConfig(BaseRerankConfig): "top_n", "rank_fields", "return_documents", + "max_tokens_per_doc", "instruction", + "truncate_prompt_tokens", + "truncation_side", + "max_tokens_per_query", ] def map_cohere_rerank_params( @@ -100,7 +115,15 @@ class HostedVLLMRerankConfig(BaseRerankConfig): if instruction is not None: mapped_params["instruction"] = instruction - return dict(mapped_params) + truncation: Final = validated_truncation_params(non_default_params) + forwarded: Final[OptionalRerankParams] = { + **mapped_params, + "max_tokens_per_doc": max_tokens_per_doc, + "truncate_prompt_tokens": truncation.truncate_prompt_tokens, + "truncation_side": truncation.truncation_side, + "max_tokens_per_query": truncation.max_tokens_per_query, + } + return dict(forwarded) def validate_environment( self, @@ -138,6 +161,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): if "documents" not in optional_rerank_params: raise ValueError("documents is required for Hosted VLLM rerank") + truncation: Final = HostedVLLMRerankTruncationParams.model_validate(optional_rerank_params) rerank_request: Final = RerankRequest( model=model, query=optional_rerank_params["query"], @@ -146,6 +170,10 @@ class HostedVLLMRerankConfig(BaseRerankConfig): rank_fields=optional_rerank_params.get("rank_fields", None), return_documents=optional_rerank_params.get("return_documents", None), instruction=optional_rerank_params.get("instruction", None), + max_tokens_per_doc=truncation.max_tokens_per_doc, + truncate_prompt_tokens=truncation.truncate_prompt_tokens, + truncation_side=truncation.truncation_side, + max_tokens_per_query=truncation.max_tokens_per_query, ) return rerank_request.model_dump(exclude_none=True) diff --git a/litellm/llms/langflow/a2a.py b/litellm/llms/langflow/a2a.py index cae750d586e..060dc0a4d05 100644 --- a/litellm/llms/langflow/a2a.py +++ b/litellm/llms/langflow/a2a.py @@ -1,28 +1,9 @@ -import hashlib from typing import Any, Final - -def get_session_id_from_a2a_params(params: dict[str, Any]) -> str | None: - message: Final = params.get("message", {}) - if isinstance(message, dict): - return message.get("contextId") - return getattr(message, "contextId", None) - - -def scope_session_to_principal(session_id: str, principal: str | None) -> str: - """ - Bind a client-supplied A2A contextId to the authenticated principal. - - Without this, two distinct keys authorized for the same LangFlow agent could - set the same contextId and read/append to each other's LangFlow memory. The - principal is hashed (it is already a hashed token) so the raw value is never - sent to the LangFlow backend, while the original contextId is kept as a - suffix for operator-side correlation. - """ - if not principal: - return session_id - principal_prefix: Final = hashlib.sha256(principal.encode("utf-8")).hexdigest()[:16] - return f"{principal_prefix}-{session_id}" +from litellm.a2a_protocol.utils import ( + get_session_id_from_a2a_params, + scope_session_to_principal, +) def merge_a2a_session_into_litellm_params( diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py index 34f0cd854c4..4f3c366d8c1 100644 --- a/litellm/llms/milvus/vector_stores/transformation.py +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -1,9 +1,14 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx -import litellm -from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig +from litellm.llms.base_llm.vector_store.transformation import ( + BaseQueryEmbeddingVectorStoreConfig, + VectorStoreEmbeddingExecutor, +) from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( @@ -19,6 +24,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -36,7 +42,7 @@ MILVUS_OPTIONAL_PARAMS: Final = { } -class MilvusVectorStoreConfig(BaseVectorStoreConfig): +class MilvusVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig): """ Configuration for Milvus Vector Store @@ -117,77 +123,79 @@ class MilvusVectorStoreConfig(BaseVectorStoreConfig): def transform_search_vector_store_request( self, vector_store_id: str, - query: str | list[str], + query: str | Sequence[str], vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, api_base: str, litellm_logging_obj: LiteLLMLoggingObj, - litellm_params: dict, - extra_body: dict[str, Any] | None = None, - ) -> tuple[str, dict[str, Any]]: - """ - Transform search request for Azure AI Search API + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + router: Router | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: + query_text: Final = self.query_text(query) + query_vector: Final = self.embed_query(query_text, litellm_params, embedding_executor, router) + return self._search_request( + vector_store_id, + query_text, + query_vector, + vector_store_search_optional_params, + api_base, + litellm_logging_obj, + litellm_params, + ) - Generates embeddings using litellm.embeddings and constructs Azure AI Search request - """ - # Convert query to string if it's a list - if isinstance(query, list): - query = " ".join(query) + async def atransform_search_vector_store_request( + self, + vector_store_id: str, + query: str | Sequence[str], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: LiteLLMLoggingObj, + litellm_params: Mapping[str, object], + extra_body: Mapping[str, object] | None = None, + router: Router | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, + ) -> tuple[str, dict[str, object]]: + query_text: Final = self.query_text(query) + query_vector: Final = await self.aembed_query(query_text, litellm_params, embedding_executor, router) + return self._search_request( + vector_store_id, + query_text, + query_vector, + vector_store_search_optional_params, + api_base, + litellm_logging_obj, + litellm_params, + ) - # Get embedding model from litellm_params (required) - embedding_model: Final = litellm_params.get("litellm_embedding_model") - if not embedding_model: - raise ValueError( - "embedding_model is required in litellm_params for Milvus. You can call any litellm embedding model." - "Example: litellm_params['embedding_model'] = 'azure/text-embedding-3-large'" + @staticmethod + def _search_request( + vector_store_id: str, + query_text: str, + query_vector: Sequence[float], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: LiteLLMLoggingObj, + litellm_params: Mapping[str, object], + ) -> tuple[str, dict[str, object]]: + scope: Final = { + key: value + for key, value in ( + ("dbName", litellm_params.get("milvus_db_name")), + ("partitionNames", litellm_params.get("milvus_partition_names")), ) - - embedding_config: Final = litellm_params.get("litellm_embedding_config", {}) - if not embedding_config: - raise ValueError( - "embedding_config is required in litellm_params for Milvus. You can call any litellm embedding model." - "Example: litellm_params['embedding_config'] = {'api_base': 'https://krris-mh44uf7y-eastus2.cognitiveservices.azure.com/', 'api_key': 'os.environ/AZURE_API_KEY', 'api_version': '2025-09-01'}" - ) - - # Get top_k (number of results to return) - # Generate embedding for the query using litellm.embeddings - try: - embedding_response: Final = litellm.embedding( - model=embedding_model, - input=[query], - **embedding_config, - ) - query_vector: Final = embedding_response.data[0]["embedding"] - except Exception as e: - raise Exception(f"Failed to generate embedding for query: {e}") - - # Azure AI Search endpoint for search - index_name: Final = vector_store_id # vector_store_id is the index name - url: Final = f"{api_base}/v2/vectordb/entities/search" - - # Build the request body for Azure AI Search with vector search - request_body: Final[dict[str, Any]] = { - "collectionName": index_name, + if value + } + litellm_logging_obj.model_call_details["input"] = query_text + litellm_logging_obj.model_call_details["embedding_model"] = litellm_params.get("litellm_embedding_model") + return f"{api_base}/v2/vectordb/entities/search", { + "collectionName": vector_store_id, "data": [query_vector], "annsField": "book_intro_vector", **vector_store_search_optional_params, + **scope, } - db_name: Final = litellm_params.get("milvus_db_name") - if db_name: - request_body["dbName"] = db_name - - partition_names: Final = litellm_params.get("milvus_partition_names") - if partition_names: - request_body["partitionNames"] = partition_names - - ######################################################### - # Update logging object with details of the request - ######################################################### - litellm_logging_obj.model_call_details["input"] = query - litellm_logging_obj.model_call_details["embedding_model"] = embedding_model - - return url, request_body - def transform_search_vector_store_response( self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj ) -> VectorStoreSearchResponse: diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index de626b468f0..181894646e3 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -423,6 +423,7 @@ class OllamaChatConfig(BaseConfig): class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): started_reasoning_content: bool = False finished_reasoning_content: bool = False + seen_tool_calls: bool = False def _is_function_call_complete(self, function_args: str | dict) -> bool: if isinstance(function_args, dict): @@ -468,6 +469,7 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): # process tool calls - if complete function arg - add id to tool call tool_calls: Final = chunk["message"].get("tool_calls") if tool_calls is not None: + self.seen_tool_calls = True for tool_call in tool_calls: function_args = tool_call.get("function").get("arguments") if function_args is not None and len(function_args) > 0: @@ -508,9 +510,10 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): if chunk["done"] is True: finish_reason = chunk.get("done_reason") or "stop" - # Override finish_reason when tool_calls are present + # Override finish_reason when tool_calls appeared in any chunk # Fixes: https://github.com/BerriAI/litellm/issues/18922 - if tool_calls is not None: + # Fixes: https://github.com/BerriAI/litellm/issues/34692 + if self.seen_tool_calls: finish_reason = "tool_calls" choices = [ StreamingChoices( diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 96a5ed663fc..d41c8557d72 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -26,6 +26,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, + StreamingScanKey, StreamTransformSink, ) from litellm.llms.base_llm.guardrail_translation.utils import ( @@ -39,6 +40,8 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( role_out_of_guardrail_scope, scoped_structured_message_indices, stream_item_field, + stream_item_fingerprint, + stream_item_items, ) from litellm.main import stream_chunk_builder from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -503,12 +506,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): """Block-only streaming path: run the guardrail so an in-flight BLOCK can terminate the stream. Text rewrites are not propagated to the client here (see ``_process_streaming_transform`` for the incremental_diff path).""" - # check if the stream has ended - has_stream_ended = False - for chunk in responses_so_far: - if chunk.choices and chunk.choices[0].finish_reason is not None: - has_stream_ended = True - break + has_stream_ended: Final = self._first_choice_has_finished(responses_so_far) if has_stream_ended: # convert to model response @@ -706,8 +704,33 @@ class OpenAIChatCompletionsHandler(BaseTranslation): indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback) } + def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: + chunks: Final = tuple(chunk for chunk in responses_so_far if isinstance(chunk, ModelResponseStream)) + stream_ended: Final = self._first_choice_has_finished(responses_so_far) + return StreamingScanKey( + texts=tuple(self._combine_streaming_texts(chunks).values()), + tool_calls=self._streamed_tool_call_fingerprints(responses_so_far) if stream_ended else (), + stream_ended=stream_ended, + ) + + @staticmethod + def _streamed_tool_call_fingerprints(responses_so_far: Sequence[object]) -> tuple[str, ...]: + return tuple( + stream_item_fingerprint(tool_call) + for chunk in responses_so_far + for choice in _stream_chunk_choices(chunk) + for tool_call in stream_item_items(stream_item_field(choice, "delta"), "tool_calls") + ) + + @staticmethod + def _first_choice_has_finished(responses_so_far: Sequence[object]) -> bool: + first_choices: Final = tuple( + choices[0] for choices in (_stream_chunk_choices(chunk) for chunk in responses_so_far) if choices + ) + return any(stream_item_field(choice, "finish_reason") is not None for choice in first_choices) + def _combine_streaming_texts( - self, responses_so_far: list["ModelResponseStream"] + self, responses_so_far: Sequence["ModelResponseStream"] ) -> dict[tuple[int, int | None], str]: """ Combine all streaming chunks into complete text per choice. diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 4d774f6f165..bcd4ea43243 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -305,14 +305,16 @@ class BaseOpenAILLM: # Get unified SSL configuration ssl_config: Final = get_ssl_configuration() + transport: Final = AsyncHTTPHandler._create_async_transport( + ssl_context=(ssl_config if isinstance(ssl_config, ssl.SSLContext) else None), + ssl_verify=ssl_config if isinstance(ssl_config, bool) else None, + shared_session=shared_session, + ) return httpx.AsyncClient( verify=ssl_config, - transport=AsyncHTTPHandler._create_async_transport( - ssl_context=(ssl_config if isinstance(ssl_config, ssl.SSLContext) else None), - ssl_verify=ssl_config if isinstance(ssl_config, bool) else None, - shared_session=shared_session, - ), + transport=transport, + mounts=AsyncHTTPHandler._create_httpx_proxy_mounts(transport, verify=ssl_config, cert=None), follow_redirects=True, ) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 1530c154e93..a0db7aadb9e 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -28,6 +28,7 @@ Output: response.output is List[GenericResponseOutputItem] where each has: - text: str """ +import copy import time import uuid from collections.abc import Mapping, Sequence @@ -36,7 +37,6 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall -from openai.types.responses.tool_param import FunctionToolParam from pydantic import BaseModel, TypeAdapter from typing_extensions import ReadOnly, TypedDict @@ -44,11 +44,17 @@ from litellm._logging import verbose_proxy_logger from litellm.completion_extras.litellm_responses_transformation.transformation import ( OpenAiResponsesToChatCompletionStreamIterator, ) -from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.llms.base_llm.guardrail_translation.base_translation import ( + BaseTranslation, + StreamingScanKey, +) from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_responses_stream_usage, stream_item_field, + stream_item_fingerprint, + stream_item_items, ) +from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -62,7 +68,6 @@ from litellm.types.llms.openai import ( ContentPartDonePartOutputText, ErrorEvent, ErrorEventError, - OpenAIMcpServerTool, OutputItemAddedEvent, OutputItemDoneEvent, OutputTextDeltaEvent, @@ -157,23 +162,31 @@ class OpenAIResponsesHandler(BaseTranslation): Handles both string input and list of message objects. """ input_data: Final[str | ResponseInputParam | None] = data.get("input") - tools_to_check: Final[list[ChatCompletionToolParam]] = [] if input_data is None: return data structured_messages: Final = self.get_structured_messages(data) + raw_tools: Final = data.get("tools") + original_tools: Final[tuple[Mapping[str, object], ...]] = ( + tuple(raw_tools) if isinstance(raw_tools, list) else () + ) + flattened_tool_groups: Final = tuple( + form.chat_tools for form in LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(original_tools) + ) + flattened_tools: Final = tuple( + cast(ChatCompletionToolParam, tool) # cast-ok: mcp tools ride along in the guardrail's tool list + for group in flattened_tool_groups + for tool in group + ) + tools_to_check: Final[list[ChatCompletionToolParam]] = list( # mutable-ok: guardrail inputs want a list + copy.deepcopy(flattened_tools) + ) # Handle simple string input if isinstance(input_data, str): inputs = GenericGuardrailAPIInputs(texts=[input_data]) - original_tools: list[dict[str, object]] = [] - - # Extract and transform tools if present - if "tools" in data and data["tools"]: - original_tools = list(data["tools"]) - self._extract_and_transform_tools(data["tools"], tools_to_check) - if tools_to_check: - inputs["tools"] = tools_to_check + if tools_to_check: + inputs["tools"] = tools_to_check if structured_messages: inputs["structured_messages"] = structured_messages # Include model information if available @@ -189,7 +202,9 @@ class OpenAIResponsesHandler(BaseTranslation): ) guardrailed_texts = guardrailed_inputs.get("texts", []) data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data - self._apply_guardrailed_tools_to_data(data, original_tools, guardrailed_inputs.get("tools")) + self._apply_guardrailed_tools_to_data( + data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools") + ) verbose_proxy_logger.debug("OpenAI Responses API: Processed string input") return data @@ -200,7 +215,6 @@ class OpenAIResponsesHandler(BaseTranslation): texts_to_check: Final[list[str]] = [] images_to_check: Final[list[str]] = [] task_mappings: Final[list[tuple[int, int | None]]] = [] - original_tools_list: Final[list[dict[str, object]]] = list(data.get("tools") or []) # Step 1: Extract all text content, images, and tools for msg_idx, message in enumerate(input_data): @@ -212,10 +226,6 @@ class OpenAIResponsesHandler(BaseTranslation): task_mappings=task_mappings, ) - # Extract and transform tools if present - if "tools" in data and data["tools"]: - self._extract_and_transform_tools(data["tools"], tools_to_check) - # Step 2: Apply guardrail to all texts in batch if texts_to_check: inputs = GenericGuardrailAPIInputs(texts=texts_to_check) @@ -238,9 +248,7 @@ class OpenAIResponsesHandler(BaseTranslation): guardrailed_texts = guardrailed_inputs.get("texts", []) self._apply_guardrailed_tools_to_data( - data, - original_tools_list, - guardrailed_inputs.get("tools"), + data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools") ) # Step 3: Map guardrail responses back to original input structure @@ -267,73 +275,18 @@ class OpenAIResponsesHandler(BaseTranslation): names.append(str(tool["server_label"])) return names - def _extract_and_transform_tools( - self, - tools: list[FunctionToolParam | OpenAIMcpServerTool], - tools_to_check: list[ChatCompletionToolParam], - ) -> None: - """ - Extract and transform tools from Responses API format to Chat Completion format. - - Uses the LiteLLM transformation function to convert Responses API tools - to Chat Completion tools that can be passed to guardrails. - """ - if tools is not None and isinstance(tools, list): - # Transform Responses API tools to Chat Completion tools - ( - transformed_tools, - _, - ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(tools) - tools_to_check.extend(cast(list[ChatCompletionToolParam], transformed_tools)) - - def _remap_tools_to_responses_api_format(self, guardrailed_tools: list[Any]) -> list[dict[str, object]]: - """ - Remap guardrail-returned tools (Chat Completion format) back to - Responses API request tool format. - """ - return LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools( - guardrailed_tools - ) - - def _merge_tools_after_guardrail( - self, - original_tools: list[dict[str, object]], - remapped: list[dict[str, object]], - ) -> list[dict[str, object]]: - """ - Merge remapped guardrailed tools with original tools that were not sent - to the guardrail (e.g. web_search, web_search_preview), preserving order. - Tools a guardrail appended (``remapped`` longer than ``original_tools``) - have no original slot and are kept so an injected tool is not dropped. - """ - if not original_tools: - return remapped - result: Final[list[dict[str, object]]] = [] - j = 0 - for tool in original_tools: - if isinstance(tool, dict) and tool.get("type") in ( - "web_search", - "web_search_preview", - ): - result.append(tool) - else: - if j < len(remapped): - result.append(remapped[j]) - j += 1 - # Keep guardrail-appended tools that matched no original slot above. - result.extend(remapped[j:]) - return result - def _apply_guardrailed_tools_to_data( self, data: dict, - original_tools: list[dict[str, object]], - guardrailed_tools: list[ChatCompletionToolParam] | None, + original_tools: Sequence[Mapping[str, object]], + flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]], + guardrailed_tools: Sequence[ChatCompletionToolParam] | None, ) -> None: - """Remap guardrailed tools to Responses API format and merge with original, then set data['tools'].""" - if guardrailed_tools is not None: - remapped: Final = self._remap_tools_to_responses_api_format(guardrailed_tools) - data["tools"] = self._merge_tools_after_guardrail(original_tools, remapped) + if guardrailed_tools is None: + return + data["tools"] = list( # mutable-ok: downstream wants a list # rebind-ok: in-place request rewrite + merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools) + ) def _extract_input_text_and_images( self, @@ -645,18 +598,55 @@ class OpenAIResponsesHandler(BaseTranslation): ) return responses_so_far - def _check_streaming_has_ended(self, responses_so_far: Sequence[ResponsesStreamChunk]) -> bool: + def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool: """ Check if the streaming has ended. """ if not responses_so_far: return False - terminal_types: Final = { - ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value, - ResponsesAPIStreamEvents.RESPONSE_FAILED.value, - ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value, - } - return responses_so_far[-1].get("type") in terminal_types + terminal_types: Final = frozenset( + ( + ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value, + ResponsesAPIStreamEvents.RESPONSE_FAILED.value, + ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value, + ) + ) + return stream_item_field(responses_so_far[-1], "type") in terminal_types + + def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: + if not responses_so_far or not hasattr(responses_so_far[-1], "get"): + return None + last_event: Final = responses_so_far[-1] + last_event_type: Final = stream_item_field(last_event, "type") + if last_event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE.value: + return None + if last_event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value: + return self._completed_response_scan_key(stream_item_field(last_event, "response")) + return StreamingScanKey( + texts=(self.get_streaming_string_so_far(responses_so_far),), + stream_ended=self._check_streaming_has_ended(responses_so_far), + ) + + @staticmethod + def _completed_response_scan_key(response: object) -> StreamingScanKey: + output_items: Final = stream_item_items(response, "output") + message_items: Final = tuple( + item for item in output_items if stream_item_field(item, "type") != "function_call" + ) + return StreamingScanKey( + texts=tuple( + text + for item in message_items + for part in stream_item_items(item, "content") + if isinstance(text := stream_item_field(part, "text"), str) and text + ), + tool_calls=tuple( + stream_item_fingerprint(item) + for item in output_items + if stream_item_field(item, "type") == "function_call" + ), + stream_ended=True, + ) def build_stream_error_items( self, @@ -681,7 +671,7 @@ class OpenAIResponsesHandler(BaseTranslation): ), ) - def get_streaming_string_so_far(self, responses_so_far: Sequence[ResponsesStreamChunk]) -> str: + def get_streaming_string_so_far(self, responses_so_far: Sequence[object]) -> str: """ Get the string so far from the responses so far. @@ -693,12 +683,16 @@ class OpenAIResponsesHandler(BaseTranslation): """ keyed_events: Final = tuple( ( - (event.get("item_id"), event.get("output_index"), event.get("content_index")), - event.get("text"), - event.get("delta"), + ( + stream_item_field(event, "item_id"), + stream_item_field(event, "output_index"), + stream_item_field(event, "content_index"), + ), + stream_item_field(event, "text"), + stream_item_field(event, "delta"), ) for event in responses_so_far - if isinstance(event.get("text"), str) or isinstance(event.get("delta"), str) + if isinstance(stream_item_field(event, "text"), str) or isinstance(stream_item_field(event, "delta"), str) ) def part_text(part_key: tuple[object, object, object]) -> str: diff --git a/litellm/llms/openai/responses/guardrail_translation/tool_merge.py b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py new file mode 100644 index 00000000000..b596adfad6f --- /dev/null +++ b/litellm/llms/openai/responses/guardrail_translation/tool_merge.py @@ -0,0 +1,182 @@ +from collections.abc import Iterable, Mapping, Sequence +from itertools import accumulate, chain, groupby +from types import MappingProxyType +from typing import Final, TypeAlias + +from pydantic import BaseModel, TypeAdapter, ValidationError + +from litellm._logging import verbose_logger +from litellm.responses.litellm_completion_transformation.transformation import ( + NAMESPACE_DESCRIPTION_SEPARATOR, + LiteLLMCompletionResponsesConfig, +) + +Tool: TypeAlias = Mapping[str, object] +IndexedKey: TypeAlias = tuple[str, int] + +_TOOL_ADAPTER: Final = TypeAdapter(dict[str, object]) +_CHAT_TOOL_TOP_LEVEL_KEYS: Final = frozenset({"type", "function"}) + + +def _as_tool(value: object) -> Tool | None: + candidate: Final = value.model_dump(exclude_unset=True) if isinstance(value, BaseModel) else value + try: + return _TOOL_ADAPTER.validate_python(candidate) + except ValidationError: + return None + + +def _validated_tools(values: Iterable[object]) -> tuple[Tool, ...]: + validated: Final = tuple(map(_as_tool, values)) + dropped: Final = sum(tool is None for tool in validated) + if dropped: + verbose_logger.warning("Dropping %d guardrail-returned tools that are not objects", dropped) + return tuple(tool for tool in validated if tool is not None) + + +def _is_function(tool: Tool) -> bool: + return tool.get("type") == "function" + + +def _chat_tool_key(tool: Tool) -> str: + tool_type: Final = str(tool.get("type") or "") + function: Final = _as_tool(tool.get("function")) + if function is not None: + return f"{tool_type}:{function.get('name') or ''}" + return f"{tool_type}:{tool.get('server_label') or tool.get('name') or ''}" + + +def _indexed_keys(tools: Sequence[Tool]) -> tuple[IndexedKey, ...]: + keys: Final = tuple(_chat_tool_key(tool) for tool in tools) + positions_by_key: Final = groupby(sorted(range(len(keys)), key=keys.__getitem__), key=keys.__getitem__) + ordinal_by_position: Final = MappingProxyType( + {position: ordinal for _, positions in positions_by_key for ordinal, position in enumerate(positions)} + ) + return tuple((key, ordinal_by_position[position]) for position, key in enumerate(keys)) + + +def _namespace_members(namespace: Tool) -> tuple[Tool, ...]: + members: Final = namespace.get("tools") + if not isinstance(members, Sequence) or isinstance(members, (str, bytes)): + return () + return tuple(member for member in map(_as_tool, members) if member is not None) + + +def _function_fields(tool: Tool) -> Tool: + function: Final = _as_tool(tool.get("function")) + return function if function is not None else MappingProxyType({}) + + +def _without_namespace_prefix(key: str, value: object, prefix: str) -> object: + if key != "description" or not isinstance(value, str) or not value.startswith(prefix): + return value + return value[len(prefix) :] + + +def _rebuilt_member(member: Tool, flattened: Tool, guardrailed: Tool, namespace_description: str) -> Tool: + flattened_function: Final = _function_fields(flattened) + prefix: Final = f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}" if namespace_description else "" + changed_function: Final = MappingProxyType( + { + key: _without_namespace_prefix(key, value, prefix) + for key, value in _function_fields(guardrailed).items() + if flattened_function.get(key) != value + } + ) + changed_extras: Final = MappingProxyType( + { + key: value + for key, value in guardrailed.items() + if key not in _CHAT_TOOL_TOP_LEVEL_KEYS and flattened.get(key) != value + } + ) + return {**member, **changed_extras, **changed_function} # mutable-ok: json.dumps rejects MappingProxyType + + +def _rebuilt_function_members( + function_members: Sequence[Tool], + flattened_group: Sequence[Tool], + group_keys: Sequence[IndexedKey], + guardrailed_by_key: Mapping[IndexedKey, Tool], + namespace_description: str, +) -> tuple[Tool | None, ...]: + return tuple( + None + if key not in guardrailed_by_key + else member + if guardrailed_by_key[key] == flattened + else _rebuilt_member(member, flattened, guardrailed_by_key[key], namespace_description) + for member, flattened, key in zip(function_members, flattened_group, group_keys) + ) + + +def _rebuilt_namespace( + original: Tool, + members: Sequence[Tool], + flattened_group: Sequence[Tool], + group_keys: Sequence[IndexedKey], + guardrailed_by_key: Mapping[IndexedKey, Tool], +) -> tuple[Tool, ...]: + namespace_description: Final = str(original.get("description") or "") + rebuilt_functions: Final = iter( + _rebuilt_function_members( + tuple(member for member in members if _is_function(member)), + flattened_group, + group_keys, + guardrailed_by_key, + namespace_description, + ) + ) + rebuilt_members: Final = tuple( + rebuilt + for rebuilt in (next(rebuilt_functions) if _is_function(member) else member for member in members) + if rebuilt is not None + ) + if not rebuilt_members: + return () + return ({**original, "tools": list(rebuilt_members)},) # mutable-ok: json.dumps needs a plain dict and list + + +def _merged_original( + original: Tool, + flattened_group: Sequence[Tool], + group_keys: Sequence[IndexedKey], + guardrailed_by_key: Mapping[IndexedKey, Tool], +) -> tuple[Tool, ...]: + if not group_keys: + return (original,) + guardrailed_group: Final = tuple(guardrailed_by_key[key] for key in group_keys if key in guardrailed_by_key) + if guardrailed_group == tuple(flattened_group): + return (original,) + members: Final = _namespace_members(original) if original.get("type") == "namespace" else () + if members and sum(map(_is_function, members)) == len(flattened_group): + return _rebuilt_namespace(original, members, flattened_group, group_keys, guardrailed_by_key) + if not guardrailed_group: + return () + return tuple( + LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools(guardrailed_group) + ) + + +def merge_guardrailed_tools( + original_tools: Sequence[Tool], + flattened_groups: Sequence[Sequence[Tool]], + guardrailed_tools: Iterable[object], +) -> tuple[Tool, ...]: + guardrailed: Final = _validated_tools(guardrailed_tools) + flattened_keys: Final = _indexed_keys(tuple(chain.from_iterable(flattened_groups))) + guardrailed_keys: Final = _indexed_keys(guardrailed) + guardrailed_by_key: Final = MappingProxyType(dict(zip(guardrailed_keys, guardrailed))) + group_ends: Final = tuple(accumulate(len(group) for group in flattened_groups)) + group_key_slices: Final = tuple( + flattened_keys[end - len(group) : end] for group, end in zip(flattened_groups, group_ends) + ) + merged_originals: Final = chain.from_iterable( + _merged_original(original, group, group_keys, guardrailed_by_key) + for original, group, group_keys in zip(original_tools, flattened_groups, group_key_slices) + ) + owned_keys: Final = frozenset(flattened_keys) + appended: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools( + tuple(tool for key, tool in zip(guardrailed_keys, guardrailed) if key not in owned_keys) + ) + return tuple(chain(merged_originals, appended)) diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py index f6c093f2e2a..4e925494039 100644 --- a/litellm/llms/openai/vector_stores/transformation.py +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -21,6 +21,7 @@ from litellm.utils import add_openai_metadata if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -99,6 +100,7 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, + router: "Router | None" = None, ) -> tuple[str, dict]: encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url: Final = f"{api_base}/{encoded_vector_store_id}/search" diff --git a/litellm/llms/openai_like/README.md b/litellm/llms/openai_like/README.md index e9aaafe48a1..e1409b81c35 100644 --- a/litellm/llms/openai_like/README.md +++ b/litellm/llms/openai_like/README.md @@ -54,7 +54,10 @@ That's it! The provider will be automatically loaded and available. "constraints": { "temperature_max": 1.0, "temperature_min": 0.0, - "temperature_min_with_n_gt_1": 0.3 + "temperature_min_with_n_gt_1": 0.3, + // /v1/messages providers only: keep Anthropic cache_control extensions + // such as ttl instead of stripping them down to {"type": ...} + "cache_control_ttl": true }, // Optional: Special handling flags diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 11dc236064d..ac99617521c 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -1,11 +1,13 @@ from typing import Any, Final import litellm +from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.openai_like.json_loader import SimpleProviderConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" @@ -19,10 +21,17 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): ``"/v1/messages"``. The inbound Anthropic payload (system, cache_control, thinking, tools, ...) is forwarded essentially unchanged to ``{api_base}/v1/messages``, so Anthropic-only features that the - Anthropic->OpenAI translation would otherwise drop are preserved. Response - parsing and streaming are inherited from the native Anthropic config. + Anthropic->OpenAI translation would otherwise drop are preserved. The one + exception is ``cache_control``, whose Anthropic-only extensions (``ttl``) + are stripped unless the deployment opts in with + ``model_info.cache_control_ttl: true``. Response parsing and streaming are + inherited from the native Anthropic config. """ + def __init__(self, cache_control_ttl: bool = False) -> None: + super().__init__() + self._cache_control_ttl: Final = cache_control_ttl + def validate_anthropic_messages_environment( self, headers: dict[str, str], @@ -53,6 +62,35 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): def should_filter_anthropic_beta_headers(self) -> bool: return False + def supports_cache_control_ttl(self) -> bool: + return self._cache_control_ttl + + def transform_anthropic_messages_request( + self, + model: str, + messages: list[dict], # mutable-ok: matches dict-typed base signature + anthropic_messages_optional_request_params: dict, # mutable-ok: matches dict-typed base signature + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: matches dict-typed base signature + ) -> dict: # mutable-ok: matches dict-typed base signature + """ + Anthropic ignores prompt-caching hints it cannot honor, but strict + non-Anthropic implementations of the Messages API 400 the whole request + on Anthropic-only ``cache_control`` extensions (``cache_control.ttl: 1h + is not supported``), so unless the provider declares ttl support the + hints are reduced to their portable ``{"type": ...}`` core. + """ + request: Final = super().transform_anthropic_messages_request( + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + if self.supports_cache_control_ttl(): + return request + return normalize_cache_control_in_anthropic_payload(request) + def get_complete_url( self, api_base: str | None, @@ -81,7 +119,7 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig): """ def __init__(self, provider: SimpleProviderConfig): - super().__init__() + super().__init__(cache_control_ttl=bool(provider.constraints.get("cache_control_ttl"))) self._provider = provider @property diff --git a/litellm/llms/pg_vector/vector_stores/transformation.py b/litellm/llms/pg_vector/vector_stores/transformation.py index e4b06c36bf4..9de1f589ae4 100644 --- a/litellm/llms/pg_vector/vector_stores/transformation.py +++ b/litellm/llms/pg_vector/vector_stores/transformation.py @@ -8,6 +8,7 @@ from litellm.types.vector_stores import VectorStoreSearchOptionalRequestParams if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -80,6 +81,7 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, + router: "Router | None" = None, ) -> tuple[str, dict]: encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url: Final = f"{api_base}/{encoded_vector_store_id}/search" diff --git a/litellm/llms/ragflow/vector_stores/transformation.py b/litellm/llms/ragflow/vector_stores/transformation.py index 282cb7a92a7..ffa6c9e1076 100644 --- a/litellm/llms/ragflow/vector_stores/transformation.py +++ b/litellm/llms/ragflow/vector_stores/transformation.py @@ -17,6 +17,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -92,6 +93,7 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, + router: "Router | None" = None, ) -> tuple[str, dict]: """RAGFlow vector stores are management-only, search is not supported.""" raise NotImplementedError("RAGFlow vector stores support dataset management only, not search/retrieval") diff --git a/litellm/llms/s3_vectors/vector_stores/transformation.py b/litellm/llms/s3_vectors/vector_stores/transformation.py index 5be35ae4148..733358381fe 100644 --- a/litellm/llms/s3_vectors/vector_stores/transformation.py +++ b/litellm/llms/s3_vectors/vector_stores/transformation.py @@ -1,8 +1,8 @@ -import re from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.caching._embedding_router import resolve_embedding_router from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.types.router import GenericLiteLLMParams @@ -18,6 +18,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -58,13 +59,20 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): return headers def get_complete_url(self, api_base: str | None, litellm_params: dict) -> str: - aws_region_name: Final = litellm_params.get("aws_region_name") - if not aws_region_name: - raise ValueError("aws_region_name is required for S3 Vectors") - if not re.match(r"^[a-z][a-z0-9-]*$", aws_region_name): - raise ValueError("Invalid aws_region_name format") + # Resolve region the same way the ingestion path does: + # dynamic param -> AWS_REGION_NAME -> AWS_REGION -> default (us-west-2) + aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(litellm_params.get("aws_region_name")) return f"https://s3vectors.{aws_region_name}.api.aws" + def _resolve_query_embedding_router(self, embedding_model: str, router: "Router | None") -> "Router | None": + """Return the router iff it serves ``embedding_model`` as a deployment.""" + if router is None: + return None + model_list: Final = [ + dict(m) for m in (router.get_model_list() or ()) + ] # mutable-ok: resolve_embedding_router requires list[dict] + return resolve_embedding_router(embedding_model=embedding_model, llm_router=router, llm_model_list=model_list) + def transform_search_vector_store_request( self, vector_store_id: str, @@ -74,6 +82,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, + router: "Router | None" = None, ) -> tuple[str, dict]: """Sync version - generates embedding synchronously.""" # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name @@ -99,10 +108,16 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): # Generate embedding for the query embedding_model: Final = litellm_params.get("embedding_model", "text-embedding-3-small") + embedding_router: Final = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router) import litellm as litellm_module - embedding_response: Final = litellm_module.embedding(model=embedding_model, input=[query]) + embedding_input: Final = [query] # mutable-ok: the embedding API takes list input + embedding_response: Final = ( + embedding_router.embedding(model=embedding_model, input=embedding_input) + if embedding_router is not None + else litellm_module.embedding(model=embedding_model, input=embedding_input) + ) query_embedding: Final = embedding_response.data[0]["embedding"] url: Final = f"{api_base}/QueryVectors" @@ -128,6 +143,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: dict[str, Any] | None = None, + router: "Router | None" = None, ) -> tuple[str, dict]: """Async version - generates embedding asynchronously.""" # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name @@ -153,10 +169,16 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): # Generate embedding for the query asynchronously embedding_model: Final = litellm_params.get("embedding_model", "text-embedding-3-small") + embedding_router: Final = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router) import litellm as litellm_module - embedding_response: Final = await litellm_module.aembedding(model=embedding_model, input=[query]) + embedding_input: Final = [query] # mutable-ok: the embedding API takes list input + embedding_response: Final = ( + await embedding_router.aembedding(model=embedding_model, input=embedding_input) + if embedding_router is not None + else await litellm_module.aembedding(model=embedding_model, input=embedding_input) + ) query_embedding: Final = embedding_response.data[0]["embedding"] url: Final = f"{api_base}/QueryVectors" diff --git a/litellm/llms/valkey/vector_stores/transformation.py b/litellm/llms/valkey/vector_stores/transformation.py index 3cbfca0f1a9..b250f71cf3f 100644 --- a/litellm/llms/valkey/vector_stores/transformation.py +++ b/litellm/llms/valkey/vector_stores/transformation.py @@ -15,7 +15,10 @@ import httpx from pydantic import BaseModel, ConfigDict import litellm -from litellm.llms.base_llm.vector_store.transformation import BaseDirectVectorStoreConfig +from litellm.llms.base_llm.vector_store.transformation import ( + BaseDirectVectorStoreConfig, + VectorStoreEmbeddingExecutor, +) from litellm.llms.valkey.common_utils import build_valkey_url, pack_vector from litellm.types.utils import EmbeddingResponse from litellm.types.vector_stores import ( @@ -213,6 +216,7 @@ class ValkeyVectorStoreConfig(BaseDirectVectorStoreConfig): vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, litellm_logging_obj: "LiteLLMLoggingObj", litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None = None, timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: params: Final = _ValkeySearchParams.model_validate(litellm_params) @@ -222,10 +226,18 @@ class ValkeyVectorStoreConfig(BaseDirectVectorStoreConfig): embedding_field=params.embedding_field, text_field=params.text_field, ) - embedding_response: Final = self.embedding_fn( - model=params.require_embedding_model(), - input=[query_text], # mutable-ok: litellm.embedding's input contract is a list - **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + embedding_response: Final = ( + embedding_executor.embed( + params.require_embedding_model(), + query_text, + params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG, + ) + if embedding_executor is not None + else self.embedding_fn( + model=params.require_embedding_model(), + input=[query_text], # mutable-ok: the injected embedding callable requires list input + **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + ) ) vec_params: Final = {"vec": pack_vector(embedding_response.data[0]["embedding"])} # mutable-ok: redis-py API @@ -252,6 +264,7 @@ class ValkeyVectorStoreConfig(BaseDirectVectorStoreConfig): vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, litellm_logging_obj: "LiteLLMLoggingObj", litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None = None, timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: params: Final = _ValkeySearchParams.model_validate(litellm_params) @@ -261,10 +274,18 @@ class ValkeyVectorStoreConfig(BaseDirectVectorStoreConfig): embedding_field=params.embedding_field, text_field=params.text_field, ) - embedding_response: Final = await self.aembedding_fn( - model=params.require_embedding_model(), - input=[query_text], # mutable-ok: litellm.embedding's input contract is a list - **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + embedding_response: Final = ( + await embedding_executor.aembed( + params.require_embedding_model(), + query_text, + params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG, + ) + if embedding_executor is not None + else await self.aembedding_fn( + model=params.require_embedding_model(), + input=[query_text], # mutable-ok: the injected embedding callable requires list input + **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + ) ) vec_params: Final = {"vec": pack_vector(embedding_response.data[0]["embedding"])} # mutable-ok: redis-py API diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d8b1e7ba17c..69fe5678de9 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -949,7 +949,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # For Gemini 3+ models, use thinkingLevel instead of thinkingBudget if model and VertexGeminiConfig._is_gemini_3_or_newer(model): if thinking_enabled: - if thinking_budget is None or thinking_budget == 0: + if thinking_budget == 0: params["includeThoughts"] = False else: params["includeThoughts"] = True diff --git a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py index 2603552152d..b57a87c3325 100644 --- a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py +++ b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py @@ -177,8 +177,9 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig): content_item = {"type": "image_url", "image_url": document_url} # Build DeepSeek OCR request + provider_model: Final = model if model.startswith("deepseek-ai/") else f"deepseek-ai/{model}" data: Final = { - "model": "deepseek-ai/" + model, + "model": provider_model, "messages": [{"role": "user", "content": [content_item]}], } diff --git a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py index 5c250fc1a7e..36b57e7c995 100644 --- a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py @@ -21,6 +21,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -161,6 +162,7 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Mapping[str, object] | None = None, + router: "Router | None" = None, ) -> tuple[str, dict[str, object]]: """ Transform search request for Vertex AI RAG API diff --git a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py index 0bcf16ee06f..f0812e3ed9f 100644 --- a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py @@ -25,6 +25,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -245,6 +246,7 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Mapping[str, object] | None = None, + router: "Router | None" = None, ) -> tuple[str, dict[str, object]]: """ Transform a search request for the Vertex AI Search (Discovery Engine) API. diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index ef03e61a858..7579bc8c02e 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -177,6 +177,10 @@ class VertexAIAnthropicConfig(AnthropicConfig): # Restore original model name for any other processing model = original_model + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model=original_model, optional_params=optional_params, custom_llm_provider="vertex_ai" + ) + return optional_params def transform_response( diff --git a/litellm/main.py b/litellm/main.py index c4c5bbefc4f..0128e4defe5 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -141,7 +141,6 @@ from litellm.utils import ( convert_to_model_response_object, create_pretrained_tokenizer, create_tokenizer, - get_api_key, get_llm_provider, get_model_info, get_non_default_completion_params, @@ -8637,6 +8636,16 @@ def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Opti hidden_params["response_cost"] = response_cost +def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: + if logging_obj is None: + return + if isinstance(getattr(usage, "cost", None), (int, float)): + return + computed_cost: Final = logging_obj._response_cost_calculator(result=response) + if isinstance(computed_cost, (int, float)) and computed_cost > 0: + setattr(usage, "cost", computed_cost) + + def stream_chunk_builder( chunks: list, messages: list | None = None, @@ -8731,12 +8740,7 @@ def stream_chunk_builder( ) break - if litellm.include_cost_in_streaming_usage and logging_obj is not None: - setattr( - usage, - "cost", - logging_obj._response_cost_calculator(result=response), - ) + _stamp_streaming_usage_cost(usage, response, logging_obj) _set_stream_builder_response_cost(response, logging_obj) processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) @@ -8915,10 +8919,7 @@ def stream_chunk_builder( ) break - # Add cost to usage object if include_cost_in_streaming_usage is True - if litellm.include_cost_in_streaming_usage and logging_obj is not None: - setattr(usage, "cost", logging_obj._response_cost_calculator(result=response)) - + _stamp_streaming_usage_cost(usage, response, logging_obj) _set_stream_builder_response_cost(response, logging_obj) processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a3cfb300ea6..d8a8f84b032 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9643,7 +9643,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2.5-Flash": { "input_cost_per_image_token": 1.75e-06, @@ -9656,7 +9657,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2e": { "deprecation_date": "2026-08-15", @@ -10155,7 +10157,9 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "cache_read_input_token_cost": 1.45e-07, + "supports_prompt_caching": true }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", @@ -10169,18 +10173,20 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true - }, - "azure_ai/deepseek-v4-flash-0731": { + "supports_tool_choice": true, "cache_read_input_token_cost": 2.8e-08, + "supports_prompt_caching": true + }, + "azure_ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, + "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.1e-07, + "output_cost_per_token": 1.32e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_prompt_caching": true, @@ -10400,11 +10406,13 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supports_function_calling": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true }, "azure_ai/kimi-k2.6": { "deprecation_date": "2027-04-16", @@ -10415,7 +10423,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k2-6-in-microsoft-foundry/4513125", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supported_modalities": [ "text", "image" @@ -10426,7 +10434,9 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1.6e-07, + "supports_prompt_caching": true }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, @@ -12110,7 +12120,7 @@ "max_output_tokens": 2048, "max_tokens": 2048, "mode": "chat", - "output_cost_per_token": 2.65e-06, + "output_cost_per_token": 6e-07, "supports_pdf_input": true }, "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": { @@ -23514,6 +23524,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "vertex_ai/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -25351,6 +25418,65 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "rpm": 2000, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "tpm": 800000, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -25759,6 +25885,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_audio_token": 7e-07, @@ -29098,16 +29281,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29119,6 +29305,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29161,16 +29348,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29182,6 +29372,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29225,16 +29416,19 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, "cache_creation_input_token_cost_flex": 1.25e-06, "cache_creation_input_token_cost_priority": 5e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, @@ -29246,6 +29440,7 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, "output_cost_per_token_above_272k_tokens_flex": 9e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, @@ -29288,16 +29483,19 @@ "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, "cache_creation_input_token_cost_flex": 1.25e-07, "cache_creation_input_token_cost_priority": 5e-07, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_above_272k_tokens_flex": 2e-07, + "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, @@ -29309,6 +29507,7 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, "output_cost_per_token_above_272k_tokens_flex": 9e-07, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, @@ -29548,7 +29747,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, @@ -29602,7 +29804,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -29751,7 +29956,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, @@ -29800,7 +30008,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, @@ -29849,7 +30060,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, @@ -29898,7 +30111,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, @@ -30834,17 +31049,18 @@ }, "gpt-realtime-2": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, - "output_cost_per_token": 1.6e-05, + "output_cost_per_token": 2.4e-05, "supported_endpoints": [ "/v1/realtime" ], @@ -30908,8 +31124,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -30941,7 +31157,7 @@ "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", - "max_input_tokens": 128000, + "max_input_tokens": 32000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", @@ -32877,6 +33093,88 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "meta/muse-spark-1.3": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "meta", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.25e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.0025, + "search_context_size_low": 0.0025, + "search_context_size_medium": 0.0025 + }, + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_minimal_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "meta/muse-spark-1.3-contributor": { + "cache_read_input_token_cost": 2e-09, + "input_cost_per_token": 1e-07, + "litellm_provider": "meta", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.0025, + "search_context_size_low": 0.0025, + "search_context_size_medium": 0.0025 + }, + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_minimal_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "meta_llama/Llama-3.3-70B-Instruct": { "litellm_provider": "meta_llama", "max_input_tokens": 128000, @@ -33705,19 +34003,21 @@ "source": "https://mistral.ai/pricing#api-pricing" }, "mistral/magistral-medium-latest": { - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-2506": { "deprecation_date": "2025-11-30", @@ -33736,19 +34036,21 @@ "supports_tool_choice": true }, "mistral/magistral-small-latest": { - "input_cost_per_token": 5e-07, + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", + "output_cost_per_token": 6e-07, + "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-1-2-2509": { "deprecation_date": "2026-07-31", @@ -33880,16 +34182,21 @@ "supports_vision": true }, "mistral/mistral-medium": { - "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 8.1e-06, + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/mistral-medium-2312": { "deprecation_date": "2025-06-16", @@ -41340,13 +41647,13 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/Qwen/Qwen3.8-2.4T-A95B": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, "max_tokens": 1010000, "mode": "chat", - "output_cost_per_token": 6.25e-06, + "output_cost_per_token": 6e-06, "source": "https://docs.together.ai/docs/serverless-models", "supports_prompt_caching": true }, @@ -41956,6 +42263,70 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024 }, + "us-gov.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us-gov.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, @@ -45849,6 +46220,26 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, + "voyage/rerank-3": { + "input_cost_per_token": 5e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, + "voyage/rerank-3-lite": { + "input_cost_per_token": 2e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, "voyage/voyage-2": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", @@ -47100,6 +47491,27 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-build-latest": { + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "xai", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://docs.x.ai/developers/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "xai/grok-4.6": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, @@ -57420,6 +57832,34 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/glm-5p3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/accounts/fireworks/models/inkling": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 4.05e-06, + "source": "https://fireworks.ai/models/fireworks/inkling", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-8b": { "input_cost_per_token": 1e-07, "output_cost_per_token": 0.0, @@ -57479,5 +57919,542 @@ "supported_endpoints": [ "/v1/audio/transcriptions" ] + }, + "scaleway/glm-5.2": { + "input_cost_per_token": 1.8e-06, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 5.5e-06, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": false + }, + "scaleway/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": false + }, + "azure_ai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "deprecation_date": "2026-10-03", + "input_cost_per_token": 9.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-west-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_above_272k_tokens": 5.28e-06, + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-06, + "cache_read_input_token_cost": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-07, + "output_cost_per_token": 1.584e-05, + "output_cost_per_token_above_272k_tokens": 2.376e-05 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-07, + "input_cost_per_token_above_272k_tokens": 5.28e-07, + "cache_creation_input_token_cost": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-07, + "cache_read_input_token_cost": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-08, + "output_cost_per_token": 1.584e-06, + "output_cost_per_token_above_272k_tokens": 2.376e-06 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.3": { + "use_openai_responses_path": true, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/", + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.4e-07 + }, + "bedrock_mantle/us-gov-east-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "azure/us-gov/gpt-5.1": { + "cache_read_input_token_cost": 1.71875e-07, + "default_reasoning_effort": "none", + "input_cost_per_token": 1.71875e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.375e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us-gov/o3-mini": { + "cache_read_input_token_cost": 7.57e-07, + "input_cost_per_token": 1.513e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 6.05e-06, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "azure/us-gov/text-embedding-3-large": { + "input_cost_per_token": 1.63e-07, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "azure/us-gov/text-embedding-3-small": { + "input_cost_per_token": 2.5e-08, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/openai/whisper": { + "input_cost_per_second": 7.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "cloudflare/@cf/openai/whisper-large-v3-turbo": { + "input_cost_per_second": 8.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper-large-v3-turbo/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] } } diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index b918f013700..b260ec6e06f 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -194,6 +194,12 @@ def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool: return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS +def _rust_ocr_enabled(prepared_request: _PreparedOCRRequest) -> bool: + raw_request_override: Final = prepared_request.litellm_params.get("rust") + request_override: Final = raw_request_override if isinstance(raw_request_override, bool) else None + return rust_ocr_bridge.rust_ocr_enabled(request_override=request_override) + + def _rust_bridge_optional_params( prepared_request: _PreparedOCRRequest, resolve_secret: Callable[[str], str | None], @@ -422,7 +428,7 @@ async def aocr( custom_llm_provider = prepared.custom_llm_provider completion_kwargs.update({"model": model, "custom_llm_provider": custom_llm_provider}) - if _rust_ocr_supported(prepared) and rust_ocr_bridge.rust_ocr_enabled(): + if _rust_ocr_supported(prepared) and _rust_ocr_enabled(prepared): from litellm.secret_managers.main import get_secret_str rust_response: Final = await _run_rust_aocr( @@ -694,7 +700,7 @@ def ocr( custom_llm_provider = prepared.custom_llm_provider completion_kwargs.update({"model": model, "custom_llm_provider": custom_llm_provider}) - if _rust_ocr_supported(prepared) and rust_ocr_bridge.rust_ocr_enabled(): + if _rust_ocr_supported(prepared) and _rust_ocr_enabled(prepared): from litellm.secret_managers.main import get_secret_str rust_response: Final = _run_rust_ocr( diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index c4bd03fb1c3..7076683f294 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -113,10 +113,8 @@ class AsyncPassthroughStreamingResponse(AsyncGenerator[bytes, bytes]): ) ) - # Compliant: Save a strong reference to prevent GC self._background_tasks.add(task) - # Remove the task from the set when it finishes to avoid memory leaks task.add_done_callback(self._background_tasks.discard) except Exception as e: # noqa: BLE001 # Safe catch-all for verbose logging verbose_logger.exception( @@ -578,7 +576,6 @@ def llm_passthrough_route( else: return response except Exception as e: - # provider_config is guaranteed non-None here due to the earlier guard assert provider_config is not None raise base_llm_http_handler._handle_error( e=e, diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 425f82794e6..66d4aedba06 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -1149,10 +1149,11 @@ class MCPRequestHandler: would miss a real outage wrapped inside it.""" from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler - if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e): + outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(e) + if outage is not None: raise HTTPException( status_code=503, - detail="Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + detail=PrismaDBExceptionHandler.database_unavailable_message(outage), ) from None @staticmethod diff --git a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py index 09a3703e904..35a30127e27 100644 --- a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py +++ b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py @@ -101,18 +101,29 @@ class _ResolvedKey: key: "UserAPIKeyAuth" -_KeyResolutionFailure = Literal["no_active_key", "unavailable", "unresolvable"] +_KeyResolutionFailure = Literal["no_active_key", "unavailable", "faulted", "unresolvable"] """Why a token request yielded no active litellm key, kept distinct so a caller statuses each truthfully instead of blaming the client for a gateway problem: - ``no_active_key``: none was presented, or the presented key is unknown / blocked / expired (the caller's request is at fault) - ``unavailable``: the auth database was transiently unreachable while resolving (retryable) +- ``faulted``: the auth database's query engine reported a fault that retrying will not clear (still a + 503, but the wording must not tell the operator to wait) - ``unresolvable``: the gateway cannot resolve identity right now (no DB connection, or an unexpected error) -- a gateway fault, not the caller's The classification mirrors admission's ``_reload_admitted_key`` so the mint (ingress) and admission (egress) never disagree on the status of the same outage.""" +def _database_failure(exc: Exception) -> Literal["unavailable", "faulted"]: + from litellm.proxy.db.exception_handler import ( # noqa: PLC0415 # inline import avoids a module-load circular import + PrismaDBExceptionHandler, + ) + + fault: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(exc) or exc + return "faulted" if PrismaDBExceptionHandler.is_permanent_database_fault(fault) else "unavailable" + + async def _resolve_active_litellm_key(request: Request) -> "_ResolvedKey | _KeyResolutionFailure": """Resolve the presented litellm key to an active key record, or say precisely why not. @@ -170,7 +181,7 @@ async def _reload_active_key_by_hash(key_hash: str) -> "_ResolvedKey | _KeyResol return "no_active_key" except Exception as exc: # noqa: BLE001 # classify: a DB outage is retryable, anything else is an opaque gateway fault if PrismaDBExceptionHandler.is_database_service_unavailable_error(exc): - return "unavailable" + return _database_failure(exc) verbose_logger.debug( "_reload_active_key_by_hash: unexpected key-resolution error (%s)", type(exc).__name__, @@ -225,8 +236,9 @@ async def load_active_user_by_id(user_id: str) -> "LiteLLM_UserTable | _KeyResol except (ProxyException, HTTPException): return "no_active_key" except Exception as exc: # noqa: BLE001 # a DB outage is retryable; a missing user (get_user_object's wrapped ValueError) or any other resolution failure fails closed as no_active_key, never a 500 - if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(exc): - return "unavailable" + outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(exc) + if outage is not None: + return _database_failure(outage) verbose_logger.debug("_reload_active_user_by_id: user-resolution error (%s)", type(exc).__name__) return "no_active_key" if user_object is None: @@ -383,6 +395,7 @@ _BridgeMintError = Literal[ "no_identity", "invalid_refresh", "identity_unavailable", + "identity_faulted", "identity_unresolvable", "not_configured", "no_upstream_token", @@ -433,6 +446,13 @@ def _bridge_mint_error_response(error: _BridgeMintError) -> JSONResponse: "temporarily_unavailable", "the authentication database is temporarily unreachable; retry shortly", ) + case "identity_faulted": + status, code, desc = ( + 503, + "temporarily_unavailable", + "the authentication database reported a fault that is not a transient outage; " + "retrying will not help until the gateway deployment is repaired", + ) case "identity_unresolvable": status, code, desc = ( 500, @@ -485,6 +505,8 @@ def _key_resolution_failure_to_mint_error(failure: _KeyResolutionFailure) -> _Br return "no_identity" case "unavailable": return "identity_unavailable" + case "faulted": + return "identity_faulted" case "unresolvable": return "identity_unresolvable" case _: @@ -569,6 +591,8 @@ def _refresh_key_failure_to_mint_error(failure: _KeyResolutionFailure) -> _Bridg return "invalid_refresh" case "unavailable": return "identity_unavailable" + case "faulted": + return "identity_faulted" case "unresolvable": return "identity_unresolvable" case _: diff --git a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py b/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py index a43e762a456..c7b0045dde5 100644 --- a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py +++ b/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py @@ -150,11 +150,18 @@ _CLIENT_RECORD_DEBUG_KEY: Final = "gateway_dcr_client" _CONNECT_FLOW_DEBUG_KEY: Final = "gateway_connect_flow" _AUTH_CODE_DEBUG_KEY: Final = "gateway_authorization_code" -ReloadUserFailure = Literal["unresolvable", "unavailable", "no_active_key"] +ReloadUserFailure = Literal["unresolvable", "unavailable", "faulted", "no_active_key"] ReloadUser = Callable[[str], Awaitable[ReloadUserFailure | None]] """Injected live-user revalidation (the token endpoint's mirror of admission): -``None`` means the user is active; ``unavailable`` is a retryable DB outage; anything -else fails the grant closed.""" +``None`` means the user is active; ``unavailable`` is a retryable DB outage; ``faulted`` is +a DB fault retrying will not clear (still 503, worded so nobody just waits); anything else +fails the grant closed.""" + +_DB_UNAVAILABLE_DESCRIPTION: Final = "the gateway database is unavailable; retry" +_DB_FAULTED_DESCRIPTION: Final = ( + "the gateway database reported a fault that is not a transient outage; " + "retrying will not help until the gateway deployment is repaired" +) PROXY_API_AUDIENCE: Final[SessionAudience] = "proxy_api" """The audience a native client (``lite login --pkce``, a Go CLI) asks for by sending the @@ -659,7 +666,9 @@ def _set_flow_cookie(response: Response, request: Request, handle: str, flow: _C def _consent_lookup_failure_response(failure: ReloadUserFailure) -> Response: match failure: case "unavailable": - return _oauth_error(503, "temporarily_unavailable", "the gateway database is unavailable; retry") + return _oauth_error(503, "temporarily_unavailable", _DB_UNAVAILABLE_DESCRIPTION) + case "faulted": + return _oauth_error(503, "temporarily_unavailable", _DB_FAULTED_DESCRIPTION) case "unresolvable": return _oauth_error(500, "server_error", "the gateway is not configured to resolve users") case "no_active_key": @@ -962,7 +971,9 @@ def _reload_failure_response(failure: ReloadUserFailure) -> Response: ``ReloadUserFailure`` member is a type error here rather than silently 400ing.""" match failure: case "unavailable": - return _oauth_error(503, "temporarily_unavailable", "the gateway database is unavailable; retry") + return _oauth_error(503, "temporarily_unavailable", _DB_UNAVAILABLE_DESCRIPTION) + case "faulted": + return _oauth_error(503, "temporarily_unavailable", _DB_FAULTED_DESCRIPTION) case "unresolvable": return _oauth_error(500, "server_error", "the gateway is not configured to resolve users") case "no_active_key": @@ -981,7 +992,7 @@ def _mint_failure_response(failure: ProxyCredentialMintFailure) -> Response: return _oauth_error( 400, "invalid_grant", "this user belongs to a team; sign in again and pick the team for this credential" ) - case "unavailable" | "unresolvable" | "no_active_key": + case "unavailable" | "faulted" | "unresolvable" | "no_active_key": return _reload_failure_response(failure) case _: assert_never(failure) @@ -1297,8 +1308,8 @@ async def introspect_gateway_token( if peeked == "claimed": return _inactive_introspection_response() failure: Final = await reload_user(opened.principal.user_id) - if failure == "unavailable": - return _oauth_error(503, "temporarily_unavailable", "the gateway database is unavailable; retry") + if failure == "unavailable" or failure == "faulted": + return _reload_failure_response(failure) if failure is not None: return _inactive_introspection_response() return _active_introspection_response(opened) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py index 84f714db449..6f1e5baa109 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_endpoint.py @@ -121,6 +121,23 @@ class TokenEndpointClient: return Ok(ExchangedToken(access_token=parsed.access_token, expires_in=parsed.expires_in)) +class _KeyGuard: + """The per-key single-flight lock plus the invalidation generation that lock protects. + + Both live on one object so their lifetimes cannot diverge. `get_or_compute` binds the guard to + a local for its whole critical section, which keeps the weak map's entry alive for as long as + that compute could still write; an `invalidate` overlapping the compute therefore reaches the + very same object and its bump is guaranteed to be observed. Conversely a guard nobody holds is + collectible precisely because no write is outstanding for it to fence. + """ + + __slots__ = ("__weakref__", "generation", "lock") + + def __init__(self) -> None: + self.lock = asyncio.Lock() + self.generation = 0 + + class ExchangedTokenCache: """Memoizes the final token string per key, single-flighting concurrent misses on one lock.""" @@ -129,7 +146,7 @@ class ExchangedTokenCache: max_size_in_memory=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE, default_ttl=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL, ) - self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = weakref.WeakValueDictionary() + self._guards: weakref.WeakValueDictionary[str, _KeyGuard] = weakref.WeakValueDictionary() async def get_or_compute( self, @@ -144,28 +161,50 @@ class ExchangedTokenCache: guaranteeing the token it gets back was minted for the *current* inputs: a stored entry whose fingerprint differs reads as a miss and is re-minted over. That keeps eviction addressable without the key having to encode the credential material it protects. + + An `invalidate` landing while `compute` is in flight wins over that compute's write. The + token is still returned to the caller it was minted for, but it is not stored, so the next + resolution re-mints rather than serving a bearer that predates the invalidation for the + rest of its TTL. """ cached = self._get(cache_key, fingerprint) if cached is not None: return Ok(cached) - async with self._lock(cache_key): + guard = self._guard(cache_key) + async with guard.lock: cached = self._get(cache_key, fingerprint) if cached is not None: return Ok(cached) + generation = guard.generation match await compute(): case Ok(token): - self._cache.set_cache( # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped - cache_key, - (fingerprint, token.access_token), - ttl=_cache_ttl_seconds(token.expires_in), - ) + if guard.generation == generation: + self._store(cache_key, fingerprint, token) return Ok(token.access_token) case Error(err): return Error(err) def invalidate(self, cache_key: str) -> None: - """Evict one cached token so the next `get_or_compute` re-mints (e.g. after an upstream 401).""" + """Evict one cached token so the next `get_or_compute` re-mints (e.g. after an upstream 401). + + Bumping the guard's generation is what makes the eviction stick against a compute already + awaiting the token endpoint: that compute snapshotted the old generation and so skips its + write. No guard means no compute is in flight, since an in-flight one pins its own. + + Stays synchronous: callers invalidate from plain `def`s. + """ self._cache.delete_cache(cache_key) # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped + guard = self._guards.get(cache_key) + if guard is None: + return + guard.generation += 1 + + def _store(self, cache_key: str, fingerprint: str, token: ExchangedToken) -> None: + self._cache.set_cache( # pyright: ignore[reportUnknownMemberType] # InMemoryCache is untyped + cache_key, + (fingerprint, token.access_token), + ttl=_cache_ttl_seconds(token.expires_in), + ) def _get(self, cache_key: str, fingerprint: str) -> str | None: """The stored token, or None when absent or minted for different inputs. @@ -180,12 +219,12 @@ class ExchangedTokenCache: return None return token if stored_fingerprint == fingerprint else None - def _lock(self, cache_key: str) -> asyncio.Lock: - lock = self._locks.get(cache_key) - if lock is None: - lock = asyncio.Lock() - self._locks[cache_key] = lock - return lock + def _guard(self, cache_key: str) -> _KeyGuard: + guard = self._guards.get(cache_key) + if guard is None: + guard = _KeyGuard() + self._guards[cache_key] = guard + return guard def _cache_ttl_seconds(expires_in: int | None) -> int: diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index d1ef73a15cd..37474f85fe7 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -74,7 +74,13 @@ _MCP_GUARDRAIL_REJECTIONS: Final = ( ) -def _connection_error_message(exc: BaseException) -> str: +def _connection_error_message(exc: BaseException, url: str | None, timeout_seconds: float) -> str: + if isinstance(exc, TimeoutError): + return ( + f"Failed to connect to MCP server: no response from {url or 'the server'} " + f"within {timeout_seconds:.0f}s. Check that the LiteLLM proxy can reach this URL " + "from its network (DNS, egress rules, firewalls) and that the server answers MCP requests." + ) if isinstance(exc, httpx.LocalProtocolError): return ( "Failed to connect to MCP server: a request header is malformed. " @@ -92,6 +98,9 @@ def _connection_error_message(exc: BaseException) -> str: if MCP_AVAILABLE: + from mcp.types import Tool as MCPTool + + from litellm.experimental_mcp_client.client import MCPClient from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, global_mcp_server_manager, @@ -876,7 +885,6 @@ if MCP_AVAILABLE: return (), classify_list_exception(e) return tools_result, ServerListOk(tool_count=len(tools_result)) - # Query all servers the user has access to queried_servers: Final = tuple( server for server in map(global_mcp_server_manager.get_mcp_server_by_id, allowed_server_ids) @@ -1141,12 +1149,18 @@ if MCP_AVAILABLE: scopes: Final[list[str] | None] = scopes_raw if isinstance(scopes_raw, list) else None return client_id, client_secret, scopes + async def _list_tools_within(client: MCPClient, deadline: float) -> list[MCPTool] | None: + with anyio.move_on_after(deadline): + return await client.list_tools(raise_on_error=True) + return None + async def _execute_with_mcp_client( request: NewMCPServerRequest, operation: Callable[..., Awaitable[Mapping[str, object]]], mcp_auth_header: str | dict[str, str] | None = None, oauth2_headers: dict[str, str] | None = None, raw_headers: dict[str, str] | None = None, + timeout_seconds: float = MCP_TOOL_LISTING_TIMEOUT, ) -> Mapping[str, object]: """ Create a temporary MCP client from *request*, run *operation*, and return the result. @@ -1162,6 +1176,10 @@ if MCP_AVAILABLE: oauth2_headers: Headers extracted from the incoming request (may contain the litellm API key — must NOT be forwarded for M2M servers). raw_headers: Raw request headers forwarded for stdio env construction. + timeout_seconds: Cap on OAuth discovery, connect, handshake, and *operation* + combined. Defaults to ``MCP_TOOL_LISTING_TIMEOUT`` (30s, below common LB + timeouts) so an unreachable upstream yields this endpoint's JSON error + instead of an opaque load-balancer 504 with an empty body. Returns: The dict returned by *operation*, or an error dict on failure. @@ -1252,15 +1270,16 @@ if MCP_AVAILABLE: static_headers=request.static_headers, ) - client: Final = await global_mcp_server_manager._create_mcp_client( - server=server_model, - mcp_auth_header=mcp_auth_header, - extra_headers=merged_headers, - stdio_env=stdio_env, - cred_provider=preview_cred_provider, - ) + with anyio.fail_after(timeout_seconds): + client: Final = await global_mcp_server_manager._create_mcp_client( + server=server_model, + mcp_auth_header=mcp_auth_header, + extra_headers=merged_headers, + stdio_env=stdio_env, + cred_provider=preview_cred_provider, + ) - return await operation(client) + return await operation(client) except (KeyboardInterrupt, SystemExit, asyncio.CancelledError): raise @@ -1269,7 +1288,7 @@ if MCP_AVAILABLE: return { "status": "error", "error": True, - "message": _connection_error_message(e), + "message": _connection_error_message(e, request.url, timeout_seconds), } async def _preview_openapi_tools(spec_path: str) -> dict: @@ -1422,9 +1441,7 @@ if MCP_AVAILABLE: getattr(client, "timeout", MCP_CLIENT_TIMEOUT) or MCP_CLIENT_TIMEOUT, MCP_TOOL_LISTING_TIMEOUT, ) - list_tools_result = None # rebind-ok: set inside the timeout scope below - with anyio.move_on_after(listing_deadline): - list_tools_result = await client.list_tools(raise_on_error=True) # rebind-ok: fills the init above + list_tools_result: Final = await _list_tools_within(client, listing_deadline) if list_tools_result is None: verbose_logger.warning( "MCP tools/list preview timed out after %s seconds while paginating upstream tools", diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 989b08b929a..af3ff6714af 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -3855,6 +3855,13 @@ if MCP_AVAILABLE: and server.auth_type == MCPAuth.oauth2_token_exchange and oauth2_headers and len(mcp_servers or []) == 1 + and server.server_id + in frozenset( + allowed.server_id + for allowed in await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, mcp_servers=mcp_servers, client_ip=client_ip + ) + ) ): await global_mcp_server_manager.preflight_token_exchange( server=server, diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py index f79765f6d01..af02c11ad86 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_search.py +++ b/litellm/proxy/_experimental/mcp_server/tool_search.py @@ -2,20 +2,31 @@ from __future__ import annotations import json from collections.abc import Mapping, Sequence +from dataclasses import dataclass from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, TypedDict, assert_never +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, TypedDict -from typing_extensions import ReadOnly, Required +from pydantic import ValidationError +from typing_extensions import ReadOnly, Required, assert_never import litellm from litellm.proxy.agent_endpoints.agent_search import DEFAULT_AGENT_SEARCH_TOP_K +from litellm.proxy.common_utils.semantic_text_index import ( + Embedder, + EmbeddingFailed, + SemanticTextIndex, + router_embedder, +) +from litellm.types.mcp import MCPToolSearchSettings if TYPE_CHECKING: - from mcp.types import CallToolResult + from mcp.types import CallToolResult, Tool from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import UserAPIKeyAuth +MCP_TOOL_SEARCH_SETTINGS_KEY: Final[str] = "mcp_tool_search" MCP_TOOL_SEARCH_TOOL_NAME: Final[str] = "mcp_tool_search" MCP_TOOL_CALL_TOOL_NAME: Final[str] = "mcp_tool_call" AGENT_SEARCH_TOOL_NAME: Final[str] = "agent_search" @@ -29,17 +40,91 @@ def coerce_top_k(value: Any, default: int = 5) -> int: return default -def search_tools(query: str, tools: list[dict[str, Any]], top_k: int = 5) -> list[dict[str, Any]]: +class ToolSearchResult(TypedDict, total=False): + name: Required[ReadOnly[str]] + description: Required[ReadOnly[str]] + inputSchema: Required[ReadOnly[Mapping[str, object]]] + score: ReadOnly[float] + + +@dataclass(frozen=True, slots=True) +class SemanticToolRanker: + embed: Embedder + embedding_model: str + index: SemanticTextIndex + + +global_mcp_tool_search_index: Final = SemanticTextIndex() + + +def mcp_tool_search_settings() -> MCPToolSearchSettings | ValidationError: + try: + return MCPToolSearchSettings.model_validate(litellm.mcp_tool_search or {}) + except ValidationError as exc: + return exc + + +def _tool_result(tool: Tool) -> ToolSearchResult: + return {"name": tool.name, "description": tool.description or "", "inputSchema": tool.inputSchema} + + +def _scored_result(tool: Tool, score: float) -> ToolSearchResult: + return {"name": tool.name, "description": tool.description or "", "inputSchema": tool.inputSchema, "score": score} + + +def _tool_text(tool: Tool) -> str: + return "\n".join(part for part in (tool.name, tool.description or "") if part) + + +def _keyword_score(query: str, tool: Tool) -> float: + haystack: Final = _tool_text(tool).lower() + return float(sum(1 for token in query.lower().split() if token in haystack)) + + +def _split_core_tools(tools: Sequence[Tool], core_tools: Sequence[str]) -> tuple[tuple[Tool, ...], tuple[Tool, ...]]: + by_name: Final = MappingProxyType({tool.name: tool for tool in tools}) + core: Final = tuple(by_name[name] for name in dict.fromkeys(core_tools) if name in by_name) + rest: Final = tuple(tool for tool in tools if tool.name not in frozenset(core_tools)) + return core, rest + + +def _top_hits( + tools: Sequence[Tool], scores: Sequence[float], minimum: float, limit: int +) -> tuple[tuple[float, Tool], ...]: + hits: Final = ((score, tool) for score, tool in zip(scores, tools, strict=True) if score >= minimum) + return tuple(sorted(hits, key=lambda hit: hit[0], reverse=True)[:limit]) + + +def search_tools(query: str, tools: Sequence[Tool], top_k: int = 5) -> tuple[ToolSearchResult, ...]: + """Keyword fallback used when no embedding model is configured: one point per query token found in the tool.""" if not query: - return [] - tokens: Final = query.lower().split() + return () + scores: Final = tuple(_keyword_score(query, tool) for tool in tools) + return tuple(_tool_result(tool) for _, tool in _top_hits(tools, scores, minimum=1.0, limit=top_k)) - def _score(tool: dict[str, Any]) -> int: - haystack: Final = (tool.get("name", "") + " " + tool.get("description", "")).lower() - return sum(1 for t in tokens if t in haystack) - scored: Final = ((s, tool) for tool in tools if (s := _score(tool)) > 0) - return [tool for _, tool in sorted(scored, key=lambda x: x[0], reverse=True)[:top_k]] +async def search_mcp_tools( + query: str, + tools: Sequence[Tool], + top_k: int, + settings: MCPToolSearchSettings, + ranker: SemanticToolRanker | None, +) -> tuple[ToolSearchResult, ...] | EmbeddingFailed: + """Core tools the caller can access come first, then up to `top_k` ranked matches from the remaining tools.""" + core, rest = _split_core_tools(tools, settings.core_tools) + limit: Final = min(top_k, settings.top_k) + core_results: Final = tuple(_tool_result(tool) for tool in core) + if ranker is None: + return (*core_results, *search_tools(query, rest, limit)) + if not query: + return core_results + scores: Final = await ranker.index.scores( + query, tuple(_tool_text(tool) for tool in rest), ranker.embed, ranker.embedding_model + ) + if isinstance(scores, EmbeddingFailed): + return scores + hits: Final = _top_hits(rest, scores, minimum=settings.similarity_threshold, limit=limit) + return (*core_results, *(_scored_result(tool, score) for score, tool in hits)) class _ToolParamSchema(TypedDict, total=False): @@ -66,11 +151,17 @@ def _json_array(*items: str) -> Sequence[str]: _MCP_TOOL_SEARCH_DEFINITION: Final[VirtualToolDefinition] = { "name": MCP_TOOL_SEARCH_TOOL_NAME, - "description": "Search for MCP tools by keyword. Returns top matching tools with names, descriptions, and input schemas.", + "description": ( + "Search for MCP tools by describing what you need. " + "Returns top matching tools with names, descriptions, and input schemas." + ), "inputSchema": { "type": "object", "properties": { - "query": {"type": "string", "description": "Keywords to search for in tool names and descriptions."}, + "query": { + "type": "string", + "description": "What the tool should do, matched against names and descriptions.", + }, "top_k": {"type": "integer", "description": "Maximum number of results to return.", "default": 5}, }, "required": _json_array("query"), @@ -165,10 +256,28 @@ async def handle_mcp_tool_search( oauth2_headers: dict[str, str] | None = None, raw_headers: dict[str, str] | None = None, ) -> CallToolResult: - from mcp.types import CallToolResult, TextContent - from litellm.proxy._experimental.mcp_server.server import _list_mcp_tools + from litellm.proxy.proxy_server import llm_router + settings: Final = mcp_tool_search_settings() + if isinstance(settings, ValidationError): + return _text_tool_result( + f"litellm_settings.{MCP_TOOL_SEARCH_SETTINGS_KEY} is invalid: {settings}", is_error=True + ) + if settings.embedding_model is not None and llm_router is None: + return _text_tool_result( + f"litellm_settings.{MCP_TOOL_SEARCH_SETTINGS_KEY}.embedding_model needs a model_list so it can be called", + is_error=True, + ) + ranker: Final = ( + SemanticToolRanker( + embed=router_embedder(llm_router, settings.embedding_model, user_api_key_dict), + embedding_model=settings.embedding_model, + index=global_mcp_tool_search_index, + ) + if settings.embedding_model is not None and llm_router is not None + else None + ) mcp_listing: Final = await _list_mcp_tools( user_api_key_auth=user_api_key_dict, mcp_servers=mcp_servers, @@ -178,17 +287,10 @@ async def handle_mcp_tool_search( oauth2_headers=oauth2_headers, raw_headers=raw_headers, ) - mcp_tools: Final = mcp_listing.tools - tools: Final = [ - { - "name": t.name, - "description": t.description or "", - "inputSchema": t.inputSchema, - } - for t in mcp_tools - ] - results: Final = search_tools(query, tools, top_k) - return CallToolResult(content=[TextContent(type="text", text=json.dumps(results))], isError=False) + results: Final = await search_mcp_tools(query, mcp_listing.tools, top_k, settings, ranker) + if isinstance(results, EmbeddingFailed): + return _text_tool_result(results.reason, is_error=True) + return _text_tool_result(json.dumps(results), is_error=False) async def handle_mcp_tool_call( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 18714256a8f..849e54c65aa 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2508,6 +2508,14 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="max batch input file size in MB for /v1/files uploads with purpose=batch, if a file is larger than this size it will be rejected before being forwarded to the provider", ) + max_file_size_mb: int | None = Field( + None, + description="max file size in MB for /v1/files uploads, for any purpose, if a file is larger than this size it will be rejected before being forwarded to the provider", + ) + blocked_file_extensions: tuple[str, ...] | None = Field( + None, + description="file extensions (e.g. ['.exe', '.sh']) rejected on /v1/files uploads, for any purpose, matched case-insensitively against the uploaded filename", + ) max_response_size_mb: int | None = Field( None, description="max response size in MB, if a response is larger than this size it will be rejected", @@ -2586,6 +2594,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.", ) + missing_session_id: Literal["generate", "reject"] | None = Field( + None, + description="What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.", + ) enable_public_model_hub: bool = Field( default=False, description="Public model hub for users to see what models they have access to, supported openai params, etc.", @@ -2696,6 +2708,40 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="List of MCP server fields that must be filled in for a submission to pass standards checks (e.g. ['description', 'source_url', 'alias']).", ) + password_policy_min_length: int | None = Field( + None, + description=( + "Minimum length required for a locally-managed user's password. Default is 12; " + "a value below 8 is floored to 8 rather than weakening the requirement further." + ), + ) + password_policy_require_uppercase: bool | None = Field( + None, + description="If True (default), a locally-managed user's password must contain an uppercase letter.", + ) + password_policy_require_lowercase: bool | None = Field( + None, + description="If True (default), a locally-managed user's password must contain a lowercase letter.", + ) + password_policy_require_numbers: bool | None = Field( + None, + description="If True (default), a locally-managed user's password must contain a number.", + ) + password_policy_require_special_characters: bool | None = Field( + None, + description="If True (default), a locally-managed user's password must contain a special (non-alphanumeric) character.", + ) + disable_password_login_when_sso_enabled: bool | None = Field( + None, + description=( + "If True and SSO is configured (MICROSOFT_CLIENT_ID, GOOGLE_CLIENT_ID, " + "GENERIC_CLIENT_ID, or SAML_IDP_METADATA_URL/XML), disables username/password " + "login on /login, /v2/login, and /v3/login so SSO is the only way to reach the " + "Admin UI. An admin locked out of the UI can still administer the proxy over the " + "API with the master key; unset this setting and restart the proxy to restore " + "UI username/password login. Default is False." + ), + ) disable_budget_reservation: bool | None = Field( None, description=( diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 31b05320cd3..28882484db4 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -1019,4 +1019,6 @@ async def invoke_agent_a2a( ) except Exception: pass + if isinstance(e, litellm.BadRequestError): + return _jsonrpc_error(body.get("id"), -32602, e.message, 400) return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {e}", 500) diff --git a/litellm/proxy/agent_endpoints/agent_registry.py b/litellm/proxy/agent_endpoints/agent_registry.py index 144de52d0d2..c7b6bca72cf 100644 --- a/litellm/proxy/agent_endpoints/agent_registry.py +++ b/litellm/proxy/agent_endpoints/agent_registry.py @@ -6,8 +6,12 @@ from datetime import datetime, timezone from types import MappingProxyType from typing import TYPE_CHECKING, Final, NamedTuple, Protocol, TypedDict +from pydantic import TypeAdapter, ValidationError + import litellm +from litellm.constants import REDACTED_BY_LITELM_STRING from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.proxy.management_helpers.object_permission_utils import ( handle_update_object_permission_common, ) @@ -52,6 +56,9 @@ class AgentRecord(Protocol): @property def agent_name(self) -> str: ... + @property + def litellm_params(self) -> Mapping[str, object] | None: ... + @property def object_permission_id(self) -> str | None: ... @@ -121,6 +128,188 @@ def _dump_agent_params(raw: Mapping[str, object]) -> dict[str, object]: return dict(raw) if raw else {} +_AGENT_PARAMS_MASKER: Final = SensitiveDataMasker() +_REDACT_AGENT_PARAMS_MAX_DEPTH: Final = 10 +_AGENT_PARAMS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter( + dict[str, object] +) # mutable-ok: safe_dumps() and AgentResponse.litellm_params both require a real dict, not a Mapping +_AGENT_PARAMS_SEQUENCE_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) +_EMPTY_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) + + +def redact_sensitive_agent_litellm_params(litellm_params: object, _depth: int = 0) -> object: + """ + Replace credential-bearing values in an agent's litellm_params with + ``REDACTED_BY_LITELM_STRING`` while preserving non-secret keys (``model``, + ``is_public``, rate-limit config). Used so list/get/create/update + responses never echo a stored provider credential back to the caller. + + Handles a plain dict, a JSON-serialized string (some callers hold the + in-memory registry's params that way), and ``None`` at the top level; + anything else is passed through. Recursion depth is bounded to match the + convention documented in ``tests/code_coverage_tests/recursive_detector.py``. + """ + if litellm_params is None: + return None + if isinstance(litellm_params, str): + if _depth >= _REDACT_AGENT_PARAMS_MAX_DEPTH: + return REDACTED_BY_LITELM_STRING + try: + parsed_params: Final = _AGENT_PARAMS_ADAPTER.validate_json(litellm_params) + except ValidationError: + return REDACTED_BY_LITELM_STRING + return json.dumps(_redact_agent_params_tree(parsed_params, _depth + 1)) + return _redact_agent_params_tree(litellm_params, _depth) + + +def _redact_agent_params_tree(value: object, _depth: int) -> object: + """Structural recursion over an already-parsed litellm_params value: a + dict redacts sensitive keys and recurses into the rest, a list redacts + each element (so a secret nested inside a list of provider configs is + still caught), and anything else -- including a plain string leaf, which + must never be re-interpreted as a JSON blob -- passes through unchanged. + """ + if _depth >= _REDACT_AGENT_PARAMS_MAX_DEPTH: + return REDACTED_BY_LITELM_STRING + if isinstance(value, list): + typed_items: Final = _AGENT_PARAMS_SEQUENCE_ADAPTER.validate_python(value) + return tuple(_redact_agent_params_tree(item, _depth + 1) for item in typed_items) + if not isinstance(value, dict): + return value + typed_params: Final = _AGENT_PARAMS_ADAPTER.validate_python(value) + return { + key: ( + REDACTED_BY_LITELM_STRING + if _AGENT_PARAMS_MASKER.is_sensitive_key(key) + else _redact_agent_params_tree(nested_value, _depth + 1) + ) + for key, nested_value in typed_params.items() + } # mutable-ok: consumed by json.dumps()/AgentResponse.litellm_params, both of which require a real dict + + +def parse_agent_litellm_params(value: object) -> Mapping[str, object]: + """Normalize a stored litellm_params column to a read-only mapping. + + The prisma Json column comes back as either an already-parsed dict or a + JSON string depending on the read path, so handle both rather than + assuming one. Only ever read from (merge-source lookups), never mutated + or re-serialized directly, so a read-only view is enough here. + """ + if isinstance(value, str): + try: + return _AGENT_PARAMS_ADAPTER.validate_json(value) + except ValidationError: + return _EMPTY_LITELLM_PARAMS + if isinstance(value, Mapping): + try: + return _AGENT_PARAMS_ADAPTER.validate_python(value) + except ValidationError: + return _EMPTY_LITELLM_PARAMS + return _EMPTY_LITELLM_PARAMS + + +_MISSING_AGENT_PARAM: Final = object() +_RESTORE_AGENT_PARAMS_MAX_DEPTH: Final = 10 + + +def _restore_redacted_nested_value(incoming_value: object, existing_value: object, _depth: int) -> object: + """Recurse into a non-sensitively-named dict/list value so a secret + nested underneath it (e.g. inside a list of per-provider configs) is + still restored, not just top-level keys. Mirrors the shapes + ``redact_sensitive_agent_litellm_params`` recurses into on read, so + restore and redact stay symmetric. + + List elements are paired with the existing list by position: with no + stable per-element identity in an arbitrary ``dict[str, object]`` schema, + index is the same correspondence every other part of this restore (and + the endpoints' existing full-replace-on-PUT semantics) already assumes. + This correctly preserves a masked secret across an ordinary edit of that + same entry's other fields; it does not protect against a caller who both + reorders/resizes the list AND echoes back a masked marker in the same + request, which is a known, narrow limitation (see LIT-6736 PR discussion) + rather than a cross-entry credential leak in the common case. + + A value collapsed to the flat marker by the read side's depth cap is + recovered wholesale from ``existing_value`` (rather than the marker + string itself getting persisted) whenever ``existing_value`` isn't + already that same flat marker. Depth-bounded like its read-side + counterpart; a value at the cap is returned unchanged rather than + corrupted. + """ + if incoming_value == REDACTED_BY_LITELM_STRING and existing_value != REDACTED_BY_LITELM_STRING: + return existing_value + if _depth >= _RESTORE_AGENT_PARAMS_MAX_DEPTH: + return incoming_value + if isinstance(incoming_value, Mapping): + typed_incoming_map: Final = _AGENT_PARAMS_ADAPTER.validate_python(incoming_value) + existing_map: Final = ( + _AGENT_PARAMS_ADAPTER.validate_python(existing_value) + if isinstance(existing_value, Mapping) + else _EMPTY_LITELLM_PARAMS + ) + return _restore_redacted_litellm_params(typed_incoming_map, existing_map, _depth + 1) + if isinstance(incoming_value, (list, tuple)): + typed_incoming_seq: Final = _AGENT_PARAMS_SEQUENCE_ADAPTER.validate_python(incoming_value) + existing_seq: Final = ( + _AGENT_PARAMS_SEQUENCE_ADAPTER.validate_python(existing_value) + if isinstance(existing_value, (list, tuple)) + else () + ) + return tuple( + _restore_redacted_nested_value( + item, + existing_seq[index] if index < len(existing_seq) else None, + _depth + 1, + ) + for index, item in enumerate(typed_incoming_seq) + ) + return incoming_value + + +def _resolved_agent_param_value( + key: str, + incoming: Mapping[str, object], + existing: Mapping[str, object], + _depth: int, +) -> object: + """The value ``key`` should end up with in a restored litellm_params, or + ``_MISSING_AGENT_PARAM`` when it should be dropped entirely.""" + if key in incoming: + value: Final = incoming[key] + if _AGENT_PARAMS_MASKER.is_sensitive_key(key): + return existing.get(key, _MISSING_AGENT_PARAM) if value == REDACTED_BY_LITELM_STRING else value + return _restore_redacted_nested_value(value, existing.get(key), _depth) + if _AGENT_PARAMS_MASKER.is_sensitive_key(key): + return existing.get(key, _MISSING_AGENT_PARAM) + return _MISSING_AGENT_PARAM + + +def _restore_redacted_litellm_params( + incoming: Mapping[str, object], + existing: Mapping[str, object], + _depth: int = 0, +) -> dict[str, object]: + """Restore the real credential behind any litellm_params value the caller + echoed back as ``REDACTED_BY_LITELM_STRING``, and behind any sensitive key + omitted entirely, so an edit to an unrelated field never overwrites (or + silently drops) a stored provider credential -- the UI never has to + read-and-resend a secret to keep it. Recurses into nested dicts and lists + so a secret nested under a non-sensitively-named key is restored too. + + A sensitive key given a real (non-marker) value, including an explicit + empty string, is treated as a deliberate update -- that's how a caller + clears a credential. Non-sensitive keys always take the incoming value + (recursed into), matching the endpoints' existing full-replace-on-PUT / + merge-on-PATCH semantics for everything that isn't a secret. + """ + all_keys: Final = frozenset(incoming) | frozenset(existing) + return { + key: value + for key in all_keys + if (value := _resolved_agent_param_value(key, incoming, existing, _depth)) is not _MISSING_AGENT_PARAM + } # mutable-ok: fed to safe_dumps() for JSON-column storage, which requires a real dict + + class GrantMigrationResult(NamedTuple): rewritten: int missed: int @@ -301,9 +490,14 @@ class AgentRegistry: try: agent_name: Final = agent.get("agent_name") - # Serialize litellm_params + # Serialize litellm_params. A create has no stored row to restore a + # secret behind, so a sensitive key submitted as the redaction + # marker (e.g. a stray client re-post) is dropped rather than + # persisted as the literal placeholder string. litellm_params_obj: Final = agent.get("litellm_params", {}) - litellm_params_dict: Final[dict[str, object]] = _dump_agent_params(litellm_params_obj) + litellm_params_dict: Final = _restore_redacted_litellm_params( + _dump_agent_params(litellm_params_obj), _EMPTY_LITELLM_PARAMS + ) litellm_params: Final[str] = safe_dumps(litellm_params_dict) # Serialize agent_card_params @@ -410,8 +604,14 @@ class AgentRegistry: update_data: Final[dict[str, object]] = {} if augment_agent.get("agent_name"): update_data["agent_name"] = augment_agent.get("agent_name") - if augment_agent.get("litellm_params"): - update_data["litellm_params"] = safe_dumps(augment_agent.get("litellm_params")) + if "litellm_params" in agent: + existing_litellm_params: Final = parse_agent_litellm_params(existing_agent.get("litellm_params")) + update_data["litellm_params"] = safe_dumps( + _restore_redacted_litellm_params( + _dump_agent_params(agent.get("litellm_params") or _EMPTY_LITELLM_PARAMS), + existing_litellm_params, + ) + ) if augment_agent.get("agent_card_params"): update_data["agent_card_params"] = safe_dumps(augment_agent.get("agent_card_params")) @@ -474,9 +674,22 @@ class AgentRegistry: try: agent_name: Final = agent.get("agent_name") + # A PUT fully replaces litellm_params from the request body, so the + # existing row is read up front to restore any sensitive key the + # caller echoed back redacted (or omitted) rather than persisting + # the marker -- or nothing -- over the real stored credential. + existing_row: Final = await agents_table(prisma_client).find_unique( + where={"agent_id": agent_id} # mutable-ok: prisma's query builder rejects a Mapping/MappingProxyType + ) + existing_litellm_params: Final = parse_agent_litellm_params( + existing_row.litellm_params if existing_row is not None else None + ) + # Serialize litellm_params litellm_params_obj: Final = agent.get("litellm_params", {}) - litellm_params_dict: Final[dict[str, object]] = _dump_agent_params(litellm_params_obj) + litellm_params_dict: Final = _restore_redacted_litellm_params( + _dump_agent_params(litellm_params_obj), existing_litellm_params + ) litellm_params: Final[str] = safe_dumps(litellm_params_dict) # Serialize agent_card_params @@ -512,9 +725,8 @@ class AgentRegistry: update_data[rate_field] = _val if agent.get("object_permission") is not None: - existing_agent: Final = await agents_table(prisma_client).find_unique(where={"agent_id": agent_id}) existing_object_permission_id: Final = ( - existing_agent.object_permission_id if existing_agent is not None else None + existing_row.object_permission_id if existing_row is not None else None ) agent_copy: Final = dict(agent) object_permission_id: Final = await handle_update_object_permission_common( diff --git a/litellm/proxy/agent_endpoints/agent_search.py b/litellm/proxy/agent_endpoints/agent_search.py index 46ab36d7b72..76e3fe6c5ad 100644 --- a/litellm/proxy/agent_endpoints/agent_search.py +++ b/litellm/proxy/agent_endpoints/agent_search.py @@ -2,17 +2,18 @@ from __future__ import annotations -import math -from collections.abc import Awaitable, Mapping, Sequence +from collections.abc import Sequence from dataclasses import dataclass -from itertools import chain -from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Protocol, TypeAlias +from typing import TYPE_CHECKING, Final, TypeAlias -from openai import OpenAIError from pydantic import BaseModel, ConfigDict, ValidationError -from litellm.exceptions import BudgetExceededError +from litellm.proxy.common_utils.semantic_text_index import ( + Embedder, + EmbeddingFailed, + SemanticTextIndex, + router_embedder, +) from litellm.types.agents import AgentResponse if TYPE_CHECKING: @@ -21,12 +22,6 @@ if TYPE_CHECKING: DEFAULT_AGENT_SEARCH_TOP_K: Final = 5 -Vector: TypeAlias = tuple[float, ...] - - -class Embedder(Protocol): - def __call__(self, texts: Sequence[str]) -> Awaitable[Sequence[Vector]]: ... - @dataclass(frozen=True, slots=True) class AgentSearchHit: @@ -67,18 +62,6 @@ class _SearchableCard(BaseModel): skills: tuple[_SearchableSkill, ...] = () -class _EmbeddingItem(BaseModel): - model_config = ConfigDict(frozen=True, extra="ignore") - - embedding: tuple[float, ...] - - -class _EmbeddingData(BaseModel): - model_config = ConfigDict(frozen=True, extra="ignore") - - data: tuple[_EmbeddingItem, ...] - - class AgentSearchResult(BaseModel): model_config = ConfigDict(frozen=True) @@ -117,110 +100,21 @@ def agent_search_result(hit: AgentSearchHit) -> AgentSearchResult: ) -def cosine_similarity(left: Vector, right: Vector) -> float: - dot: Final = sum(a * b for a, b in zip(left, right, strict=True)) - norms: Final = math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right)) - return dot / norms if norms else 0.0 - - -def embedding_spend_metadata(user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: - from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup - - return { # mutable-ok: the router mutates the metadata dict it is handed - **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict), - "user_api_key": user_api_key_dict.api_key, - } - - -def router_embedder(router: Router, embedding_model: str, user_api_key_dict: UserAPIKeyAuth) -> Embedder: - async def embed(texts: Sequence[str]) -> Sequence[Vector]: - batch: Final = list(texts) # mutable-ok: Router.aembedding accepts only str | list input - response: Final = await router.aembedding( - model=embedding_model, input=batch, metadata=embedding_spend_metadata(user_api_key_dict) - ) - return tuple(item.embedding for item in _EmbeddingData.model_validate(response.model_dump()).data) - - return embed - - -_NO_VECTORS: Final[Mapping[str, Vector]] = MappingProxyType({}) - - -async def _embed_all(embed: Embedder, texts: Sequence[str]) -> tuple[Vector, ...] | AgentSearchEmbeddingFailed: - try: - vectors: Final = tuple(await embed(texts)) - except (OpenAIError, ValueError, BudgetExceededError) as exc: - return AgentSearchEmbeddingFailed(reason=f"embedding the search query failed: {exc}") - if len(vectors) != len(texts): - return AgentSearchEmbeddingFailed( - reason=f"embedding model returned {len(vectors)} vectors for {len(texts)} inputs" - ) - return vectors - - -@dataclass(frozen=True, slots=True) -class _Embedded: - query_vector: Vector - vectors: Mapping[str, Vector] - - -def _same_dimension(query_vector: Vector, vectors: Mapping[str, Vector], texts: Sequence[str]) -> bool: - return all(len(vectors[text]) == len(query_vector) for text in texts) - - -async def _embed_query_and_agents( - embed: Embedder, query: str, texts: Sequence[str], cached: Mapping[str, Vector] -) -> _Embedded | AgentSearchEmbeddingFailed: - missing: Final = tuple(dict.fromkeys(text for text in texts if text not in cached)) - embedded: Final = await _embed_all(embed, (query, *missing)) - if isinstance(embedded, AgentSearchEmbeddingFailed): - return embedded - vectors: Final = MappingProxyType(dict(chain(cached.items(), zip(missing, embedded[1:], strict=True)))) - if _same_dimension(embedded[0], vectors, texts): - return _Embedded(query_vector=embedded[0], vectors=vectors) - unique: Final = tuple(dict.fromkeys(texts)) - reembedded: Final = await _embed_all(embed, (query, *unique)) - if isinstance(reembedded, AgentSearchEmbeddingFailed): - return reembedded - return _Embedded( - query_vector=reembedded[0], vectors=MappingProxyType(dict(zip(unique, reembedded[1:], strict=True))) - ) - - class AgentSearchIndex: """Caches one vector per distinct agent text per embedding model, so repeat searches only embed the query.""" def __init__(self) -> None: - self._vectors: Mapping[str, Mapping[str, Vector]] = MappingProxyType({}) - - def _merged(self, embedding_model: str, embedded: _Embedded) -> Mapping[str, Vector]: - kept: Final = { - text: vector - for text, vector in self._vectors.get(embedding_model, _NO_VECTORS).items() - if len(vector) == len(embedded.query_vector) - } - return MappingProxyType({**kept, **embedded.vectors}) + self._index: Final = SemanticTextIndex() async def search( self, query: str, agents: Sequence[AgentResponse], top_k: int, embed: Embedder, embedding_model: str ) -> AgentSearchHits | AgentSearchEmbeddingFailed: - if not agents: - return AgentSearchHits(hits=()) texts: Final = tuple(agent_search_text(agent) for agent in agents) - cached: Final = self._vectors.get(embedding_model, _NO_VECTORS) - embedded: Final = await _embed_query_and_agents(embed, query, texts, cached) - if isinstance(embedded, AgentSearchEmbeddingFailed): - return embedded - if not _same_dimension(embedded.query_vector, embedded.vectors, texts): - return AgentSearchEmbeddingFailed( - reason=f"embedding model {embedding_model} returned vectors of mixed dimensions" - ) - self._vectors = MappingProxyType({**self._vectors, embedding_model: self._merged(embedding_model, embedded)}) + scores: Final = await self._index.scores(query, texts, embed, embedding_model) + if isinstance(scores, EmbeddingFailed): + return AgentSearchEmbeddingFailed(reason=scores.reason) ranked: Final = sorted( - ( - AgentSearchHit(agent=agent, score=cosine_similarity(embedded.query_vector, embedded.vectors[text])) - for agent, text in zip(agents, texts, strict=True) - ), + (AgentSearchHit(agent=agent, score=score) for agent, score in zip(agents, scores, strict=True)), key=lambda hit: hit.score, reverse=True, ) diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index b6c41a17503..3b8151d1064 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -13,14 +13,13 @@ import os import uuid from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Annotated, Final, TypedDict, assert_never +from typing import Annotated, Final, TypedDict from fastapi import APIRouter, Depends, HTTPException, Query, Request -from typing_extensions import ReadOnly, Required +from typing_extensions import ReadOnly, Required, assert_never import litellm from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.litellm_logging import _get_masked_values from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._types import ( CommonProxyErrors, @@ -33,6 +32,10 @@ from litellm.proxy.a2a.agent_card import ( merge_agent_card, normalize_protocol_version, ) +from litellm.proxy.agent_endpoints.agent_registry import ( + parse_agent_litellm_params, + redact_sensitive_agent_litellm_params, +) from litellm.proxy.agent_endpoints.agent_search import ( DEFAULT_AGENT_SEARCH_TOP_K, AgentSearchEmbeddingFailed, @@ -139,25 +142,37 @@ async def _attach_keys_to_agents(agents: Sequence[AgentResponse], prisma_client) agent.keys = matched_keys or None +def _redact_agent_litellm_params_dict( + litellm_params: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: AgentResponse.litellm_params is declared as a plain dict, not Mapping + """Type-narrowing wrapper: a dict in always yields a dict back from + ``redact_sensitive_agent_litellm_params``, which the function's general + (possible-JSON-string, possibly-None) signature can't express.""" + return dict( # mutable-ok: AgentResponse.litellm_params is declared as a plain dict, not Mapping + parse_agent_litellm_params(redact_sensitive_agent_litellm_params(litellm_params)) + ) + + def _redact_sensitive_agent_fields( agents: Sequence[AgentResponse], + *, + is_admin: bool, ) -> list[AgentResponse]: """ - Return copies of the given agents with sensitive configuration fields - redacted. The original objects are not modified. + Return copies of the given agents with credential-bearing litellm_params + values replaced by a fixed marker (never returned to ANY caller, + admin included) and, for non-admin callers, virtual-key and header + fields stripped entirely. The original objects are not modified. """ redacted: Final[list[AgentResponse]] = [] for agent in agents: copy = agent.model_copy(deep=True) - copy.static_headers = None - copy.extra_headers = None - copy.keys = None + if not is_admin: + copy.static_headers = None + copy.extra_headers = None + copy.keys = None if copy.litellm_params: - copy.litellm_params = _get_masked_values( - copy.litellm_params, - unmasked_length=4, - number_of_asterisks=4, - ) + copy.litellm_params = _redact_agent_litellm_params_dict(copy.litellm_params) redacted.append(copy) return redacted @@ -345,13 +360,13 @@ async def get_agents( global_agent_registry.ids_for_agent(agent.agent_id).isdisjoint(litellm.public_agent_groups) ) - # Redact sensitive fields for non-admin users + # litellm_params secrets are always redacted; keys/headers stay + # admin-only. is_admin: Final = ( user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value ) - if not is_admin: - returned_agents = _redact_sensitive_agent_fields(returned_agents) + returned_agents = _redact_sensitive_agent_fields(returned_agents, is_admin=is_admin) if health_check: agents_with_url: Final = [agent for agent in returned_agents if (agent.agent_card_params or {}).get("url")] @@ -505,7 +520,9 @@ async def create_agent( "Failed to register agent '%s' (ID: %s) in memory: %s", agent_name, agent_id, reg_error ) - return result + # The caller is a proxy admin (enforced above); litellm_params + # secrets are still never echoed back in the response. + return _redact_sensitive_agent_fields((result,), is_admin=True)[0] except HTTPException: raise @@ -578,13 +595,13 @@ async def get_agent_by_id( await _attach_keys_to_agents([agent], prisma_client) - # Redact sensitive fields for non-admin users + # litellm_params secrets are always redacted; keys/headers stay + # admin-only. is_admin = ( user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value ) - if not is_admin: - agent = _redact_sensitive_agent_fields([agent])[0] + agent = _redact_sensitive_agent_fields((agent,), is_admin=is_admin)[0] return agent except HTTPException: @@ -688,7 +705,7 @@ async def update_agent( "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id ) - return result + return _redact_sensitive_agent_fields((result,), is_admin=True)[0] except HTTPException: raise except Exception as e: @@ -791,7 +808,7 @@ async def patch_agent( "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id ) - return result + return _redact_sensitive_agent_fields((result,), is_admin=True)[0] except HTTPException: raise except Exception as e: diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 5703c6cd5e8..0db4cbc3bf2 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -4705,6 +4705,13 @@ async def is_valid_fallback_model( return True +# The shape abbreviate_api_key writes into LiteLLM_VerificationToken.key_name. The +# last four characters are only barred from being whitespace or a control code, +# because a custom key's can be anything else, punctuation and non-ASCII included; +# a real key is at least MINIMUM_CUSTOM_KEY_LENGTH long, so it never fullmatches. +_MASKED_KEY_NAME_RE: Final = re.compile(r"sk-\.\.\.(?:[^\s\x00-\x1f\x7f-\x9f]{4})?") + + def _apply_budget_exceeded_throttle(valid_token: UserAPIKeyAuth) -> bool: """ Throttle an over-budget key instead of blocking it, when the key opted in @@ -4785,10 +4792,15 @@ async def _virtual_key_max_budget_check( if math.isfinite(valid_token.max_budget) and spend >= valid_token.max_budget: if _apply_budget_exceeded_throttle(valid_token): return - # name the key in the error so operators don't have to reverse-map - # spend back to a key; key_name is the masked form (last 4 chars) + # This message is returned to the caller, and key_name has no enforced + # shape (a direct DB write bypasses abbreviate_api_key), so echo it only + # when it still looks masked and fall back to the alias otherwise. key_label: Final = valid_token.key_alias or "key" - key_descriptor: Final = f"{key_label} ({valid_token.key_name})" if valid_token.key_name else key_label + key_descriptor: Final = ( + f"{key_label} ({valid_token.key_name})" + if valid_token.key_name and _MASKED_KEY_NAME_RE.fullmatch(valid_token.key_name) + else key_label + ) raise litellm.BudgetExceededError( current_cost=spend, max_budget=valid_token.max_budget, diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 64878a480a7..b36c8a038fc 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -61,9 +61,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException: return e if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): return ProxyException( - message=( - "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." - ), + message=PrismaDBExceptionHandler.database_unavailable_message(e), type=ProxyErrorTypes.no_db_connection, param="None", code=status.HTTP_503_SERVICE_UNAVAILABLE, diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 89b2c92cdfd..d6007a2d56e 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1,3 +1,4 @@ +import importlib.util import os import re import sys @@ -1402,7 +1403,7 @@ def is_pass_through_provider_route(route: str) -> bool: return False -def _has_user_setup_sso() -> bool: +def has_user_setup_sso() -> bool: """ Check if the user has set up single sign-on (SSO). @@ -1425,6 +1426,63 @@ def _has_user_setup_sso() -> bool: ) +def _is_google_ready() -> bool: + return bool(os.getenv("GOOGLE_CLIENT_ID")) and bool(os.getenv("GOOGLE_CLIENT_SECRET")) + + +def _is_microsoft_ready() -> bool: + return ( + bool(os.getenv("MICROSOFT_CLIENT_ID")) + and bool(os.getenv("MICROSOFT_CLIENT_SECRET")) + and bool(os.getenv("MICROSOFT_TENANT")) + ) + + +def _is_generic_oauth_ready() -> bool: + return ( + bool(os.getenv("GENERIC_CLIENT_ID")) + and bool(os.getenv("GENERIC_CLIENT_SECRET")) + and bool(os.getenv("GENERIC_AUTHORIZATION_ENDPOINT")) + and bool(os.getenv("GENERIC_TOKEN_ENDPOINT")) + and bool(os.getenv("GENERIC_USERINFO_ENDPOINT")) + ) + + +def _is_saml_ready() -> bool: + if not (os.getenv("SAML_IDP_METADATA_URL") or os.getenv("SAML_IDP_METADATA_XML")): + return False + # SAML's runtime (python3-saml) is an optional dependency; the SAML + # handler itself fails closed on every request when it is missing + # (SAMLAuthHandler raises before touching the IdP), so metadata alone + # is not "ready" either. find_spec raises ModuleNotFoundError (rather + # than returning None) when the top-level package is absent entirely, + # so this must not be a bare boolean expression or every password + # login would 500 on a deployment that configured SAML metadata + # without installing the optional extra. + try: + return importlib.util.find_spec("onelogin.saml2.auth") is not None + except ModuleNotFoundError: + return False + + +def is_sso_provider_fully_configured() -> bool: + """Whether ANY configured SSO provider has every companion setting it + needs to actually authenticate a user, not merely a client id. + + A lone ``MICROSOFT_CLIENT_ID`` with no secret or tenant makes + ``has_user_setup_sso()`` return True while every real sign-in attempt + fails, so a gate that BLOCKS the password fallback (unlike the UI + discovery use of ``has_user_setup_sso()``, where a dead login button is + merely confusing) must check readiness here, or it can lock every admin + out with no way to sign in at all. Checks every provider independently + (mirroring ``/sso/readiness``'s per-provider requirements) rather than + stopping at the first one with a client id set, so a stray leftover + client id for an unused provider can never mask a different, fully + configured provider that would otherwise satisfy this gate. + """ + return _is_google_ready() or _is_microsoft_ready() or _is_generic_oauth_ready() or _is_saml_ready() + + def get_customer_user_header_from_mapping(user_id_mapping) -> list | None: """Return the header_name mapped to CUSTOMER role, if any (dict-based).""" if not user_id_mapping: diff --git a/litellm/proxy/auth/ip_address_utils.py b/litellm/proxy/auth/ip_address_utils.py index 558ea54495f..c2614b85016 100644 --- a/litellm/proxy/auth/ip_address_utils.py +++ b/litellm/proxy/auth/ip_address_utils.py @@ -6,8 +6,11 @@ External callers (public IPs) only see servers with available_on_public_internet """ import ipaddress +import os +from collections.abc import Mapping from dataclasses import dataclass from typing import Any, Final +from urllib.parse import urlparse from fastapi import Request from pydantic import TypeAdapter, ValidationError @@ -137,7 +140,7 @@ class IPAddressUtils: @staticmethod def is_request_from_trusted_proxy( request: Request, - general_settings: dict[str, Any] | None = None, + general_settings: Mapping[str, Any] | None = None, ) -> bool: """ Return True if X-Forwarded-* headers on this request should be trusted. @@ -190,6 +193,36 @@ class IPAddressUtils: trusted_networks: Final = IPAddressUtils.parse_trusted_proxy_networks(trusted_ranges) return IPAddressUtils.is_trusted_proxy(direct_ip, trusted_networks) + @staticmethod + def is_request_https( + request: Request, + general_settings: Mapping[str, Any] | None = None, + ) -> bool: + """ + Whether this request's PUBLIC-facing origin is HTTPS, for deciding + whether a cookie set on the response should be marked ``Secure``. + + litellm only sees a plain-HTTP hop whenever TLS terminates at a + reverse proxy, so ``request.url.scheme`` alone cannot answer this in + that deployment shape. Resolved from the first trusted signal: + 1. ``PROXY_BASE_URL`` (operator-declared public origin). + 2. ``X-Forwarded-Proto``, only when the request's direct peer is a + configured trusted proxy -- see ``is_request_from_trusted_proxy``. + An untrusted caller cannot spoof this header to strip Secure. + 3. The request's own literal scheme (direct TLS termination, or no + reverse proxy in front of litellm). + """ + configured_base_url: Final = os.environ.get("PROXY_BASE_URL", "").strip() + if configured_base_url: + return urlparse(configured_base_url).scheme == "https" + + if IPAddressUtils.is_request_from_trusted_proxy(request, general_settings=general_settings): + forwarded_proto: Final = request.headers.get("X-Forwarded-Proto") + if forwarded_proto: + return forwarded_proto.split(",")[0].strip().lower() == "https" + + return request.url.scheme == "https" + @staticmethod def extract_client_ip_from_xff_hops( xff_header: str, diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index fba95972944..8d4f6f81363 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -7,7 +7,9 @@ login endpoints (e.g., /login and /v2/login). import os import secrets +from collections.abc import Mapping from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import Final, Literal, cast import jwt @@ -24,6 +26,7 @@ from litellm.proxy._types import ( UpdateUserRequest, UserAPIKeyAuth, ) +from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, @@ -111,6 +114,7 @@ async def authenticate_user( password: str, master_key: str | None, prisma_client: PrismaClient | None, + general_settings: Mapping[str, object] = MappingProxyType({}), ) -> LoginResult: """ Authenticate a user and generate an API key for UI access. @@ -124,13 +128,40 @@ async def authenticate_user( password: Password from the login form master_key: Master key for the proxy (required) prisma_client: Prisma database client (optional) + general_settings: Proxy general_settings, checked for + `disable_password_login_when_sso_enabled` Returns: LoginResult: Object containing authentication data Raises: - ProxyException: If authentication fails or required configuration is missing + ProxyException: If authentication fails or required configuration is missing, + or if username/password login is disabled while SSO is configured + + Recovery: an admin locked out of the UI by + `disable_password_login_when_sso_enabled` can still administer the proxy over + the API with the master key (Authorization: Bearer ), which never + goes through this function. To restore UI username/password login, unset the + setting in config.yaml (or the DB-persisted general_settings) and restart the + proxy; this is a deliberate, auditable config change rather than a hidden + bypass. + + The gate below requires the SSO provider to be FULLY configured (every + companion secret/endpoint an actual sign-in needs), not merely that a + client id is present, so an incomplete SSO setup can never disable the + only working login path. """ + if general_settings.get("disable_password_login_when_sso_enabled") is True and is_sso_provider_fully_configured(): + raise ProxyException( + message=( + "Username/password login is disabled because SSO is configured " + "and 'disable_password_login_when_sso_enabled' is set. Sign in via SSO." + ), + type=ProxyErrorTypes.auth_error, + param="disable_password_login_when_sso_enabled", + code=403, + ) + if master_key is None: raise ProxyException( message="Master Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml. https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.", diff --git a/litellm/proxy/auth/password_policy.py b/litellm/proxy/auth/password_policy.py new file mode 100644 index 00000000000..ab7a565894a --- /dev/null +++ b/litellm/proxy/auth/password_policy.py @@ -0,0 +1,92 @@ +"""Password-strength policy enforcement for locally-managed proxy users. + +Applied at every path that persists a new or changed password for a DB-backed +user (``/user/update``, ``/user/bulk_update``, and the invitation onboarding +claim flow), so the strength bar is configured in one place instead of +per-endpoint. +""" + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from litellm.proxy._types import ProxyErrorTypes, ProxyException + +DEFAULT_MIN_LENGTH: Final = 12 +MIN_ALLOWED_LENGTH: Final = 8 + + +def _has_uppercase(password: str) -> bool: + return any(ch.isupper() for ch in password) + + +def _has_lowercase(password: str) -> bool: + return any(ch.islower() for ch in password) + + +def _has_digit(password: str) -> bool: + return any(ch.isdigit() for ch in password) + + +def _has_special_character(password: str) -> bool: + """Unicode-aware: a letter or digit from ANY script counts as + alphanumeric, not just ASCII, so an accented letter (e.g. the second + character of "Passwörd1234") cannot be miscounted as the required + special character the way an ASCII-only `[^A-Za-z0-9]` regex would.""" + return any(not ch.isalnum() for ch in password) + + +@dataclass(frozen=True, slots=True) +class PasswordPolicy: + min_length: int + require_uppercase: bool + require_lowercase: bool + require_numbers: bool + require_special_characters: bool + + +def _configured_min_length(general_settings: Mapping[str, object]) -> int: + """The configured minimum, floored at MIN_ALLOWED_LENGTH so a nonpositive + or too-low override (a typo, or `0`/`false` coercing through) cannot + silently disable the length requirement rather than merely relaxing it.""" + min_length_setting: Final = general_settings.get("password_policy_min_length") + if isinstance(min_length_setting, bool) or not isinstance(min_length_setting, (int, float)): + return DEFAULT_MIN_LENGTH + return max(MIN_ALLOWED_LENGTH, int(min_length_setting)) + + +def get_password_policy(general_settings: Mapping[str, object]) -> PasswordPolicy: + return PasswordPolicy( + min_length=_configured_min_length(general_settings), + require_uppercase=general_settings.get("password_policy_require_uppercase", True) is not False, + require_lowercase=general_settings.get("password_policy_require_lowercase", True) is not False, + require_numbers=general_settings.get("password_policy_require_numbers", True) is not False, + require_special_characters=( + general_settings.get("password_policy_require_special_characters", True) is not False + ), + ) + + +def _policy_violations(password: str, policy: PasswordPolicy) -> tuple[str, ...]: + checks: Final = ( + (len(password) < policy.min_length, f"be at least {policy.min_length} characters long"), + (policy.require_uppercase and not _has_uppercase(password), "include an uppercase letter"), + (policy.require_lowercase and not _has_lowercase(password), "include a lowercase letter"), + (policy.require_numbers and not _has_digit(password), "include a number"), + (policy.require_special_characters and not _has_special_character(password), "include a special character"), + ) + return tuple(message for failed, message in checks if failed) + + +def validate_password_policy(password: str, general_settings: Mapping[str, object]) -> None: + """Raise ``ProxyException`` (400) if ``password`` fails the configured policy.""" + policy: Final = get_password_policy(general_settings) + violations: Final = _policy_violations(password, policy) + if not violations: + return + raise ProxyException( + message="Password does not meet the required policy: must " + ", ".join(violations) + ".", + type=ProxyErrorTypes.validation_error, + param="password", + code=400, + ) diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 2fad9f933c1..a96d3fb9c85 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -100,9 +100,10 @@ class CliPollData(TypedDict, total=False): class CliSsoStartData(TypedDict): - login_id: str - poll_secret: str - user_code: str + login_id: ReadOnly[str] + poll_secret: ReadOnly[str] + user_code: ReadOnly[str] + verification_uri_complete: ReadOnly[NotRequired[str]] class CliAuthResult(TypedDict): @@ -860,11 +861,22 @@ def login(ctx: click.Context, config_claude: bool, pkce: bool) -> None: poll_secret: Final = cli_sso_flow["poll_secret"] user_code: Final = cli_sso_flow["user_code"] - sso_url = f"{base_url}/sso/key/generate?" + urlencode({"source": LITELLM_CLI_SOURCE_IDENTIFIER, "key": key_id}) + browser_prefills_code: Final = isinstance(cli_sso_flow.get("verification_uri_complete"), str) + sso_url: Final = f"{base_url}/sso/key/generate?" + urlencode( + ( + ("source", LITELLM_CLI_SOURCE_IDENTIFIER), + ("key", key_id), + *((("user_code", user_code),) if browser_prefills_code else ()), + ) + ) click.echo(f"Opening browser to: {sso_url}") click.echo("Please complete the SSO authentication in your browser...") - click.echo(f"Verification code: {user_code}") + click.echo( + f"Verification code: {user_code} (pre-filled in the browser, check it matches)" + if browser_prefills_code + else f"Verification code: {user_code}" + ) click.echo(f"Session ID: {key_id}") # Open browser diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 05ddef822f1..fc83c1ddeed 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3,7 +3,6 @@ import contextlib import json import logging import math -import traceback from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Mapping, Sequence from datetime import datetime from functools import lru_cache @@ -18,7 +17,7 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse from starlette.types import Receive, Scope, Send import litellm -from litellm._logging import _redact_string, verbose_proxy_logger +from litellm._logging import redact_internal_details_from_client_message, verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import ( DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE, @@ -3417,7 +3416,7 @@ class ProxyBaseLLMRequestProcessing: else: _code = status.HTTP_500_INTERNAL_SERVER_ERROR raise ProxyException( - message=getattr(e, "message", error_msg), + message=redact_internal_details_from_client_message(getattr(e, "message", error_msg)), type=getattr(e, "type", "None"), param=getattr(e, "param", "None"), openai_code=getattr(e, "code", None), @@ -3629,10 +3628,8 @@ class ProxyBaseLLMRequestProcessing: if isinstance(e, HTTPException): raise e - error_traceback: Final = _redact_string(traceback.format_exc()) - error_msg: Final = f"{e}\n\n{error_traceback}" proxy_exception: Final = ProxyException( - message=getattr(e, "message", error_msg), + message=redact_internal_details_from_client_message(getattr(e, "message", str(e))), type=getattr(e, "type", "None"), param=getattr(e, "param", "None"), code=getattr(e, "status_code", 500), diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index 9fd24162f7e..213a697b3dd 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -10,13 +10,36 @@ legacy internal names with `general_settings.use_team_public_model_name: false`. from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast if TYPE_CHECKING: from litellm.router import Router +def configured_display_names( + entries: Sequence[tuple[str, str]], + llm_router: Router | None, +) -> Mapping[str, str]: + """response_id -> configured `model_info.display_name` for the listing entries + that have one. + + Metadata is looked up by each entry's internal lookup id (so team-scoped rows + resolve), while the returned map is keyed by the public response id the + Anthropic-shaped listing is built from. Entries without a configured name are + omitted so the listing falls back to the id itself. + """ + if llm_router is None: + return MappingProxyType({}) + resolved: Final = ( + (response_id, llm_router.get_configured_display_name(lookup_id)) for response_id, lookup_id in entries + ) + return MappingProxyType( + {response_id: display_name for response_id, display_name in resolved if display_name is not None} + ) + + class TeamModelNameTranslator: """Translates internal team routing keys to their public names for the model listing/retrieve responses. Stateless; the live router and general_settings diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 1682cf12f4e..47f69732e95 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -7,7 +7,9 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from enum import Enum from types import MappingProxyType -from typing import Final, Literal, Protocol, TypeVar, assert_never +from typing import Final, Literal, Protocol, TypeVar + +from typing_extensions import assert_never import litellm from litellm._logging import verbose_proxy_logger diff --git a/litellm/proxy/common_utils/semantic_text_index.py b/litellm/proxy/common_utils/semantic_text_index.py new file mode 100644 index 00000000000..0820459af49 --- /dev/null +++ b/litellm/proxy/common_utils/semantic_text_index.py @@ -0,0 +1,142 @@ +"""Embedding-similarity ranking over short texts with a per-model vector cache, shared by agent search and MCP tool search.""" + +from __future__ import annotations + +import math +from collections.abc import Awaitable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias + +from openai import OpenAIError +from pydantic import BaseModel, ConfigDict + +from litellm.exceptions import BudgetExceededError + +if TYPE_CHECKING: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.router import Router + +Vector: TypeAlias = tuple[float, ...] + + +class Embedder(Protocol): + def __call__(self, texts: Sequence[str]) -> Awaitable[Sequence[Vector]]: ... + + +@dataclass(frozen=True, slots=True) +class EmbeddingFailed: + reason: str + + +class _EmbeddingItem(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + embedding: tuple[float, ...] + + +class _EmbeddingData(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + data: tuple[_EmbeddingItem, ...] + + +def cosine_similarity(left: Vector, right: Vector) -> float: + dot: Final = sum(a * b for a, b in zip(left, right, strict=True)) + norms: Final = math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right)) + return dot / norms if norms else 0.0 + + +def embedding_spend_metadata(user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: # mutable-ok: router mutates it + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + return { # mutable-ok: the router mutates the metadata dict it is handed + **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict), + "user_api_key": user_api_key_dict.api_key, + } + + +def router_embedder(router: Router, embedding_model: str, user_api_key_dict: UserAPIKeyAuth) -> Embedder: + async def embed(texts: Sequence[str]) -> Sequence[Vector]: + batch: Final = list(texts) # mutable-ok: Router.aembedding accepts only str | list input + response: Final = await router.aembedding( + model=embedding_model, input=batch, metadata=embedding_spend_metadata(user_api_key_dict) + ) + return tuple(item.embedding for item in _EmbeddingData.model_validate(response.model_dump()).data) + + return embed + + +_NO_VECTORS: Final[Mapping[str, Vector]] = MappingProxyType({}) + + +async def _embed_all(embed: Embedder, texts: Sequence[str]) -> tuple[Vector, ...] | EmbeddingFailed: + try: + vectors: Final = tuple(await embed(texts)) + except (OpenAIError, ValueError, BudgetExceededError) as exc: + return EmbeddingFailed(reason=f"embedding the search query failed: {exc}") + if len(vectors) != len(texts): + return EmbeddingFailed(reason=f"embedding model returned {len(vectors)} vectors for {len(texts)} inputs") + return vectors + + +@dataclass(frozen=True, slots=True) +class _Embedded: + query_vector: Vector + vectors: Mapping[str, Vector] + + +def _same_dimension(query_vector: Vector, vectors: Mapping[str, Vector], texts: Sequence[str]) -> bool: + return all(len(vectors[text]) == len(query_vector) for text in texts) + + +async def _embed_query_and_texts( + embed: Embedder, query: str, texts: Sequence[str], cached: Mapping[str, Vector] +) -> _Embedded | EmbeddingFailed: + missing: Final = tuple(dict.fromkeys(text for text in texts if text not in cached)) + embedded: Final = await _embed_all(embed, (query, *missing)) + if isinstance(embedded, EmbeddingFailed): + return embedded + vectors: Final = MappingProxyType(dict(chain(cached.items(), zip(missing, embedded[1:], strict=True)))) + if _same_dimension(embedded[0], vectors, texts): + return _Embedded(query_vector=embedded[0], vectors=vectors) + unique: Final = tuple(dict.fromkeys(texts)) + reembedded: Final = await _embed_all(embed, (query, *unique)) + if isinstance(reembedded, EmbeddingFailed): + return reembedded + return _Embedded( + query_vector=reembedded[0], vectors=MappingProxyType(dict(zip(unique, reembedded[1:], strict=True))) + ) + + +class SemanticTextIndex: + """Caches one vector per distinct text per embedding model, so repeat searches only embed the query.""" + + def __init__(self) -> None: + self._vectors: Mapping[str, Mapping[str, Vector]] = MappingProxyType({}) + + def _merged(self, embedding_model: str, embedded: _Embedded) -> Mapping[str, Vector]: + kept: Final = MappingProxyType( + { + text: vector + for text, vector in self._vectors.get(embedding_model, _NO_VECTORS).items() + if len(vector) == len(embedded.query_vector) + } + ) + return MappingProxyType({**kept, **embedded.vectors}) + + async def scores( + self, query: str, texts: Sequence[str], embed: Embedder, embedding_model: str + ) -> tuple[float, ...] | EmbeddingFailed: + """Cosine similarity of `query` to each entry of `texts`, in the same order.""" + if not texts: + return () + cached: Final = self._vectors.get(embedding_model, _NO_VECTORS) + embedded: Final = await _embed_query_and_texts(embed, query, texts, cached) + if isinstance(embedded, EmbeddingFailed): + return embedded + if not _same_dimension(embedded.query_vector, embedded.vectors, texts): + return EmbeddingFailed(reason=f"embedding model {embedding_model} returned vectors of mixed dimensions") + self._vectors = MappingProxyType({**self._vectors, embedding_model: self._merged(embedding_model, embedded)}) + return tuple(cosine_similarity(embedded.query_vector, embedded.vectors[text]) for text in texts) diff --git a/litellm/proxy/container_endpoints/endpoints.py b/litellm/proxy/container_endpoints/endpoints.py index 4a088140725..eaa3db336a9 100644 --- a/litellm/proxy/container_endpoints/endpoints.py +++ b/litellm/proxy/container_endpoints/endpoints.py @@ -208,7 +208,7 @@ async def list_containers( # Read query parameters query_params: Final = dict(request.query_params) - data: Final[dict[str, Any]] = {"query_params": query_params} + data: Final[dict[str, Any]] = {"query_params": query_params, "model": query_params.get("model")} # Extract custom_llm_provider using priority chain custom_llm_provider: Final = ( diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 5502543b926..ef1c4a66203 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -1,4 +1,4 @@ -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Iterator from typing import Any, Final, TypeVar from litellm._logging import verbose_proxy_logger @@ -9,10 +9,43 @@ from litellm.proxy._types import ( ) from litellm.secret_managers.main import str_to_bool -# Bounds the __cause__/__context__ walk in is_database_service_unavailable_error_in_chain. +# Bounds the __cause__/__context__ walk in find_database_service_unavailable_error_in_chain. # Real exception chains are a few links deep; the cap also makes the walk cycle-safe. _MAX_EXCEPTION_CHAIN_DEPTH: Final = 20 +_TRANSIENT_DB_UNAVAILABLE_MESSAGE: Final = ( + "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." +) + + +def _exception_chain(e: BaseException) -> Iterator[BaseException]: + current = e # rebind-ok: advances one link per iteration of the bounded walk + for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH): + yield current + following = current.__cause__ or current.__context__ + if following is None: + return + current = following + + +def _database_service_unavailable_errors(e: BaseException) -> tuple[Exception, ...]: + return tuple( + link + for link in _exception_chain(e) + if isinstance(link, Exception) and PrismaDBExceptionHandler.is_database_service_unavailable_error(link) + ) + + +def _exception_types(*candidates: object) -> tuple[type[BaseException], ...]: + """Keep only the real exception classes among ``candidates``. + + The predicates below resolve prisma's error classes at call time, so a test + that swaps ``sys.modules["prisma"]`` for a ``MagicMock`` hands them mocks, + and ``isinstance`` against a mock raises ``TypeError`` instead of answering + False. Dropping the non-types lets the call fall through to the other checks. + """ + return tuple(c for c in candidates if isinstance(c, type) and issubclass(c, BaseException)) + class PrismaDBExceptionHandler: """ @@ -59,7 +92,7 @@ class PrismaDBExceptionHandler: if isinstance(e, DB_CONNECTION_ERROR_TYPES): return True - if isinstance(e, prisma.engine.errors.EngineConnectionError): + if isinstance(e, _exception_types(prisma.engine.errors.EngineConnectionError)): return True return isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection @@ -81,7 +114,7 @@ class PrismaDBExceptionHandler: """ import prisma - data_layer_errors: Final = ( + data_layer_errors: Final = _exception_types( prisma.errors.DataError, prisma.errors.UniqueViolationError, prisma.errors.ForeignKeyViolationError, @@ -94,7 +127,7 @@ class PrismaDBExceptionHandler: return False if isinstance(e, DB_CONNECTION_ERROR_TYPES): return True - if isinstance(e, prisma.errors.PrismaError): + if isinstance(e, _exception_types(prisma.errors.PrismaError)): return True if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection: return True @@ -138,13 +171,13 @@ class PrismaDBExceptionHandler: return True if isinstance( e, - ( + _exception_types( prisma.errors.ClientNotConnectedError, prisma.errors.HTTPClientClosedError, ), ): return True - if isinstance(e, prisma.errors.PrismaError): + if isinstance(e, _exception_types(prisma.errors.PrismaError)): error_message: Final = str(e).lower() connection_keywords: Final = ( "can't reach database server", @@ -171,7 +204,7 @@ class PrismaDBExceptionHandler: """True iff ``e`` is a Postgres deadlock (P2034 / 40P01) surfaced through prisma.""" import prisma - if not isinstance(e, prisma.errors.PrismaError): + if not isinstance(e, _exception_types(prisma.errors.PrismaError)): return False if getattr(e, "code", None) == "P2034": return True @@ -202,7 +235,7 @@ class PrismaDBExceptionHandler: """ import prisma - if isinstance(e, prisma.errors.PrismaError): + if isinstance(e, _exception_types(prisma.errors.PrismaError)): return False tb = getattr(e, "__traceback__", None) while tb is not None: @@ -268,6 +301,51 @@ class PrismaDBExceptionHandler: ), ) + @staticmethod + def is_permanent_database_fault(e: Exception) -> bool: + """True for a service-unavailable failure that will not clear on its + own: an engine-layer ``PrismaError`` (missing or version-skewed engine + binary, engine error status, misused transaction) that is neither the + transient ``EngineConnectionError`` nor a reconnectable transport failure. + + Picks only the wording of a 503, never whether one is sent; + ``is_database_service_unavailable_error`` stays the status gate. + """ + if PrismaDBExceptionHandler.is_database_connection_error(e): + return False + if PrismaDBExceptionHandler.is_database_transport_error(e): + return False + return PrismaDBExceptionHandler.is_database_infrastructure_error(e) + + @staticmethod + def database_unavailable_message(e: Exception) -> str: + """The 503 detail for a service-unavailable database failure: retry + guidance for a transient outage, a pointer at the deployment for a + fault that retrying cannot fix. A permanent fault anywhere in the + exception chain wins, since the transport error that surfaced it is + not what blocks recovery.""" + fault: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(e) or e + if not PrismaDBExceptionHandler.is_permanent_database_fault(fault): + return _TRANSIENT_DB_UNAVAILABLE_MESSAGE + return ( + "Service Unavailable, the authentication database query engine reported " + f"{type(fault).__name__}, which is not a transient outage and will not clear by retrying. " + "The proxy deployment needs attention." + ) + + @staticmethod + def find_database_service_unavailable_error_in_chain(e: BaseException) -> Exception | None: + """The exception in the ``__cause__`` / ``__context__`` chain that + ``is_database_service_unavailable_error`` accepts, or ``None``. Callers + that word a response by the kind of outage need the wrapped database + error itself, not just the fact that one is present. A permanent fault + outranks a transient one wherever it sits in the chain: a reconnect that + dies on a missing engine binary raises the transport error last, but the + binary is what keeps the database down.""" + outages: Final = _database_service_unavailable_errors(e) + permanent: Final = next(filter(PrismaDBExceptionHandler.is_permanent_database_fault, outages), None) + return permanent if permanent is not None else next(iter(outages), None) + @staticmethod def is_database_service_unavailable_error_in_chain(e: BaseException) -> bool: """Like ``is_database_service_unavailable_error`` but also walks the @@ -285,14 +363,7 @@ class PrismaDBExceptionHandler: The walk is depth-bounded, which also makes it cycle-safe. """ - current: BaseException | None = e - for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH): - if not isinstance(current, Exception): - return False - if PrismaDBExceptionHandler.is_database_service_unavailable_error(current): - return True - current = current.__cause__ or current.__context__ - return False + return PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(e) is not None @staticmethod def handle_db_exception(e: Exception): diff --git a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py index 64414b90ae7..c2053693f2e 100644 --- a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -14,7 +14,7 @@ router: Final = APIRouter() @router.get("/.well-known/litellm-ui-config", response_model=UiDiscoveryEndpoints) @router.get("/litellm/.well-known/litellm-ui-config", response_model=UiDiscoveryEndpoints) # if mounted at root path async def get_ui_config(): - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso from litellm.proxy.proxy_server import general_settings from litellm.proxy.utils import get_proxy_base_url, get_server_root_path @@ -28,7 +28,7 @@ async def get_ui_config(): or general_settings.get("hide_default_credentials_hint", False) is True ) - sso_configured: Final = _has_user_setup_sso() + sso_configured: Final = has_user_setup_sso() from litellm.proxy.proxy_server import proxy_config diff --git a/litellm/proxy/guardrails/anthropic_sse.py b/litellm/proxy/guardrails/anthropic_sse.py index 50c05daee11..28220f09f00 100644 --- a/litellm/proxy/guardrails/anthropic_sse.py +++ b/litellm/proxy/guardrails/anthropic_sse.py @@ -13,6 +13,19 @@ from typing import Final from litellm.types.utils import Choices, ModelResponse +_ANTHROPIC_EVENT_TYPES: Final = frozenset( + { + "message_start", + "message_delta", + "message_stop", + "content_block_start", + "content_block_delta", + "content_block_stop", + "ping", + "error", + } +) + def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool: return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks) @@ -30,23 +43,43 @@ def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None: return None -def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None: +def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]: from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, ) + return tuple( + event_data + for event in AnthropicPassthroughLoggingHandler._split_sse_chunk_into_events(sse_stream) # pyright: ignore[reportPrivateUsage] # same parser the assembler uses + if (event_data := AnthropicPassthroughLoggingHandler._extract_sse_data(event)) is not None # pyright: ignore[reportPrivateUsage] # same parser the assembler uses; a private import beats forking SSE parsing + ) + + +def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None: return next( ( message - for event in AnthropicPassthroughLoggingHandler._split_sse_chunk_into_events(sse_stream) # pyright: ignore[reportPrivateUsage] # same parser the assembler uses - if (event_data := AnthropicPassthroughLoggingHandler._extract_sse_data(event)) is not None # pyright: ignore[reportPrivateUsage] # same parser the assembler uses; a private import beats forking SSE parsing - and event_data.get("type") == "message_start" - and isinstance(message := event_data.get("message"), dict) + for event_data in _parsed_sse_events(sse_stream) + if event_data.get("type") == "message_start" and isinstance(message := event_data.get("message"), dict) ), None, ) +def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool: + """Whether raw SSE frames are Anthropic Messages events. + + ``is_raw_sse_stream`` only says the chunks are unparsed bytes, and ``/v1/messages`` is not the + only endpoint that streams those: the Google ``:streamGenerateContent`` route marks its own + stream raw too. Reading its frames as Anthropic ones would refuse the response in a wire format + its client cannot parse, so the surface is decided on the event types actually present. + """ + sse_stream: Final = _joined_sse_stream(all_chunks) + if sse_stream is None: + return False + return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream)) + + def assemble_anthropic_sse_stream( all_chunks: Sequence[object], *, restore_identity: bool = False ) -> ModelResponse | None: @@ -111,6 +144,27 @@ def anthropic_sse_error_frames(message: str) -> tuple[bytes, ...]: ) +def is_sse_error_stream(all_chunks: Sequence[object]) -> bool: + """Whether the buffered stream carries nothing but error frames. + + post_call guardrails run in a chain, so a hook can be handed the terminal error frames an + earlier guardrail emitted when it blocked. Those carry no message to assemble, and replacing + them would hide the refusal the client is owed. Covers both wire forms a guardrail emits: the + Anthropic ``error`` event and the chat-completions ``{"error": ...}`` payload. + """ + if not all(isinstance(chunk, (str, bytes)) for chunk in all_chunks): + # A stream mixing typed chunks with an error frame still carries content to scan, and the + # frames-only join below would drop exactly the part that has to be scanned + return False + sse_stream: Final = _joined_sse_stream(all_chunks) + if sse_stream is None: + return False + events: Final = _parsed_sse_events(sse_stream) + return len(events) > 0 and all( + event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events + ) + + def anthropic_sse_chunks_from_response(assembled: ModelResponse) -> tuple[bytes, ...]: from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 2b04828f0f2..3d2ed641a30 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -514,12 +514,26 @@ async def update_guardrail( guardrail_name: Final = result.get("guardrail_name", "Unknown") try: - IN_MEMORY_GUARDRAIL_HANDLER.update_in_memory_guardrail( - guardrail_id=guardrail_id, guardrail=cast(Guardrail, result) - ) + IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(guardrail=cast(Guardrail, result)) verbose_proxy_logger.info( "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id ) + except (ValueError, TypeError) as update_error: + # The new config is invalid (a raising guardrail __init__): + # reinitialize_guardrail already restored the previous live instance, but + # update_guardrail_in_db above already persisted the rejected config to + # the DB. Roll that back too, so the DB and the live guardrail never + # disagree about what's actually enforcing, and surface the rejection to + # the caller instead of a misleading 200. + await GUARDRAIL_REGISTRY.update_guardrail_in_db( + guardrail_id=guardrail_id, + guardrail=existing_guardrail, + prisma_client=prisma_client, + ) + raise HTTPException( + status_code=422, + detail=f"Invalid guardrail configuration, update rejected: {update_error}", + ) from update_error except Exception as update_error: verbose_proxy_logger.warning( "Immediate sync: Failed to update '%s' (ID: %s) in memory: %s", diff --git a/litellm/proxy/guardrails/guardrail_hooks/alice/alice.py b/litellm/proxy/guardrails/guardrail_hooks/alice/alice.py index 27018769909..9cabac2d0fa 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/alice/alice.py +++ b/litellm/proxy/guardrails/guardrail_hooks/alice/alice.py @@ -8,6 +8,7 @@ import json import os from collections.abc import Mapping +from itertools import islice from typing import ( TYPE_CHECKING, Any, # noqa: TID251 # **kwargs forwards verbatim to CustomGuardrail.__init__; see ruff-strict.toml @@ -341,19 +342,18 @@ def _json_safe( if depth >= _MAX_DEPTH or id(value) in seen: return None - nested: Final = seen | {id(value)} # mutable-ok: one-shot set literal, unioned into a frozenset immediately + nested: Final = seen | frozenset((id(value),)) if isinstance(value, dict): - out: dict[str, object] = {} # mutable-ok: bounded accumulator local to this call, never escapes as-is - for key, item in list(value.items())[:_MAX_ITEMS]: # mutable-ok: list() only to slice an unordered view - if isinstance(key, str) and key not in strip_keys: - out[key] = _json_safe(item, depth + 1, nested, strip_keys) - return out + return { + key: _json_safe(item, depth + 1, nested, strip_keys) + for key, item in islice(value.items(), _MAX_ITEMS) + if isinstance(key, str) and key not in strip_keys + } if isinstance(value, (list, tuple, set, frozenset)): return [ # mutable-ok: return value is a one-shot list, discarded by the caller after use - _json_safe(item, depth + 1, nested, strip_keys) - for item in list(value)[:_MAX_ITEMS] # mutable-ok: list() only to slice an unordered view + _json_safe(item, depth + 1, nested, strip_keys) for item in islice(value, _MAX_ITEMS) ] dump: Final = getattr(value, "model_dump", None) diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/__init__.py index 5e75b7d4d94..c88e6e97a96 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/__init__.py @@ -1,8 +1,8 @@ from typing import TYPE_CHECKING, Final -from litellm.types.guardrails import GuardrailEventHooks, SupportedGuardrailIntegrations +from litellm.types.guardrails import SupportedGuardrailIntegrations -from .crowdstrike_aidr import CrowdStrikeAIDRHandler +from .crowdstrike_aidr import CrowdStrikeAIDRHandler, streaming_params_from_litellm_params if TYPE_CHECKING: from litellm.types.guardrails import Guardrail, LitellmParams @@ -15,17 +15,16 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" if not guardrail_name: raise ValueError("CrowdStrike AIDR guardrail name is required") + streaming_params: Final = streaming_params_from_litellm_params(litellm_params) _crowdstrike_aidr_callback: Final = CrowdStrikeAIDRHandler( guardrail_name=guardrail_name, api_base=litellm_params.api_base, api_key=litellm_params.api_key, - # Exclude during_call to prevent duplicate input events - event_hook=[ - GuardrailEventHooks.pre_call.value, - GuardrailEventHooks.post_call.value, - ], + event_hook=litellm_params.mode, default_on=litellm_params.default_on, fail_on_error=litellm_params.fail_on_error, + streaming_end_of_stream_only=streaming_params.streaming_end_of_stream_only, + streaming_sampling_rate=streaming_params.streaming_sampling_rate, ) litellm.logging_callback_manager.add_litellm_callback(_crowdstrike_aidr_callback) diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py index c8284fac440..f7f500b1adc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py @@ -24,8 +24,11 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, ) -from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.guardrails import GuardrailEventHooks, LitellmParams from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionToolParam +from litellm.types.proxy.guardrails.guardrail_hooks.crowdstrike_aidr import ( + CrowdStrikeAIDRGuardrailConfigModelOptionalParams, +) from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: @@ -153,6 +156,21 @@ def _merge_metadata_bags(request_data: Mapping[str, Any]) -> Mapping[str, Any] | return merged if present else None +def streaming_params_from_litellm_params( + litellm_params: LitellmParams, +) -> CrowdStrikeAIDRGuardrailConfigModelOptionalParams: + extras: Final[Mapping[str, object]] = litellm_params.model_extra or {} + nested: Final = litellm_params.optional_params + optional_params: Final[Mapping[str, object]] = {} if nested is None else nested.model_dump() + return CrowdStrikeAIDRGuardrailConfigModelOptionalParams.model_validate( + { + name: value + for name in CrowdStrikeAIDRGuardrailConfigModelOptionalParams.model_fields + if (value := optional_params.get(name, extras.get(name))) is not None + } + ) + + def _messages_since_last_assistant( messages: Sequence[AllMessageValues], ) -> _FilteredMessages: @@ -241,6 +259,8 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): api_key: str | None = None, api_base: str | None = None, fail_on_error: bool | None = True, + streaming_end_of_stream_only: bool | None = None, + streaming_sampling_rate: int | None = None, **kwargs, ) -> None: """ @@ -250,10 +270,19 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): guardrail_name (str): The name of the guardrail instance. api_key (str | None): The CrowdStrike AIDR API key. Reads from CS_AIDR_TOKEN env var if None. api_base (str | None): The CrowdStrike AIDR API base URL. Reads from CS_AIDR_BASE_URL env var if None. + streaming_end_of_stream_only (bool | None): Scan streamed output once at end of stream instead of + every streaming_sampling_rate chunks. Defaults to False. + streaming_sampling_rate (int | None): Scan the accumulated streamed output every Nth chunk. Defaults to 5. **kwargs: Additional arguments passed to the CustomGuardrail base class. """ self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) self.fail_on_error = True if fail_on_error is None else fail_on_error + self._set_streaming_params( + CrowdStrikeAIDRGuardrailConfigModelOptionalParams( + streaming_end_of_stream_only=streaming_end_of_stream_only, + streaming_sampling_rate=streaming_sampling_rate, + ) + ) self.api_key = api_key or os.environ.get("CS_AIDR_TOKEN") if not self.api_key: @@ -274,6 +303,15 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): "Initialized CrowdStrike AIDR Guardrail: name=%s, api_base=%s", guardrail_name, self.api_base ) + def _set_streaming_params(self, streaming_params: CrowdStrikeAIDRGuardrailConfigModelOptionalParams) -> None: + self.streaming_end_of_stream_only: bool = streaming_params.streaming_end_of_stream_only or False + self.streaming_sampling_rate: int = streaming_params.streaming_sampling_rate or 5 + + @override + def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: + super().update_in_memory_litellm_params(litellm_params) + self._set_streaming_params(streaming_params_from_litellm_params(litellm_params)) + async def _call_crowdstrike_aidr_guard( self, payload: dict[str, Any], hook_name: str ) -> _GuardChatCompletionsResult: diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index d8c8c2f4974..fc881a60f43 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, TypeGuard import httpx from fastapi import HTTPException from httpx import Response as HttpxResponse +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -52,6 +53,10 @@ BYPASS_HEADER: Final = "x-headroom-bypass" HEADROOM_RETRIEVE_TOOL_NAME: Final = "headroom_retrieve" _HASH_PATTERN: Final = re.compile(r"hash=([a-f0-9]{24})") _HASH_CACHE_TTL_SECONDS: Final = 15 * 60 +# Narrows the base class's bare-dict ``request_data`` at the boundary so its +# untranslated messages can be read with concrete types (values pass through by +# reference, so this is a shallow top-level reconstruction). +_REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object]) def _is_str_object_dict(value: object) -> TypeGuard[dict[str, object]]: # guard-ok: isinstance narrows correctly; predicate is trivially correct # fmt: skip @@ -116,16 +121,119 @@ def _restore_content_shapes( return restored -def _protected_indices(messages: Sequence[Mapping[str, object]]) -> frozenset[int]: +def _tool_call_name(tool_call: Mapping[str, object]) -> str | None: + function: Final = tool_call.get("function") + if not _is_str_object_dict(function): + return None + name: Final = function.get("name") + return name if isinstance(name, str) else None + + +def _is_retrieve_tool_name(name: str | None) -> bool: + """Match the retrieve tool whether called directly or via the MCP gateway. + + Server-side the tool is ``headroom_retrieve``; exposed through LiteLLM's MCP + gateway a client calls it as ``mcp____headroom_retrieve``. + """ + return name is not None and ( + name == HEADROOM_RETRIEVE_TOOL_NAME or name.endswith(f"__{HEADROOM_RETRIEVE_TOOL_NAME}") + ) + + +def _retrieve_call_ids_in_message(message: Mapping[str, object]) -> frozenset[str]: + if message.get("role") != "assistant": + return frozenset() + tool_calls: Final = message.get("tool_calls") + if not _is_object_list(tool_calls): + return frozenset() + return frozenset( + str(tool_call["id"]) + for tool_call in tool_calls + if _is_str_object_dict(tool_call) and tool_call.get("id") and _is_retrieve_tool_name(_tool_call_name(tool_call)) + ) + + +def _anthropic_tool_use_retrieve_id(block: object) -> str | None: + if not _is_str_object_dict(block) or block.get("type") != "tool_use": + return None + name: Final = block.get("name") + call_id: Final = block.get("id") + if isinstance(name, str) and call_id is not None and _is_retrieve_tool_name(name): + return str(call_id) + return None + + +def _anthropic_retrieve_ids_in_message(message: Mapping[str, object]) -> frozenset[str]: + content: Final = message.get("content") + if not _is_object_list(content): + return frozenset() + return frozenset(call_id for block in content if (call_id := _anthropic_tool_use_retrieve_id(block)) is not None) + + +def _raw_retrieve_call_ids(messages: object) -> frozenset[str]: + """Retrieve-tool call ids read from the request's own, untranslated messages. + + The guardrail otherwise scans an OpenAI-translated view where a tool name + over 64 chars is truncated to ``{prefix}_{hash}``, which drops the + ``__headroom_retrieve`` suffix a long ``mcp____`` prefix pushes past + the limit. Tool-call ids are never truncated, so pairing the tool result to + an id read from the original request keeps the match intact. Both wire + shapes are handled: OpenAI ``tool_calls`` and Anthropic ``tool_use`` blocks. + """ + if not _is_object_list(messages): + return frozenset() + return frozenset( + call_id + for message in messages + if _is_str_object_dict(message) + for call_id in _retrieve_call_ids_in_message(message) | _anthropic_retrieve_ids_in_message(message) + ) + + +def _retrieval_result_indices( + messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset() +) -> frozenset[int]: + """Indices of tool-result rows that carry ``headroom_retrieve`` output. + + When the retrieve tool is exposed to a client that runs its own tool loop + (the LiteLLM MCP gateway path), the client executes the call and sends the + recovered original content back as a tool result on the next turn. That + content is exactly what a prior compression stubbed, so compressing it again + re-derives the identical content hash: a no-op that strands the model on the + marker and loops the agent. Hold those rows back so the expansion survives. + + ``extra_retrieve_call_ids`` carries ids recovered from the untruncated + request so the pairing survives tool-name truncation (see + ``_raw_retrieve_call_ids``). + """ + retrieve_call_ids: Final = extra_retrieve_call_ids | frozenset( + call_id for message in messages for call_id in _retrieve_call_ids_in_message(message) + ) + if not retrieve_call_ids: + return frozenset() + return frozenset( + index + for index, message in enumerate(messages) + if message.get("role") in ("tool", "function") and str(message.get("tool_call_id")) in retrieve_call_ids + ) + + +def _protected_indices( + messages: Sequence[Mapping[str, object]], extra_retrieve_call_ids: frozenset[str] = frozenset() +) -> frozenset[int]: """Indices headroom must not send to the compression service. ``get_protected_indices`` is litellm's own compression policy: the system - rows, the last user row, the last assistant row. It is expanded over whole + rows, the last user row, the last assistant row. Rows carrying just-retrieved + ``headroom_retrieve`` output are added so re-compression can't collapse them + back to the marker they were expanded from. The union is expanded over whole tool exchanges the way ``compress()`` expands it, so a protected assistant tool call cannot end up answered by a marker standing in for the result the model just asked for. """ - protected: Final = frozenset(get_protected_indices(messages)) + protected: Final = frozenset(get_protected_indices(messages)) | _retrieval_result_indices( + messages, extra_retrieve_call_ids + ) return protected | frozenset( index for group in group_tool_exchanges(messages) @@ -634,7 +742,11 @@ class HeadroomGuardrail(CustomGuardrail): # /v1/compress grows a field for sending the live turn as the retrieval # query without compressing it: query-aware compression reads the newest # user message, so it is withheld here at some cost to history ranking. - protected_indices: Final = _protected_indices(messages) + # request_data is a bare dict on the base signature; narrow it before + # reading the untranslated messages so long tool names can be recovered. + raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages") + raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages) + protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids) compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices] if not compressible: return inputs diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index d187b5b12e9..7d88a037f4f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -1,4 +1,5 @@ from collections.abc import AsyncGenerator, Mapping, Sequence +from enum import Enum, auto from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -27,12 +28,25 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.guardrails.anthropic_sse import ( + anthropic_sse_chunks_from_response, + anthropic_sse_error_frames, + assemble_anthropic_sse_stream, + is_anthropic_sse_stream, + is_raw_sse_stream, + is_sse_error_stream, +) from litellm.proxy.guardrails.guardrail_hooks.model_armor.file_scanning import ( MODEL_ARMOR_MAX_FILE_SIZE_BYTES, plan_file_scans, ) from litellm.types.guardrails import GuardrailEventHooks, LitellmParams -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionToolCallChunk, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) from litellm.types.utils import ( CallTypes, CallTypesLiteral, @@ -41,10 +55,33 @@ from litellm.types.utils import ( ModelResponse, ModelResponseStream, StandardLoggingGuardrailInformation, + TextCompletionResponse, ) GUARDRAIL_NAME: Final = "model_armor" +# Only these carry the finished output; response.created carries an empty body +_RESPONSES_TERMINAL_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete", "response.failed"}) + +# Every event whose ``delta`` is model output already on its way to the client. Read off the event +# enum rather than listed, so an event added there cannot quietly fall out of the scan +_RESPONSES_DELTA_EVENT_TYPES: Final = frozenset( + event.value for event in ResponsesAPIStreamEvents if event.value.endswith(".delta") +) + +# What makes two delta events part of the same field of the turn, rather than two fields that merely +# streamed next to each other +_RESPONSES_DELTA_FIELD_ATTRS: Final = ("type", "item_id", "output_index", "content_index", "summary_index") + + +class _StreamSurface(Enum): + """Wire format of a buffered streaming response, which decides how it is read and how it is refused.""" + + CHAT_COMPLETIONS = auto() + ANTHROPIC_MESSAGES = auto() + RESPONSES = auto() + OPAQUE_SSE = auto() + class ModelArmorAPIError(Exception): """Model Armor API failure (non-2xx), distinct from a content-block decision so @@ -322,19 +359,9 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): else: return {"modelResponseData": {"byteItem": {"byteDataType": file_type, "byteData": base64_data}}} - def _should_block_content(self, armor_response: dict, allow_sanitization: bool = False) -> bool: + def _should_block_content(self, armor_response: Mapping[str, Any], allow_sanitization: bool = False) -> bool: """Check if Model Armor response indicates content should be blocked, including both inspectResult and deidentifyResult.""" - sanitization_result: Final = armor_response.get("sanitizationResult", {}) - filter_results: Final = sanitization_result.get("filterResults", {}) - - # filterResults can be a dict (named keys) or a list (array of filter result dicts) - filter_result_items = [] - if isinstance(filter_results, dict): - filter_result_items = list(filter_results.values()) - elif isinstance(filter_results, list): - filter_result_items = filter_results - - for filt in filter_result_items: + for filt in self._filter_result_items(armor_response): # Check RAI, PI/Jailbreak, Malicious URI, CSAM, Virus scan as before if filt.get("raiFilterResult", {}).get("matchState") == "MATCH_FOUND": return True @@ -358,22 +385,12 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): # Fallback dict code removed; all cases handled above return False - def _get_sanitized_content(self, armor_response: dict) -> str | None: + def _get_sanitized_content(self, armor_response: Mapping[str, Any]) -> str | None: """ Get the sanitized content from a Model Armor response, if available. Looks for sanitized text in deidentifyResult, and falls back to root-level fields if not found. """ - result: Final = armor_response.get("sanitizationResult", {}) - filter_results: Final = result.get("filterResults", {}) - - # filterResults can be a dict (single filter) or a list (multiple filters) - filters: Final = ( - list(filter_results.values()) - if isinstance(filter_results, dict) - else filter_results - if isinstance(filter_results, list) - else [] - ) + filters: Final = self._filter_result_items(armor_response) # Prefer sanitized text from deidentifyResult if present for filter_entry in filters: @@ -397,6 +414,61 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): # Fallback: if Model Armor put sanitized text at the root, use it return armor_response.get("sanitizedText") or armor_response.get("text") + @staticmethod + def _filter_result_items(armor_response: Mapping[str, Any]) -> Sequence[Any]: + """Every filter result in a scan response. + + filterResults is a dict of named filters on most templates and a list on some, so both + shapes are flattened to the same list of filter entries. + """ + filter_results: Final = armor_response.get("sanitizationResult", {}).get("filterResults", {}) + if isinstance(filter_results, dict): + return list(filter_results.values()) + if isinstance(filter_results, list): + return filter_results + return [] + + def _has_deidentify_match(self, armor_response: Mapping[str, Any]) -> bool: + """Whether an SDP de-identify filter matched, i.e. Model Armor owes this response a redaction.""" + for filter_entry in self._filter_result_items(armor_response): + sdp = filter_entry.get("sdpFilterResult") + if sdp and sdp.get("deidentifyResult", {}).get("matchState") == "MATCH_FOUND": + return True + return False + + def _resolve_streaming_outcome( + self, + armor_response: Mapping[str, Any], + assembled_response: object, + content: str, + ) -> tuple[bool, str | None]: + """Whether to block the buffered stream, and the rewrite to emit when it is not blocked. + + A de-identify match only reaches here unblocked because masking is on, so the redaction it + stands for has to be both resolvable and emittable. Where it is neither, the buffered + original still carries what Model Armor matched on, so this fails closed instead of + releasing it. + """ + if self._should_block_content(armor_response, allow_sanitization=self.mask_response_content): + return True, None + if not self.mask_response_content: + return False, None + + sanitized_content: Final = self._get_sanitized_content(armor_response) + if not sanitized_content: + # No rewrite to apply. Harmless unless a match is outstanding, in which case applying + # nothing would hand back the very content that matched + return self._has_deidentify_match(armor_response), None + if sanitized_content == content: + return False, None + if not isinstance(assembled_response, ModelResponse): + verbose_proxy_logger.warning( + "Model Armor: sanitized content cannot be re-emitted on this streaming endpoint, " + "blocking the response instead" + ) + return True, None + return False, sanitized_content + @staticmethod def _append_armor_response(existing: object, armor_response: Mapping[str, object]) -> object: """Accumulate scan responses so a later text scan does not drop an earlier file scan. @@ -831,6 +903,185 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): return response + @staticmethod + def _is_terminal_error_stream(all_chunks: Sequence[object]) -> bool: + """Whether the buffered stream is only the refusal an earlier guardrail in the chain emitted. + + post_call guardrails are composed, so this hook can be handed the terminal error items a + preceding one produced. They carry no message to scan, and replacing them would hide the + refusal the client is owed. + """ + if all(getattr(chunk, "type", None) == "error" for chunk in all_chunks): + return True + return is_sse_error_stream(all_chunks) + + @staticmethod + def _classify_stream(all_chunks: Sequence[object]) -> _StreamSurface: + """Wire format the buffered chunks belong to.""" + if is_raw_sse_stream(all_chunks): + return ( + _StreamSurface.ANTHROPIC_MESSAGES if is_anthropic_sse_stream(all_chunks) else _StreamSurface.OPAQUE_SSE + ) + if any( + isinstance(event_type := getattr(chunk, "type", None), str) and event_type.startswith("response.") + for chunk in all_chunks + ): + return _StreamSurface.RESPONSES + return _StreamSurface.CHAT_COMPLETIONS + + @staticmethod + def _final_responses_api_response(all_chunks: Sequence[object]) -> ResponsesAPIResponse | None: + """Response body carried by a terminal ``/v1/responses`` event. + + A stream cut short before it completes has to read as unassembled rather than as a clean + empty response: ``response.created`` also carries a body, but an empty one, and scanning + that would release every buffered delta unscanned. + """ + return next( + ( + body + for chunk in reversed(all_chunks) + if getattr(chunk, "type", None) in _RESPONSES_TERMINAL_EVENT_TYPES + and isinstance(body := getattr(chunk, "response", None), ResponsesAPIResponse) + ), + None, + ) + + @staticmethod + def _responses_api_response_text(response: ResponsesAPIResponse) -> str: + """Text to scan in a Responses API response, tool-call arguments included. + + Tool calls are folded in because ``get_content_from_model_response`` folds them into what + the chat surface scans, and a Responses turn can carry its whole payload in them. + """ + from litellm.llms.openai.responses.guardrail_translation.handler import ( + OpenAIResponsesHandler, + ) + + texts: Final[list[str]] = [] # mutable-ok: the shared extractor below appends into caller-owned lists + tool_calls: Final[list[ChatCompletionToolCallChunk]] = [] # mutable-ok: the same extractor's tool-call sink + handler: Final = OpenAIResponsesHandler() + for output_idx, output_item in enumerate(response.output or ()): + handler._extract_output_text_and_images( # pyright: ignore[reportPrivateUsage] # the shared Responses output extractor; forking it would duplicate per-item parsing + output_item=output_item, + output_idx=output_idx, + texts_to_check=texts, + images_to_check=[], # mutable-ok: the extractor's images sink, unused here + task_mappings=[], # mutable-ok: the extractor's task-mapping sink, unused here + tool_calls_to_check=tool_calls, + ) + return "".join((*texts, *(json.dumps(tool_call) for tool_call in tool_calls))) + + def _extract_streaming_content(self, assembled_response: object) -> str: + """Text to scan from an assembled stream, for every endpoint shape this hook serves.""" + if isinstance(assembled_response, ResponsesAPIResponse): + return self._responses_api_response_text(assembled_response) + return self._extract_content_from_response(assembled_response) + + @staticmethod + def _responses_delta_field(chunk: object) -> tuple[str, ...]: + """Which field of the turn a delta event belongs to.""" + return tuple(str(getattr(chunk, attr, None)) for attr in _RESPONSES_DELTA_FIELD_ATTRS) + + @staticmethod + def _responses_delta_field_texts(all_chunks: Sequence[object]) -> tuple[str, ...]: + """Text each field of a ``/v1/responses`` turn has already spelled out in its delta events. + + One field's deltas are joined as they streamed, since a finding can be split across them, + and separate fields stay apart, so a reasoning summary running into the visible answer + cannot spell out a finding that neither of them carries. + """ + deltas: Final = tuple( + (ModelArmorGuardrail._responses_delta_field(chunk), delta) + for chunk in all_chunks + if getattr(chunk, "type", None) in _RESPONSES_DELTA_EVENT_TYPES + and isinstance(delta := getattr(chunk, "delta", None), str) + ) + return tuple( + "".join(delta for field, delta in deltas if field == streamed_field) + for streamed_field in dict.fromkeys(field for field, _ in deltas) + ) + + def _streaming_content_to_scan( + self, + assembled_response: object, + all_chunks: Sequence[object], + surface: _StreamSurface, + ) -> str: + """Text to scan for a buffered stream, which is everything the client is about to receive. + + A ``/v1/responses`` stream also spells out reasoning summaries and tool-call arguments in + delta events that its terminal body never repeats, so every delta field the body does not + already carry is scanned after it. + """ + content: Final = self._extract_streaming_content(assembled_response) + if surface is not _StreamSurface.RESPONSES: + return content + unscanned: Final = tuple(text for text in self._responses_delta_field_texts(all_chunks) if text not in content) + return "\n".join(part for part in (content, *unscanned) if part) + + @staticmethod + def _apply_sanitized_content(assembled_response: ModelResponse, sanitized_content: str) -> None: + """Replace every non-empty choice message with the Model Armor sanitized text.""" + for choice in assembled_response.choices: + if isinstance(choice, Choices) and choice.message.content: + choice.message.content = sanitized_content + + @staticmethod + def _assemble_chat_completion_stream( + all_chunks: list[object], # mutable-ok: stream_chunk_builder only accepts a mutable list + ) -> ModelResponse | TextCompletionResponse | None: + """Assemble chat-completion chunks, returning ``None`` when they cannot be assembled.""" + from litellm.main import stream_chunk_builder + + try: + return stream_chunk_builder(chunks=all_chunks) + except Exception as exc: + verbose_proxy_logger.warning("Model Armor: chat-completion stream assembly failed (%s)", exc) + return None + + def _assemble_stream( + self, all_chunks: Sequence[object], surface: _StreamSurface + ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None: + """Assemble the buffered stream into the scannable response its surface produces.""" + if surface is _StreamSurface.ANTHROPIC_MESSAGES: + return assemble_anthropic_sse_stream(all_chunks, restore_identity=True) + if surface is _StreamSurface.RESPONSES: + return self._final_responses_api_response(all_chunks) + if surface is _StreamSurface.OPAQUE_SSE: + return None + return self._assemble_chat_completion_stream(list(all_chunks)) + + @staticmethod + def _error_payload(exc: HTTPException) -> Mapping[str, object]: + """Error object for a terminal stream item, carrying the status the frame would otherwise lose.""" + detail: Final = exc.detail if isinstance(exc.detail, Mapping) else {"message": str(exc.detail)} + error_value: Final = detail.get("error", detail) + return { + **(dict(error_value) if isinstance(error_value, Mapping) else {"message": str(error_value)}), + "code": str(exc.status_code), + } + + @staticmethod + def _build_responses_error_items(exc: HTTPException) -> Sequence[object] | None: + """Responses API error events for a failure discovered after the stream started.""" + from litellm.llms.openai.responses.guardrail_translation.handler import ( + OpenAIResponsesHandler, + ) + + return OpenAIResponsesHandler().build_stream_error_items(exc, responses_so_far=None) + + def _stream_error_items(self, exc: HTTPException, *, surface: _StreamSurface) -> Sequence[object]: + """Frame a guardrail failure as terminal stream items in this endpoint's wire format.""" + payload: Final = self._error_payload(exc) + if surface is _StreamSurface.ANTHROPIC_MESSAGES: + return anthropic_sse_error_frames(str(payload.get("message", ""))) + if surface is _StreamSurface.RESPONSES and (responses_items := self._build_responses_error_items(exc)): + return responses_items + # Also the fallback when a surface cannot frame its own error: create_response() reads the + # status back out of this form, so the refusal keeps its code instead of arriving as a 200 + return (f"data: {json.dumps({'error': payload})}\n\n",) + async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -840,97 +1091,125 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): """Process streaming response chunks.""" from litellm.llms.base_llm.base_model_iterator import MockResponseIterator - from litellm.main import stream_chunk_builder + from litellm.proxy.common_utils.callback_utils import ( + add_guardrail_to_applied_guardrails_header, + ) # Collect all chunks - all_chunks: Final[list[ModelResponseStream]] = [] + all_chunks: Final[list[Any]] = [] async for chunk in response: all_chunks.append(chunk) + if not all_chunks or self._is_terminal_error_stream(all_chunks): + for chunk in all_chunks: + yield chunk + return + + surface: Final = self._classify_stream(all_chunks) + # Build complete response - assembled_response: Final = stream_chunk_builder(chunks=all_chunks) + assembled_response: Final = self._assemble_stream(all_chunks, surface) - if isinstance(assembled_response, ModelResponse): - # Extract content - content: Final = self._extract_content_from_response(assembled_response) + if assembled_response is None: + if not self.optional_params.get("fail_on_error", True): + verbose_proxy_logger.warning( + "Model Armor: streamed response could not be assembled for scanning, " + "forwarding it unscanned because fail_on_error is disabled" + ) + for chunk in all_chunks: + yield chunk + return - if content: - try: - # Check with Model Armor - armor_response: Final = await self.make_model_armor_request( - content=content, - source="model_response", - request_data=request_data, - ) + # Forwarding an unscannable stream would silently disable the guardrail, so fail closed + add_guardrail_to_applied_guardrails_header(request_data=request_data, guardrail_name=self.guardrail_name) + for error_item in self._stream_error_items( + HTTPException( + status_code=500, + detail=f"{self.guardrail_name}: streamed response could not be assembled for scanning, blocking it", + ), + surface=surface, + ): + yield error_item + return - # Attach Model Armor response & status to this request's metadata to avoid race conditions - if isinstance(request_data, dict): - _, metadata = get_or_create_metadata_bucket(request_data) - metadata["_model_armor_response"] = self._build_logging_response(armor_response) - metadata["_model_armor_status"] = ( - "blocked" if self._should_block_content(armor_response) else "success" - ) + # Extract content + content: Final = self._streaming_content_to_scan( + assembled_response=assembled_response, all_chunks=all_chunks, surface=surface + ) - # Add guardrail to applied_guardrails BEFORE potential blocking - # This ensures guardrail is recorded even when it blocks the request - from litellm.proxy.common_utils.callback_utils import ( - add_guardrail_to_applied_guardrails_header, - ) + if not content: + verbose_proxy_logger.debug("Model Armor: No text content in streaming response, skipping guardrail") + for chunk in all_chunks: + yield chunk + return - add_guardrail_to_applied_guardrails_header( - request_data=request_data, guardrail_name=self.guardrail_name - ) + try: + # Check with Model Armor + armor_response: Final = await self.make_model_armor_request( + content=content, + source="model_response", + request_data=request_data, + ) - # Check if blocked - if self._should_block_content(armor_response): - raise HTTPException( - status_code=400, - detail=self._build_block_error_detail( - "Streaming response blocked by Model Armor", - armor_response, - ), - ) + # Decide the outcome before recording it. Mirrors the non-streaming sibling: with + # masking on, a de-identify match is a redaction to apply rather than a refusal, but + # that only holds while the redaction can actually be delivered + blocked, sanitized_content = self._resolve_streaming_outcome( + armor_response=armor_response, + assembled_response=assembled_response, + content=content, + ) - # Apply sanitization if enabled - if self.mask_response_content: - sanitized_content: Final = self._get_sanitized_content(armor_response) - if sanitized_content and sanitized_content != content: - # Update assembled response - for choice in assembled_response.choices: - if isinstance(choice, Choices): - if choice.message.content: - choice.message.content = sanitized_content + # Attach Model Armor response & status to this request's metadata to avoid race conditions + if isinstance(request_data, dict): + _, metadata = get_or_create_metadata_bucket(request_data) + metadata["_model_armor_response"] = self._build_logging_response(armor_response) + metadata["_model_armor_status"] = "blocked" if blocked else "success" - # Return sanitized stream - mock_response: Final = MockResponseIterator(model_response=assembled_response) - async for chunk in mock_response: - yield chunk - return + # Add guardrail to applied_guardrails BEFORE potential blocking + # This ensures guardrail is recorded even when it blocks the request + add_guardrail_to_applied_guardrails_header(request_data=request_data, guardrail_name=self.guardrail_name) - except ModelArmorAPIError as e: - if self.optional_params.get("fail_on_error", True): - error_obj = {"message": e.detail, "code": "500"} - yield f"data: {json.dumps({'error': error_obj})}\n\n" - return - except HTTPException as e: - # Yield error as SSE event so create_response() detects it and - # returns a proper JSON error response with the correct status code. - # (Raising from a generator hits create_response's generic except → 500.) - detail: Final = e.detail if isinstance(e.detail, dict) else {"message": str(e.detail)} - error_value: Final = detail.get("error", detail) - if isinstance(error_value, dict): - error_obj = dict(error_value) - else: - error_obj = {"message": str(error_value)} - error_obj["code"] = str(e.status_code) - yield f"data: {json.dumps({'error': error_obj})}\n\n" + if blocked: + raise HTTPException( + status_code=400, + detail=self._build_block_error_detail( + "Streaming response blocked by Model Armor", + armor_response, + ), + ) + + if sanitized_content is not None and isinstance(assembled_response, ModelResponse): + self._apply_sanitized_content(assembled_response, sanitized_content) + + # Return sanitized stream + if surface is _StreamSurface.ANTHROPIC_MESSAGES: + for sse_chunk in anthropic_sse_chunks_from_response(assembled_response): + yield sse_chunk return - except Exception as e: - verbose_proxy_logger.error("Model Armor streaming error: %s", str(e), exc_info=True) - if self.optional_params.get("fail_on_error", True): - raise - else: - verbose_proxy_logger.debug("Model Armor: No text content in streaming response, skipping guardrail") + mock_response: Final = MockResponseIterator(model_response=assembled_response) + async for chunk in mock_response: + yield chunk + return + + except ModelArmorAPIError as e: + if self.optional_params.get("fail_on_error", True): + for error_item in self._stream_error_items( + HTTPException(status_code=500, detail=e.detail), surface=surface + ): + yield error_item + return + except HTTPException as e: + # Yield the error as a terminal stream item so create_response() detects it and returns + # a proper JSON error response with the correct status code. Raising from a generator + # instead hits create_response's generic except and becomes a 500. + for error_item in self._stream_error_items(e, surface=surface): + yield error_item + return + except Exception as e: + verbose_proxy_logger.error("Model Armor streaming error: %s", str(e), exc_info=True) + if self.optional_params.get("fail_on_error", True): + raise # Return original chunks if no sanitization needed for chunk in all_chunks: diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index da51a905ae3..70ea21320ee 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -1633,6 +1633,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): Update the guardrails litellm params in memory """ super().update_in_memory_litellm_params(litellm_params) + if self.apply_to_output: + self.output_parse_pii = False if litellm_params.pii_entities_config: self.pii_entities_config = litellm_params.pii_entities_config if litellm_params.presidio_score_thresholds: diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 46b00829b74..c6b8df1b493 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -36,6 +36,7 @@ if TYPE_CHECKING: from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, + StreamingScanKey, ) # Call types that stream JSON-RPC events (A2A); guardrail HTTPException is emitted as in-stream error @@ -54,6 +55,9 @@ class _EndpointTranslation(Protocol): @property def process_output_streaming_response(self) -> "Callable[..., Awaitable[object]]": ... + @property + def get_streaming_scan_key(self) -> "Callable[[Sequence[object]], StreamingScanKey | None]": ... + @property def build_block_sse_chunks(self) -> "Callable[..., Sequence[bytes] | None]": ... @@ -70,6 +74,12 @@ def _chunk_choices(item: object) -> Sequence[object]: return choices +def _is_redundant_scan(scan_key: "StreamingScanKey | None", last_scan_key: "StreamingScanKey | None") -> bool: + if scan_key is None: + return False + return scan_key == last_scan_key or scan_key.has_nothing_to_scan + + class _StreamTerminated(Exception): """Internal signal that the incremental transform stream has already emitted its terminal chunks (block message or in-stream error) and must stop.""" @@ -1011,6 +1021,7 @@ class UnifiedLLMGuardrails(CustomLogger): # Drives how a block terminates the stream: continue the in-progress # message (True) vs emit a standalone block message (False, buffered). chunks_yielded = False + last_scan_key: StreamingScanKey | None = None # rebind-ok: replaced after every scan round async for item in response: chunk_counter += 1 @@ -1052,6 +1063,19 @@ class UnifiedLLMGuardrails(CustomLogger): # Process chunk based on sampling rate if chunk_counter % sampling_rate == 0: + endpoint_translation = endpoint_guardrail_translation_mappings[CallTypes(call_type)]() + scan_key = endpoint_translation.get_streaming_scan_key(responses_so_far) + if _is_redundant_scan(scan_key, last_scan_key): + verbose_proxy_logger.debug( + "Skipping streaming chunk %s for guardrail %s: nothing new to scan since the last round", + chunk_counter, + guardrail_to_apply.guardrail_name, + ) + chunks_yielded = True + responses_yielded.append(item) + yield item + continue + verbose_proxy_logger.debug( "Processing streaming chunk %s (sampling_rate=%s) with guardrail %s", chunk_counter, @@ -1067,8 +1091,6 @@ class UnifiedLLMGuardrails(CustomLogger): # string, permanently losing this chunk's content. original_item = copy.deepcopy(item) - endpoint_translation = endpoint_guardrail_translation_mappings[CallTypes(call_type)]() - try: await endpoint_translation.process_output_streaming_response( responses_so_far=responses_so_far, @@ -1110,6 +1132,8 @@ class UnifiedLLMGuardrails(CustomLogger): ): yield error_item return + if scan_key is not None: + last_scan_key = scan_key chunks_yielded = True responses_yielded.append(original_item) yield original_item @@ -1136,6 +1160,18 @@ class UnifiedLLMGuardrails(CustomLogger): # preserve the list, not clone every chunk (deepcopy would double # peak memory for large responses). buffered_items: Final = list(responses_so_far) if buffer_until_moderated else None + end_scan_key: Final = endpoint_translation.get_streaming_scan_key(responses_so_far) + if _is_redundant_scan(end_scan_key, last_scan_key): + verbose_proxy_logger.debug( + "Skipping end-of-stream scan for guardrail %s: the last sampled round already scanned it all", + guardrail_to_apply.guardrail_name, + ) + for buffered_item in buffered_items or (): + yield buffered_item + for pending_item in pending_end_of_stream_items: + responses_yielded.append(pending_item) + yield pending_item + return try: await endpoint_translation.process_output_streaming_response( diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 76dea1b7784..16369abbfb0 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -2,6 +2,7 @@ from typing import Any, Final import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import CommonProxyErrors from litellm.types.guardrails import * @@ -85,7 +86,7 @@ def initialize_lakera_v2(litellm_params: LitellmParams, guardrail: Guardrail): return _lakera_v2_callback -def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail): +def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail) -> tuple[CustomGuardrail, ...]: from litellm.proxy.guardrails.guardrail_hooks.presidio import ( _OPTIONAL_PresidioPIIMasking, ) @@ -94,7 +95,7 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail): run_input: Final = filter_scope in ("input", "both") run_output: Final = filter_scope in ("output", "both") - def _make_presidio_callback(**overrides): + def _make_presidio_callback(**overrides) -> CustomGuardrail: params: Final = dict( guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, @@ -120,27 +121,27 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail): litellm.logging_callback_manager.add_litellm_callback(callback) return callback - primary_callback = None - - if run_input: - primary_callback = _make_presidio_callback() - - if litellm_params.output_parse_pii: - _make_presidio_callback( - output_parse_pii=True, - event_hook=GuardrailEventHooks.post_call.value, - ) - - if run_output: - output_callback: Final = _make_presidio_callback( + input_callback: Final = _make_presidio_callback() if run_input else None + unmask_output_callback: Final = ( + _make_presidio_callback( + output_parse_pii=True, + event_hook=GuardrailEventHooks.post_call.value, + ) + if run_input and litellm_params.output_parse_pii + else None + ) + mask_output_callback: Final = ( + _make_presidio_callback( apply_to_output=True, event_hook=GuardrailEventHooks.post_call.value, output_parse_pii=False, ) - if primary_callback is None: - primary_callback = output_callback - - return primary_callback + if run_output + else None + ) + return tuple( + callback for callback in (input_callback, unmask_output_callback, mask_output_callback) if callback is not None + ) def initialize_hide_secrets(litellm_params: LitellmParams, guardrail: Guardrail): diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index dc13c09dd38..003576ba555 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -3,10 +3,10 @@ import asyncio import importlib import os -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence from datetime import datetime, timezone from itertools import chain, count -from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, cast +from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, TypeAlias, cast from pydantic import ValidationError @@ -90,6 +90,8 @@ guardrail_initializer_registry: Final = { CONFIG_GUARDRAIL_ID_NAMESPACE: Final = uuid.UUID("625f63f4-935a-50e5-98b5-fbe77babc74a") +GuardrailCallbacks: TypeAlias = tuple[CustomGuardrail, ...] + guardrail_class_registry: Final[dict[str, type[CustomGuardrail]]] = { SupportedGuardrailIntegrations.BEDROCK.value: BedrockGuardrail, SupportedGuardrailIntegrations.GRAYSWAN.value: GraySwanGuardrail, @@ -424,6 +426,41 @@ def _apply_configured_bool_overrides(instance: CustomGuardrail, litellm_params: instance.scan_raw_request = bool(litellm_params.scan_raw_request) +def _as_callback_tuple( + initialized: CustomGuardrail | Sequence[CustomGuardrail] | None, +) -> GuardrailCallbacks: + if initialized is None: + return () + if isinstance(initialized, (list, tuple)): + return tuple(initialized) + return (initialized,) + + +def _configure_callback_scoping( + custom_guardrail_callback: CustomGuardrail, guardrail_name: str, litellm_params: LitellmParams +) -> None: + for scoping_param in ( + "skip_system_message_in_guardrail", + "skip_tool_message_in_guardrail", + "scan_only_tool_results", + ): + setattr(custom_guardrail_callback, scoping_param, getattr(litellm_params, scoping_param, None)) + scan_only_tool_results_enabled: Final = effective_scan_only_tool_results_for_guardrail(custom_guardrail_callback) + if scan_only_tool_results_enabled and not custom_guardrail_callback.supports_scan_only_tool_results(): + raise ValueError( + f"Guardrail {guardrail_name}: scan_only_tool_results is enabled, but this " + "guardrail's role filtering never scans tool results, so no request content would ever " + "be scanned. Remove scan_only_tool_results or the guardrail's role-filtering option." + ) + if scan_only_tool_results_enabled and effective_skip_tool_message_for_guardrail(custom_guardrail_callback): + raise ValueError( + f"Guardrail {guardrail_name}: scan_only_tool_results and " + "skip_tool_message_in_guardrail are enabled together, which excludes every message from " + "scanning, so no request content would ever be scanned. Remove one of the two." + ) + _apply_configured_bool_overrides(custom_guardrail_callback, litellm_params) + + class InMemoryGuardrailHandler: """ Class that handles initializing guardrails and adding them to the CallbackManager @@ -440,6 +477,8 @@ class InMemoryGuardrailHandler: Guardrail id to CustomGuardrail object mapping """ + self.guardrail_id_to_sibling_callbacks: dict[str, GuardrailCallbacks] = {} # mutable-ok: per-id registry + self._sources: dict[str, Literal["db", "config"]] = {} """ Guardrail id to provenance marker. "db" entries are reconciled against @@ -474,7 +513,6 @@ class InMemoryGuardrailHandler: self._sources[guardrail_id] = source return self.IN_MEMORY_GUARDRAILS[guardrail_id] - custom_guardrail_callback: CustomGuardrail | None = None litellm_params_data: Final = guardrail["litellm_params"] verbose_proxy_logger.debug("litellm_params= %s", litellm_params_data) @@ -498,54 +536,15 @@ class InMemoryGuardrailHandler: if guardrail_type is None: raise ValueError("guardrail_type is required") - initializer: Final = guardrail_initializer_registry.get(guardrail_type) - - if initializer: - # Try to call with llm_router first, fall back to without if it fails - import inspect - - sig: Final = inspect.signature(initializer) - if "llm_router" in sig.parameters: - custom_guardrail_callback = initializer( - litellm_params, - guardrail, - llm_router, - ) - else: - custom_guardrail_callback = initializer(litellm_params, guardrail) - elif isinstance(guardrail_type, str) and "." in guardrail_type: - custom_guardrail_callback = self.initialize_custom_guardrail( - guardrail=guardrail, - guardrail_type=guardrail_type, - litellm_params=litellm_params, - config_file_path=config_file_path, - ) - else: - raise ValueError(f"Unsupported guardrail: {guardrail_type}") - - if custom_guardrail_callback is not None: - for scoping_param in ( - "skip_system_message_in_guardrail", - "skip_tool_message_in_guardrail", - "scan_only_tool_results", - ): - setattr(custom_guardrail_callback, scoping_param, getattr(litellm_params, scoping_param, None)) - scan_only_tool_results_enabled: Final = effective_scan_only_tool_results_for_guardrail( - custom_guardrail_callback - ) - if scan_only_tool_results_enabled and not custom_guardrail_callback.supports_scan_only_tool_results(): - raise ValueError( - f"Guardrail {guardrail['guardrail_name']}: scan_only_tool_results is enabled, but this " - "guardrail's role filtering never scans tool results, so no request content would ever " - "be scanned. Remove scan_only_tool_results or the guardrail's role-filtering option." - ) - if scan_only_tool_results_enabled and effective_skip_tool_message_for_guardrail(custom_guardrail_callback): - raise ValueError( - f"Guardrail {guardrail['guardrail_name']}: scan_only_tool_results and " - "skip_tool_message_in_guardrail are enabled together, which excludes every message from " - "scanning, so no request content would ever be scanned. Remove one of the two." - ) - _apply_configured_bool_overrides(custom_guardrail_callback, litellm_params) + created_callbacks: Final = self._create_callbacks( + guardrail=guardrail, + guardrail_type=guardrail_type, + litellm_params=litellm_params, + config_file_path=config_file_path, + llm_router=llm_router, + ) + for custom_guardrail_callback in created_callbacks: + _configure_callback_scoping(custom_guardrail_callback, guardrail["guardrail_name"], litellm_params) parsed_guardrail: Final = Guardrail( guardrail_id=guardrail.get("guardrail_id"), @@ -556,11 +555,44 @@ class InMemoryGuardrailHandler: # store references to the guardrail in memory self.IN_MEMORY_GUARDRAILS[guardrail_id] = parsed_guardrail - self.guardrail_id_to_custom_guardrail[guardrail_id] = custom_guardrail_callback + self.guardrail_id_to_custom_guardrail[guardrail_id] = created_callbacks[0] if created_callbacks else None + self.guardrail_id_to_sibling_callbacks[guardrail_id] = created_callbacks[1:] self._sources[guardrail_id] = source return parsed_guardrail + def _create_callbacks( + self, + guardrail: Guardrail, + guardrail_type: str, + litellm_params: LitellmParams, + config_file_path: str | None, + llm_router: Optional["Router"], + ) -> GuardrailCallbacks: + initializer: Final = guardrail_initializer_registry.get(guardrail_type) + if initializer: + import inspect + + sig: Final = inspect.signature(initializer) + if "llm_router" in sig.parameters: + return _as_callback_tuple(initializer(litellm_params, guardrail, llm_router)) + return _as_callback_tuple(initializer(litellm_params, guardrail)) + if isinstance(guardrail_type, str) and "." in guardrail_type: + return _as_callback_tuple( + self.initialize_custom_guardrail( + guardrail=guardrail, + guardrail_type=guardrail_type, + litellm_params=litellm_params, + config_file_path=config_file_path, + ) + ) + raise ValueError(f"Unsupported guardrail: {guardrail_type}") + + def _tracked_callbacks(self, guardrail_id: str) -> GuardrailCallbacks: + primary: Final = self.guardrail_id_to_custom_guardrail.get(guardrail_id) + siblings: Final = self.guardrail_id_to_sibling_callbacks.get(guardrail_id, ()) + return (() if primary is None else (primary,)) + siblings + def initialize_custom_guardrail( self, guardrail: Guardrail, @@ -630,10 +662,15 @@ class InMemoryGuardrailHandler: self.IN_MEMORY_GUARDRAILS[guardrail_id] = guardrail self._sources[guardrail_id] = source - custom_guardrail_callback: Final = self.guardrail_id_to_custom_guardrail.get(guardrail_id) - if custom_guardrail_callback: - updated_litellm_params: Final = cast(LitellmParams, guardrail.get("litellm_params", {})) - custom_guardrail_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params) + tracked_callbacks: Final = self._tracked_callbacks(guardrail_id) + if not tracked_callbacks: + return + updated_litellm_params: Final = cast(LitellmParams, guardrail.get("litellm_params", {})) + tracked_callbacks[0].update_in_memory_litellm_params(litellm_params=updated_litellm_params) + for sibling_callback in tracked_callbacks[1:]: + sibling_stage = sibling_callback.event_hook + sibling_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params) + sibling_callback.event_hook = sibling_stage def delete_in_memory_guardrail(self, guardrail_id: str) -> None: """ @@ -648,11 +685,11 @@ class InMemoryGuardrailHandler: self.IN_MEMORY_GUARDRAILS.pop(guardrail_id, None) self._sources.pop(guardrail_id, None) - custom_guardrail_callback: Final = self.guardrail_id_to_custom_guardrail.pop(guardrail_id, None) - if custom_guardrail_callback is None: - return - - litellm.logging_callback_manager.remove_callback_from_all_lists(custom_guardrail_callback) + tracked_callbacks: Final = self._tracked_callbacks(guardrail_id) + self.guardrail_id_to_custom_guardrail.pop(guardrail_id, None) + self.guardrail_id_to_sibling_callbacks.pop(guardrail_id, None) + for custom_guardrail_callback in tracked_callbacks: + litellm.logging_callback_manager.remove_callback_from_all_lists(custom_guardrail_callback) def list_in_memory_guardrails(self) -> list[Guardrail]: """ @@ -789,11 +826,12 @@ class InMemoryGuardrailHandler: Removes old callback from litellm.callbacks and creates fresh instance. If the new config fails to initialize (e.g. an invalid on_flagged - combination), the previous instance is restored rather than left - deleted: initialize_guardrail's own ValueError/TypeError propagate - uncaught, so a caller reaching this point after already deleting the - old instance would otherwise leave the guardrail providing no - protection at all, not merely "still enforcing the old config." + combination or an invalid regex), the previous instance is restored + rather than left deleted, and the failure is re-raised as ValueError so + every init failure reaches callers as one exception type: a caller + reaching this point after already deleting the old instance would + otherwise leave the guardrail providing no protection at all, not + merely "still enforcing the old config." """ guardrail_id: Final = guardrail.get("guardrail_id") if not guardrail_id: @@ -812,7 +850,7 @@ class InMemoryGuardrailHandler: # that was enforcing must never fail open because an update was bad. try: return self.initialize_guardrail(guardrail=guardrail, config_file_path=config_file_path, source=source) - except Exception: + except Exception as init_error: if previous_guardrail is not None: verbose_proxy_logger.exception( "Reinitializing guardrail %s with updated params failed; restoring the previous configuration", @@ -824,7 +862,7 @@ class InMemoryGuardrailHandler: ) except Exception: # noqa: BLE001 # the original failure must propagate even if the restore breaks verbose_proxy_logger.exception("Restoring previous guardrail %s also failed", guardrail_id) - raise + raise ValueError(f"Guardrail initialization failed: {init_error}") from init_error def sync_guardrail_from_db(self, guardrail: Guardrail, config_file_path: str | None = None) -> Guardrail | None: """ diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index c5d10b2749b..efaaab277a9 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -1,6 +1,6 @@ import json import time -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import Final @@ -199,18 +199,10 @@ async def build_model_max_budget_usage( ) for budget_model, budget_config in budgets ) - batched: Final = await cache.async_batch_get_cache( - keys=list(spend_keys) # mutable-ok: async_batch_get_cache annotates keys as list, so one must exist here - ) - # async_batch_get_cache returns None if it fails internally, and its result is - # index-aligned with `keys` otherwise. An unusable result reads as a miss, - # which is what a never-written counter already reads as. - current_spends: Final = ( - tuple(batched) if isinstance(batched, list) and len(batched) == len(budgets) else (None,) * len(budgets) - ) + current_spends: Final = await _current_window_spends(cache=cache, spend_keys=spend_keys) return { budget_model: { - "current_spend": round(_as_spend(current_spend), 4), + "current_spend": round(current_spend, 4), "budget_limit": budget_config.max_budget, "time_period": budget_config.budget_duration, } @@ -218,6 +210,22 @@ async def build_model_max_budget_usage( } +async def _current_window_spends(cache: DualCache, spend_keys: Sequence[str]) -> tuple[float, ...]: + """Redis holds the window total across replicas; the in-memory copy is one replica's share.""" + keys: Final = list(spend_keys) # mutable-ok: both batch readers annotate their key argument as list + redis_cache: Final = cache.redis_cache + if redis_cache is not None: + shared: Final = await redis_cache.async_batch_get_cache(key_list=keys) + return tuple(_as_spend(shared.get(key)) for key in keys) + # async_batch_get_cache returns None if it fails internally, and its result is + # index-aligned with `keys` otherwise. An unusable result reads as a miss, + # which is what a never-written counter already reads as. + batched: Final = await cache.async_batch_get_cache(keys=keys) + if not isinstance(batched, list) or len(batched) != len(keys): + return (0.0,) * len(keys) + return tuple(_as_spend(current_spend) for current_spend in batched) + + def _usable_budget_config(raw_budget_config: object) -> BudgetConfig | None: try: budget_config: Final = BudgetConfig.model_validate(raw_budget_config) @@ -404,7 +412,10 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): return current_spend + _as_spend(await self._cached_spend(legacy_spend_key)) async def _cached_spend(self, spend_key: str) -> float | None: - return await self.dual_cache.async_get_cache(key=spend_key) + redis_cache: Final = self.dual_cache.redis_cache + if redis_cache is None: + return await self.dual_cache.async_get_cache(key=spend_key) + return await redis_cache.async_get_cache(key=spend_key) async def async_filter_deployments( self, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 20f83085286..1d440448c2f 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -16,6 +16,7 @@ from starlette.datastructures import Headers import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging +from litellm._uuid import uuid from litellm.constants import ( CONSUMED_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, @@ -23,6 +24,7 @@ from litellm.constants import ( OTEL_SERVICE_NAME_METADATA_KEYS, PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + SESSION_ID_GENERATED_METADATA_KEY, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -40,6 +42,7 @@ from litellm.proxy._types import ( AddTeamCallback, CommonProxyErrors, LitellmDataForBackendLLMCall, + LiteLLMRoutes, LitellmUserRoles, ProxyErrorTypes, ProxyException, @@ -47,6 +50,8 @@ from litellm.proxy._types import ( TeamCallbackMetadata, UserAPIKeyAuth, ) +from litellm.proxy.auth.auth_utils import get_request_route +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.callback_utils import ( decrypt_callback_vars, get_metadata_variable_name_from_kwargs, @@ -715,6 +720,50 @@ def _get_anthropic_session_id_from_metadata(metadata: object) -> str | None: return session_id +def _is_llm_inference_route(request: Request) -> bool: + route: Final = get_request_route(request) + return RouteChecks.is_llm_api_route(route=route) and not RouteChecks.check_route_access( + route=route, allowed_routes=LiteLLMRoutes.mcp_routes.value + ) + + +def apply_missing_session_id_policy( + data: dict[str, object], # mutable-ok: stamps session ids in place on the request body the pipeline threads through + _metadata_variable_name: str, + general_settings: Mapping[str, object] | None, + request: Request, +) -> None: + policy: Final = general_settings.get("missing_session_id") if general_settings else None + if policy is None or not _is_llm_inference_route(request): + return + metadata: Final = data.get(_metadata_variable_name) + if not isinstance(metadata, dict): + return + if data.get("litellm_session_id") or metadata.get("session_id"): + return + match policy: + case "generate": + session_id: Final = str(data.get("litellm_trace_id") or metadata.get("trace_id") or uuid.uuid4()) + data["litellm_session_id"] = session_id # rebind-ok: data is an out-param + data.setdefault("litellm_trace_id", session_id) + metadata["session_id"] = session_id + metadata[SESSION_ID_GENERATED_METADATA_KEY] = True + case "reject": + raise ProxyException( + message=( + "Request has no session id. Send an `x-litellm-session-id` header or `metadata.session_id`. " + "Required by `general_settings.missing_session_id: reject`." + ), + type=ProxyErrorTypes.bad_request_error, + param="session_id", + code=400, + ) + case _: + verbose_proxy_logger.warning( + "Ignoring unknown general_settings.missing_session_id=%r; expected 'generate' or 'reject'", policy + ) + + def is_claude_code_user_agent(user_agent: str) -> bool: """Claude Code identifies itself as ``claude-cli/ ...``; the IDE extensions and the Agent SDK run through the same CLI and share that prefix.""" @@ -1818,6 +1867,12 @@ async def add_litellm_data_to_request( data=data, _metadata_variable_name=_metadata_variable_name, ) + apply_missing_session_id_policy( + data=data, + _metadata_variable_name=_metadata_variable_name, + general_settings=general_settings, + request=request, + ) # Expose request headers under the metadata field for guardrails (fixes #17477) if _metadata_variable_name in data and isinstance(data[_metadata_variable_name], dict): diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index c08ca5b7783..5326074ad3c 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -27,6 +27,7 @@ from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.proxy._types import * from litellm.proxy.auth.auth_checks import get_team_object, get_user_object +from litellm.proxy.auth.password_policy import validate_password_policy from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.user_api_key_cache import ( object_permission_cache_key, @@ -154,9 +155,10 @@ def _team_membership_table( return team_membership_table -def _hash_password_in_dict(data: dict) -> None: - """Hash password field in-place if present.""" +def _hash_password_in_dict(data: dict, general_settings: Mapping[str, object]) -> None: + """Validate and hash password field in-place if present.""" if "password" in data and data["password"] is not None: + validate_password_policy(data["password"], general_settings) data["password"] = hash_password(data["password"]) @@ -500,7 +502,7 @@ async def new_user( ``` """ try: - from litellm.proxy.proxy_server import _license_check, prisma_client + from litellm.proxy.proxy_server import _license_check, general_settings, prisma_client if prisma_client is None: raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) @@ -548,7 +550,7 @@ async def new_user( # generate_key_helper_fn only forwards object_permission_id, so without this the entitlement # the caller sent would be dropped on the floor. data_json = await _set_object_permission(data_json=data_json, prisma_client=prisma_client) - _hash_password_in_dict(data_json) + _hash_password_in_dict(data_json, general_settings) teams = data.teams if teams is None: teams = check_if_default_team_set() @@ -1405,7 +1407,7 @@ async def _update_single_user_helper( Returns the updated user data or raises an exception on failure. """ - from litellm.proxy.proxy_server import litellm_proxy_admin_name, prisma_client + from litellm.proxy.proxy_server import general_settings, litellm_proxy_admin_name, prisma_client if prisma_client is None: raise Exception("Not connected to DB!") @@ -1420,7 +1422,7 @@ async def _update_single_user_helper( data_json: Final[dict] = user_request.model_dump(exclude_unset=True) non_default_values = _update_internal_user_params(data_json=data_json, data=user_request) - _hash_password_in_dict(non_default_values) + _hash_password_in_dict(non_default_values, general_settings) existing_user_row: BaseModel | None = None if user_request.user_id: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c3403cf477c..d7d20d168b5 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3785,15 +3785,22 @@ async def info_key_fn_v2( @router.get("/key/info", tags=["key management"], dependencies=[Depends(user_api_key_auth)]) @management_endpoint_wrapper async def info_key_fn( - key: str | None = fastapi.Query(default=None, description="Key in the request parameters"), + key: str | None = fastapi.Query( + default=None, + description=( + "Key to look up. Pass the key's sha256 hash so the raw key stays out of URLs and access " + "logs. Example key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa'" + ), + ), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ Retrieve information about a key. Parameters: - - key: str | None (query parameter) - The key to look up. Accepts the plaintext key or its hash. - Defaults to the key in the Authorization header. + - key: str | None (query parameter) - The key to look up. Accepts the plaintext key or its hash; + prefer the hash, since a query parameter is recorded verbatim by any HTTP access log in front + of the proxy. Defaults to the key in the Authorization header. Returns: - key: str - The key that was looked up, echoed back as it was passed in @@ -3825,7 +3832,7 @@ async def info_key_fn( Example Curl: ``` - curl -X GET "http://0.0.0.0:4000/key/info?key=sk-test-example-key-123" \ + curl -X GET "http://0.0.0.0:4000/key/info?key=d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa" \ -H "Authorization: Bearer sk-1234" ``` diff --git a/litellm/proxy/management_endpoints/sso/saml_sso.py b/litellm/proxy/management_endpoints/sso/saml_sso.py index 3e67b211f62..466b100ea1f 100644 --- a/litellm/proxy/management_endpoints/sso/saml_sso.py +++ b/litellm/proxy/management_endpoints/sso/saml_sso.py @@ -36,6 +36,7 @@ from pydantic import ValidationError from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache +from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.management_endpoints.types import CustomOpenID, get_litellm_user_role from litellm.proxy.utils import get_custom_url @@ -131,7 +132,7 @@ class SAMLAuthHandler: @staticmethod def _is_https(request: Request) -> bool: - return SAMLAuthHandler._base_url(request).startswith("https") + return IPAddressUtils.is_request_https(request) @staticmethod def _acs_url(request: Request) -> str: diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 606569c5b8b..1feefa5725d 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -89,9 +89,10 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken, get_user_object from litellm.proxy.auth.auth_utils import ( _get_request_ip_address, - _has_user_setup_sso, + has_user_setup_sso, ) from litellm.proxy.auth.handle_jwt import JWTHandler +from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.admin_ui_utils import ( admin_ui_disabled, @@ -1118,7 +1119,7 @@ async def google_login( request=request, ) if sso_redirect is not None: - _persist_return_to_cookie(sso_redirect, return_to) + _persist_return_to_cookie(sso_redirect, return_to, request) return sso_redirect from fastapi.responses import HTMLResponse @@ -1138,7 +1139,7 @@ async def google_login( # helper the SSO branch uses, so /login can resume the connect flow instead of dead-ending at the # dashboard. One implementation → the two sign-in branches cannot diverge (and the login form always # renders, since the helper never raises on a bad return_to). - _persist_return_to_cookie(form_response, return_to) + _persist_return_to_cookie(form_response, return_to, request) return form_response @@ -2617,7 +2618,7 @@ async def get_ui_settings(request: Request): _proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None) _logout_url: Final = os.getenv("PROXY_LOGOUT_URL", None) _api_doc_base_url: Final = os.getenv("LITELLM_UI_API_DOC_BASE_URL", None) - _is_sso_enabled: Final = _has_user_setup_sso() + _is_sso_enabled: Final = has_user_setup_sso() disable_expensive_db_queries: Final = ( proxy_state.get_proxy_state_variable("spend_logs_row_count") > MAX_SPENDLOG_ROWS_TO_QUERY ) @@ -2741,6 +2742,7 @@ async def _sso_return_to_redirect( jwt_token: str, redis_usage_cache, user_api_key_cache, + request: Request, ) -> RedirectResponse | None: """Resolve the post-SSO redirect for a ``return_to``, or None to fall through to the dashboard. @@ -2759,7 +2761,7 @@ async def _sso_return_to_redirect( if _is_same_origin_return_path(return_to): redirect_response = RedirectResponse(url=return_to, status_code=303) - redirect_response.set_cookie(key="token", value=jwt_token) + set_session_token_cookie(redirect_response, request, jwt_token) redirect_response.delete_cookie("litellm_cp_return_to") return redirect_response @@ -2782,7 +2784,25 @@ async def _sso_return_to_redirect( return None -def _persist_return_to_cookie(response: Response, return_to: str | None) -> None: +def set_session_token_cookie(response: Response, request: Request, jwt_token: str) -> None: + """Set the ``token`` session cookie shared by every sign-in path. + + Not HttpOnly: the dashboard reads this cookie via ``document.cookie`` to + populate its own Authorization headers (see + ``ui/litellm-dashboard/src/utils/cookieUtils.ts``), so marking it + HttpOnly would break login. Secure is still required whenever the public + origin is HTTPS, resolved the same trust-aware way as every other + litellm cookie.""" + response.set_cookie( + key="token", + value=jwt_token, + secure=IPAddressUtils.is_request_https(request), + httponly=False, + samesite="lax", + ) + + +def _persist_return_to_cookie(response: Response, return_to: str | None, request: Request) -> None: """Best-effort: persist a SAFE ``return_to`` on ``response`` as the one-shot ``litellm_cp_return_to`` cookie so ANY sign-in path — SSO / Okta / generic OR the username/password form — can resume there afterwards. THIS is the single source of truth, called by every sign-in branch so they cannot @@ -2803,6 +2823,7 @@ def _persist_return_to_cookie(response: Response, return_to: str | None) -> None max_age=600, httponly=True, samesite="lax", + secure=IPAddressUtils.is_request_https(request), ) @@ -3079,8 +3100,11 @@ class SSOAuthenticationHandler: # incoming request is HTTP (local dev). Without # ``Secure`` the cookie is sent over plain HTTP, # letting a network observer read and replay the - # state value and bypass this protection. - secure_flag: Final = request is None or request.url.scheme == "https" + # state value and bypass this protection. Trust-aware: + # honors PROXY_BASE_URL / a trusted reverse proxy's + # X-Forwarded-Proto instead of only the literal scheme + # litellm sees on the wire. + secure_flag: Final = request is None or IPAddressUtils.is_request_https(request) redirect_response.set_cookie( key="litellm_oauth_state", value=state_value, @@ -3628,6 +3652,7 @@ class SSOAuthenticationHandler: jwt_token=jwt_token, redis_usage_cache=redis_usage_cache, user_api_key_cache=user_api_key_cache, + request=request, ) if return_to_redirect is not None: return return_to_redirect @@ -3636,7 +3661,7 @@ class SSOAuthenticationHandler: litellm_dashboard_ui += "?login=success" verbose_proxy_logger.info("Redirecting to %s", litellm_dashboard_ui) redirect_response: Final = RedirectResponse(url=litellm_dashboard_ui, status_code=303) - redirect_response.set_cookie(key="token", value=jwt_token) + set_session_token_cookie(redirect_response, request, jwt_token) return redirect_response @staticmethod diff --git a/litellm/proxy/management_helpers/access_group_key_sync.py b/litellm/proxy/management_helpers/access_group_key_sync.py index 5d43cb29978..c9f93fae0d9 100644 --- a/litellm/proxy/management_helpers/access_group_key_sync.py +++ b/litellm/proxy/management_helpers/access_group_key_sync.py @@ -38,6 +38,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_checks import ( _delete_cache_access_object, # pyright: ignore[reportPrivateUsage] # the access-group endpoints reach for this same cache primitive ) +from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient from litellm.repositories.table_repositories import AccessGroupRepository @@ -72,8 +73,9 @@ _REPOINT_KEY_SQL: Final = ( def _raw_executor(prisma_client: object) -> _RawExecutor: - """Narrow the untyped Prisma client down to the raw-query call this module makes.""" - return AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client + """Narrow the untyped Prisma client down to the raw-query call this module makes, pinned to the writer.""" + db: Final = AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client + return WriterPinnedClient(db).db # pyright: ignore[reportAny, reportReturnType] # untyped Prisma client behind the pin async def _invalidate_access_group_cache(access_group_id: str) -> None: diff --git a/litellm/proxy/openai_files_endpoints/batch_file_validation.py b/litellm/proxy/openai_files_endpoints/batch_file_validation.py index 0aee5e8cc54..a41bd36d510 100644 --- a/litellm/proxy/openai_files_endpoints/batch_file_validation.py +++ b/litellm/proxy/openai_files_endpoints/batch_file_validation.py @@ -2,7 +2,9 @@ import json from collections.abc import Iterator from dataclasses import dataclass from itertools import chain -from typing import BinaryIO, Final, NoReturn, assert_never +from typing import BinaryIO, Final, NoReturn + +from typing_extensions import assert_never from litellm.proxy._types import ProxyException diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 9bc90260de1..bf07f4748ef 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -70,6 +70,15 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( validate_managed_files_requirement, validate_managed_id_requirement, ) +from litellm.proxy.openai_files_endpoints.general_upload_validation import ( + MB, + check_blocked_extension, + check_unsafe_filename, + check_upload_file_size, + coerce_optional_int_setting, + coerce_optional_str_list_setting, + raise_upload_validation_failure, +) from litellm.proxy.utils import ProxyLogging, is_known_model from litellm.repositories.table_repositories import ManagedFileRepository from litellm.router import Router @@ -397,13 +406,23 @@ async def create_file( # descriptor and its disk blocks until the collector runs. spools: Final[list[BinaryIO]] = [] # mutable-ok: filled as the scan opens handles try: + unsafe_filename_failure: Final = check_unsafe_filename(file.filename) + if unsafe_filename_failure is not None: + raise_upload_validation_failure(unsafe_filename_failure) + + max_file_size_mb: Final = coerce_optional_int_setting(general_settings.get("max_file_size_mb")) + # Batch uploads can be gigabytes. Starlette has already spooled the upload # to disk, so stream from that handle instead of reading it into memory. - # Other uploads are small and stay in-memory bytes. + # Other uploads stay in-memory bytes, bounded to max_file_size_mb (plus one + # byte, to still tell "exactly at the limit" from "over it") when it is set, + # so an oversized upload cannot be read to completion before it is rejected. file_source: bytes | BinaryIO if purpose == "batch": await file.seek(0) file_source = file.file + elif max_file_size_mb is not None and max_file_size_mb > 0: + file_source = await file.read(max_file_size_mb * MB + 1) else: file_source = await file.read() custom_llm_provider = ( @@ -442,6 +461,15 @@ async def create_file( # Cast purpose to OpenAIFilesPurpose type purpose = cast(OpenAIFilesPurpose, purpose) + general_size_failure: Final = check_upload_file_size(file_source, max_file_size_mb) + if general_size_failure is not None: + raise_upload_validation_failure(general_size_failure) + + blocked_extensions: Final = coerce_optional_str_list_setting(general_settings.get("blocked_file_extensions")) + blocked_extension_failure: Final = check_blocked_extension(file.filename, blocked_extensions) + if blocked_extension_failure is not None: + raise_upload_validation_failure(blocked_extension_failure) + if purpose == "batch": batch_file_failure: Final = await asyncio.to_thread( check_batch_file_upload, diff --git a/litellm/proxy/openai_files_endpoints/general_upload_validation.py b/litellm/proxy/openai_files_endpoints/general_upload_validation.py new file mode 100644 index 00000000000..8c59a520272 --- /dev/null +++ b/litellm/proxy/openai_files_endpoints/general_upload_validation.py @@ -0,0 +1,152 @@ +""" +Upload validation applied to every purpose at POST /v1/files. + +batch_file_validation.py checks the JSONL shape of purpose="batch" uploads; this +module applies the same fast-fail-before-forwarding shape (size cap, blocked +extensions, path-traversal filenames) regardless of purpose. +""" + +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO, Final, NoReturn + +from typing_extensions import assert_never + +from litellm.proxy._types import ProxyException +from litellm.proxy.common_utils.path_utils import safe_filename + +MB: Final = 1024 * 1024 + + +def coerce_optional_int_setting(raw: object) -> int | None: + """A general_settings value declared as an optional integer, e.g. max_file_size_mb. + + bool is an int subclass, so an explicit isinstance(raw, bool) exclusion is needed + or a YAML `true`/`false` would silently pass as 1/0. + """ + if raw is None: + return None + if isinstance(raw, int) and not isinstance(raw, bool): + return raw + raise TypeError(f"expected an integer, got {raw!r}") + + +def coerce_optional_str_list_setting(raw: object) -> tuple[str, ...]: + """A general_settings value declared as an optional list of strings, e.g. blocked_file_extensions.""" + if raw is None: + return () + if not isinstance(raw, list) or not all(isinstance(item, str) for item in raw): + raise TypeError(f"expected a list of strings, got {raw!r}") + return tuple(raw) + + +@dataclass(frozen=True, slots=True) +class UploadedFileTooLarge: + size_bytes: int + limit_mb: int + + +@dataclass(frozen=True, slots=True) +class UploadedFileBlockedExtension: + extension: str + + +@dataclass(frozen=True, slots=True) +class UploadedFileUnsafeFilename: + filename: str + + +UploadValidationFailure = UploadedFileTooLarge | UploadedFileBlockedExtension | UploadedFileUnsafeFilename + + +def _file_size_bytes(file_source: bytes | BinaryIO) -> int: + if isinstance(file_source, bytes): + return len(file_source) + original_position: Final = file_source.tell() + file_source.seek(0, 2) + size: Final = file_source.tell() + file_source.seek(original_position) + return size + + +def check_upload_file_size( + file_source: bytes | BinaryIO, + max_file_size_mb: int | None, +) -> UploadedFileTooLarge | None: + if max_file_size_mb is None or max_file_size_mb <= 0: + return None + size_bytes: Final = _file_size_bytes(file_source) + if size_bytes > max_file_size_mb * MB: + return UploadedFileTooLarge(size_bytes=size_bytes, limit_mb=max_file_size_mb) + return None + + +def check_blocked_extension( + filename: str | None, + blocked_extensions: tuple[str, ...], +) -> UploadedFileBlockedExtension | None: + if not blocked_extensions or not filename: + return None + try: + extension: Final = Path(safe_filename(filename)).suffix.lower() + except ValueError: + return None + # The uploaded name's extension is normalized above; blocked_extensions comes + # straight from config.yaml or the DB and is normalized here too, so a + # differently-cased entry (".EXE") still catches a lowercase upload. + normalized_blocked: Final = frozenset(item.lower() for item in blocked_extensions) + if extension and extension in normalized_blocked: + return UploadedFileBlockedExtension(extension=extension) + return None + + +def check_unsafe_filename(filename: str | None) -> UploadedFileUnsafeFilename | None: + """Reject a filename before it can influence any storage path or backend call. + + Only flags a genuine traversal component ("..") or a null byte, so an ordinary + name like "report.v2.pdf" or ".env" is never rejected. + """ + if not filename: + return None + if "\x00" in filename: + return UploadedFileUnsafeFilename(filename=filename) + normalized: Final = filename.replace("\\", "/") + if any(part == ".." for part in normalized.split("/")): + return UploadedFileUnsafeFilename(filename=filename) + return None + + +def raise_upload_validation_failure(failure: UploadValidationFailure) -> NoReturn: + match failure: + case UploadedFileTooLarge(size_bytes=size_bytes, limit_mb=limit_mb): + raise ProxyException( + message=( + f"Uploaded file exceeds the configured max_file_size_mb of {limit_mb} MB " + f"(read stopped at {size_bytes / MB:.1f} MB). The file was not forwarded to the provider." + ), + type="invalid_request_error", + param="file", + code=413, + ) + case UploadedFileBlockedExtension(extension=extension): + raise ProxyException( + message=( + f"File extension '{extension}' is blocked by this proxy's blocked_file_extensions " + "setting. The file was not forwarded to the provider." + ), + type="invalid_request_error", + param="file", + code=400, + ) + case UploadedFileUnsafeFilename(filename=filename): + raise ProxyException( + message=( + f"Filename '{filename}' is not allowed: directory traversal sequences are not " + "permitted in uploaded file names. The file was not forwarded to the provider." + ), + type="invalid_request_error", + param="file", + code=400, + ) + case _: + assert_never(failure) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 78d8ce296b8..b48b8d81494 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1731,7 +1731,7 @@ def get_vertex_ai_allowed_incoming_headers(request: Request) -> dict: def get_vertex_pass_through_handler( - call_type: Literal["discovery", "aiplatform"], # noqa: UP037 + call_type: Literal["discovery", "aiplatform"], # noqa: UP037 # ruff reports quoted Literal values here ) -> BaseVertexAIPassThroughHandler: if call_type == "discovery": return VertexAIDiscoveryPassThroughHandler() @@ -2961,7 +2961,6 @@ async def handle_gigachat_passthrough_router_model( """ from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing - # Detect streaming based on request body is_streaming: Final = request_body.get("stream", False) # pyright: ignore[reportUnknownVariableType] # request_body is dict[Unknown, Unknown] data: dict[str, Any] = await _read_request_body( @@ -2997,7 +2996,6 @@ async def handle_gigachat_passthrough_router_model( data["json"] = request_body data["custom_llm_provider"] = "gigachat" - # Remove sensitive keys from data keys: Final = [ # mutable-ok: list of keys to remove from data "gigachat_auth_url", "gigachat_access_token", diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 23932ba7c8c..ed247d52ce2 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -261,6 +261,7 @@ class ProxyInitializationHelpers: "app": "litellm.proxy.proxy_server:app", "host": host, "port": port, + "server_header": False, } if log_config is not None: print(f"Using log_config: {log_config}") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 77a80ea0052..27132c90e05 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -39,7 +39,7 @@ from typing import ( import anyio import websockets import websockets.exceptions -from pydantic import BaseModel, Json, JsonValue, ValidationError +from pydantic import BaseModel, Json, JsonValue, TypeAdapter, ValidationError from typing_extensions import NotRequired, ReadOnly, assert_never from litellm._uuid import uuid @@ -60,6 +60,7 @@ from litellm.constants import ( LITELLM_SETTINGS_SAFE_DB_OVERRIDES, LITELLM_UI_ALLOW_HEADERS, LITELLM_UI_SESSION_DURATION, + RUNTIME_UPDATABLE_ROUTER_SETTINGS, ) from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, @@ -253,6 +254,7 @@ from litellm.constants import ( PROXY_BUDGET_RESCHEDULER_MAX_TIME, PROXY_BUDGET_RESCHEDULER_MIN_TIME, PROXY_CONFIG_RELOAD_INTERVAL_SECONDS, + ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG, USER_SPEND_ALERTS_JOB_ID, WEEKLY_SPEND_REPORT_JOB_ID, ) @@ -308,6 +310,7 @@ from litellm.proxy.auth.model_checks import ( get_mcp_server_ids, get_team_models, ) +from litellm.proxy.auth.password_policy import validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( _fetch_global_spend_with_event_coordination, user_api_key_auth, @@ -352,7 +355,10 @@ from litellm.proxy.common_utils.load_config_utils import ( get_file_contents_from_s3, ) from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations -from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator +from litellm.proxy.common_utils.model_listing_utils import ( + TeamModelNameTranslator, + configured_display_names, +) from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, ) @@ -2275,7 +2281,7 @@ user_api_key_cache: UserApiKeyCache = UserApiKeyCache( ) spend_counter_cache: Final = DualCache(default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value) cli_sso_session_cache: Final = DualCache(default_in_memory_ttl=CLI_SSO_SESSION_TTL_SECONDS) -model_max_budget_limiter: Final = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=user_api_key_cache) +model_max_budget_limiter: Final = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=spend_counter_cache) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) redis_usage_cache: RedisCache | None = None # redis cache used for tracking spend, tpm/rpm limits polling_via_cache_enabled: Literal["all"] | list[str] | bool = False @@ -4738,6 +4744,63 @@ class ProxyConfig: ) return coordination_redis_cache + @staticmethod + async def _init_coordination_redis_env_fallback(litellm_settings: Mapping[str, object]) -> RedisCache | None: + """ + Last-resort coordination Redis, tried after an explicit + `general_settings.coordination_redis` block and `litellm_settings.cache` + have both had a chance to resolve one. Without this, a deployment that + only exports REDIS_HOST/REDIS_PORT (no cache block, no coordination_redis + block) gets NO cross-pod coordination at all: spend counters, budget-window + enforcement, and the reset_spend cache-eviction broadcast all silently stay + per-pod local, so a key reset on one pod never clears another pod's stale + enforcement. + + Unlike the explicit block and cache-backend paths (a deliberate opt-in, so a + bad connection target or a malformed REDIS_CLUSTER_NODES/REDIS_SENTINEL_NODES + value should fail loudly), this one is inferred from bare env vars that may be + set for an unrelated reason -- e.g. a REDIS_HOST left over from a different + job/service, or a REDIS_CLUSTER_NODES value nothing here ever asked to be + parsed. Wrongly guessing "coordination available" must not turn a previously + harmless in-memory-only proxy into one that fails to boot or raises on every + cache write, so a malformed value or a failed/slow ping are both treated the + same as no REDIS_* vars at all. + """ + try: + env_coordination_redis_cache: Final = _build_redis_usage_cache_from_environment() + except Exception as e: # noqa: BLE001 # a malformed inferred Redis env var must not block startup + verbose_proxy_logger.warning( + "coordination_redis: could not build a Redis client from REDIS_* environment variables " + "(%s); cross-pod coordination stays in-memory. Set general_settings.coordination_redis " + "explicitly to require it.", + e, + ) + return None + if env_coordination_redis_cache is None: + return None + try: + reachable: Final = await asyncio.wait_for(env_coordination_redis_cache.ping(), timeout=2.0) + except Exception as e: # noqa: BLE001 # an unreachable inferred Redis must not block startup or writes + verbose_proxy_logger.warning( + "coordination_redis: REDIS_* environment variables named a Redis that is not reachable " + "(%s); cross-pod coordination stays in-memory. Set general_settings.coordination_redis " + "explicitly to require it.", + e, + ) + return None + if not reachable: + return None + _attach_redis_usage_cache( + env_coordination_redis_cache, + enable_redis_auth_cache=litellm_settings.get("enable_redis_auth_cache", False) is True, + ) + verbose_proxy_logger.info( + "coordination_redis: using a standalone Redis built from REDIS_* " + "environment variables for usage tracking, rate limiting, and " + "cross-pod coordination." + ) + return env_coordination_redis_cache + def _init_cache( self, cache_params: dict, @@ -5406,6 +5469,13 @@ class ProxyConfig: reset_audit_log_callback_cache() _in_memory_loggers[:] = [cb for cb in _in_memory_loggers if not isinstance(cb, S3V2Logger)] + if redis_usage_cache is None: + env_coordination_redis_cache: Final = await self._init_coordination_redis_env_fallback( + litellm_settings=litellm_settings + ) + if env_coordination_redis_cache is not None: + _set_redis_usage_cache(env_coordination_redis_cache) + ## GENERAL SERVER SETTINGS (e.g. master key,..) # do this after initializing litellm, to ensure sentry logging works for proxylogging general_settings = config.get("general_settings", {}) if general_settings is None: @@ -5710,13 +5780,9 @@ class ProxyConfig: router_settings: Final = config.get("router_settings", None) if router_settings and isinstance(router_settings, dict): - # model list and search_tools already set - exclude_args: Final = { - "model_list", - "search_tools", - } - - available_args: Final = [x for x in litellm.Router.get_valid_args() if x not in exclude_args] + available_args: Final = [ + x for x in litellm.Router.get_valid_args() if x not in ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG + ] for k, v in router_settings.items(): if k in available_args: @@ -6643,6 +6709,12 @@ class ProxyConfig: if "max_batch_file_size_mb" not in self._yaml_general_settings_keys: general_settings["max_batch_file_size_mb"] = _general_settings.get("max_batch_file_size_mb") + if "max_file_size_mb" not in self._yaml_general_settings_keys: + general_settings["max_file_size_mb"] = _general_settings.get("max_file_size_mb") + + if "blocked_file_extensions" not in self._yaml_general_settings_keys: + general_settings["blocked_file_extensions"] = _general_settings.get("blocked_file_extensions") + ## ALERTING ARGS ## if "alerting_args" in _general_settings: general_settings["alerting_args"] = _general_settings["alerting_args"] @@ -10223,7 +10295,8 @@ async def model_list( # The internal routing key drives the metadata/fallback lookup, while the # public name is what the client sees as the model id. model_data = [] - for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings): + admin_entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings) + for response_id, lookup_id in admin_entries: model_info = create_model_info_response( model_id=lookup_id, provider="openai", @@ -10236,7 +10309,10 @@ async def model_list( if wants_anthropic_format: admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above - return create_anthropic_model_list_response(admin_listing) + return create_anthropic_model_list_response( + admin_listing, + display_names=configured_display_names(admin_entries, llm_router), + ) return dict( data=model_data, @@ -10267,7 +10343,8 @@ async def model_list( # The internal routing key drives the metadata/fallback lookup, while the # public name is what the client sees as the model id. model_data = [] - for response_id, lookup_id in TeamModelNameTranslator.listing_entries(all_models, llm_router, settings): + entries: Final = TeamModelNameTranslator.listing_entries(all_models, llm_router, settings) + for response_id, lookup_id in entries: model_info = create_model_info_response( model_id=lookup_id, provider="openai", @@ -10280,7 +10357,10 @@ async def model_list( if wants_anthropic_format: listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above - return create_anthropic_model_list_response(listing) + return create_anthropic_model_list_response( + listing, + display_names=configured_display_names(entries, llm_router), + ) return dict( data=model_data, @@ -15227,6 +15307,7 @@ async def login(request: Request): password=password, master_key=master_key, prisma_client=prisma_client, + general_settings=general_settings, ) # Create UI token object @@ -15248,7 +15329,10 @@ async def login(request: Request): # authorize round-trip), mirroring the SSO callback; otherwise land on the dashboard. Gated by # _is_same_origin_return_path (strictly relative path) so it can never be an open redirect, and the # one-shot cookie is cleared after use. - from litellm.proxy.management_endpoints.ui_sso import _sso_return_to_redirect + from litellm.proxy.management_endpoints.ui_sso import ( + _sso_return_to_redirect, + set_session_token_cookie, + ) # Resume through the SAME resumer the SSO callback uses, rather than a second, narrower arm. # _persist_return_to_cookie stores both shapes it accepts (a relative same-origin path AND a @@ -15265,6 +15349,7 @@ async def login(request: Request): jwt_token=jwt_token, redis_usage_cache=redis_usage_cache, user_api_key_cache=user_api_key_cache, + request=request, ) except Exception: # noqa: BLE001 # resuming must NEVER block a completed sign-in # The symmetric half of _persist_return_to_cookie's "never raises" contract. The resumer @@ -15279,7 +15364,7 @@ async def login(request: Request): # Create redirect response with cookie redirect_response: Final = RedirectResponse(url=litellm_dashboard_ui, status_code=303) - redirect_response.set_cookie(key="token", value=jwt_token) + set_session_token_cookie(redirect_response, request, jwt_token) if cp_return_to: redirect_response.delete_cookie(key="litellm_cp_return_to") return redirect_response @@ -15289,6 +15374,7 @@ async def login(request: Request): async def login_v2(request: Request): global premium_user, general_settings, master_key from litellm.proxy.auth.login_utils import authenticate_user, create_ui_token_object, encode_ui_session_jwt + from litellm.proxy.management_endpoints.ui_sso import set_session_token_cookie from litellm.proxy.utils import get_custom_url try: @@ -15301,6 +15387,7 @@ async def login_v2(request: Request): password=password, master_key=master_key, prisma_client=prisma_client, + general_settings=general_settings, ) returned_ui_token_object: Final = create_ui_token_object( @@ -15322,7 +15409,7 @@ async def login_v2(request: Request): content={"redirect_url": litellm_dashboard_ui, "token": jwt_token}, status_code=status.HTTP_200_OK, ) - json_response.set_cookie(key="token", value=jwt_token) + set_session_token_cookie(json_response, request, jwt_token) return json_response except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.login_v2(): Exception occurred - %s", e) @@ -15371,6 +15458,7 @@ async def login_v3(request: Request): password=password, master_key=master_key, prisma_client=prisma_client, + general_settings=general_settings, ) returned_ui_token_object: Final = create_ui_token_object( @@ -15421,6 +15509,8 @@ async def login_v3(request: Request): @router.post("/v3/login/exchange", include_in_schema=False) # exchange single-use opaque code for JWT async def login_v3_exchange(request: Request): + from litellm.proxy.management_endpoints.ui_sso import set_session_token_cookie + try: if not general_settings.get("control_plane_url"): raise ProxyException( @@ -15467,7 +15557,7 @@ async def login_v3_exchange(request: Request): }, status_code=status.HTTP_200_OK, ) - json_response.set_cookie(key="token", value=cached_data["token"]) + set_session_token_cookie(json_response, request, cached_data["token"]) return json_response except ProxyException: raise @@ -15740,6 +15830,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request): detail={"error": "Invalid onboarding session for invitation link."}, ) + validate_password_policy(data.password, general_settings) hashed_pw: Final = hash_password(data.password) current_time = litellm.utils.get_utc_datetime() async with prisma_client.db.tx() as tx: @@ -16207,6 +16298,7 @@ async def invitation_delete( ) async def update_config( config_info: ConfigYAML, + request: Request, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -16222,6 +16314,26 @@ async def update_config( if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException(status_code=403, detail="Only proxy admins can update config") + request_body: Final[Mapping[str, JsonValue]] = TypeAdapter(Mapping[str, JsonValue]).validate_python( + await request.json() + ) + raw_router_settings: Final = request_body.get("router_settings") + if isinstance(raw_router_settings, dict): + supported_router_settings: Final = RUNTIME_UPDATABLE_ROUTER_SETTINGS | ( + frozenset(litellm.Router.get_valid_args()) - ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG + ) + unsupported_router_settings: Final = sorted(set(raw_router_settings) - supported_router_settings) + if unsupported_router_settings: + raise HTTPException( + status_code=400, + detail={ + "error": ( + f"Unsupported router settings: {', '.join(unsupported_router_settings)} " + "are not valid router settings" + ) + }, + ) + if prisma_client is None: raise Exception("No DB Connected") @@ -16323,11 +16435,19 @@ async def update_config( ) # router_settings: merge existing + request, request wins. - if config_info.router_settings is not None: + if isinstance(raw_router_settings, dict): existing = await _read_section("router_settings") before_router_settings: Final = copy.deepcopy(existing) - updates = config_info.router_settings.dict(exclude_none=True) - new_router_settings: Final = {**existing, **updates} + typed_router_settings: Final = ( + config_info.router_settings.dict(exclude_none=True) if config_info.router_settings is not None else {} + ) + raw_router_settings_without_none: Final = { + key: value + for key, value in raw_router_settings.items() + if key not in typed_router_settings and value is not None + } + router_settings_updates: Final = {**typed_router_settings, **raw_router_settings_without_none} + new_router_settings: Final = {**existing, **router_settings_updates} await _upsert_section("router_settings", new_router_settings) asyncio.create_task( create_config_audit_log( @@ -16374,6 +16494,8 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro "global_max_parallel_requests": "Integer", "max_request_size_mb": "Integer", "max_batch_file_size_mb": "Integer", + "max_file_size_mb": "Integer", + "blocked_file_extensions": "List", "max_response_size_mb": "Integer", "proxy_config_reload_interval_seconds": "Integer", "pass_through_endpoints": "PydanticModel", diff --git a/litellm/proxy/public_endpoints/agent_create_fields.json b/litellm/proxy/public_endpoints/agent_create_fields.json index 36484cc1065..cc2fc17d759 100644 --- a/litellm/proxy/public_endpoints/agent_create_fields.json +++ b/litellm/proxy/public_endpoints/agent_create_fields.json @@ -107,7 +107,9 @@ "required": true, "field_type": "text", "default_value": null, - "include_in_litellm_params": false + "include_in_litellm_params": false, + "validation_pattern": "^arn:aws[a-zA-Z0-9-]*:bedrock-agentcore:[a-z0-9-]+:[0-9]{12}:runtime/.+$", + "validation_message": "Enter the complete Bedrock AgentCore runtime ARN, including the runtime ID after \"runtime/\" (e.g. arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime)." } ], "litellm_params_template": { diff --git a/ui/litellm-dashboard/src/autorouter_presets.json b/litellm/proxy/public_endpoints/autorouter_presets.json similarity index 83% rename from ui/litellm-dashboard/src/autorouter_presets.json rename to litellm/proxy/public_endpoints/autorouter_presets.json index b977bd484ac..c2b13b81542 100644 --- a/ui/litellm-dashboard/src/autorouter_presets.json +++ b/litellm/proxy/public_endpoints/autorouter_presets.json @@ -17,6 +17,7 @@ "classification_mode": "every_request", "session_affinity": false, "modality_routing": false, + "modality_pin_override": false, "deployment_affinity": true } }, @@ -35,6 +36,7 @@ "classification_mode": "every_request", "session_affinity": false, "modality_routing": false, + "modality_pin_override": false, "deployment_affinity": true } }, @@ -63,24 +65,29 @@ "classification_mode": "every_request", "session_affinity": false, "modality_routing": false, + "modality_pin_override": false, "deployment_affinity": true } }, "openai_family": { "label": "OpenAI Family", - "description": "Routes across the GPT model family: gpt-5.4-nano for simple queries, gpt-5.4-mini for medium, gpt-5.4 for complex, o3 for reasoning-heavy requests.", + "description": "Routes across the GPT model family: Luna for simple queries, Terra for medium, Sol for complex, Sol at xhigh thinking for reasoning.", "complexity_router_config": { "tiers": { - "SIMPLE": ["gpt-5.4-nano"], - "MEDIUM": ["gpt-5.4-mini"], - "COMPLEX": ["gpt-5.4"], - "REASONING": ["o3"] + "SIMPLE": ["gpt-5.6-luna"], + "MEDIUM": ["gpt-5.6-terra"], + "COMPLEX": ["gpt-5.6-sol"], + "REASONING": ["gpt-5.6-sol"] + }, + "tier_model_configs": { + "REASONING": [{ "model_name": "gpt-5.6-sol", "litellm_params": { "reasoning_effort": "xhigh" } }] }, "classifier_type": "heuristic", "escalation_keywords": ["LITELLM ESCALATE"], "classification_mode": "every_request", "session_affinity": false, "modality_routing": false, + "modality_pin_override": false, "deployment_affinity": true } } diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 4d58a974bb8..94a59828451 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -1,11 +1,13 @@ +import asyncio import json import os import re -from collections.abc import Awaitable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from importlib.resources import files from typing import TYPE_CHECKING, Final, Protocol from fastapi import APIRouter, HTTPException, Request +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict import litellm @@ -28,6 +30,7 @@ from litellm.types.proxy.management_endpoints.model_management_endpoints import ) from litellm.types.proxy.public_endpoints.public_endpoints import ( AgentCreateInfo, + AutoRouterPresetRecord, ComplexityScorerDefaults, ProviderCreateInfo, PublicModelHubInfo, @@ -464,6 +467,86 @@ async def get_litellm_blog_posts(): return BlogPostsResponse(posts=posts) +_AUTOROUTER_PRESETS_ADAPTER: Final = TypeAdapter(dict[str, AutoRouterPresetRecord]) + + +def _load_bundled_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord]: + raw: Final = json.loads( + files("litellm.proxy.public_endpoints").joinpath("autorouter_presets.json").read_text(encoding="utf-8") + ) + return _AUTOROUTER_PRESETS_ADAPTER.validate_python(raw) + + +async def _fetch_remote_autorouter_presets(url: str) -> Mapping[str, AutoRouterPresetRecord]: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.UI) + response: Final = await client.get(url, timeout=5.0) + response.raise_for_status() + presets: Final = _AUTOROUTER_PRESETS_ADAPTER.validate_python(response.json()) + if not presets: + raise ValueError("remote auto-router preset catalog is empty") + return presets + + +async def _resolve_autorouter_presets( + url: str, + fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]], +) -> Mapping[str, AutoRouterPresetRecord]: + if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true": + return _load_bundled_autorouter_presets() + try: + return await fetch(url) + except Exception as e: + verbose_logger.warning( + "LiteLLM: failed to fetch auto-router presets from %s: %s. Serving the bundled catalog for the life of this process.", + url, + str(e), + ) + return _load_bundled_autorouter_presets() + + +class _AutoRouterPresetsCache: + presets: Mapping[str, AutoRouterPresetRecord] | None = None + lock: asyncio.Lock | None = None + + +async def get_autorouter_presets( + url: str, + fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]] = _fetch_remote_autorouter_presets, +) -> Mapping[str, AutoRouterPresetRecord]: + cached: Final = _AutoRouterPresetsCache.presets + if cached is not None: + return cached + if _AutoRouterPresetsCache.lock is None: + _AutoRouterPresetsCache.lock = asyncio.Lock() + async with _AutoRouterPresetsCache.lock: + held: Final = _AutoRouterPresetsCache.presets + if held is not None: + return held + resolved: Final = await _resolve_autorouter_presets(url=url, fetch=fetch) + _AutoRouterPresetsCache.presets = resolved + return resolved + + +@router.get( + "/public/autorouter_presets", + tags=["public", "auto router"], # mutable-ok: FastAPI route tags take a list + response_model=dict[str, AutoRouterPresetRecord], +) +async def get_public_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord]: + """ + Return the auto-router preset catalog the dashboard's template picker renders. + + Resolved once per process, like the model cost map: fetched from ``litellm.autorouter_presets_url`` + (override with ``LITELLM_AUTOROUTER_PRESETS_URL``) on the first request, falling back to the + catalog bundled with the package on any failure. Set ``LITELLM_LOCAL_AUTOROUTER_PRESETS=True`` + to serve the bundled catalog only. A restart picks up a newly published catalog. + """ + return await get_autorouter_presets(url=litellm.autorouter_presets_url) + + @router.get( "/public/endpoints", tags=["public"], diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index db574f859b3..e144ff965ae 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -9,6 +9,7 @@ Provides: import base64 import json from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import orjson @@ -19,6 +20,9 @@ from starlette.datastructures import UploadFile import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + LiteLLM_ManagedVectorStore, +) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.proxy._types import * from litellm.proxy.auth.auth_utils import is_request_body_safe @@ -36,6 +40,10 @@ from litellm.proxy.rag_endpoints.upload_security import ( RejectedUpload, validate_upload, ) +from litellm.proxy.vector_store_endpoints.endpoints import ( + build_request_data_from_managed_vector_store, + reject_caller_embedding_selection_params, +) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, ) @@ -120,12 +128,21 @@ def _collect_vector_store_ids_from_payload(payload: object) -> set[str]: async def _authorize_nested_vector_store_ids( payload: object, user_api_key_dict: UserAPIKeyAuth, -) -> None: - for vector_store_id in sorted(_collect_vector_store_ids_from_payload(payload)): - await assert_user_can_access_vector_store_id( - vector_store_id=vector_store_id, - user_api_key_dict=user_api_key_dict, - ) +) -> Mapping[str, LiteLLM_ManagedVectorStore]: + """Authorize every nested vector store id and return the managed stores it resolved.""" + return MappingProxyType( + { + vector_store_id: store + for vector_store_id in sorted(_collect_vector_store_ids_from_payload(payload)) + if ( + store := await assert_user_can_access_vector_store_id( + vector_store_id=vector_store_id, + user_api_key_dict=user_api_key_dict, + ) + ) + is not None + } + ) def _build_file_metadata_entry( @@ -700,11 +717,27 @@ async def rag_query( status_code=400, detail={"error": "retrieval_config must contain 'vector_store_id'"}, ) - await _authorize_nested_vector_store_ids( + reject_caller_embedding_selection_params(payload=retrieval_config, source="retrieval_config") + resolved_stores: Final = await _authorize_nested_vector_store_ids( payload=retrieval_config, user_api_key_dict=user_api_key_dict, ) + # Merge litellm-managed vector store params (provider, region, embedding + # model, credentials, ...) from the registry: the same source the direct + # /vector_stores/{id}/search endpoint uses. Store-managed keys win on + # conflict so callers cannot override the store's provider or credentials. + managed_store: Final = resolved_stores.get(retrieval_config["vector_store_id"]) + store_data: Final = ( + build_request_data_from_managed_vector_store(managed_store) + if managed_store is not None + else MappingProxyType({}) + ) + merged_retrieval_config: Final = { + **retrieval_config, + **store_data, + } # mutable-ok: litellm.aquery requires a plain dict payload + # Add litellm data request_data: dict[str, object] = {} request_data = await add_litellm_data_to_request( @@ -716,13 +749,18 @@ async def rag_query( proxy_config=proxy_config, ) - verbose_proxy_logger.debug("RAG Query - model: %s, retrieval_config: %s", model, retrieval_config) + verbose_proxy_logger.debug( + "RAG Query - model: %s, vector_store_id: %s, custom_llm_provider: %s", + model, + retrieval_config["vector_store_id"], + merged_retrieval_config.get("custom_llm_provider"), + ) # Call query response: Final = await litellm.aquery( model=model, messages=messages, - retrieval_config=retrieval_config, + retrieval_config=merged_retrieval_config, rerank=rerank, stream=stream, router=llm_router, diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 41c65b1d5c5..bb7dfafb297 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -55,6 +55,10 @@ router: Final = APIRouter() SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000 +_SESSION_GROUP_KEY_SQL: Final = "COALESCE(NULLIF(session_id, ''), request_id), api_key" +_MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" +_AGENT_CALL_TYPE_SQL: Final = "'asend_message'" + _INTERNAL_HEALTH_CHECK_API_KEYS: Final = ( LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, hash_token(token=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME), @@ -144,21 +148,16 @@ class _DailyTagSpendRow(TypedDict): total_spend: float -class _SessionCountAggregate(TypedDict): - session_id: int - - -class _SessionCountRow(TypedDict): - session_id: str - _count: _SessionCountAggregate - - class _SessionSpendRow(TypedDict): session_id: str + api_key: ReadOnly[str] + session_total_count: ReadOnly[int] session_total_spend: float mcp_tool_call_count: int mcp_tool_call_spend: float session_cache_hit_count: ReadOnly[int] + session_llm_count: ReadOnly[int] + session_agent_count: ReadOnly[int] class _SpendSumAggregate(TypedDict, total=False): @@ -242,18 +241,6 @@ async def _count_spend_logs(prisma_client: PrismaClient, where: Mapping[str, obj return await _spend_logs_table(prisma_client).count(where=where) -async def _count_logs_per_session( - prisma_client: PrismaClient, session_ids: Sequence[str | None] -) -> Sequence[_SessionCountRow]: - """Count spend log rows per session for the given session ids.""" - rows: Final = await _spend_logs_table(prisma_client).group_by( - by=["session_id"], - where={"session_id": {"in": session_ids}}, - count={"session_id": True}, - ) - return cast(Sequence[_SessionCountRow], rows) # cast-ok: group_by(count=) shape is fixed by the by/count args - - async def _find_team_row(prisma_client: PrismaClient, team_id: str) -> _SupportsModelDump | None: """Read a single team row as a Prisma model instance.""" return await _team_table(prisma_client).find_unique(where={"team_id": team_id}) @@ -1204,7 +1191,10 @@ async def get_global_spend_report( ), api_key: str | None = fastapi.Query( default=None, - description="View spend for a specific api_key. Example api_key='sk-1234", + description=( + "View spend for a specific api_key. Pass the key's sha256 hash so the raw key stays " + "out of URLs and access logs. Example api_key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa'" + ), ), internal_user_id: str | None = fastapi.Query( default=None, @@ -1685,7 +1675,11 @@ async def get_key_spend_report( api_key: Annotated[ str | None, fastapi.Query( - description="View spend for a specific api_key. Proxy admin only; other callers are scoped to their own key." + description=( + "View spend for a specific api_key. Proxy admin only; other callers are scoped to their " + "own key. Pass the key's sha256 hash so the raw key stays out of URLs and access logs. " + "Example api_key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa'" + ) ), ] = None, ) -> Sequence[Mapping[str, object]]: @@ -2283,6 +2277,10 @@ async def ui_view_spend_logs( default=False, description="Exclude LiteLLM internal health check requests from results", ), + group_by_session: bool = fastapi.Query( + default=False, + description="Paginate over sessions instead of raw logs: one representative row per session, total counts sessions", + ), ): """ View spend logs with pagination support. @@ -2637,12 +2635,16 @@ async def ui_view_spend_logs( else: _order_expr = order_column + joined_conditions: Final = " AND ".join(sql_conditions) + session_grouping: Final = group_by_session is True + count_group_clause: Final = f"GROUP BY {_SESSION_GROUP_KEY_SQL}" if session_grouping else "" count_query: Final = f""" SELECT COUNT(*) AS total_count FROM ( SELECT 1 FROM "LiteLLM_SpendLogs" - WHERE {" AND ".join(sql_conditions)} + WHERE {joined_conditions} + {count_group_clause} LIMIT ${p} ) AS bounded_matches """ @@ -2653,21 +2655,36 @@ async def ui_view_spend_logs( total_is_capped: Final = raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP total_records: Final = SPEND_LOGS_PAGINATION_COUNT_CAP if total_is_capped else raw_total - sql_query: Final = f""" - SELECT - request_id, call_type, api_key, spend, total_tokens, + select_columns: Final = """request_id, call_type, api_key, spend, total_tokens, prompt_tokens, completion_tokens, "startTime", "endTime", "completionStartTime", model, model_id, model_group, custom_llm_provider, api_base, "user", metadata, cache_hit, cache_key, request_tags, team_id, organization_id, end_user, requester_ip_address, session_id, status, mcp_namespaced_tool_name, agent_id, - COALESCE(request_duration_ms, (EXTRACT(EPOCH FROM ("endTime" - "startTime")) * 1000)::INTEGER) AS request_duration_ms + COALESCE(request_duration_ms, (EXTRACT(EPOCH FROM ("endTime" - "startTime")) * 1000)::INTEGER) AS request_duration_ms""" + sql_query: Final = ( + f""" + SELECT * FROM ( + SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) + {select_columns} + FROM "LiteLLM_SpendLogs" + WHERE {joined_conditions} + ORDER BY {_SESSION_GROUP_KEY_SQL}, call_type IN {_MCP_CALL_TYPES_SQL}, "startTime" DESC + ) AS session_representatives + ORDER BY {_order_expr} {_sql_dir}{_nulls_clause}, request_id + LIMIT ${p} OFFSET ${p + 1} + """ + if session_grouping + else f""" + SELECT + {select_columns} FROM "LiteLLM_SpendLogs" - WHERE {" AND ".join(sql_conditions)} + WHERE {joined_conditions} ORDER BY {_order_expr} {_sql_dir}{_nulls_clause} LIMIT ${p} OFFSET ${p + 1} """ + ) sql_params.extend([page_size, skip]) data: Final = await prisma_client.db.query_raw(sql_query, *sql_params) @@ -2945,7 +2962,7 @@ async def view_spend_logs( Example Request for specific api_key ``` - curl -X GET "http://0.0.0.0:8000/spend/logs?api_key=sk-test-example-key-123" \ + curl -X GET "http://0.0.0.0:8000/spend/logs?api_key=d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa" \ -H "Authorization: Bearer sk-1234" ``` @@ -4068,11 +4085,12 @@ async def _build_ui_spend_logs_response( Build the paginated response for the UI spend-logs endpoint. When ``enrich_session_counts`` is ``True`` (the default for the v1/UI - endpoint), each row is enriched with ``session_total_count`` so the - frontend knows which sessions are expandable (multi-call sessions). - For every row that carries a ``session_id``, a single ``GROUP BY`` query - fetches the total number of logs in each referenced session. Rows without - a ``session_id`` default to ``1``. + endpoint), each row is enriched with ``session_total_count`` plus spend + and call-type aggregates so the frontend knows which sessions are + expandable (multi-call sessions). One ``GROUP BY (session_id, api_key)`` + query serves every referenced session, keyed per api key so two callers + reusing a session id never see each other's totals. Rows without a + ``session_id`` default to ``1``. When ``enrich_session_counts`` is ``False`` (v2 endpoint), rows are serialised without the extra query. @@ -4094,7 +4112,6 @@ async def _build_ui_spend_logs_response( A dict with ``data`` (enriched rows), ``total``, ``page``, ``page_size``, ``total_pages``, and ``total_is_capped``. """ - count_map: dict[str, int] = {} if enrich_session_counts: session_ids: Final[Sequence[str | None]] = list( { @@ -4103,15 +4120,8 @@ async def _build_ui_spend_logs_response( if (row.get("session_id") if isinstance(row, dict) else getattr(row, "session_id", None)) } ) - if session_ids: - # NOTE: This GROUP BY runs on every v1/UI page load. The IN clause - # is bounded by page_size (typically 25-50 distinct session IDs). - # If performance degrades at scale, consider short-lived caching or - # folding the count into the main query via a window function. - counts: Final = await _count_logs_per_session(prisma_client, session_ids) - count_map = {r["session_id"]: r["_count"]["session_id"] for r in counts if r.get("session_id")} - session_spend_map: dict[str, dict[str, int | float]] = {} + session_spend_map: dict[tuple[str, str], dict[str, int | float]] = {} if enrich_session_counts and session_ids: from prisma.errors import PrismaError @@ -4123,38 +4133,46 @@ async def _build_ui_spend_logs_response( { (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) for row in data - if (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) + if (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) is not None } ) rows: Final[Sequence[_SessionSpendRow]] = await _query_raw( prisma_client, - """ - SELECT session_id, + f""" + SELECT session_id, api_key, + COUNT(*)::int AS session_total_count, COALESCE(SUM(spend), 0)::double precision AS session_total_spend, COUNT(*) FILTER ( - WHERE call_type IN ('call_mcp_tool', 'list_mcp_tools') + WHERE call_type IN {_MCP_CALL_TYPES_SQL} )::int AS mcp_tool_call_count, COALESCE(SUM(spend) FILTER ( - WHERE call_type IN ('call_mcp_tool', 'list_mcp_tools') + WHERE call_type IN {_MCP_CALL_TYPES_SQL} ), 0)::double precision AS mcp_tool_call_spend, - COUNT(*) FILTER (WHERE LOWER(cache_hit) = 'true')::int AS session_cache_hit_count + COUNT(*) FILTER (WHERE LOWER(cache_hit) = 'true')::int AS session_cache_hit_count, + COUNT(*) FILTER ( + WHERE call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} + )::int AS session_llm_count, + COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count FROM "LiteLLM_SpendLogs" WHERE session_id = ANY($1::text[]) AND api_key = ANY($2::text[]) - GROUP BY session_id + GROUP BY session_id, api_key """, session_ids, authorized_api_keys, ) session_spend_map = { - row["session_id"]: { + (row["session_id"], row["api_key"]): { + "session_total_count": int(row.get("session_total_count") or 0), "session_total_spend": float(row.get("session_total_spend") or 0.0), "mcp_tool_call_count": int(row.get("mcp_tool_call_count") or 0), "mcp_tool_call_spend": float(row.get("mcp_tool_call_spend") or 0.0), "session_cache_hit_count": int(row.get("session_cache_hit_count") or 0), + "session_llm_count": int(row.get("session_llm_count") or 0), + "session_agent_count": int(row.get("session_agent_count") or 0), } for row in rows - if row.get("session_id") + if row.get("session_id") and row.get("api_key") is not None } except PrismaError: verbose_proxy_logger.debug( @@ -4167,14 +4185,17 @@ async def _build_ui_spend_logs_response( for row in data: row_dict = dict(row) if isinstance(row, dict) else row.model_dump() sid = row_dict.get("session_id") - row_dict["session_total_count"] = count_map.get(sid, 1) if sid else 1 - session_stats = session_spend_map.get(sid) if sid else None + row_api_key = row_dict.get("api_key") + session_stats = session_spend_map.get((sid, row_api_key)) if sid and row_api_key is not None else None + row_dict["session_total_count"] = int(session_stats["session_total_count"]) if session_stats else 1 if session_stats: row_dict["session_total_spend"] = session_stats["session_total_spend"] if session_stats["mcp_tool_call_count"]: row_dict["mcp_tool_call_count"] = session_stats["mcp_tool_call_count"] row_dict["mcp_tool_call_spend"] = session_stats["mcp_tool_call_spend"] row_dict["session_cache_hit_count"] = session_stats["session_cache_hit_count"] + row_dict["session_llm_count"] = session_stats["session_llm_count"] + row_dict["session_agent_count"] = session_stats["session_agent_count"] enriched.append(row_dict) response_data: list = enriched else: diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 52258602581..c12d071dd36 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -21,6 +21,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_keys +from litellm.proxy._experimental.mcp_server.tool_search import MCP_TOOL_SEARCH_SETTINGS_KEY from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.config_resolvers.sso import ( @@ -38,6 +39,7 @@ from litellm.repositories.table_repositories import ( UISettingsRepository, ) from litellm.repositories.team_repository import TeamRepository +from litellm.types.mcp import MCPToolSearchSettings from litellm.types.proxy.management_endpoints.ui_sso import ( DefaultTeamSSOParams, SSOConfig, @@ -448,6 +450,10 @@ class MCPSemanticFilterSettingsResponse(SettingsResponse): """Response model for MCP semantic filter settings""" +class MCPToolSearchSettingsResponse(SettingsResponse): + """Response model for native MCP tool search settings""" + + @router.get( "/get/allowed_ips", tags=["Budget & Spend Tracking"], @@ -835,7 +841,7 @@ async def update_default_team_member_budget(teams: list[NewUserRequestTeam], use async def _update_litellm_setting( - settings: DefaultInternalUserParams | DefaultTeamSSOParams | MCPSemanticFilterSettings, + settings: DefaultInternalUserParams | DefaultTeamSSOParams | MCPSemanticFilterSettings | MCPToolSearchSettings, settings_key: str, success_message: str, user_api_key_dict: UserAPIKeyAuth, @@ -861,7 +867,7 @@ async def _update_litellm_setting( detail={"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."}, ) - in_memory_var: Final = settings.model_dump(exclude_none=True) + in_memory_var: Final = settings.model_dump(mode="json", exclude_none=True) # Load existing config first, then set in-memory value after, # because get_config() may overwrite litellm. with stale DB values @@ -1359,6 +1365,59 @@ async def update_mcp_semantic_filter_settings( return result +@router.get( + "/get/mcp_tool_search_settings", + tags=["Settings"], # mutable-ok: FastAPI's route decorator only accepts a list + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI's route decorator only accepts a list + response_model=MCPToolSearchSettingsResponse, +) +async def get_mcp_tool_search_settings( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> Mapping[str, object]: + """ + Get the `litellm_settings.mcp_tool_search` configuration used by the native `mcp_tool_search` virtual tool. + """ + from litellm.proxy.proxy_server import prisma_client, proxy_config + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected. Please connect a database.") + + config: Final = await proxy_config.get_config() + + return await _get_settings_with_schema( + settings_key=MCP_TOOL_SEARCH_SETTINGS_KEY, + settings_class=MCPToolSearchSettings, + config=config, + ) + + +@router.patch( + "/update/mcp_tool_search_settings", + tags=["Settings"], # mutable-ok: FastAPI's route decorator only accepts a list + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI's route decorator only accepts a list +) +async def update_mcp_tool_search_settings( + settings: MCPToolSearchSettings, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> Mapping[str, object]: + """ + Update `litellm_settings.mcp_tool_search` in the database. + Settings will be picked up by all pods within approximately 10 seconds via background polling. + """ + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail="Only proxy admins can update MCP tool search settings.", + ) + + return await _update_litellm_setting( + settings=settings, + settings_key=MCP_TOOL_SEARCH_SETTINGS_KEY, + success_message="MCP tool search settings updated successfully. Changes will be applied across all pods within 10 seconds.", + user_api_key_dict=user_api_key_dict, + ) + + UI_SETTINGS_CACHE_KEY: Final = "ui_settings:settings_dict" UI_SETTINGS_CACHE_TTL: Final = 600 # 10 minutes @@ -1594,7 +1653,10 @@ async def update_ui_settings( tags=["UI Theme Settings"], dependencies=[Depends(user_api_key_auth)], ) -async def upload_logo(file: UploadFile = File(...)): +async def upload_logo( + file: UploadFile = File(...), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ Upload a custom logo for the admin UI. Accepts image files (PNG, JPG, JPEG, SVG) and stores them for use in the UI. @@ -1602,6 +1664,12 @@ async def upload_logo(file: UploadFile = File(...)): import os from pathlib import Path + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail="Only proxy admins can upload a UI logo.", + ) + # Validate file type allowed_extensions: Final = {".png", ".jpg", ".jpeg", ".svg"} file_extension: Final = Path(file.filename or "").suffix.lower() @@ -1612,9 +1680,11 @@ async def upload_logo(file: UploadFile = File(...)): detail=f"Invalid file type. Allowed types: {', '.join(allowed_extensions)}", ) - # Validate file size (max 5MB) - file_content: Final = await file.read() - if len(file_content) > 5 * 1024 * 1024: # 5MB + # Read bounded to one byte past the limit, so an oversized upload is never + # fully buffered in memory before being rejected. + max_logo_size_bytes: Final = 5 * 1024 * 1024 + file_content: Final = await file.read(max_logo_size_bytes + 1) + if len(file_content) > max_logo_size_bytes: raise HTTPException(status_code=400, detail="File size too large. Maximum size is 5MB.") # Create uploads directory if it doesn't exist diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 051d36c4d0f..cab2bd6d9db 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1524,6 +1524,7 @@ class ProxyLogging: prompt_label=data.pop("prompt_label", None) or {}, prompt_version=data.pop("prompt_version", None) or {}, request_kwargs=data, + injected_for_every_deployment=True, ) data.update(optional_params) diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index a59d7a277cc..1feda0b0bb5 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import ( Annotated, Any, # noqa: TID251 # jsonify_object in proxy/utils.py is annotated with a bare dict @@ -14,9 +16,6 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.utils import jsonify_object -from litellm.proxy.vector_store_endpoints.management_endpoints import ( - _resolve_embedding_config, -) from litellm.proxy.vector_store_endpoints.utils import ( assert_proxy_admin_for_vector_store_index_management, assert_user_can_access_vector_store, @@ -27,11 +26,48 @@ from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry router: Final = APIRouter() + +BLOCKED_QUERY_EMBEDDING_SELECTION_PARAMS: Final = frozenset( + { + "embedding_model", + "litellm_embedding_model", + "litellm_embedding_config", + "litellm_credential_name", + } +) + + +def reject_caller_embedding_selection_params(payload: Mapping[str, object], source: str) -> None: + blocked: Final = sorted(BLOCKED_QUERY_EMBEDDING_SELECTION_PARAMS & payload.keys()) + if blocked: + raise HTTPException( + status_code=400, + detail={ + "error": f"'{blocked[0]}' cannot be set in {source}. " + "Embedding configuration comes from the vector store's server-side registration." + }, + ) + + ######################################################## # OpenAI Compatible Endpoints ######################################################## +def build_request_data_from_managed_vector_store( + vector_store: LiteLLM_ManagedVectorStore, +) -> Mapping[str, object]: + top_level: Final = MappingProxyType( + { + key: vector_store.get(key) + for key in ("custom_llm_provider", "litellm_credential_name") + if key in vector_store + } + ) + litellm_params: Final = vector_store.get("litellm_params") or MappingProxyType({}) + return MappingProxyType({**top_level, **litellm_params}) + + async def _update_request_data_with_litellm_managed_vector_store_registry( data: dict, vector_store_id: str, @@ -51,47 +87,14 @@ async def _update_request_data_with_litellm_managed_vector_store_registry( vector_store_to_run: Final[LiteLLM_ManagedVectorStore | None] = await get_litellm_managed_vector_store( vector_store_id=vector_store_id ) - if vector_store_to_run is not None: - if user_api_key_dict is not None: - await assert_user_can_access_vector_store( - vector_store=vector_store_to_run, - user_api_key_dict=user_api_key_dict, - ) - - if "custom_llm_provider" in vector_store_to_run: - data["custom_llm_provider"] = vector_store_to_run.get("custom_llm_provider") - - if "litellm_credential_name" in vector_store_to_run: - data["litellm_credential_name"] = vector_store_to_run.get("litellm_credential_name") - - if "litellm_params" in vector_store_to_run: - litellm_params = vector_store_to_run.get("litellm_params", {}) or {} - # Resolve ``litellm_embedding_config`` here, at request-handling - # time, instead of at row-creation time. The resolved - # ``api_key`` / ``api_base`` / ``api_version`` lives only in - # this per-request ``data`` dict and is never persisted. - # Legacy rows that already carry a resolved (cleartext) - # ``litellm_embedding_config`` skip the lookup and pass through - # unchanged so the embed call keeps working. - embedding_model: Final = litellm_params.get("litellm_embedding_model") - if embedding_model and not litellm_params.get("litellm_embedding_config"): - from litellm.proxy.proxy_server import prisma_client - - resolved_config: Final = await _resolve_embedding_config( - embedding_model=embedding_model, prisma_client=prisma_client - ) - if resolved_config: - # Build a fresh dict via spread instead of mutating - # ``litellm_params`` in place — the registry hands back - # a reference to its cached object, so an in-place - # update would persist the resolved cleartext into the - # in-memory cache for the lifetime of the process. - litellm_params = { - **litellm_params, - "litellm_embedding_config": resolved_config, - } - data.update(litellm_params) - return data + if vector_store_to_run is None: + return data + if user_api_key_dict is not None: + await assert_user_can_access_vector_store( + vector_store=vector_store_to_run, + user_api_key_dict=user_api_key_dict, + ) + return {**data, **build_request_data_from_managed_vector_store(vector_store_to_run)} @router.post( @@ -130,6 +133,7 @@ async def vector_store_search( ) data = await _read_request_body(request=request) + reject_caller_embedding_selection_params(payload=data, source="the search request body") data["vector_store_id"] = vector_store_id # Check for legacy vector store registry (non-managed vector stores) diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 183a03cc13c..c928398a87f 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -18,11 +18,8 @@ if TYPE_CHECKING: from prisma.models import LiteLLM_ManagedVectorStoresTable as _VectorStoreRow from litellm.proxy.utils import PrismaClient - from litellm.router import Router - import litellm from litellm._logging import verbose_proxy_logger -from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import REDACTED_BY_LITELM_STRING from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker @@ -32,13 +29,10 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user from litellm.proxy.vector_store_endpoints.utils import can_user_access_vector_store -from litellm.repositories.model_repository import ModelRepository from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import ManagedVectorStoresRepository -from litellm.secret_managers.main import get_secret from litellm.types.vector_stores import ( LiteLLM_ManagedVectorStore, LiteLLM_ManagedVectorStoreListResponse, @@ -64,28 +58,6 @@ _LITELLM_PARAMS_MASKER: Final = SensitiveDataMasker() _REDACT_LITELLM_PARAMS_MAX_DEPTH: Final = 10 -# Use-time embedding-config resolution runs on every vector-store request -# whose persisted row carries only a model reference (the post-fix shape). -# Without a cache, that's one ``litellm_proxymodeltable.find_first`` per -# request — the no-DB-in-critical-path rule. Hold the resolved config in -# memory for a short TTL so a hot model name pays the DB lookup at most -# once per ``_EMBEDDING_CONFIG_CACHE_TTL`` seconds. Cleartext credentials -# only ever live in process memory (never persisted, never echoed in -# management responses), so the cache doesn't widen the disclosure surface. -_EMBEDDING_CONFIG_CACHE_TTL: Final = 60 -_EMBEDDING_CONFIG_CACHE_MAX_SIZE: Final = 256 -_embedding_config_cache: InMemoryCache | None = None - - -def _get_embedding_config_cache() -> InMemoryCache: - global _embedding_config_cache - if _embedding_config_cache is None: - _embedding_config_cache = InMemoryCache( - max_size_in_memory=_EMBEDDING_CONFIG_CACHE_MAX_SIZE, - default_ttl=_EMBEDDING_CONFIG_CACHE_TTL, - ) - return _embedding_config_cache - def _redact_sensitive_litellm_params(litellm_params: Any, _depth: int = 0) -> Any: """ @@ -155,235 +127,6 @@ async def _fetch_and_authorize_vector_store( return typed -def _resolve_embedding_config_from_router(embedding_model: str, llm_router) -> dict[str, object] | None: - """ - Resolve embedding config from router's config-defined models. - - Config-defined models (from proxy_config.yaml) are stored in the router's model_list, - not in the database. This function looks up the model in the router and extracts - api_key, api_base, and api_version from the deployment's litellm_params. - - Args: - embedding_model: The embedding model string (e.g., "text-embedding-ada-002" or "azure/text-embedding-3-large") - llm_router: The LiteLLM router instance - - Returns: - Dictionary with api_key, api_base, and api_version if model found, None otherwise - """ - if not embedding_model or llm_router is None: - return None - - # Extract model name candidates - could be "text-embedding-ada-002" or "azure/text-embedding-3-large" - # Try exact match first, then try without provider prefix - model_name_candidates: Final = [embedding_model] - if "/" in embedding_model: - # If it has a provider prefix, also try without it - _, model_name = embedding_model.split("/", 1) - model_name_candidates.append(model_name) - - # Try to find model in router - for model_name in model_name_candidates: - try: - # Try to get deployment by model group name (model_name in config) - deployment = llm_router.get_deployment_by_model_group_name(model_group_name=model_name) - - if deployment is not None and deployment.litellm_params is not None: - litellm_params = deployment.litellm_params - - # Build embedding config from model params - embedding_config: dict[str, object] = {} - - # Extract api_key - api_key = getattr(litellm_params, "api_key", None) - if api_key: - # Handle os.environ/ prefix - if isinstance(api_key, str) and api_key.startswith("os.environ/"): - api_key = get_secret(api_key) - embedding_config["api_key"] = api_key - - # Extract api_base - api_base = getattr(litellm_params, "api_base", None) - if api_base: - # Handle os.environ/ prefix - if isinstance(api_base, str) and api_base.startswith("os.environ/"): - api_base = get_secret(api_base) - embedding_config["api_base"] = api_base - - # Extract api_version - api_version = getattr(litellm_params, "api_version", None) - if api_version: - embedding_config["api_version"] = api_version - - project_id = getattr(litellm_params, "project_id", None) - if project_id: - embedding_config["project_id"] = project_id - - # Only return config if we have at least api_key or api_base - if embedding_config: - verbose_proxy_logger.debug( - "Resolved embedding config from router model %s: %s", model_name, list(embedding_config.keys()) - ) - return embedding_config - except Exception as e: - verbose_proxy_logger.debug("Error resolving embedding config from router for model %s: %s", model_name, e) - continue - - return None - - -async def _resolve_embedding_config_from_db( - embedding_model: str, prisma_client: "PrismaClient" -) -> dict[str, object] | None: - """ - Resolve embedding config from database model configuration. - - If litellm_embedding_model is provided but litellm_embedding_config is not, - this function looks up the model in the database and extracts api_key, api_base, - and api_version from the model's litellm_params to build the embedding config. - - Args: - embedding_model: The embedding model string (e.g., "text-embedding-ada-002" or "azure/text-embedding-3-large") - prisma_client: The Prisma client instance - - Returns: - Dictionary with api_key, api_base, and api_version if model found, None otherwise - """ - if not embedding_model: - return None - - # Extract model name - could be "text-embedding-ada-002" or "azure/text-embedding-3-large" - # Try to find model by exact match first, then try without provider prefix - model_name_candidates: Final = [embedding_model] - if "/" in embedding_model: - # If it has a provider prefix, also try without it - _, model_name = embedding_model.split("/", 1) - model_name_candidates.append(model_name) - - # Try to find model in database - for model_name in model_name_candidates: - try: - db_model = await ModelRepository(prisma_client).table.find_first(where={"model_name": model_name}) - - if db_model and db_model.litellm_params: - # Extract litellm_params (could be dict or JSON string) - model_params = db_model.litellm_params - if isinstance(model_params, str): # pyright: ignore[reportUnnecessaryIsInstance] # prisma Json is str - model_params = json.loads(model_params) - - # Decrypt values from database (similar to how proxy_server.py does it) - # Values stored in DB are encrypted, so we need to decrypt them first - decrypted_params = {} - if isinstance(model_params, dict): - for k, v in model_params.items(): - if isinstance(v, str): - # Decrypt value - returns original value if decryption fails or no key is set - decrypted_value = decrypt_value_helper(value=v, key=k, return_original_value=True) - decrypted_params[k] = decrypted_value - else: - decrypted_params[k] = v - else: - decrypted_params = model_params - - # Build embedding config from model params - embedding_config = {} - - # Extract api_key - api_key = decrypted_params.get("api_key") - if api_key: - # Handle os.environ/ prefix (after decryption, values may be os.environ/ prefixed) - if isinstance(api_key, str) and api_key.startswith("os.environ/"): - api_key = get_secret(api_key) - embedding_config["api_key"] = api_key - - # Extract api_base - api_base = decrypted_params.get("api_base") - if api_base: - # Handle os.environ/ prefix (after decryption, values may be os.environ/ prefixed) - if isinstance(api_base, str) and api_base.startswith("os.environ/"): - api_base = get_secret(api_base) - embedding_config["api_base"] = api_base - - # Extract api_version - api_version = decrypted_params.get("api_version") - if api_version: - embedding_config["api_version"] = api_version - - # Only return config if we have at least api_key or api_base - if embedding_config: - verbose_proxy_logger.debug( - "Resolved embedding config from database model %s: %s", - model_name, - list(embedding_config.keys()), - ) - return embedding_config - except Exception as e: - verbose_proxy_logger.debug("Error resolving embedding config for model %s: %s", model_name, e) - continue - - return None - - -async def _resolve_embedding_config( - embedding_model: str, prisma_client: "PrismaClient | None", llm_router: "Router | None" = None -) -> dict[str, object] | None: - """ - Resolve embedding config from either router (config-defined) or database models. - - This function first checks the router for config-defined models, then falls back - to the database. This allows users to use models defined in either location. - - Results are cached in process memory for ``_EMBEDDING_CONFIG_CACHE_TTL`` - seconds so the request-handling path doesn't hit the database on every - vector-store call. Negative results (model not found) are intentionally - not cached to avoid blocking a freshly-added model behind the TTL. - - Args: - embedding_model: The embedding model string (e.g., "text-embedding-ada-002" or "azure/text-embedding-3-large") - prisma_client: The Prisma client instance - llm_router: The LiteLLM router instance (optional, will be imported if not provided) - - Returns: - Dictionary with api_key, api_base, and api_version if model found, None otherwise - """ - if not embedding_model: - return None - - cache: Final = _get_embedding_config_cache() - cached: Final = cache.get_cache(embedding_model) - if cached is not None: - return cached - - # Import llm_router if not provided - if llm_router is None: - try: - from litellm.proxy.proxy_server import llm_router - except ImportError: - llm_router = None - - # First try to resolve from router (config-defined models) - if llm_router is not None: - router_config = _resolve_embedding_config_from_router(embedding_model=embedding_model, llm_router=llm_router) - if router_config: - verbose_proxy_logger.debug("Resolved embedding config from router for model %s", embedding_model) - cache.set_cache(embedding_model, router_config) - return router_config - - # Fall back to database - if prisma_client is not None: - db_config: Final = await _resolve_embedding_config_from_db( - embedding_model=embedding_model, prisma_client=prisma_client - ) - if db_config: - verbose_proxy_logger.debug("Resolved embedding config from database for model %s", embedding_model) - cache.set_cache(embedding_model, db_config) - return db_config - - verbose_proxy_logger.debug( - "Could not resolve embedding config for model %s from router or database", embedding_model - ) - return None - - ######################################################## # Helper Functions ######################################################## @@ -469,10 +212,9 @@ async def create_vector_store_in_db( # (``api_key``, ``api_base``, ``api_version``) into this row. That # exposed every env-stored embedding-model credential on the # ``/vector_store/{new,info,update,list}`` responses. Keep the user's - # raw ``litellm_embedding_model`` reference; resolution now happens in - # ``_update_request_data_with_litellm_managed_vector_store_registry`` - # at request-handling time so the cleartext config exists only in - # per-request memory and never reaches the database. + # raw ``litellm_embedding_model`` reference; each search embeds the + # query through the router at request time, so the credentials stay + # on the deployment and never reach the database. if litellm_params: litellm_params_dict: Final = GenericLiteLLMParams(**litellm_params).model_dump(exclude_none=True) data_to_create["litellm_params"] = safe_dumps(litellm_params_dict) @@ -862,11 +604,9 @@ async def update_vector_store( # Handle litellm_params if provided. As with the create path, the # embedding-config auto-resolve previously persisted cleartext - # credentials into the row; resolution now happens at request- - # handling time in - # ``_update_request_data_with_litellm_managed_vector_store_registry`` - # so this row only ever stores the user-supplied - # ``litellm_embedding_model`` reference. + # credentials into the row; each search now embeds the query + # through the router at request time, so this row only ever stores + # the user-supplied ``litellm_embedding_model`` reference. if "litellm_params" in update_data: _input_litellm_params: Final[dict] = update_data.get("litellm_params", {}) or {} litellm_params_dict: Final = GenericLiteLLMParams(**_input_litellm_params).model_dump(exclude_none=True) diff --git a/litellm/rag/main.py b/litellm/rag/main.py index 7bc1a6a52a3..94bfc305a6a 100644 --- a/litellm/rag/main.py +++ b/litellm/rag/main.py @@ -14,6 +14,7 @@ import contextvars from collections.abc import Coroutine, Iterator from contextlib import contextmanager from functools import partial +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx @@ -50,6 +51,21 @@ INGESTION_REGISTRY: Final[dict[str, type[BaseRAGIngestion]]] = { "vertex_ai": VertexAIRAGIngestion, } +# Only these retrieval_config keys are forwarded to vector_stores.asearch as +# provider-specific params. The explicit allowlist keeps caller-controlled +# connection overrides (api_base, api_key, ...) away from the search call, +# where they could redirect store credentials to an attacker-chosen host. +_FORWARDABLE_RETRIEVAL_CONFIG_KEYS: Final = frozenset( + { + "aws_region_name", + "vector_bucket_name", + "embedding_model", + "litellm_embedding_model", + "litellm_embedding_config", + "litellm_credential_name", + } +) + def get_ingestion_class(provider: str) -> type[BaseRAGIngestion]: """ @@ -224,13 +240,20 @@ async def _execute_query_pipeline( raise ValueError("No query found in messages for RAG query") # 2. Search vector store + # Forward allowlisted provider retrieval_config extras (region, embedding + # model, bucket, credential refs) to the search call; kwargs win on conflict. + provider_search_params: Final = MappingProxyType( + {k: v for k, v in retrieval_config.items() if k in _FORWARDABLE_RETRIEVAL_CONFIG_KEYS} + ) + forwarded_search_params: Final = MappingProxyType({**provider_search_params, **kwargs}) with _suppressed_sub_call_billing(): search_response: Final = await litellm.vector_stores.asearch( vector_store_id=retrieval_config["vector_store_id"], query=query_text, max_num_results=retrieval_config.get("top_k", 10), custom_llm_provider=retrieval_config.get("custom_llm_provider", "openai"), - **kwargs, + router=router, + **forwarded_search_params, ) search_provider: Final = retrieval_config.get("custom_llm_provider", "openai") diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index c8f7842aebf..37ca989b8d3 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -6,6 +6,7 @@ from typing import Any, Final, Literal import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler @@ -43,10 +44,23 @@ async def arerank( """ Async: Reranks a list of documents based on their relevance to the query """ + _custom_llm_provider: str | None = ( + None # rebind-ok: set by the declared-provider guard or the get_llm_provider unpack; read in the except + ) try: loop: Final = asyncio.get_event_loop() kwargs["arerank"] = True + declared_provider: Final = declared_authenticating_provider(model, custom_llm_provider) + if declared_provider is not None: + _custom_llm_provider = declared_provider # rebind-ok: see pre-declaration above + else: + _, _custom_llm_provider, _, _ = litellm.get_llm_provider( # rebind-ok: see pre-declaration above + model=model, + custom_llm_provider=custom_llm_provider, + api_base=kwargs.get("api_base", None), + ) + func: Final = partial( rerank, model, @@ -70,7 +84,11 @@ async def arerank( response = init_response return response except Exception as e: - raise e + raise exception_type( + model=model, + custom_llm_provider=_custom_llm_provider or custom_llm_provider, + original_exception=e, + ) @client @@ -115,6 +133,7 @@ def rerank( model_info: Final = kwargs.get("model_info", None) user: Final = kwargs.get("user", None) client: Final = kwargs.get("client", None) + _custom_llm_provider: str | None = None # rebind-ok: set by the get_llm_provider unpack; read in the except try: _is_async: Final = kwargs.pop("arerank", False) is True optional_params: Final = GenericLiteLLMParams(**kwargs) @@ -127,7 +146,7 @@ def rerank( ( model, - _custom_llm_provider, + _custom_llm_provider, # rebind-ok: see pre-declaration above dynamic_api_key, dynamic_api_base, ) = litellm.get_llm_provider( @@ -538,4 +557,8 @@ def rerank( return response except Exception as e: verbose_logger.error("Error in rerank: %s", e) - raise exception_type(model=model, custom_llm_provider=custom_llm_provider, original_exception=e) + raise exception_type( + model=model, + custom_llm_provider=_custom_llm_provider or custom_llm_provider, + original_exception=e, + ) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index db1c3acbefb..bc25f4fffb1 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1169,16 +1169,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _emit_response_completed_event(self, litellm_model_response: ModelResponse) -> ResponseCompletedEvent | None: if litellm_model_response: - # Add cost to usage object if include_cost_in_streaming_usage is True - if litellm.include_cost_in_streaming_usage and self.litellm_logging_obj is not None: - usage: Final[object] = getattr(litellm_model_response, "usage", None) - if usage is not None: - setattr( - usage, - "cost", - self.litellm_logging_obj._response_cost_calculator(result=litellm_model_response), - ) - # Transform the response responses_api_response: Final = ( LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 5f3e88bb12f..b2d1a69e0d8 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -6,6 +6,7 @@ import json import re import uuid from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -102,6 +103,15 @@ from .custom_tools import ( NamespaceNameMap: TypeAlias = Mapping[str, tuple[str, str]] NamespaceTool: TypeAlias = Mapping[str, object] ResponseTools: TypeAlias = Sequence[Mapping[str, object]] | None +ChatToolParam: TypeAlias = ChatCompletionToolParam | OpenAIMcpServerTool +NAMESPACE_DESCRIPTION_SEPARATOR: Final = "\n\n" + + +@dataclass(frozen=True, slots=True) +class ResponsesToolChatForm: + chat_tools: tuple[ChatToolParam, ...] + web_search_options: OpenAIWebSearchOptions | None + if TYPE_CHECKING: from openai.types.responses.response_apply_patch_tool_call import ( @@ -1771,7 +1781,7 @@ class LiteLLMCompletionResponsesConfig: tool_name: Final = str(namespace_tool.get("name") or "") raw_description: Final = str(namespace_tool.get("description") or "") description: Final = ( - f"{namespace_description}\n\n{raw_description}" + f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}{raw_description}" if nested and namespace_description and raw_description else namespace_description if nested and namespace_description @@ -1837,9 +1847,78 @@ class LiteLLMCompletionResponsesConfig: + ", ".join(sorted(conflicting_tool_names)) ) + @staticmethod + def _responses_tool_to_chat_form(tool: Mapping[str, object]) -> ResponsesToolChatForm: + tool_type: Final = tool.get("type") + if tool_type == "mcp": + return ResponsesToolChatForm(chat_tools=(cast(OpenAIMcpServerTool, tool),), web_search_options=None) + if tool_type == "web_search_preview" or tool_type == "web_search": + _search_context_size: Final[Literal["low", "medium", "high"]] = cast( + Literal["low", "medium", "high"], tool.get("search_context_size") + ) + _user_location: Final[OpenAIWebSearchUserLocation | None] = cast( + OpenAIWebSearchUserLocation | None, + tool.get("user_location") or None, + ) + return ResponsesToolChatForm( + chat_tools=(), + web_search_options=OpenAIWebSearchOptions( + search_context_size=_search_context_size, + user_location=_user_location, + ), + ) + if tool_type == "function": + typed_tool: Final = cast(FunctionToolParam, tool) + raw_parameters: Final = typed_tool.get("parameters", {}) or {} + parameters: Final = ( + {**raw_parameters} # mutable-ok: json.dumps rejects MappingProxyType + if "type" in raw_parameters + else {**raw_parameters, "type": "object"} # mutable-ok: json.dumps rejects MappingProxyType + ) + chat_completion_tool: Final[dict[str, object]] = { + "type": "function", + "function": { + "name": typed_tool.get("name") or "", + "description": typed_tool.get("description") or "", + "parameters": parameters, + "strict": typed_tool.get("strict", False) or False, + }, + } + if tool.get("cache_control"): + chat_completion_tool["cache_control"] = tool.get("cache_control") + if tool.get("defer_loading"): + chat_completion_tool["defer_loading"] = tool.get("defer_loading") + if tool.get("allowed_callers"): + chat_completion_tool["allowed_callers"] = tool.get("allowed_callers") + if tool.get("input_examples"): + chat_completion_tool["input_examples"] = tool.get("input_examples") + return ResponsesToolChatForm( + chat_tools=(cast(ChatCompletionToolParam, chat_completion_tool),), web_search_options=None + ) + if tool_type == "namespace": + return ResponsesToolChatForm( + chat_tools=LiteLLMCompletionResponsesConfig._namespace_chat_tools(tool), web_search_options=None + ) + if tool_type == "custom": + converted: Final = convert_custom_tool_to_function_tool(tool) + return ResponsesToolChatForm(chat_tools=() if converted is None else (converted,), web_search_options=None) + if tool_type in ("computer_use", "image_generation", "shell"): + verbose_logger.warning( + "Dropping Responses API tool of type '%s': it has no Chat Completions " + "equivalent and the target provider would reject the request.", + tool_type, + ) + return ResponsesToolChatForm(chat_tools=(), web_search_options=None) + return ResponsesToolChatForm(chat_tools=(cast(ChatToolParam, tool),), web_search_options=None) + + @staticmethod + def responses_tools_to_chat_forms(tools: ResponseTools) -> tuple[ResponsesToolChatForm, ...]: + LiteLLMCompletionResponsesConfig._validate_namespace_name_collisions(tools) + return tuple(LiteLLMCompletionResponsesConfig._responses_tool_to_chat_form(tool) for tool in tools or ()) + @staticmethod def transform_responses_api_tools_to_chat_completion_tools( - tools: list[FunctionToolParam | OpenAIMcpServerTool] | None, + tools: ResponseTools, ) -> tuple[ list[ChatCompletionToolParam | OpenAIMcpServerTool], OpenAIWebSearchOptions | None, @@ -1849,73 +1928,16 @@ class LiteLLMCompletionResponsesConfig: """ if tools is None: return [], None - LiteLLMCompletionResponsesConfig._validate_namespace_name_collisions(tools) - chat_completion_tools: Final[list[ChatCompletionToolParam | OpenAIMcpServerTool]] = [] - web_search_options: OpenAIWebSearchOptions | None = None - for tool in tools: - if tool.get("type") == "mcp": - chat_completion_tools.append(cast(OpenAIMcpServerTool, tool)) - elif tool.get("type") == "web_search_preview" or tool.get("type") == "web_search": - _search_context_size: Literal["low", "medium", "high"] = cast( - Literal["low", "medium", "high"], tool.get("search_context_size") - ) - _user_location: OpenAIWebSearchUserLocation | None = cast( - OpenAIWebSearchUserLocation | None, - tool.get("user_location") or None, - ) - web_search_options = OpenAIWebSearchOptions( - search_context_size=_search_context_size, - user_location=_user_location, - ) - elif tool.get("type") == "function": - typed_tool = cast(FunctionToolParam, tool) - # Ensure parameters has "type": "object" as required by providers like Anthropic - parameters = dict(typed_tool.get("parameters", {}) or {}) - if not parameters or "type" not in parameters: - parameters["type"] = "object" - chat_completion_tool: dict[str, object] = { - "type": "function", - "function": { - "name": typed_tool.get("name") or "", - "description": typed_tool.get("description") or "", - "parameters": parameters, - "strict": typed_tool.get("strict", False) or False, - }, - } - if tool.get("cache_control"): - chat_completion_tool["cache_control"] = tool.get("cache_control") - if tool.get("defer_loading"): - chat_completion_tool["defer_loading"] = tool.get("defer_loading") - if tool.get("allowed_callers"): - chat_completion_tool["allowed_callers"] = tool.get("allowed_callers") - if tool.get("input_examples"): - chat_completion_tool["input_examples"] = tool.get("input_examples") - chat_completion_tools.append(cast(ChatCompletionToolParam, chat_completion_tool)) - elif tool.get("type") == "namespace": - chat_completion_tools.extend(LiteLLMCompletionResponsesConfig._namespace_chat_tools(tool)) - elif tool.get("type") == "custom": - converted = convert_custom_tool_to_function_tool(tool) - if converted is not None: - chat_completion_tools.append(converted) - else: - _tool_type = tool.get("type") - if _tool_type in ("computer_use", "image_generation", "shell"): - # Drop unsupported Responses-API-only tool types that have no - # Chat Completions equivalent. Passing them through verbatim - # causes providers to reject the request with "'function' is a - # required property". - verbose_logger.warning( - "Dropping Responses API tool of type '%s': it has no Chat Completions " - "equivalent and the target provider would reject the request.", - _tool_type, - ) - continue - chat_completion_tools.append(cast(ChatCompletionToolParam | OpenAIMcpServerTool, tool)) - return chat_completion_tools, web_search_options + forms: Final = LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(tools) + web_search_options: Final = next( + (form.web_search_options for form in reversed(forms) if form.web_search_options is not None), + None, + ) + return [chat_tool for form in forms for chat_tool in form.chat_tools], web_search_options @staticmethod def transform_chat_completion_tool_params_to_responses_api_tools( - chat_completion_tools: list[ChatCompletionToolParam | OpenAIMcpServerTool] | None, + chat_completion_tools: Sequence[Mapping[str, object]] | None, ) -> list[dict[str, object]]: """ Transform Chat Completion tool params (e.g. from guardrail output) back to @@ -1926,9 +1948,6 @@ class LiteLLMCompletionResponsesConfig: return [] result: Final[list[dict[str, object]]] = [] for tool in chat_completion_tools: - if not isinstance(tool, dict): - result.append(tool) - continue if tool.get("type") == "function": fn = cast(_ToolFunctionDefinition, tool.get("function") or {}) parameters = dict(fn.get("parameters", {}) or {}) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 82abac3e772..f271655f5e3 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -17,6 +17,7 @@ from typing_extensions import TypeIs import litellm from litellm.constants import ( + EMPTY_MAPPING, LITELLM_MAX_STREAMING_DURATION_SECONDS, STREAM_SSE_DONE_STRING, ) @@ -273,6 +274,9 @@ class BaseResponsesAPIStreamingIterator: self._hidden_params["additional_headers"] = process_response_headers( self.response.headers or {} ) # GUARANTEE OPENAI HEADERS IN RESPONSE + self._raw_response_headers: Mapping[str, str] = MappingProxyType( + dict(self.response.headers or {}) # mutable-ok: immediately frozen by MappingProxyType + ) def _check_max_streaming_duration(self) -> None: """Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS.""" @@ -407,23 +411,7 @@ class BaseResponsesAPIStreamingIterator: openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED, ): self.completed_response = openai_responses_api_chunk - # Add cost to usage object if include_cost_in_streaming_usage is True - if litellm.include_cost_in_streaming_usage and self.logging_obj is not None: - response_obj: Final[ResponsesAPIResponse | None] = getattr( - openai_responses_api_chunk, "response", None - ) - if response_obj: - usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None) - if usage_obj is not None: - try: - cost: Final[float | None] = self.logging_obj._response_cost_calculator( - result=response_obj - ) - if cost is not None: - setattr(usage_obj, "cost", cost) - except Exception: - # Best-effort usage cost annotation should not break stream replay. - pass + _stamp_responses_usage_cost(getattr(openai_responses_api_chunk, "response", None), self.logging_obj) if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED: self._handle_logging_failed_response() @@ -462,6 +450,7 @@ class BaseResponsesAPIStreamingIterator: except Exception: # Fallback to original if serialization fails pass + self._restore_provider_response_headers(logging_response) end_time: Final = datetime.now() if is_async: @@ -496,6 +485,41 @@ class BaseResponsesAPIStreamingIterator: ) self._run_post_success_hooks(end_time=end_time) + def _restore_provider_response_headers(self, logging_response: object) -> None: + """Re-apply the provider's response headers to the copy handed to logging callbacks. + + ``model_validate(model_dump())`` above drops pydantic private attributes, so the + ``_hidden_params`` the provider transform set on the nested response are lost. Returns early + when that copy fell back to the original event, so logging-only state never lands on the + object the caller is iterating. + """ + if logging_response is self.completed_response: + return + target: Final[object] = getattr(logging_response, "response", None) + existing_hidden: Final[object] = getattr(target, "_hidden_params", None) + if not isinstance(existing_hidden, Mapping): + return + existing: Final[Mapping[str, object]] = existing_hidden + source_hidden: Final[object] = getattr( + getattr(self.completed_response, "response", None), "_hidden_params", None + ) + source: Final[Mapping[str, object]] = source_hidden if isinstance(source_hidden, Mapping) else EMPTY_MAPPING + processed: Final[object] = source.get("additional_headers") or self._hidden_params.get("additional_headers") + raw: Final[object] = source.get("headers") or self._raw_response_headers + headers: Final[Mapping[str, object]] = processed if isinstance(processed, Mapping) else EMPTY_MAPPING + raw_headers: Final[Mapping[str, object]] = raw if isinstance(raw, Mapping) else EMPTY_MAPPING + # rebuild by value and let existing keys win: sharing the source dicts would alias what the proxy + # splats into the client's HTTP headers, and copying non-header keys would carry response_cost + setattr( # noqa: B010 # target is typed object here, so a plain attribute store does not type check + target, + "_hidden_params", + { # mutable-ok: the cost calculator writes optional_params into _hidden_params + "additional_headers": {**headers}, # mutable-ok: fresh copy, logging callbacks may mutate it + "headers": {**raw_headers}, # mutable-ok: fresh copy, logging callbacks may mutate it + **existing, + }, + ) + def _handle_logging_completed_response(self): """Base implementation - should be overridden by subclasses""" @@ -1274,6 +1298,24 @@ def _add_text_like_part_events( ) +def _stamp_responses_usage_cost( + response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None +) -> None: + if response_obj is None or logging_obj is None: + return + usage_obj: Final[ResponseAPIUsage | None] = getattr(response_obj, "usage", None) + if usage_obj is None: + return + if isinstance(getattr(usage_obj, "cost", None), (int, float)): + return + try: + cost: Final[float | None] = logging_obj._response_cost_calculator(result=response_obj) + except Exception: + return + if isinstance(cost, (int, float)) and cost > 0: + setattr(usage_obj, "cost", cost) + + def build_synthetic_response_events( *, transformed: ResponsesAPIResponse, @@ -1281,15 +1323,7 @@ def build_synthetic_response_events( chunk_size: int, ) -> list[ResponsesAPIStreamingResponse]: openai_types: Final = _get_openai_response_types() - if litellm.include_cost_in_streaming_usage and logging_obj is not None: - usage_obj: Final = transformed.usage if hasattr(transformed, "usage") else None - if usage_obj is not None: - try: - cost: Final[float | None] = logging_obj._response_cost_calculator(result=transformed) - if cost is not None: - setattr(usage_obj, "cost", cost) - except Exception: - pass + _stamp_responses_usage_cost(transformed, logging_obj) events: Final[list[ResponsesAPIStreamingResponse]] = [ _build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed), diff --git a/litellm/router.py b/litellm/router.py index 23d8907fb49..303b22c9484 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -30,7 +30,7 @@ import anyio import httpx import openai from openai import AsyncOpenAI -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import overload import litellm @@ -50,6 +50,7 @@ from litellm.constants import ( DEFAULT_HEALTH_CHECK_INTERVAL, DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER, DEFAULT_MAX_LRU_CACHE_SIZE, + RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger @@ -84,6 +85,10 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( mask_credentials_in_payload, mask_sensitive_structure, ) +from litellm.llms.base_llm.vector_store.transformation import ( + RouterVectorStoreEmbeddingExecutor, + vector_store_request_metadata, +) from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_strategy.least_busy import LeastBusyLoggingHandler @@ -145,8 +150,10 @@ from litellm.router_utils.fallback_event_handlers import ( _check_non_standard_fallback_format, clear_pre_routing_selection, fallback_lookup_groups, + fallbacks_disabled_for_request, get_fallback_model_group_for_lookup_groups, get_pre_routing_selection, + record_disable_fallbacks, record_pre_routing_selection, run_async_fallback, ) @@ -353,6 +360,15 @@ _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" +_CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") +_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600 + +_RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS: Final[Mapping[str, type[CustomLogger]]] = MappingProxyType( + { + "prompt_caching": PromptCachingDeploymentCheck, + "enforce_model_rate_limits": ModelRateLimitingCheck, + } +) def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool: @@ -375,6 +391,28 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) return False +_NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({}) +_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +def _with_router_resolved_session_model(session: object, model_name: str) -> Mapping[str, Mapping[str, object]]: + """ + Realtime client-secret requests carry the model inside ``session`` as well, and the caller's copy of it still + holds the pre-routing model group name, so it has to follow the deployment the router just picked. + + Returns kwargs to merge into the downstream call, empty when there is no session model to resolve. + """ + try: + typed_session: Final = _SESSION_ADAPTER.validate_python(session) + except ValidationError: + return _NO_SESSION_KWARGS + if "model" not in typed_session: + return _NO_SESSION_KWARGS + return MappingProxyType( + {"session": {**typed_session, "model": model_name}} # mutable-ok: callees deepcopy and JSON-dump session + ) + + # Router._aanthropic_messages_streaming_iterator buffers lifecycle chunks # until real content commits the primary stream; a hostile or slow-starting # upstream that never emits content or an error could otherwise grow that @@ -780,6 +818,10 @@ class Router: self.cache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache() ) # use a dual cache (Redis+In-Memory) for tracking cooldowns, usage, etc. + self._claude_code_session_router_cache: DualCache = DualCache( + redis_cache=redis_cache, + in_memory_cache=InMemoryCache(), + ) ### SCHEDULER ### self.scheduler = Scheduler(polling_interval=polling_interval, redis_cache=redis_cache) @@ -1101,8 +1143,8 @@ class Router: ``` and caching to just work. """ - if self.cache.redis_cache is None: - self.cache.redis_cache = cache + self.cache.attach_redis_cache(cache) + self._claude_code_session_router_cache.attach_redis_cache(cache) # Maps a routing strategy string to the attribute on `self` that holds # the default group's strategy selector for that strategy. (The selectors @@ -2072,11 +2114,39 @@ class Router: if _callback is None: continue + if self.optional_callbacks is not None and any( + isinstance(callback, type(_callback)) for callback in self.optional_callbacks + ): + continue if self.optional_callbacks is None: self.optional_callbacks = [] self.optional_callbacks.append(_callback) litellm.logging_callback_manager.add_litellm_callback(_callback) + def set_optional_pre_call_checks(self, optional_pre_call_checks: OptionalPreCallChecks | None) -> None: + if optional_pre_call_checks is None: + return + requested: Final = frozenset(optional_pre_call_checks) + for name, callback_cls in _RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS.items(): + if name not in requested: + self._remove_optional_callbacks_of_type(callback_cls) + self.add_optional_pre_call_checks(optional_pre_call_checks) + + def _remove_optional_callbacks_of_type(self, callback_cls: type[CustomLogger]) -> None: + if self.optional_callbacks is None or not any(type(cb) is callback_cls for cb in self.optional_callbacks): + return + self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls] + if any( + router is not self and any(type(cb) is callback_cls for cb in (router.optional_callbacks or [])) + for router in tuple(_live_routers) + ): + return + for cb in tuple(litellm.callbacks): + if type(cb) is callback_cls: + litellm.logging_callback_manager.remove_callback_from_list_by_object( + litellm.callbacks, cb, require_self=False + ) + def print_deployment(self, deployment: dict): """ returns a copy of the deployment with the api key masked @@ -2324,7 +2394,7 @@ class Router: @overload async def acompletion( self, model: str, messages: list[AllMessageValues], stream: Literal[True, False] = False, **kwargs - ) -> CustomStreamWrapper | ModelResponse: + ) -> CustomStreamWrapper | ModelResponse: ... # fmt: on @@ -4007,6 +4077,7 @@ class Router: prompt_variables=prompt_variables, prompt_label=prompt_label, request_kwargs=kwargs, + injected_for_every_deployment=True, ) # Filter out prompt management specific parameters from data before merging @@ -4894,6 +4965,7 @@ class Router: "caching": self.cache_responses, **kwargs, "model": model_name, + **_with_router_resolved_session_model(kwargs.get("session"), model_name), } # Only set custom_llm_provider if it's not None if custom_llm_provider is not None: @@ -5123,7 +5195,7 @@ class Router: if not has_generated_content and error_event is None else None ) - if refusal_stop_details is not None and self._has_content_policy_fallback(model, initial_kwargs): + if refusal_stop_details is not None and self._refusal_fallback_available(model, initial_kwargs): refusal_error = safeguard_refusal_error(model=model, stop_details=refusal_stop_details) raise MidStreamFallbackError( message=refusal_error.message, @@ -6374,8 +6446,6 @@ class Router: "responses", "generate_content", "generate_content_stream", - "vector_store_search", - "vector_store_create", "ocr", "search", "video_generation", @@ -6399,6 +6469,8 @@ class Router: return sync_wrapper if call_type in ( + "vector_store_search", + "vector_store_create", "vector_store_retrieve", "vector_store_list", "vector_store_update", @@ -6410,11 +6482,29 @@ class Router: client: object | None = None, **kwargs, ): - if custom_llm_provider and "custom_llm_provider" not in kwargs: - kwargs["custom_llm_provider"] = custom_llm_provider - if kwargs.get("model"): - return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs) - return original_function(**kwargs) + provider_kwargs: Final = ( + MappingProxyType({**kwargs, "custom_llm_provider": custom_llm_provider}) + if custom_llm_provider and "custom_llm_provider" not in kwargs + else MappingProxyType(kwargs) + ) + search_kwargs: Final = ( + MappingProxyType( + { + **provider_kwargs, + "_direct_vector_store_embedding_executor": RouterVectorStoreEmbeddingExecutor( + router=self, + metadata=self._vector_store_request_metadata(kwargs), + ), + } + ) + if call_type == "vector_store_search" + else provider_kwargs + ) + if search_kwargs.get("model"): + return self._generic_api_call_with_fallbacks(original_function=original_function, **search_kwargs) + if call_type == "vector_store_search": + return original_function(**MappingProxyType({**search_kwargs, "router": self})) + return original_function(**search_kwargs) return vector_store_sync_wrapper @@ -6587,10 +6677,22 @@ class Router: "avector_store_update", "avector_store_delete", ): + vector_store_kwargs: Final = ( + { # mutable-ok: the async routed request requires dynamic keyword arguments + **kwargs, + "_direct_vector_store_embedding_executor": RouterVectorStoreEmbeddingExecutor( + router=self, + metadata=self._vector_store_request_metadata(kwargs), + ), + } + if call_type == "avector_store_search" + else kwargs + ) return await self._init_vector_store_api_endpoints( original_function=original_function, custom_llm_provider=custom_llm_provider, - **kwargs, + call_type=call_type, + **vector_store_kwargs, ) elif call_type in ("afile_delete", "afile_content"): return await self._ageneric_api_call_with_fallbacks( @@ -6626,10 +6728,15 @@ class Router: return async_wrapper + @staticmethod + def _vector_store_request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: + return vector_store_request_metadata(kwargs) + async def _init_vector_store_api_endpoints( self, original_function: Callable, custom_llm_provider: str | None = None, + call_type: str | None = None, **kwargs, ): """ @@ -6648,6 +6755,13 @@ class Router: **kwargs, ) + # For search, pass the router so provider transforms can resolve + # router-managed embedding models (e.g. S3 Vectors query embeddings). + # The merge also overrides any client-supplied `router` key. + if call_type == "avector_store_search": + search_kwargs: Final = MappingProxyType({**kwargs, "router": self}) + return await original_function(**search_kwargs) + # Otherwise, call the original function directly return await original_function(**kwargs) @@ -6664,7 +6778,10 @@ class Router: metadata. When present, decode the ID, replace ``container_id`` with the upstream value, and route through ``_ageneric_api_call_with_fallbacks`` so deployment credentials (e.g. regional ``api_base`` for Azure) match - :meth:`_init_responses_api_endpoints`. Otherwise call the handler directly. + :meth:`_init_responses_api_endpoints`. Create/list calls carry no container ID, so + they route through the deployment named by ``model`` when the caller passes one, + falling back to the direct call when no deployment matches. Otherwise call the + handler directly with global provider credentials. """ if custom_llm_provider and "custom_llm_provider" not in kwargs: kwargs["custom_llm_provider"] = custom_llm_provider @@ -6696,6 +6813,14 @@ class Router: **kwargs, ) + requested_model: Final = kwargs.get("model") + if isinstance(requested_model, str) and requested_model.strip(): + return await self._ageneric_api_call_with_fallbacks( + original_function=original_function, + passthrough_on_no_deployment=True, + **kwargs, + ) + return await original_function(**kwargs) async def _init_responses_api_endpoints( @@ -7143,6 +7268,7 @@ class Router: _fallback_metadata["original_model_group"] = model_group include_fallback_errors: Final = kwargs.get("include_fallback_errors", False) is True disable_fallbacks: Final[bool | None] = kwargs.pop("disable_fallbacks", False) + record_disable_fallbacks(kwargs, disable_fallbacks is True) fallbacks: Final[list | None] = kwargs.get("fallbacks", self.fallbacks) context_window_fallbacks: list | None = kwargs.get("context_window_fallbacks", self.context_window_fallbacks) content_policy_fallbacks: list | None = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) @@ -8008,6 +8134,29 @@ class Router: ) return False + def _refusal_fallback_available(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: + """ + Whether a safeguard refusal can actually be recovered by the dispatcher. A configured + content-policy list is authoritative; with none configured at all, the dispatcher falls + through to the generic fallbacks lookup, so the gate mirrors that reachability and arms + on a resolving generic chain (tier first, then the requested group, then "*"). + """ + if fallbacks_disabled_for_request(kwargs): + return False + content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) + if content_policy_fallbacks is not None: + return self._has_content_policy_fallback(model_group, kwargs) + if self._has_default_fallbacks(): + return True + fallbacks: Final = kwargs.get("fallbacks", self.fallbacks) + if fallbacks is None: + return False + resolved, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=fallbacks, + lookup_groups=fallback_lookup_groups(kwargs, model_group), + ) + return resolved is not None + def _should_raise_content_policy_error(self, model: str, response: ModelResponse, kwargs: dict) -> bool: """ Determines if a content policy error should be raised. @@ -8039,7 +8188,7 @@ class Router: return False if get_safeguard_refusal_stop_details(response) is None: return False - return self._has_content_policy_fallback(model, kwargs) + return self._refusal_fallback_available(model, kwargs) def _get_healthy_deployments(self, model: str, parent_otel_span: Span | None): _all_deployments: list = [] @@ -9746,6 +9895,26 @@ class Router: coerce_token_limit(model_info.get("max_output_tokens")), ) + def get_configured_display_name(self, model_name: str) -> "str | None": + """ + Return the display_name explicitly configured in a concrete deployment's + model_info for model_name, via O(1) index lookup. + + Returns None for wildcard-expanded or unknown names, and treats a + non-string or empty configured value as absent rather than failing the + listing. Like get_configured_token_limits, this never triggers pattern + matching or deep copies, so it is safe to call per listed model on the + /v1/models hot path. + """ + deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + if deployment is None: + return None + + display_name: Final = deployment.model_info.get("display_name") + if isinstance(display_name, str) and display_name.strip(): + return display_name + return None + def get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None ) -> dict[str, Any] | None: @@ -11331,27 +11500,6 @@ class Router: """ Update the router settings. """ - # only the following settings are allowed to be configured - _allowed_settings: Final = [ - "routing_strategy_args", - "routing_strategy", - "routing_groups", - "allowed_fails", - "cooldown_time", - "num_retries", - "timeout", - "max_retries", - "retry_after", - "fallbacks", - "context_window_fallbacks", - "retry_policy", - "model_group_retry_policy", - "model_group_alias", - "enable_weighted_failover", - "enable_tag_filtering", - "tag_routing_prefix", - ] - _int_settings: Final = [ "timeout", "num_retries", @@ -11364,13 +11512,15 @@ class Router: rebuild_routing_groups = False relink_lar1_from_args = False for var in kwargs: - if var in _allowed_settings: + if var in RUNTIME_UPDATABLE_ROUTER_SETTINGS: if var in _int_settings: _casted_value = int(kwargs[var]) setattr(self, var, _casted_value) elif var == "routing_groups": self._routing_groups_input = kwargs[var] rebuild_routing_groups = True + elif var == "optional_pre_call_checks": + self.set_optional_pre_call_checks(kwargs[var]) elif var == "retry_policy": value = kwargs[var] if isinstance(value, dict): @@ -12565,6 +12715,100 @@ class Router: return None return candidates[0] + @staticmethod + def _request_header(request_kwargs: Mapping[str, object], header_name: str) -> str | None: + proxy_server_request: Final = request_kwargs.get("proxy_server_request") + if not isinstance(proxy_server_request, Mapping): + return None + headers: Final = proxy_server_request.get("headers") + if not isinstance(headers, Mapping): + return None + return next( + ( + value + for key, value in headers.items() + if isinstance(key, str) and key.lower() == header_name and isinstance(value, str) + ), + None, + ) + + def _claude_code_session_router_cache_key(self, request_kwargs: Mapping[str, object]) -> str | None: + session_id: Final = self._request_header(request_kwargs, "x-claude-code-session-id") + if session_id is None or _CLAUDE_CODE_SESSION_ID_RE.fullmatch(session_id) is None: + return None + metadata_name: Final = "litellm_metadata" if "litellm_metadata" in request_kwargs else "metadata" + metadata: Final = request_kwargs.get(metadata_name) + if not isinstance(metadata, Mapping): + return None + caller_scope: Final = metadata.get("user_api_key_hash") + if not isinstance(caller_scope, str) or not caller_scope: + return None + return f"claude_code_session_router:v1:{caller_scope}:{session_id}" + + async def _delete_claude_code_session_router_binding(self, cache_key: str) -> None: + try: + await self._claude_code_session_router_cache.async_delete_cache(key=cache_key) + except Exception as e: # noqa: BLE001 # cache cleanup must not fail an otherwise routable request + verbose_router_logger.warning( + "Failed to delete Claude Code session router binding; the binding may remain until its TTL expires: %s", + e, + ) + + async def _get_claude_code_session_router_binding(self, cache_key: str) -> object: + session_cache: Final = self._claude_code_session_router_cache + try: + if session_cache.redis_cache is None: + return await session_cache.async_get_cache(key=cache_key) + return await session_cache.redis_cache.async_get_cache(key=cache_key) + except Exception as e: # noqa: BLE001 # an optional binding must not make routing depend on Redis + verbose_router_logger.warning( + "Failed to read Claude Code session router binding; using the requested model: %s", + e, + ) + return None + + async def _resolve_claude_code_session_router( + self, + model: str, + registered_model_name: str, + request_kwargs: Mapping[str, object], + ) -> str: + if not any((self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers)): + return registered_model_name + cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs) + if cache_key is None or not isinstance(request_kwargs, dict): + return registered_model_name + if request_kwargs.get("fallback_depth") not in (None, 0): + return registered_model_name + + agent_id: Final = self._request_header(request_kwargs, "x-claude-code-agent-id") + if agent_id is not None: + bound_model: Final = await self._get_claude_code_session_router_binding(cache_key) + if not isinstance(bound_model, str): + return registered_model_name + bound_registered_model: Final = self._get_model_from_alias(model=bound_model) or bound_model + if self._select_pre_routing_strategy(bound_registered_model, request_kwargs) is None: + await self._delete_claude_code_session_router_binding(cache_key) + return registered_model_name + await self._claude_code_session_router_cache.async_set_cache( + key=cache_key, + value=bound_model, + ttl=_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS, + ) + self._stamp_or_clear_metadata_key(request_kwargs, "model_group", bound_model) + return bound_registered_model + + if self._request_header(request_kwargs, "x-app") != "cli": + return registered_model_name + if self._select_pre_routing_strategy(registered_model_name, request_kwargs) is None: + return registered_model_name + await self._claude_code_session_router_cache.async_set_cache( + key=cache_key, + value=model, + ttl=_CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS, + ) + return registered_model_name + async def async_pre_routing_hook( self, model: str, @@ -12584,7 +12828,12 @@ class Router: the alias, since spend metadata is stamped before routing and the response carries the tier group the strategy picked. """ - registered_model_name: Final = self._get_model_from_alias(model=model) or model + requested_registered_model_name: Final = self._get_model_from_alias(model=model) or model + registered_model_name: Final = await self._resolve_claude_code_session_router( + model=model, + registered_model_name=requested_registered_model_name, + request_kwargs=request_kwargs, + ) ######################################################### # Run the routing-plugin pipeline, if any plugins are configured. @@ -13381,6 +13630,9 @@ class Router: def flush_cache(self): litellm.cache = None self.cache.flush_cache() + session_in_memory_cache: Final = self._claude_code_session_router_cache.in_memory_cache + if session_in_memory_cache is not None: + session_in_memory_cache.flush_cache() def reset(self): ## clean up on close diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index bc8df67cc28..ee51add1ca1 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -68,6 +68,36 @@ still resolve to a deployment in `model_list`; this configuration does not creat - abc ``` +### Heuristic v2 + +Set `classifier_type: heuristic_v2` to classify with the bundled calibrated +success-probability model instead of the hand-written weighted scorer + +```yaml +model_list: + - model_name: smart-router + litellm_params: + model: auto_router/complexity_router + complexity_router_config: + classifier_type: heuristic_v2 + tiers: + SIMPLE: luna + MEDIUM: terra + COMPLEX: sol + REASONING: sol-ultra +``` + +No classifier model call or per-model training data is required. The classifier +uses global tier quality, request-type quality, and similar-request cohorts from +the bundled UltraFeedback artifact. It estimates success at every tier, enforces +monotonic probabilities, and returns the first tier meeting the trained 0.75 +threshold. The existing complexity-router tier pool then selects and dispatches +a model from that tier + +Spend logs record `routing_decision.cause: heuristic_v2`, the detected request +type, and all four predicted probabilities. Existing `classifier_type: heuristic` +configurations keep the original weighted scorer unchanged + ### Renaming the tiers `tier_labels` puts your own vocabulary on the four tiers: @@ -157,6 +187,9 @@ model_list: # Replace a routed model that cannot take image input (default: false) modality_routing: true + + # Let that replacement also override a kept session pin, for image turns only (default: false) + modality_pin_override: true ``` ## Usage @@ -197,9 +230,16 @@ vision model sits below the decided tier gets the 400 and an actionable message A same-tier re-pick keeps the decision's cause and adds `modality:image` to `signals`; a tier change or default takeover records `cause: modality_escalation` with the displaced placement (`modality_escalated_from:` or `modality_displaced_default_model`). Escalations are never -pinned by session affinity, and a KEPT session pin bypasses the gate entirely: a session pinned +pinned by session affinity, and by default a KEPT session pin bypasses the gate: a session pinned to a text-only model keeps it even when an image arrives. +Add `modality_pin_override: true` to lift that last exemption. The image turn is then re-placed +the same way every other decision is, and records `cause: modality_pin_override` whether or not +the tier moved, since the model left the pin either way. The pin itself is untouched: the session +affinity write happens upstream of the gate and stores the session's own model, so the next text +turn replays the original pin and the override is never pinned in its place. It does nothing +unless `modality_routing` is also on. + ### Heuristic-first chaining `classifier_type: heuristic_first` runs the local scorer on every request and only calls the LLM @@ -244,6 +284,49 @@ except that the heuristic outcome is the one already computed rather than a seco Spend logs record `routing_decision.cause` as `heuristic_first_short_circuit` when the classifier was skipped, and `llm_classifier` when it ran, so the two are told apart per request. +### Hybrid + +`classifier_type: hybrid` also scores locally first, but it asks a different question than +`heuristic_first`. Where heuristic-first asks how CHEAP the scorer's tier is and pays for the +classifier on everything above a ceiling, hybrid asks how DECIDED the score is and pays for the +classifier only where the score lands near a tier boundary. A confident score keeps its tier at +every tier, the most expensive one included: + +```yaml +model_list: + - model_name: smart-router + litellm_params: + model: auto_router/complexity_router + complexity_router_config: + classifier_type: hybrid + hybrid_boundary_margin: 0.03 + classifier_llm_config: + model: gpt-4o-mini + tiers: + SIMPLE: gpt-4o-mini + MEDIUM: gpt-4o + COMPLEX: claude-sonnet-4 + REASONING: o1-preview +``` + +A request routes on the scorer's own tier when its score is further than `hybrid_boundary_margin` +from every active boundary. Everything else goes to the classifier: a score inside the band, where a +hair's difference would have named the adjacent tier and its model pool, and a prompt where no +dimension fired at all, which has no opinion to be confident about. `hybrid_boundary_margin` is +required for this type and rejected on the others, the same way `heuristic_first_max_tier` is +required for heuristic-first, so the two modes are told apart by the knob each one takes rather than +by a shared field that means something different per type. + +Pick the margin against the score distribution rather than by intuition. The scorer combines a small +set of discretely weighted dimensions, so achievable scores cluster on a lumpy grid instead of +spreading smoothly, and widening the margin admits whole clusters at once rather than a few more +requests. Spend logs record `routing_decision.cause` as `hybrid_short_circuit` when the classifier +was skipped and `llm_classifier` when it ran. + +Operator-defined tier sets (`tier_definitions`) are not supported here, for the same reason they are +not supported under heuristic-first: the scorer only produces the built-in tiers. Classifier failure +behaves exactly as it does under `classifier_type: llm`. + ### Reasoning Override If 2+ reasoning markers are detected in the user message, the request is promoted to the REASONING tier even when the weighted score maps lower, so complex reasoning tasks get the appropriate model. The promotion requires the score to reach `reasoning_override_min_score`, which tracks `tier_boundaries.simple_medium` unless set, so stock phrases on an otherwise trivial prompt cannot buy the top tier. Set it to `0` to promote on the markers alone. diff --git a/litellm/router_strategy/complexity_router/artifacts/ultrafeedback_tiers.json b/litellm/router_strategy/complexity_router/artifacts/ultrafeedback_tiers.json new file mode 100644 index 00000000000..4fcb599907c --- /dev/null +++ b/litellm/router_strategy/complexity_router/artifacts/ultrafeedback_tiers.json @@ -0,0 +1,4069 @@ +{ + "schema_version": 1, + "global_statistics": [ + { + "tier": 1, + "successes": 36619.0, + "observations": 45504.0 + }, + { + "tier": 2, + "successes": 59797.0, + "observations": 70062.0 + }, + { + "tier": 3, + "successes": 48604.0, + "observations": 52245.0 + }, + { + "tier": 4, + "successes": 11393.0, + "observations": 11561.0 + } + ], + "domain_statistics": [ + { + "tier": 1, + "successes": 1592.0, + "observations": 2211.0, + "request_type": "analytical_reasoning" + }, + { + "tier": 2, + "successes": 2654.0, + "observations": 3374.0, + "request_type": "analytical_reasoning" + }, + { + "tier": 3, + "successes": 2243.0, + "observations": 2481.0, + "request_type": "analytical_reasoning" + }, + { + "tier": 4, + "successes": 538.0, + "observations": 546.0, + "request_type": "analytical_reasoning" + }, + { + "tier": 1, + "successes": 750.0, + "observations": 1015.0, + "request_type": "code_generation" + }, + { + "tier": 2, + "successes": 1271.0, + "observations": 1511.0, + "request_type": "code_generation" + }, + { + "tier": 3, + "successes": 1030.0, + "observations": 1111.0, + "request_type": "code_generation" + }, + { + "tier": 4, + "successes": 233.0, + "observations": 235.0, + "request_type": "code_generation" + }, + { + "tier": 1, + "successes": 243.0, + "observations": 277.0, + "request_type": "code_understanding" + }, + { + "tier": 2, + "successes": 385.0, + "observations": 425.0, + "request_type": "code_understanding" + }, + { + "tier": 3, + "successes": 322.0, + "observations": 334.0, + "request_type": "code_understanding" + }, + { + "tier": 4, + "successes": 74.0, + "observations": 76.0, + "request_type": "code_understanding" + }, + { + "tier": 1, + "successes": 2014.0, + "observations": 2170.0, + "request_type": "factual_lookup" + }, + { + "tier": 2, + "successes": 3120.0, + "observations": 3266.0, + "request_type": "factual_lookup" + }, + { + "tier": 3, + "successes": 2612.0, + "observations": 2670.0, + "request_type": "factual_lookup" + }, + { + "tier": 4, + "successes": 540.0, + "observations": 542.0, + "request_type": "factual_lookup" + }, + { + "tier": 1, + "successes": 30571.0, + "observations": 38161.0, + "request_type": "general" + }, + { + "tier": 2, + "successes": 50037.0, + "observations": 58821.0, + "request_type": "general" + }, + { + "tier": 3, + "successes": 40460.0, + "observations": 43618.0, + "request_type": "general" + }, + { + "tier": 4, + "successes": 9565.0, + "observations": 9716.0, + "request_type": "general" + }, + { + "tier": 1, + "successes": 159.0, + "observations": 170.0, + "request_type": "technical_design" + }, + { + "tier": 2, + "successes": 282.0, + "observations": 303.0, + "request_type": "technical_design" + }, + { + "tier": 3, + "successes": 231.0, + "observations": 236.0, + "request_type": "technical_design" + }, + { + "tier": 4, + "successes": 47.0, + "observations": 47.0, + "request_type": "technical_design" + }, + { + "tier": 1, + "successes": 1290.0, + "observations": 1500.0, + "request_type": "writing" + }, + { + "tier": 2, + "successes": 2048.0, + "observations": 2362.0, + "request_type": "writing" + }, + { + "tier": 3, + "successes": 1706.0, + "observations": 1795.0, + "request_type": "writing" + }, + { + "tier": 4, + "successes": 396.0, + "observations": 399.0, + "request_type": "writing" + } + ], + "cohort_statistics": [ + { + "tier": 1, + "successes": 272.0, + "observations": 372.0, + "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 420.0, + "observations": 519.0, + "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 341.0, + "observations": 381.0, + "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 82.0, + "observations": 84.0, + "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 0.0, + "observations": 1.0, + "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 0.0, + "observations": 2.0, + "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 11.0, + "observations": 18.0, + "cohort": "analytical_reasoning|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 33.0, + "observations": 45.0, + "cohort": "analytical_reasoning|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 28.0, + "observations": 34.0, + "cohort": "analytical_reasoning|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 3.0, + "observations": 3.0, + "cohort": "analytical_reasoning|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 131.0, + "observations": 176.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 210.0, + "observations": 274.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 187.0, + "observations": 209.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 39.0, + "observations": 41.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 4.0, + "observations": 9.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 11.0, + "observations": 13.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 6.0, + "observations": 7.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 3.0, + "observations": 3.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 10.0, + "observations": 15.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 14.0, + "observations": 16.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 15.0, + "observations": 16.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 5.0, + "observations": 5.0, + "cohort": "analytical_reasoning|long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 15.0, + "observations": 20.0, + "cohort": "analytical_reasoning|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 33.0, + "observations": 38.0, + "cohort": "analytical_reasoning|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 28.0, + "observations": 29.0, + "cohort": "analytical_reasoning|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 5.0, + "observations": 5.0, + "cohort": "analytical_reasoning|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 2.0, + "cohort": "analytical_reasoning|long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 2.0, + "cohort": "analytical_reasoning|long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 26.0, + "observations": 34.0, + "cohort": "analytical_reasoning|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 39.0, + "observations": 50.0, + "cohort": "analytical_reasoning|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 42.0, + "observations": 45.0, + "cohort": "analytical_reasoning|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 11.0, + "observations": 11.0, + "cohort": "analytical_reasoning|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 3.0, + "cohort": "analytical_reasoning|long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 4.0, + "observations": 4.0, + "cohort": "analytical_reasoning|long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 452.0, + "observations": 634.0, + "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 745.0, + "observations": 962.0, + "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 634.0, + "observations": 691.0, + "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 151.0, + "observations": 153.0, + "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 3.0, + "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 4.0, + "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|medium|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 10.0, + "observations": 20.0, + "cohort": "analytical_reasoning|medium|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 30.0, + "observations": 44.0, + "cohort": "analytical_reasoning|medium|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 12.0, + "observations": 15.0, + "cohort": "analytical_reasoning|medium|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 5.0, + "observations": 5.0, + "cohort": "analytical_reasoning|medium|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 178.0, + "observations": 270.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 304.0, + "observations": 402.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 266.0, + "observations": 295.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 73.0, + "observations": 73.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 7.0, + "observations": 18.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 17.0, + "observations": 32.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 17.0, + "observations": 27.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 7.0, + "observations": 7.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 11.0, + "observations": 17.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 17.0, + "observations": 25.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 10.0, + "observations": 16.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 6.0, + "observations": 6.0, + "cohort": "analytical_reasoning|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 58.0, + "observations": 72.0, + "cohort": "analytical_reasoning|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 90.0, + "observations": 103.0, + "cohort": "analytical_reasoning|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 72.0, + "observations": 78.0, + "cohort": "analytical_reasoning|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 19.0, + "observations": 19.0, + "cohort": "analytical_reasoning|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|medium|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 2.0, + "cohort": "analytical_reasoning|medium|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|medium|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 37.0, + "observations": 56.0, + "cohort": "analytical_reasoning|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 95.0, + "observations": 110.0, + "cohort": "analytical_reasoning|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 73.0, + "observations": 82.0, + "cohort": "analytical_reasoning|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 20.0, + "observations": 20.0, + "cohort": "analytical_reasoning|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 69.0, + "observations": 83.0, + "cohort": "analytical_reasoning|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 102.0, + "observations": 110.0, + "cohort": "analytical_reasoning|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 93.0, + "observations": 98.0, + "cohort": "analytical_reasoning|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 17.0, + "observations": 17.0, + "cohort": "analytical_reasoning|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|short|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 6.0, + "observations": 6.0, + "cohort": "analytical_reasoning|short|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|short|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 21.0, + "observations": 32.0, + "cohort": "analytical_reasoning|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 45.0, + "observations": 59.0, + "cohort": "analytical_reasoning|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 45.0, + "observations": 48.0, + "cohort": "analytical_reasoning|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 9.0, + "observations": 9.0, + "cohort": "analytical_reasoning|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|short|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 2.0, + "cohort": "analytical_reasoning|short|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|short|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 7.0, + "observations": 9.0, + "cohort": "analytical_reasoning|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 8.0, + "observations": 12.0, + "cohort": "analytical_reasoning|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 11.0, + "observations": 11.0, + "cohort": "analytical_reasoning|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 5.0, + "observations": 9.0, + "cohort": "analytical_reasoning|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 9.0, + "observations": 11.0, + "cohort": "analytical_reasoning|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 8.0, + "observations": 8.0, + "cohort": "analytical_reasoning|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 136.0, + "observations": 176.0, + "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 221.0, + "observations": 267.0, + "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 172.0, + "observations": 194.0, + "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 37.0, + "observations": 39.0, + "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 6.0, + "observations": 11.0, + "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 10.0, + "observations": 15.0, + "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 7.0, + "observations": 8.0, + "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 2.0, + "observations": 2.0, + "cohort": "analytical_reasoning|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 68.0, + "observations": 84.0, + "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 103.0, + "observations": 136.0, + "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 113.0, + "observations": 119.0, + "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 21.0, + "observations": 21.0, + "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 3.0, + "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 4.0, + "observations": 4.0, + "cohort": "analytical_reasoning|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 13.0, + "observations": 16.0, + "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 27.0, + "observations": 34.0, + "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 15.0, + "observations": 16.0, + "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 6.0, + "observations": 6.0, + "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 4.0, + "observations": 5.0, + "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 4.0, + "observations": 7.0, + "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 3.0, + "observations": 3.0, + "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|very_long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 32.0, + "observations": 39.0, + "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 48.0, + "observations": 63.0, + "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 35.0, + "observations": 40.0, + "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 14.0, + "observations": 14.0, + "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 3.0, + "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 2.0, + "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 2.0, + "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "analytical_reasoning|very_long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 28.0, + "observations": 31.0, + "cohort": "code_generation|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 41.0, + "observations": 46.0, + "cohort": "code_generation|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 25.0, + "observations": 26.0, + "cohort": "code_generation|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 5.0, + "observations": 5.0, + "cohort": "code_generation|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 6.0, + "observations": 9.0, + "cohort": "code_generation|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 20.0, + "observations": 23.0, + "cohort": "code_generation|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 16.0, + "observations": 16.0, + "cohort": "code_generation|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 4.0, + "observations": 4.0, + "cohort": "code_generation|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 46.0, + "observations": 60.0, + "cohort": "code_generation|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 72.0, + "observations": 91.0, + "cohort": "code_generation|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 59.0, + "observations": 64.0, + "cohort": "code_generation|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 17.0, + "observations": 17.0, + "cohort": "code_generation|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 32.0, + "observations": 49.0, + "cohort": "code_generation|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 60.0, + "observations": 76.0, + "cohort": "code_generation|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 48.0, + "observations": 52.0, + "cohort": "code_generation|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 15.0, + "observations": 15.0, + "cohort": "code_generation|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 93.0, + "observations": 121.0, + "cohort": "code_generation|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 149.0, + "observations": 170.0, + "cohort": "code_generation|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 146.0, + "observations": 151.0, + "cohort": "code_generation|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 26.0, + "observations": 26.0, + "cohort": "code_generation|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 12.0, + "observations": 16.0, + "cohort": "code_generation|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 22.0, + "observations": 25.0, + "cohort": "code_generation|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 13.0, + "observations": 15.0, + "cohort": "code_generation|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 4.0, + "observations": 4.0, + "cohort": "code_generation|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 196.0, + "observations": 268.0, + "cohort": "code_generation|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 312.0, + "observations": 370.0, + "cohort": "code_generation|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 234.0, + "observations": 253.0, + "cohort": "code_generation|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 56.0, + "observations": 57.0, + "cohort": "code_generation|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 0.0, + "observations": 1.0, + "cohort": "code_generation|medium|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 2.0, + "cohort": "code_generation|medium|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "code_generation|medium|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 40.0, + "observations": 59.0, + "cohort": "code_generation|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 73.0, + "observations": 92.0, + "cohort": "code_generation|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 78.0, + "observations": 83.0, + "cohort": "code_generation|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 14.0, + "observations": 14.0, + "cohort": "code_generation|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 3.0, + "cohort": "code_generation|medium|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "code_generation|medium|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 106.0, + "observations": 140.0, + "cohort": "code_generation|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 220.0, + "observations": 247.0, + "cohort": "code_generation|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 167.0, + "observations": 178.0, + "cohort": "code_generation|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 43.0, + "observations": 43.0, + "cohort": "code_generation|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 4.0, + "cohort": "code_generation|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 7.0, + "observations": 9.0, + "cohort": "code_generation|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 7.0, + "observations": 7.0, + "cohort": "code_generation|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 147.0, + "observations": 199.0, + "cohort": "code_generation|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 233.0, + "observations": 269.0, + "cohort": "code_generation|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 189.0, + "observations": 209.0, + "cohort": "code_generation|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 34.0, + "observations": 35.0, + "cohort": "code_generation|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 7.0, + "observations": 10.0, + "cohort": "code_generation|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 18.0, + "observations": 22.0, + "cohort": "code_generation|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 10.0, + "observations": 10.0, + "cohort": "code_generation|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 2.0, + "observations": 2.0, + "cohort": "code_generation|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 4.0, + "cohort": "code_generation|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 5.0, + "observations": 8.0, + "cohort": "code_generation|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 7.0, + "observations": 8.0, + "cohort": "code_generation|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 2.0, + "cohort": "code_generation|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 3.0, + "observations": 5.0, + "cohort": "code_generation|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 3.0, + "observations": 3.0, + "cohort": "code_generation|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 2.0, + "observations": 2.0, + "cohort": "code_generation|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 7.0, + "observations": 9.0, + "cohort": "code_generation|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 7.0, + "observations": 8.0, + "cohort": "code_generation|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 9.0, + "observations": 9.0, + "cohort": "code_generation|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 2.0, + "observations": 2.0, + "cohort": "code_generation|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 24.0, + "observations": 33.0, + "cohort": "code_generation|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 25.0, + "observations": 45.0, + "cohort": "code_generation|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 19.0, + "observations": 27.0, + "cohort": "code_generation|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 7.0, + "observations": 7.0, + "cohort": "code_generation|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 2.0, + "cohort": "code_understanding|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 2.0, + "cohort": "code_understanding|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 6.0, + "observations": 6.0, + "cohort": "code_understanding|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 2.0, + "observations": 2.0, + "cohort": "code_understanding|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 5.0, + "cohort": "code_understanding|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 3.0, + "observations": 3.0, + "cohort": "code_understanding|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 3.0, + "cohort": "code_understanding|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "code_understanding|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 17.0, + "observations": 23.0, + "cohort": "code_understanding|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 27.0, + "observations": 37.0, + "cohort": "code_understanding|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 32.0, + "observations": 33.0, + "cohort": "code_understanding|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 3.0, + "observations": 3.0, + "cohort": "code_understanding|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 24.0, + "observations": 28.0, + "cohort": "code_understanding|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 37.0, + "observations": 44.0, + "cohort": "code_understanding|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 33.0, + "observations": 33.0, + "cohort": "code_understanding|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 7.0, + "observations": 7.0, + "cohort": "code_understanding|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 8.0, + "observations": 9.0, + "cohort": "code_understanding|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 19.0, + "observations": 21.0, + "cohort": "code_understanding|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 10.0, + "observations": 10.0, + "cohort": "code_understanding|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 4.0, + "observations": 4.0, + "cohort": "code_understanding|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 3.0, + "cohort": "code_understanding|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 3.0, + "cohort": "code_understanding|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "code_understanding|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "code_understanding|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 73.0, + "observations": 76.0, + "cohort": "code_understanding|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 107.0, + "observations": 111.0, + "cohort": "code_understanding|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 87.0, + "observations": 87.0, + "cohort": "code_understanding|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 22.0, + "observations": 22.0, + "cohort": "code_understanding|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 32.0, + "observations": 33.0, + "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 44.0, + "observations": 44.0, + "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 44.0, + "observations": 47.0, + "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 8.0, + "observations": 8.0, + "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 0.0, + "observations": 1.0, + "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 0.0, + "observations": 2.0, + "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "code_understanding|medium|code=1|math=1|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 15.0, + "observations": 16.0, + "cohort": "code_understanding|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 23.0, + "observations": 23.0, + "cohort": "code_understanding|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 12.0, + "observations": 14.0, + "cohort": "code_understanding|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 3.0, + "observations": 3.0, + "cohort": "code_understanding|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 3.0, + "cohort": "code_understanding|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "code_understanding|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 36.0, + "observations": 43.0, + "cohort": "code_understanding|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 71.0, + "observations": 74.0, + "cohort": "code_understanding|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 48.0, + "observations": 50.0, + "cohort": "code_understanding|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 12.0, + "observations": 13.0, + "cohort": "code_understanding|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 5.0, + "observations": 6.0, + "cohort": "code_understanding|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 10.0, + "observations": 10.0, + "cohort": "code_understanding|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 7.0, + "observations": 7.0, + "cohort": "code_understanding|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "code_understanding|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "code_understanding|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 0.0, + "observations": 1.0, + "cohort": "code_understanding|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "code_understanding|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 9.0, + "observations": 11.0, + "cohort": "code_understanding|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 18.0, + "observations": 23.0, + "cohort": "code_understanding|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 11.0, + "observations": 13.0, + "cohort": "code_understanding|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "code_understanding|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 3.0, + "cohort": "code_understanding|very_long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 1.0, + "cohort": "code_understanding|very_long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 13.0, + "observations": 17.0, + "cohort": "code_understanding|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 19.0, + "observations": 23.0, + "cohort": "code_understanding|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 27.0, + "observations": 28.0, + "cohort": "code_understanding|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 7.0, + "observations": 8.0, + "cohort": "code_understanding|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 49.0, + "observations": 54.0, + "cohort": "factual_lookup|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 64.0, + "observations": 75.0, + "cohort": "factual_lookup|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 80.0, + "observations": 80.0, + "cohort": "factual_lookup|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 15.0, + "observations": 15.0, + "cohort": "factual_lookup|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 9.0, + "observations": 9.0, + "cohort": "factual_lookup|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 12.0, + "observations": 14.0, + "cohort": "factual_lookup|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 9.0, + "observations": 10.0, + "cohort": "factual_lookup|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 3.0, + "observations": 3.0, + "cohort": "factual_lookup|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "factual_lookup|long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 3.0, + "observations": 3.0, + "cohort": "factual_lookup|long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 19.0, + "observations": 22.0, + "cohort": "factual_lookup|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 35.0, + "observations": 38.0, + "cohort": "factual_lookup|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 37.0, + "observations": 41.0, + "cohort": "factual_lookup|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 7.0, + "observations": 7.0, + "cohort": "factual_lookup|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 66.0, + "observations": 75.0, + "cohort": "factual_lookup|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 73.0, + "observations": 86.0, + "cohort": "factual_lookup|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 69.0, + "observations": 74.0, + "cohort": "factual_lookup|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 21.0, + "observations": 21.0, + "cohort": "factual_lookup|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "factual_lookup|long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 2.0, + "cohort": "factual_lookup|long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "factual_lookup|long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 174.0, + "observations": 181.0, + "cohort": "factual_lookup|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 204.0, + "observations": 215.0, + "cohort": "factual_lookup|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 206.0, + "observations": 210.0, + "cohort": "factual_lookup|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 42.0, + "observations": 42.0, + "cohort": "factual_lookup|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 31.0, + "observations": 37.0, + "cohort": "factual_lookup|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 41.0, + "observations": 48.0, + "cohort": "factual_lookup|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 44.0, + "observations": 45.0, + "cohort": "factual_lookup|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 10.0, + "observations": 10.0, + "cohort": "factual_lookup|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 0.0, + "observations": 1.0, + "cohort": "factual_lookup|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "factual_lookup|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "factual_lookup|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 131.0, + "observations": 142.0, + "cohort": "factual_lookup|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 162.0, + "observations": 171.0, + "cohort": "factual_lookup|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 146.0, + "observations": 151.0, + "cohort": "factual_lookup|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 24.0, + "observations": 24.0, + "cohort": "factual_lookup|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 41.0, + "observations": 46.0, + "cohort": "factual_lookup|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 55.0, + "observations": 59.0, + "cohort": "factual_lookup|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 45.0, + "observations": 46.0, + "cohort": "factual_lookup|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 9.0, + "observations": 9.0, + "cohort": "factual_lookup|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1382.0, + "observations": 1473.0, + "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 2290.0, + "observations": 2348.0, + "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 1824.0, + "observations": 1853.0, + "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 368.0, + "observations": 370.0, + "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 5.0, + "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 12.0, + "observations": 13.0, + "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 7.0, + "observations": 7.0, + "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 3.0, + "observations": 3.0, + "cohort": "factual_lookup|short|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 11.0, + "observations": 16.0, + "cohort": "factual_lookup|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 28.0, + "observations": 31.0, + "cohort": "factual_lookup|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 15.0, + "observations": 17.0, + "cohort": "factual_lookup|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 4.0, + "observations": 4.0, + "cohort": "factual_lookup|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 36.0, + "observations": 39.0, + "cohort": "factual_lookup|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 63.0, + "observations": 66.0, + "cohort": "factual_lookup|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 64.0, + "observations": 66.0, + "cohort": "factual_lookup|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 13.0, + "observations": 13.0, + "cohort": "factual_lookup|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 4.0, + "observations": 4.0, + "cohort": "factual_lookup|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 5.0, + "observations": 6.0, + "cohort": "factual_lookup|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 4.0, + "observations": 5.0, + "cohort": "factual_lookup|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "factual_lookup|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 27.0, + "observations": 31.0, + "cohort": "factual_lookup|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 29.0, + "observations": 37.0, + "cohort": "factual_lookup|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 26.0, + "observations": 29.0, + "cohort": "factual_lookup|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 11.0, + "observations": 11.0, + "cohort": "factual_lookup|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "factual_lookup|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 0.0, + "observations": 2.0, + "cohort": "factual_lookup|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "factual_lookup|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 7.0, + "observations": 8.0, + "cohort": "factual_lookup|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 17.0, + "observations": 20.0, + "cohort": "factual_lookup|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 14.0, + "observations": 14.0, + "cohort": "factual_lookup|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 2.0, + "observations": 2.0, + "cohort": "factual_lookup|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 4.0, + "observations": 4.0, + "cohort": "factual_lookup|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 6.0, + "observations": 6.0, + "cohort": "factual_lookup|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 5.0, + "observations": 5.0, + "cohort": "factual_lookup|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "factual_lookup|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 17.0, + "observations": 21.0, + "cohort": "factual_lookup|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 19.0, + "observations": 25.0, + "cohort": "factual_lookup|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 13.0, + "observations": 13.0, + "cohort": "factual_lookup|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 5.0, + "observations": 5.0, + "cohort": "factual_lookup|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2828.0, + "observations": 3552.0, + "cohort": "general|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 4621.0, + "observations": 5551.0, + "cohort": "general|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 3628.0, + "observations": 3931.0, + "cohort": "general|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 911.0, + "observations": 922.0, + "cohort": "general|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 127.0, + "observations": 406.0, + "cohort": "general|long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 250.0, + "observations": 583.0, + "cohort": "general|long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 219.0, + "observations": 396.0, + "cohort": "general|long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 98.0, + "observations": 107.0, + "cohort": "general|long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 94.0, + "observations": 127.0, + "cohort": "general|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 168.0, + "observations": 196.0, + "cohort": "general|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 135.0, + "observations": 146.0, + "cohort": "general|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 34.0, + "observations": 35.0, + "cohort": "general|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 912.0, + "observations": 1219.0, + "cohort": "general|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 1417.0, + "observations": 1736.0, + "cohort": "general|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 1163.0, + "observations": 1258.0, + "cohort": "general|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 278.0, + "observations": 283.0, + "cohort": "general|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 33.0, + "observations": 122.0, + "cohort": "general|long|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 41.0, + "observations": 150.0, + "cohort": "general|long|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 61.0, + "observations": 96.0, + "cohort": "general|long|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 24.0, + "observations": 24.0, + "cohort": "general|long|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 16.0, + "observations": 24.0, + "cohort": "general|long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 24.0, + "observations": 34.0, + "cohort": "general|long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 18.0, + "observations": 21.0, + "cohort": "general|long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 5.0, + "observations": 5.0, + "cohort": "general|long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 271.0, + "observations": 348.0, + "cohort": "general|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 488.0, + "observations": 555.0, + "cohort": "general|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 369.0, + "observations": 386.0, + "cohort": "general|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 67.0, + "observations": 71.0, + "cohort": "general|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 3.0, + "cohort": "general|long|code=1|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 0.0, + "observations": 3.0, + "cohort": "general|long|code=1|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "general|long|code=1|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 9.0, + "observations": 9.0, + "cohort": "general|long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 14.0, + "observations": 17.0, + "cohort": "general|long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 9.0, + "observations": 10.0, + "cohort": "general|long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 4.0, + "observations": 4.0, + "cohort": "general|long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 340.0, + "observations": 426.0, + "cohort": "general|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 544.0, + "observations": 635.0, + "cohort": "general|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 472.0, + "observations": 491.0, + "cohort": "general|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 120.0, + "observations": 120.0, + "cohort": "general|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "general|long|code=1|math=1|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "general|long|code=1|math=1|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "general|long|code=1|math=1|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 0.0, + "observations": 2.0, + "cohort": "general|long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 1.0, + "cohort": "general|long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 0.0, + "observations": 1.0, + "cohort": "general|long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 8332.0, + "observations": 10133.0, + "cohort": "general|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 13225.0, + "observations": 15509.0, + "cohort": "general|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 10614.0, + "observations": 11347.0, + "cohort": "general|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 2499.0, + "observations": 2531.0, + "cohort": "general|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 437.0, + "observations": 1294.0, + "cohort": "general|medium|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 863.0, + "observations": 1932.0, + "cohort": "general|medium|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 738.0, + "observations": 1269.0, + "cohort": "general|medium|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 296.0, + "observations": 325.0, + "cohort": "general|medium|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 117.0, + "observations": 173.0, + "cohort": "general|medium|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 183.0, + "observations": 252.0, + "cohort": "general|medium|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 171.0, + "observations": 191.0, + "cohort": "general|medium|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 47.0, + "observations": 52.0, + "cohort": "general|medium|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 927.0, + "observations": 1353.0, + "cohort": "general|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 1550.0, + "observations": 2023.0, + "cohort": "general|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 1273.0, + "observations": 1430.0, + "cohort": "general|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 348.0, + "observations": 354.0, + "cohort": "general|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 60.0, + "observations": 255.0, + "cohort": "general|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 143.0, + "observations": 377.0, + "cohort": "general|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 145.0, + "observations": 243.0, + "cohort": "general|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 58.0, + "observations": 61.0, + "cohort": "general|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 27.0, + "observations": 47.0, + "cohort": "general|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 30.0, + "observations": 48.0, + "cohort": "general|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 30.0, + "observations": 36.0, + "cohort": "general|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 5.0, + "observations": 5.0, + "cohort": "general|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 1119.0, + "observations": 1348.0, + "cohort": "general|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 1816.0, + "observations": 2024.0, + "cohort": "general|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 1472.0, + "observations": 1552.0, + "cohort": "general|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 301.0, + "observations": 304.0, + "cohort": "general|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "general|medium|code=1|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 1.0, + "cohort": "general|medium|code=1|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "general|medium|code=1|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 8.0, + "observations": 10.0, + "cohort": "general|medium|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 9.0, + "observations": 12.0, + "cohort": "general|medium|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 8.0, + "observations": 9.0, + "cohort": "general|medium|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "general|medium|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 349.0, + "observations": 439.0, + "cohort": "general|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 549.0, + "observations": 622.0, + "cohort": "general|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 495.0, + "observations": 526.0, + "cohort": "general|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 111.0, + "observations": 113.0, + "cohort": "general|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "general|medium|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 4.0, + "observations": 6.0, + "cohort": "general|medium|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 4.0, + "observations": 4.0, + "cohort": "general|medium|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "general|medium|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 11635.0, + "observations": 12910.0, + "cohort": "general|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 19249.0, + "observations": 20591.0, + "cohort": "general|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 15415.0, + "observations": 15867.0, + "cohort": "general|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 3362.0, + "observations": 3392.0, + "cohort": "general|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 68.0, + "observations": 155.0, + "cohort": "general|short|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 113.0, + "observations": 202.0, + "cohort": "general|short|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 97.0, + "observations": 124.0, + "cohort": "general|short|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 31.0, + "observations": 31.0, + "cohort": "general|short|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 6.0, + "observations": 11.0, + "cohort": "general|short|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 20.0, + "observations": 21.0, + "cohort": "general|short|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 23.0, + "observations": 24.0, + "cohort": "general|short|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 4.0, + "observations": 4.0, + "cohort": "general|short|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 199.0, + "observations": 270.0, + "cohort": "general|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 334.0, + "observations": 414.0, + "cohort": "general|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 309.0, + "observations": 337.0, + "cohort": "general|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 75.0, + "observations": 75.0, + "cohort": "general|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 5.0, + "cohort": "general|short|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 5.0, + "cohort": "general|short|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "general|short|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "general|short|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 2.0, + "cohort": "general|short|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "general|short|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 629.0, + "observations": 778.0, + "cohort": "general|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 1131.0, + "observations": 1286.0, + "cohort": "general|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 919.0, + "observations": 989.0, + "cohort": "general|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 227.0, + "observations": 227.0, + "cohort": "general|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 4.0, + "cohort": "general|short|code=1|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 3.0, + "observations": 3.0, + "cohort": "general|short|code=1|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 4.0, + "observations": 5.0, + "cohort": "general|short|code=1|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 2.0, + "cohort": "general|short|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 2.0, + "cohort": "general|short|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 28.0, + "observations": 37.0, + "cohort": "general|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 54.0, + "observations": 64.0, + "cohort": "general|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 41.0, + "observations": 46.0, + "cohort": "general|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 9.0, + "observations": 9.0, + "cohort": "general|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1066.0, + "observations": 1437.0, + "cohort": "general|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 1639.0, + "observations": 2007.0, + "cohort": "general|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 1397.0, + "observations": 1507.0, + "cohort": "general|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 333.0, + "observations": 341.0, + "cohort": "general|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 6.0, + "cohort": "general|very_long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 4.0, + "cohort": "general|very_long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 3.0, + "observations": 3.0, + "cohort": "general|very_long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 3.0, + "observations": 3.0, + "cohort": "general|very_long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 65.0, + "observations": 89.0, + "cohort": "general|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 119.0, + "observations": 139.0, + "cohort": "general|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 98.0, + "observations": 103.0, + "cohort": "general|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 21.0, + "observations": 21.0, + "cohort": "general|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 403.0, + "observations": 546.0, + "cohort": "general|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 700.0, + "observations": 875.0, + "cohort": "general|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 556.0, + "observations": 612.0, + "cohort": "general|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 133.0, + "observations": 135.0, + "cohort": "general|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 17.0, + "observations": 27.0, + "cohort": "general|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 19.0, + "observations": 33.0, + "cohort": "general|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 25.0, + "observations": 27.0, + "cohort": "general|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 9.0, + "observations": 9.0, + "cohort": "general|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 210.0, + "observations": 280.0, + "cohort": "general|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 307.0, + "observations": 378.0, + "cohort": "general|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 258.0, + "observations": 289.0, + "cohort": "general|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 64.0, + "observations": 65.0, + "cohort": "general|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 17.0, + "observations": 23.0, + "cohort": "general|very_long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 31.0, + "observations": 39.0, + "cohort": "general|very_long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 16.0, + "observations": 19.0, + "cohort": "general|very_long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 3.0, + "observations": 3.0, + "cohort": "general|very_long|code=1|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 202.0, + "observations": 282.0, + "cohort": "general|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 361.0, + "observations": 478.0, + "cohort": "general|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 262.0, + "observations": 310.0, + "cohort": "general|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 82.0, + "observations": 82.0, + "cohort": "general|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 4.0, + "observations": 5.0, + "cohort": "general|very_long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 9.0, + "observations": 11.0, + "cohort": "general|very_long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 4.0, + "observations": 4.0, + "cohort": "general|very_long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 19.0, + "observations": 21.0, + "cohort": "technical_design|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 23.0, + "observations": 25.0, + "cohort": "technical_design|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 28.0, + "observations": 29.0, + "cohort": "technical_design|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 5.0, + "observations": 5.0, + "cohort": "technical_design|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 2.0, + "cohort": "technical_design|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 1.0, + "cohort": "technical_design|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "technical_design|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 3.0, + "cohort": "technical_design|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 8.0, + "observations": 9.0, + "cohort": "technical_design|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 9.0, + "observations": 9.0, + "cohort": "technical_design|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 3.0, + "observations": 3.0, + "cohort": "technical_design|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 14.0, + "observations": 15.0, + "cohort": "technical_design|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 24.0, + "observations": 25.0, + "cohort": "technical_design|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 18.0, + "observations": 18.0, + "cohort": "technical_design|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 2.0, + "observations": 2.0, + "cohort": "technical_design|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 2.0, + "cohort": "technical_design|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 3.0, + "observations": 3.0, + "cohort": "technical_design|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "technical_design|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "technical_design|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 44.0, + "observations": 45.0, + "cohort": "technical_design|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 84.0, + "observations": 86.0, + "cohort": "technical_design|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 74.0, + "observations": 74.0, + "cohort": "technical_design|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 15.0, + "observations": 15.0, + "cohort": "technical_design|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 3.0, + "cohort": "technical_design|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 6.0, + "observations": 6.0, + "cohort": "technical_design|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 3.0, + "observations": 3.0, + "cohort": "technical_design|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 23.0, + "observations": 24.0, + "cohort": "technical_design|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 47.0, + "observations": 53.0, + "cohort": "technical_design|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 28.0, + "observations": 29.0, + "cohort": "technical_design|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 10.0, + "observations": 10.0, + "cohort": "technical_design|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 4.0, + "observations": 4.0, + "cohort": "technical_design|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 7.0, + "observations": 7.0, + "cohort": "technical_design|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 7.0, + "observations": 8.0, + "cohort": "technical_design|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "technical_design|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 13.0, + "observations": 13.0, + "cohort": "technical_design|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 31.0, + "observations": 31.0, + "cohort": "technical_design|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 23.0, + "observations": 23.0, + "cohort": "technical_design|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 5.0, + "observations": 5.0, + "cohort": "technical_design|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 2.0, + "cohort": "technical_design|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 3.0, + "observations": 4.0, + "cohort": "technical_design|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "technical_design|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 10.0, + "observations": 15.0, + "cohort": "technical_design|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 26.0, + "observations": 30.0, + "cohort": "technical_design|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 19.0, + "observations": 21.0, + "cohort": "technical_design|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 2.0, + "observations": 2.0, + "cohort": "technical_design|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 3.0, + "cohort": "technical_design|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 2.0, + "cohort": "technical_design|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "technical_design|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "technical_design|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 5.0, + "observations": 6.0, + "cohort": "technical_design|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 6.0, + "observations": 7.0, + "cohort": "technical_design|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 6.0, + "observations": 6.0, + "cohort": "technical_design|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "technical_design|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "technical_design|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 4.0, + "observations": 6.0, + "cohort": "technical_design|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "technical_design|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 8.0, + "observations": 8.0, + "cohort": "technical_design|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 5.0, + "observations": 6.0, + "cohort": "technical_design|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 5.0, + "observations": 5.0, + "cohort": "technical_design|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "technical_design|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 3.0, + "cohort": "technical_design|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 2.0, + "cohort": "technical_design|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 3.0, + "observations": 3.0, + "cohort": "technical_design|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 176.0, + "observations": 226.0, + "cohort": "writing|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 267.0, + "observations": 329.0, + "cohort": "writing|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 229.0, + "observations": 245.0, + "cohort": "writing|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 51.0, + "observations": 52.0, + "cohort": "writing|long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 0.0, + "observations": 2.0, + "cohort": "writing|long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 2.0, + "cohort": "writing|long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 3.0, + "observations": 3.0, + "cohort": "writing|long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|long|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 5.0, + "cohort": "writing|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 4.0, + "observations": 5.0, + "cohort": "writing|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "writing|long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 36.0, + "observations": 47.0, + "cohort": "writing|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 53.0, + "observations": 59.0, + "cohort": "writing|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 47.0, + "observations": 51.0, + "cohort": "writing|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 14.0, + "observations": 15.0, + "cohort": "writing|long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 17.0, + "observations": 21.0, + "cohort": "writing|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 27.0, + "observations": 32.0, + "cohort": "writing|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 24.0, + "observations": 24.0, + "cohort": "writing|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 3.0, + "observations": 3.0, + "cohort": "writing|long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 2.0, + "cohort": "writing|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 2.0, + "cohort": "writing|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 6.0, + "observations": 6.0, + "cohort": "writing|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 2.0, + "observations": 2.0, + "cohort": "writing|long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 541.0, + "observations": 598.0, + "cohort": "writing|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 892.0, + "observations": 997.0, + "cohort": "writing|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 728.0, + "observations": 756.0, + "cohort": "writing|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 164.0, + "observations": 165.0, + "cohort": "writing|medium|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 7.0, + "cohort": "writing|medium|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 7.0, + "observations": 15.0, + "cohort": "writing|medium|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 7.0, + "observations": 10.0, + "cohort": "writing|medium|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 4.0, + "observations": 4.0, + "cohort": "writing|medium|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 2.0, + "cohort": "writing|medium|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|medium|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|medium|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 25.0, + "observations": 35.0, + "cohort": "writing|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 41.0, + "observations": 63.0, + "cohort": "writing|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 39.0, + "observations": 46.0, + "cohort": "writing|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 12.0, + "observations": 12.0, + "cohort": "writing|medium|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 2.0, + "cohort": "writing|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 4.0, + "cohort": "writing|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|medium|code=0|math=1|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 2.0, + "cohort": "writing|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "writing|medium|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 24.0, + "observations": 27.0, + "cohort": "writing|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 49.0, + "observations": 56.0, + "cohort": "writing|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 46.0, + "observations": 49.0, + "cohort": "writing|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 8.0, + "observations": 8.0, + "cohort": "writing|medium|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 11.0, + "observations": 13.0, + "cohort": "writing|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 13.0, + "observations": 13.0, + "cohort": "writing|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 10.0, + "observations": 10.0, + "cohort": "writing|medium|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 327.0, + "observations": 353.0, + "cohort": "writing|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 508.0, + "observations": 555.0, + "cohort": "writing|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 415.0, + "observations": 432.0, + "cohort": "writing|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 92.0, + "observations": 92.0, + "cohort": "writing|short|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 2.0, + "cohort": "writing|short|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|short|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|short|code=0|math=0|mc=0|intl=1" + }, + { + "tier": 1, + "successes": 4.0, + "observations": 4.0, + "cohort": "writing|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 7.0, + "observations": 7.0, + "cohort": "writing|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 3.0, + "observations": 3.0, + "cohort": "writing|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 2.0, + "observations": 2.0, + "cohort": "writing|short|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 3.0, + "cohort": "writing|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 8.0, + "observations": 9.0, + "cohort": "writing|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 4.0, + "observations": 4.0, + "cohort": "writing|short|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 2.0, + "observations": 2.0, + "cohort": "writing|short|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 63.0, + "observations": 85.0, + "cohort": "writing|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 111.0, + "observations": 139.0, + "cohort": "writing|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 95.0, + "observations": 99.0, + "cohort": "writing|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 29.0, + "observations": 29.0, + "cohort": "writing|very_long|code=0|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 3.0, + "observations": 5.0, + "cohort": "writing|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 2.0, + "cohort": "writing|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 4.0, + "observations": 4.0, + "cohort": "writing|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 4, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|very_long|code=0|math=0|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 26.0, + "observations": 36.0, + "cohort": "writing|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 30.0, + "observations": 44.0, + "cohort": "writing|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 23.0, + "observations": 28.0, + "cohort": "writing|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 8.0, + "observations": 8.0, + "cohort": "writing|very_long|code=0|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 2.0, + "observations": 2.0, + "cohort": "writing|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 1.0, + "observations": 1.0, + "cohort": "writing|very_long|code=0|math=1|mc=1|intl=0" + }, + { + "tier": 1, + "successes": 14.0, + "observations": 14.0, + "cohort": "writing|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 14.0, + "observations": 16.0, + "cohort": "writing|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 6.0, + "observations": 7.0, + "cohort": "writing|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 4, + "successes": 3.0, + "observations": 3.0, + "cohort": "writing|very_long|code=1|math=0|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 5.0, + "observations": 5.0, + "cohort": "writing|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 2, + "successes": 7.0, + "observations": 8.0, + "cohort": "writing|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 3, + "successes": 7.0, + "observations": 7.0, + "cohort": "writing|very_long|code=1|math=1|mc=0|intl=0" + }, + { + "tier": 1, + "successes": 0.0, + "observations": 1.0, + "cohort": "writing|very_long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 2, + "successes": 2.0, + "observations": 2.0, + "cohort": "writing|very_long|code=1|math=1|mc=1|intl=0" + }, + { + "tier": 3, + "successes": 0.0, + "observations": 1.0, + "cohort": "writing|very_long|code=1|math=1|mc=1|intl=0" + } + ], + "domain_prior_mass": 200.0, + "cohort_prior_mass": 20.0, + "routing_threshold": 0.75, + "datasets": [ + { + "name": "openbmb/UltraFeedback", + "url": "https://huggingface.co/datasets/openbmb/UltraFeedback", + "license": "MIT", + "rows": 255864, + "success_definition": "UltraFeedback overall_score >= 4" + } + ], + "success_definition": "UltraFeedback overall_score >= 4", + "split_method": "sha256(prompt): 70% train, 15% validation, 15% test" +} diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 577cee0920d..a00ae6bee80 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -26,13 +26,22 @@ from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast from pydantic import BaseModel, create_model from litellm._logging import verbose_router_logger -from litellm.constants import EMPTY_MAPPING, RETURN_RAW_MODEL_NAME_METADATA_KEY +from litellm.constants import ( + EMPTY_MAPPING, + RETURN_RAW_MODEL_NAME_METADATA_KEY, + SESSION_ID_GENERATED_METADATA_KEY, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload from litellm.llms.base_llm.base_utils import type_to_response_format_param +from litellm.router_strategy.adaptive_router.classifier import classify_prompt +from litellm.router_strategy.complexity_router.tier_predictor import ( + TierSuccessPredictor, + resolve_tier_artifact, +) from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, @@ -742,7 +751,8 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo A modality escalation is transient the same way: it describes what this one call carries (an image), not what the session's traffic looks like, and pinning it would hold every following - text turn on the vision-capable model the image forced. + text turn on the vision-capable model the image forced. A modality pin override is the same + fact on a session that already holds a pin, so it must not overwrite the pin it displaced. """ return decision is None or ( decision.get("cause") @@ -751,6 +761,7 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo "plan_mode", "housekeeping", "modality_escalation", + "modality_pin_override", ) and not decision.get("context_escalated") ) @@ -790,9 +801,11 @@ class ClassificationOutcome(NamedTuple): signals: tuple[str, ...] cause: Literal[ "heuristic_scorer", + "heuristic_v2", "reasoning_override", "llm_classifier", "heuristic_first_short_circuit", + "hybrid_short_circuit", "housekeeping", "classifier_plugin", "classifier_fallback", @@ -978,6 +991,11 @@ class ComplexityRouter(CustomLogger): if llm_classifier_configured else None ) + self._tier_success_predictor: TierSuccessPredictor | None = ( + TierSuccessPredictor(resolve_tier_artifact(self.config.heuristic_v2_artifact)) + if self.config.classifier_type == "heuristic_v2" + else None + ) verbose_router_logger.debug("ComplexityRouter initialized for %s with tiers: %s", model_name, self.config.tiers) @@ -1230,6 +1248,15 @@ class ComplexityRouter(CustomLogger): return tier, weighted_score, tuple(signals), "heuristic_scorer" + def _is_near_tier_boundary(self, score: float, margin: float) -> bool: + boundaries: Final = self._effective_tier_boundaries() + active_boundaries: Final = ( + boundaries["simple_medium"], + boundaries["medium_complex"], + boundaries["complex_reasoning"], + ) + return any(abs(score - boundary) <= margin for boundary in active_boundaries) + def _effective_reasoning_override_min_score(self) -> float: """The score a request must reach before the reasoning-marker override may promote it. @@ -1350,15 +1377,37 @@ class ComplexityRouter(CustomLogger): custom tier set, and classifier_fallback otherwise decides between the heuristic scorer and default_model. The outcome's `cause` reports which path actually ran. """ + if self.config.classifier_type == "heuristic_v2": + return self._classify_with_heuristic_v2(prompt) if self.config.classifier_type == "custom": return await self._classify_with_plugin(prompt, system_prompt, request_kwargs, raw_messages) if self.config.classifier_type == "heuristic_first" and self.config.classifier_llm_config is not None: return await self._classify_heuristic_first(prompt, system_prompt, request_kwargs, messages) + if self.config.classifier_type == "hybrid" and self.config.classifier_llm_config is not None: + return await self._classify_hybrid(prompt, system_prompt, request_kwargs, messages) if self.config.classifier_type != "llm" or self.config.classifier_llm_config is None: tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages) + def _classify_with_heuristic_v2(self, prompt: str) -> ClassificationOutcome: + predictor: Final = self._tier_success_predictor + if predictor is None: + raise ValueError("heuristic v2 predictor is not configured") + request_type: Final = classify_prompt(prompt) + prediction: Final = predictor.predict(prompt, request_type) + tier: Final = TIER_SEVERITY_ORDER[prediction.required_tier - 1] + probability_signals: Final = tuple( + f"tier-probability:{candidate.value.lower()}={prediction.probabilities[index]:.6f}" + for index, candidate in enumerate(TIER_SEVERITY_ORDER, start=1) + ) + return ClassificationOutcome( + tier=tier, + score=None, + signals=(f"request-type:{request_type.value}", *probability_signals), + cause="heuristic_v2", + ) + async def _classify_heuristic_first( self, prompt: str, @@ -1387,6 +1436,29 @@ class ComplexityRouter(CustomLogger): return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="heuristic_first_short_circuit") return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored) + async def _classify_hybrid( + self, + prompt: str, + system_prompt: str | None, + request_kwargs: dict[str, Any] | None, # mutable-ok: handed to _classify_with_llm as-is + messages: Sequence[Mapping[str, object]] | None, + ) -> ClassificationOutcome: + """Score locally, and only pay for the classifier when the score sits near a tier boundary. + + Where heuristic_first asks how CHEAP the scorer's tier is, this asks how DECIDED it is, so a + confident score keeps its tier at every tier including the most expensive one. Two things make + a score undecided: landing within hybrid_boundary_margin of an active boundary, where a + hair's difference in score would have named the adjacent tier and its model pool, and firing + no dimension at all, which scores 0.0 and lands SIMPLE by default rather than by evidence. + """ + tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) + scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) + margin: Final = self.config.hybrid_boundary_margin + decided: Final = margin is not None and bool(signals) and not self._is_near_tier_boundary(score, margin) + if decided: + return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="hybrid_short_circuit") + return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored) + async def _llm_classifier_outcome( self, prompt: str, @@ -2323,8 +2395,11 @@ class ComplexityRouter(CustomLogger): """Replace a routed model that cannot accept this request's image input. The single modality owner, applied to the decided response at the hook's exits so every - routing path is covered uniformly. A KEPT session pin is exempt by design (its cause); - replacement picks and every other path are just responses. The re-placement walks + routing path is covered uniformly. A KEPT session pin is exempt by design (its cause) + unless modality_pin_override is set, in which case the image turn is re-placed and reported + as modality_pin_override while the stored pin, written upstream from the session's own + model, is left for the next text turn; replacement picks and every other path are just + responses. The re-placement walks UPWARD-ONLY from the decision's tier (so a plan-mode floor can never be undercut), picks through `_pick_model_for_tier` so routing plugins still apply, then falls to default_model (never on plugin routers, and never on a plan-floored decision, since @@ -2337,7 +2412,11 @@ class ComplexityRouter(CustomLogger): not self.config.modality_routing or not resolved_messages or response.model is None - or (decision is not None and decision.get("cause") == "session_affinity_pin") + or ( + decision is not None + and decision.get("cause") == "session_affinity_pin" + and not self.config.modality_pin_override + ) or not request_contains_image_content(resolved_messages) or self._model_accepts_image_input(response.model) ): @@ -2379,6 +2458,10 @@ class ComplexityRouter(CustomLogger): self._restamp_adaptive_choice(request_kwargs, response.model, new_model) same_tier: Final = capable is not None and decided == capable base_cause: Final = (decision.get("cause") if decision is not None else None) or "default_fallback" + # Reaching here on a kept pin means modality_pin_override is on, since the guard above + # returns otherwise. The model moved off the pin even on a same-tier repick, so reporting + # the pin's own cause would claim the session's model served a request it did not. + displaced_pin: Final = base_cause == "session_affinity_pin" displaced_default: Final = decided is None and response.model == self.config.default_model markers: Final = ( "modality:image", @@ -2388,7 +2471,7 @@ class ComplexityRouter(CustomLogger): old_signals: Final = tuple(decision.get("signals") or ()) if decision is not None else () new_decision: Final = self._build_routing_decision( routed_model=new_model, - cause=base_cause if same_tier else "modality_escalation", + cause="modality_pin_override" if displaced_pin else (base_cause if same_tier else "modality_escalation"), tier=new_tier, score=decision.get("score") if decision is not None else None, signals=(*old_signals, *markers), @@ -2646,7 +2729,7 @@ class ComplexityRouter(CustomLogger): """Resolve a client-supplied session_id.""" for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs): session_id = metadata.get("session_id") - if session_id is not None: + if session_id is not None and not metadata.get(SESSION_ID_GENERATED_METADATA_KEY): return str(session_id) return None diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 70aeecb31c6..9f2054dda01 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -14,6 +14,8 @@ from pydantic import BaseModel, ConfigDict, Field, SkipValidation, field_seriali from litellm.types.router import AdaptiveRouterWeights, ClassifierPlugin, RoutingPlugin +from .tier_predictor import TrainedTierArtifact + class ComplexityTier(str, Enum): """Complexity tiers for routing decisions.""" @@ -41,7 +43,7 @@ DEFAULT_CLASSIFICATION_RUBRIC: Final[ClassificationRubric] = ClassificationRubri # The classifier_type values that can call classifier_llm_config.model. Every consumer asking # "is the classifier model a real dependency of this router" resolves it here, including the ones # that only hold the raw config mapping and cannot reach ComplexityRouterConfig.uses_llm_classifier. -LLM_CLASSIFIER_TYPES: Final[frozenset[str]] = frozenset({"llm", "heuristic_first"}) +LLM_CLASSIFIER_TYPES: Final[frozenset[str]] = frozenset({"llm", "heuristic_first", "hybrid"}) TIER_SEVERITY_ORDER: Final[tuple[ComplexityTier, ...]] = ( @@ -625,17 +627,28 @@ class ComplexityRouterConfig(BaseModel): ) # Classifier strategy - classifier_type: Literal["heuristic", "llm", "custom", "heuristic_first"] = Field( + classifier_type: Literal["heuristic", "heuristic_v2", "llm", "custom", "heuristic_first", "hybrid"] = Field( default="heuristic", description=( - "Classification strategy: local regex/keyword scoring, an LLM call, a custom classifier " - "plugin, or 'heuristic_first', which scores locally and only pays for the LLM classifier " - "when the local scorer does not confidently land a cheap tier" + "Classification strategy: local regex/keyword scoring, the bundled trained four-tier heuristic, " + "an LLM call, a custom classifier plugin, 'heuristic_first', which scores locally and only pays " + "for the LLM classifier when the local scorer does not confidently land a cheap tier, or 'hybrid', " + "which trusts the local scorer everywhere except when its score lands near a tier boundary" + ), + ) + heuristic_v2_artifact: TrainedTierArtifact | Literal["ultrafeedback"] = Field( + default="ultrafeedback", + description=( + "Success-probability artifact used by classifier_type 'heuristic_v2'. The bundled " + "UltraFeedback artifact is selected by default; an inline trained artifact may replace it" ), ) classifier_llm_config: ClassifierLLMConfig | None = Field( default=None, - description="Configuration for the LLM classifier; required when classifier_type is 'llm' or 'heuristic_first'", + description=( + "Configuration for the LLM classifier; required when classifier_type is 'llm', " + "'heuristic_first' or 'hybrid'" + ), ) heuristic_first_max_tier: str | None = Field( default=None, @@ -650,6 +663,19 @@ class ComplexityRouterConfig(BaseModel): "may not name the highest one, since that would make the LLM classifier unreachable." ), ) + hybrid_boundary_margin: float | None = Field( + default=None, + ge=0, + le=1, + description=( + "How close to a tier boundary a heuristic score has to land before the LLM classifier breaks the " + "tie; required when classifier_type is 'hybrid' and rejected otherwise. Everything further than " + "this from every active boundary routes on the scorer's own tier with no classifier call, at any " + "tier, which is what separates 'hybrid' from 'heuristic_first' and its cheap-tier ceiling. A " + "prompt where no dimension fired still goes to the classifier, since the scorer has no opinion " + "to be near a boundary with. 0 escalates only scores sitting exactly on a boundary." + ), + ) classifier_plugin: ClassifierPlugin | None = Field( default=None, description=( @@ -857,7 +883,20 @@ class ComplexityRouterConfig(BaseModel): "a routed model explicitly declared supports_vision false (deployment model_info " "or the model cost map; unmapped names stay routable) is replaced by the nearest " "HIGHER tier holding a capable model, then default_model, else a clear 400. A kept " - "session-affinity pin still wins even when an image arrives." + "session-affinity pin still wins even when an image arrives, unless " + "modality_pin_override is also enabled." + ), + ) + modality_pin_override: bool = Field( + default=False, + description=( + "Let modality_routing replace a kept session-affinity pin on the turns that carry an " + "image. Without this, a session pinned to a text-only model fails every image turn with " + "a provider 400, since the pin is exempt from the modality gate. When enabled, such a " + "turn routes to a capable model for that request only and the stored pin is left " + "untouched, so the next text turn replays the session's own model; the override is " + "reported as cause modality_pin_override and is never itself pinned. Inert unless " + "modality_routing is also enabled." ), ) @@ -1126,6 +1165,23 @@ class ComplexityRouterConfig(BaseModel): ) return self + @model_validator(mode="after") + def _validate_hybrid_boundary_margin(self) -> "ComplexityRouterConfig": + if self.classifier_type != "hybrid": + if self.hybrid_boundary_margin is not None: + raise ValueError( + f"hybrid_boundary_margin is set but classifier_type is {self.classifier_type!r}; " + "the scorer would never consult the classifier on a near-boundary score. Set " + "classifier_type 'hybrid' or remove hybrid_boundary_margin" + ) + return self + if self.hybrid_boundary_margin is None: + raise ValueError( + "hybrid_boundary_margin is required when classifier_type is 'hybrid': without a margin no " + "score is ever near enough to a boundary to escalate, which is classifier_type 'heuristic'" + ) + return self + @field_validator("fallback_tier") @classmethod def _reject_blank_optional_text(cls, value: str | None) -> str | None: @@ -1248,10 +1304,10 @@ class ComplexityRouterConfig(BaseModel): ) if duplicated: raise ValueError(f"tier_definitions names must be unique (case-insensitive): {', '.join(duplicated)}") - if self.classifier_type in ("heuristic", "heuristic_first"): + if self.classifier_type in ("heuristic", "heuristic_v2", "heuristic_first", "hybrid"): raise ValueError( "tier_definitions requires classifier_type 'llm' or 'custom': the heuristic scorer only " - "produces the built-in tiers" + "produces the four built-in tiers, as does heuristic_v2" ) conflicts: Final = self._tier_definition_conflicts() if conflicts: diff --git a/litellm/router_strategy/complexity_router/tier_predictor.py b/litellm/router_strategy/complexity_router/tier_predictor.py new file mode 100644 index 00000000000..764f6e6ad56 --- /dev/null +++ b/litellm/router_strategy/complexity_router/tier_predictor.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final, Literal + +from pydantic import BaseModel, Field, model_validator + +from litellm.types.router import RequestType + + +class TierGlobalStatistic(BaseModel): + tier: int = Field(ge=1, le=4) + successes: float = Field(ge=0.0) + observations: float = Field(gt=0.0) + + @model_validator(mode="after") + def _successes_do_not_exceed_observations(self) -> TierGlobalStatistic: + if self.successes > self.observations: + raise ValueError("successes cannot exceed observations") + return self + + +class TierDomainStatistic(TierGlobalStatistic): + request_type: RequestType + + +class TierCohortStatistic(TierGlobalStatistic): + cohort: str = Field(min_length=1) + + +class TierDataset(BaseModel): + name: str = Field(min_length=1) + url: str = Field(min_length=1) + license: str = Field(min_length=1) + rows: int = Field(gt=0) + success_definition: str = Field(default="quality score meets the dataset success threshold", min_length=1) + + +class TrainedTierArtifact(BaseModel): + schema_version: Literal[1] = 1 + global_statistics: tuple[TierGlobalStatistic, ...] + domain_statistics: tuple[TierDomainStatistic, ...] = () + cohort_statistics: tuple[TierCohortStatistic, ...] = () + domain_prior_mass: float = Field(default=200.0, gt=0.0) + cohort_prior_mass: float = Field(default=20.0, gt=0.0) + routing_threshold: float = Field(default=0.75, ge=0.0, le=1.0) + datasets: tuple[TierDataset, ...] = () + success_definition: str = Field(default="quality score meets the dataset success threshold", min_length=1) + split_method: str = Field(default="sha256(prompt): 70% train, 15% validation, 15% test", min_length=1) + + @model_validator(mode="after") + def _statistics_are_unique(self) -> TrainedTierArtifact: + global_tiers: Final = tuple(stat.tier for stat in self.global_statistics) + if frozenset(global_tiers) != frozenset((1, 2, 3, 4)) or len(global_tiers) != 4: + raise ValueError("global statistics must contain each tier exactly once") + domain_keys: Final = tuple((stat.request_type, stat.tier) for stat in self.domain_statistics) + if len(domain_keys) != len(frozenset(domain_keys)): + raise ValueError("domain statistics must contain unique request_type and tier pairs") + cohort_keys: Final = tuple((stat.cohort, stat.tier) for stat in self.cohort_statistics) + if len(cohort_keys) != len(frozenset(cohort_keys)): + raise ValueError("cohort statistics must contain unique cohort and tier pairs") + return self + + +_CODE_PATTERN: Final = re.compile( + r"```|\b(def|class|function|python|javascript|typescript|sql|code)\b", + re.IGNORECASE, +) +_MATH_PATTERN: Final = re.compile( + r"\b(solve|calculate|equation|probability|theorem|proof|integral)\b|[$=]", + re.IGNORECASE, +) +_MULTIPLE_CHOICE_PATTERN: Final = re.compile(r"(?:^|\s)[A-D][.)]\s") +_TIERS: Final = (1, 2, 3, 4) +_BUILTIN_ARTIFACTS: Final = MappingProxyType({"ultrafeedback": "ultrafeedback_tiers.json"}) + + +def resolve_tier_artifact(artifact: TrainedTierArtifact | str) -> TrainedTierArtifact: + if isinstance(artifact, TrainedTierArtifact): + return artifact + filename: Final = _BUILTIN_ARTIFACTS.get(artifact) + if filename is None: + raise ValueError(f"unknown complexity router tier artifact: {artifact}") + path: Final = Path(__file__).with_name("artifacts") / filename + return TrainedTierArtifact.model_validate_json(path.read_text()) + + +def similarity_cohort(prompt: str, request_type: RequestType) -> str: + length: Final = len(prompt) + length_bucket: Final = ( + "short" if length < 200 else "medium" if length < 800 else "long" if length < 2000 else "very_long" + ) + code: Final = int(bool(_CODE_PATTERN.search(prompt))) + math: Final = int(bool(_MATH_PATTERN.search(prompt))) + multiple_choice: Final = int(bool(_MULTIPLE_CHOICE_PATTERN.search(prompt))) + non_ascii: Final = int(sum(ord(character) > 127 for character in prompt) / max(1, length) > 0.1) + return f"{request_type.value}|{length_bucket}|code={code}|math={math}|mc={multiple_choice}|intl={non_ascii}" + + +@dataclass(frozen=True, slots=True) +class TierPrediction: + probabilities: Mapping[int, float] + required_tier: int + + +class TierSuccessPredictor: + def __init__(self, artifact: TrainedTierArtifact) -> None: + self._artifact = artifact + self._global: Mapping[int, TierGlobalStatistic] = MappingProxyType( + {stat.tier: stat for stat in artifact.global_statistics} + ) + self._domain: Mapping[tuple[RequestType, int], TierDomainStatistic] = MappingProxyType( + {(stat.request_type, stat.tier): stat for stat in artifact.domain_statistics} + ) + self._cohort: Mapping[tuple[str, int], TierCohortStatistic] = MappingProxyType( + {(stat.cohort, stat.tier): stat for stat in artifact.cohort_statistics} + ) + + @property + def routing_threshold(self) -> float: + return self._artifact.routing_threshold + + def predict(self, prompt: str, request_type: RequestType) -> TierPrediction: + cohort: Final = similarity_cohort(prompt, request_type) + raw: Final = tuple(self._probability(tier, request_type, cohort) for tier in _TIERS) + monotonic: Final = tuple(max(raw[:index]) for index in range(1, len(raw) + 1)) + probabilities: Final[Mapping[int, float]] = MappingProxyType( + {int(tier): probability for tier, probability in zip(_TIERS, monotonic)} + ) + required_tier: Final = next( + (tier for tier in _TIERS if probabilities[tier] >= self._artifact.routing_threshold), + 4, + ) + return TierPrediction(probabilities=probabilities, required_tier=required_tier) + + def _probability(self, tier: int, request_type: RequestType, cohort: str) -> float: + global_stat: Final = self._global[tier] + global_mean: Final = (global_stat.successes + 1.0) / (global_stat.observations + 2.0) + domain_stat: Final = self._domain.get((request_type, tier)) + domain_mean: Final = self._posterior_mean(domain_stat, self._artifact.domain_prior_mass, global_mean) + cohort_stat: Final = self._cohort.get((cohort, tier)) + return self._posterior_mean(cohort_stat, self._artifact.cohort_prior_mass, domain_mean) + + @staticmethod + def _posterior_mean( + statistic: TierGlobalStatistic | None, + prior_mass: float, + prior_mean: float, + ) -> float: + if statistic is None: + return prior_mean + return (statistic.successes + prior_mass * prior_mean) / (statistic.observations + prior_mass) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 3d37ca216a7..601b32c4386 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Final import litellm from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure from litellm.router_utils.add_retry_fallback_headers import ( add_fallback_headers_to_response, @@ -231,8 +232,6 @@ def record_pre_routing_selection(request_kwargs: Mapping[str, Any] | None, selec on /v1/messages the top-level ``metadata`` dict is the provider's own request field, so a blanket write would forward the tier stamp upstream. """ - from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs - if request_kwargs is None: return bucket: Final = request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)) @@ -264,13 +263,48 @@ def get_pre_routing_selection(kwargs: Mapping[str, Any]) -> str | None: return next((selected for selected in selections if isinstance(selected, str) and selected), None) +DISABLE_FALLBACKS_METADATA_KEY: Final = "_disable_fallbacks" + + +def record_disable_fallbacks(request_kwargs: Mapping[str, Any] | None, disabled: bool) -> None: + """ + Write-or-clear the request's disable_fallbacks verdict into the router-internal metadata + bucket. The wrapper pops the raw kwarg before any downstream frame runs, so the refusal + gate (which decides whether to convert a refusal into a recoverable error) needs this + carrier to know recovery is impossible. + """ + from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs + + if request_kwargs is None: + return + bucket: Final = request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)) + if not isinstance(bucket, dict): + return + if disabled: + bucket[DISABLE_FALLBACKS_METADATA_KEY] = True + else: + bucket.pop(DISABLE_FALLBACKS_METADATA_KEY, None) + + +def fallbacks_disabled_for_request(kwargs: Mapping[str, Any]) -> bool: + """True when this request opted out of fallbacks, read from the raw kwarg (pre-pop + snapshots keep it) or the router-internal bucket the wrapper stamps after popping it.""" + if kwargs.get("disable_fallbacks") is True: + return True + buckets: Final = (kwargs.get(name) for name in _ROUTER_METADATA_BUCKETS) + return any(isinstance(bucket, dict) and bucket.get(DISABLE_FALLBACKS_METADATA_KEY) is True for bucket in buckets) + + def fallback_lookup_groups(kwargs: Mapping[str, Any], model_group: str | None) -> tuple[str, ...]: """ Ordered keys for resolving a fallback chain: the tier a pre-routing hook selected wins, - and the requested group still resolves when no tier-keyed chain exists, so configs keyed - on the router name (the documented contract) keep working behind auto-routers. + then the routed group, then the requested group. The routed group differs when Claude Code + session affinity remaps a subagent's concrete model to its bound router. """ - ordered: Final = (get_pre_routing_selection(kwargs), model_group) + metadata: Final = kwargs.get(get_metadata_variable_name_from_kwargs(kwargs)) + routed_group_value: Final = metadata.get("model_group") if isinstance(metadata, Mapping) else None + routed_group: Final = routed_group_value if isinstance(routed_group_value, str) else None + ordered: Final = (get_pre_routing_selection(kwargs), routed_group, model_group) return tuple(dict.fromkeys(group for group in ordered if group)) @@ -470,10 +504,11 @@ async def run_async_fallback( attempted: Final = ( carried_targets if isinstance(carried_targets, AttemptedFallbackTargets) else AttemptedFallbackTargets() ) - attempted.record(original_model_group) + failed_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group + attempted.record(failed_model_group) for mg in fallback_model_group: - if mg == original_model_group: + if mg == failed_model_group: continue if same_model_group_only and _get_fallback_target_model_group(mg) != original_model_group: verbose_router_logger.info( diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index b1e9dbdefa8..39d3e25aacb 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -21,7 +21,7 @@ from typing_extensions import TypedDict from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache -from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY +from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger, Span from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import AllMessageValues @@ -265,7 +265,7 @@ class DeploymentAffinityCheck(CustomLogger): @staticmethod def _get_session_id_from_metadata_dict(metadata: dict) -> str | None: session_id: Final = metadata.get("session_id") - if session_id is None: + if session_id is None or metadata.get(SESSION_ID_GENERATED_METADATA_KEY): return None return str(session_id) diff --git a/litellm/rust_bridge/__init__.py b/litellm/rust_bridge/__init__.py index 3da5b98449b..e6d8ffef48c 100644 --- a/litellm/rust_bridge/__init__.py +++ b/litellm/rust_bridge/__init__.py @@ -1,9 +1,9 @@ """LiteLLM Rust bridge package.""" +from litellm.rust_bridge.configuration import use_litellm_rust from litellm.rust_bridge.loader import ( get_native_bridge, native_bridge_available, ) -from litellm.rust_bridge.ocr import use_litellm_rust __all__ = ["get_native_bridge", "native_bridge_available", "use_litellm_rust"] diff --git a/litellm/rust_bridge/bindings.py b/litellm/rust_bridge/bindings.py new file mode 100644 index 00000000000..d16f150a2aa --- /dev/null +++ b/litellm/rust_bridge/bindings.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Final, Generic, TypeVar + +from litellm.rust_bridge.loader import get_native_bridge + +BindingT = TypeVar("BindingT") + + +class _Unset: + pass + + +_UNSET: Final = _Unset() + + +class NativeBinding(Generic[BindingT]): + """Resolve one native attribute with an explicit, resettable test override.""" + + def __init__(self, attribute: str, *, validate: Callable[[object], BindingT | None]) -> None: + self._attribute: Final = attribute + self._validate: Final = validate + self._override: BindingT | None | _Unset = _UNSET + + def load(self) -> BindingT | None: + if not isinstance(self._override, _Unset): + return self._override + native: Final = get_native_bridge() + if native is None: + return None + return self._validate(getattr(native, self._attribute, None)) + + def override(self, value: BindingT | None) -> None: + self._override = value + + def reset(self) -> None: + self._override = _UNSET + + +def native_exception_types() -> tuple[type[BaseException], type[BaseException]] | None: + native: Final = get_native_bridge() + if native is None: + return None + declined: Final = getattr(native, "RustBridgeDeclined", None) + upstream: Final = getattr(native, "RustUpstreamError", None) + if not isinstance(declined, type) or not isinstance(upstream, type): + return None + return declined, upstream diff --git a/litellm/rust_bridge/chat_completions.py b/litellm/rust_bridge/chat_completions.py index acda3086051..c599667ab17 100644 --- a/litellm/rust_bridge/chat_completions.py +++ b/litellm/rust_bridge/chat_completions.py @@ -13,7 +13,6 @@ retrying it there would bill the customer for the same work twice. from __future__ import annotations import json -import os from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Protocol @@ -27,6 +26,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo convert_to_model_response_object, ) from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned +from litellm.rust_bridge.configuration import rust_enabled from litellm.rust_bridge.loader import get_native_bridge from litellm.rust_bridge.timeouts import timeout_to_seconds from litellm.types.utils import ModelResponse @@ -44,8 +44,6 @@ _LITELLM_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) RUST_RESPONSE_HEADER: Final = "x-litellm-rust" -_TRUTHY_ENV_VALUES: Final = frozenset({"1", "true", "yes", "on"}) - class RustChatCompletions(Protocol): def __call__( @@ -181,10 +179,6 @@ def load_rust_achat_completions() -> RustAchatCompletions | None: return loaded -def _env_enables_rust() -> bool: - return os.getenv("LITELLM_RUST", "").strip().lower() in _TRUTHY_ENV_VALUES - - def _load_rust_decline() -> RustChatCompletionsDecline | None: if _STATE.decline is not None: return _STATE.decline @@ -253,8 +247,8 @@ def rust_chat_completions_accepts( return False if stream: return False - opted_in: Final = litellm_params is not None and litellm_params.get("rust") is True - if not opted_in and not _env_enables_rust(): + request_override: Final = litellm_params.get("rust") if litellm_params is not None else None + if not rust_enabled(request_override=request_override if isinstance(request_override, bool) else None): return False if _litellm_metadata_reaches_the_provider(custom_llm_provider, litellm_params): verbose_logger.debug("Rust chat completions declined (litellm metadata user_id); using the Python path") diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py new file mode 100644 index 00000000000..d54b15f060c --- /dev/null +++ b/litellm/rust_bridge/configuration.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import os +import warnings +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from litellm.rust_bridge.messages import RustAmessages, RustMessages + from litellm.rust_bridge.ocr import RustAocr, RustOcr + from litellm.rust_bridge.responses_websocket import RustResponsesWebSocketConnection + from litellm.rust_bridge.transcription import RustAtranscription, RustTranscription + +DEFAULT_RUST_ENABLED: Final = False +_TRUE_ENV_VALUES: Final = frozenset({"1", "true", "yes", "on"}) +_GLOBAL_ENV_NAME: Final = "LITELLM_RUST" +_LEGACY_OCR_ENV_NAME: Final = "LITELLM_USE_RUST_OCR" + + +class _Unset: + pass + + +_UNSET: Final = _Unset() + + +class _RustConfiguration: + def __init__(self) -> None: + self.override: bool | None = None + + +_CONFIGURATION: Final = _RustConfiguration() + + +def _parse_env_bool(value: str | None) -> bool | None: + if value is None: + return None + return value.strip().lower() in _TRUE_ENV_VALUES + + +def resolve_rust_enabled( + *, + request_override: bool | None, + process_override: bool | None, + environment_override: bool | None, + legacy_ocr_override: bool | None = None, + release_default: bool = DEFAULT_RUST_ENABLED, +) -> bool: + if request_override is not None: + return request_override + if process_override is not None: + return process_override + if environment_override is not None: + return environment_override + if legacy_ocr_override is not None: + return legacy_ocr_override + return release_default + + +def rust_enabled(*, request_override: bool | None = None) -> bool: + if request_override is not None: + return request_override + process_override: Final = _CONFIGURATION.override + if process_override is not None: + return process_override + return resolve_rust_enabled( + request_override=None, + process_override=None, + environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)), + ) + + +def rust_ocr_enabled(*, request_override: bool | None = None) -> bool: + if request_override is not None: + return request_override + process_override: Final = _CONFIGURATION.override + if process_override is not None: + return process_override + global_override: Final = _parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)) + legacy_override: Final = None if global_override is not None else _parse_env_bool(os.getenv(_LEGACY_OCR_ENV_NAME)) + if legacy_override is not None: + warnings.warn( + f"{_LEGACY_OCR_ENV_NAME} is deprecated; use {_GLOBAL_ENV_NAME} instead", + DeprecationWarning, + stacklevel=2, + ) + return resolve_rust_enabled( + request_override=None, + process_override=None, + environment_override=global_override, + legacy_ocr_override=legacy_override, + ) + + +def reset_rust_configuration() -> None: + _CONFIGURATION.override = None + + +def use_litellm_rust( + enabled: bool = True, + *, + ocr: RustOcr | None | _Unset = _UNSET, + aocr: RustAocr | None | _Unset = _UNSET, + messages: RustMessages | None | _Unset = _UNSET, + amessages: RustAmessages | None | _Unset = _UNSET, + responses_websocket: type[RustResponsesWebSocketConnection] | None | _Unset = _UNSET, + transcription: RustTranscription | None | _Unset = _UNSET, + atranscription: RustAtranscription | None | _Unset = _UNSET, +) -> None: + """Set the process override for optional Rust paths. + + Rust-only paths, including Bedrock transcription, are not controlled by this switch. + """ + _CONFIGURATION.override = enabled + bindings: Final = (ocr, aocr, messages, amessages, responses_websocket, transcription, atranscription) + if all(isinstance(binding, _Unset) for binding in bindings): + return + warnings.warn( + "Injecting Rust bridge implementations through use_litellm_rust() is deprecated; " + "use the internal bridge setters in tests", + DeprecationWarning, + stacklevel=2, + ) + + if not isinstance(ocr, _Unset) or not isinstance(aocr, _Unset): + from litellm.rust_bridge.ocr import set_rust_ocr + + if not isinstance(ocr, _Unset): + set_rust_ocr(ocr=ocr) + if not isinstance(aocr, _Unset): + set_rust_ocr(aocr=aocr) + if not isinstance(messages, _Unset) or not isinstance(amessages, _Unset): + from litellm.rust_bridge.messages import set_rust_messages + + if not isinstance(messages, _Unset): + set_rust_messages(messages=messages) + if not isinstance(amessages, _Unset): + set_rust_messages(amessages=amessages) + if not isinstance(responses_websocket, _Unset): + from litellm.rust_bridge.responses_websocket import set_rust_responses_websocket + + set_rust_responses_websocket(connection=responses_websocket) + if not isinstance(transcription, _Unset) or not isinstance(atranscription, _Unset): + from litellm.rust_bridge.transcription import configure_rust_transcription + + if not isinstance(transcription, _Unset): + configure_rust_transcription(transcription=transcription) + if not isinstance(atranscription, _Unset): + configure_rust_transcription(atranscription=atranscription) diff --git a/litellm/rust_bridge/ocr.py b/litellm/rust_bridge/ocr.py index 82297d35170..b5b0a35a498 100644 --- a/litellm/rust_bridge/ocr.py +++ b/litellm/rust_bridge/ocr.py @@ -2,16 +2,16 @@ from __future__ import annotations -import os from collections.abc import Awaitable -from typing import TYPE_CHECKING, Any, Final, Protocol, cast +from typing import Final, Protocol, cast # noqa: TID251 # native extension exposes dynamically typed callables import httpx +from litellm.rust_bridge import configuration as _configuration from litellm.rust_bridge.timeouts import timeout_to_seconds as _timeout_to_seconds -if TYPE_CHECKING: - from litellm.rust_bridge.messages import RustAmessages, RustMessages +rust_ocr_enabled = _configuration.rust_ocr_enabled +use_litellm_rust = _configuration.use_litellm_rust class RustOcr(Protocol): @@ -51,69 +51,20 @@ class _Unset: _UNSET: Final[_Unset] = _Unset() -def _env_enables_rust_ocr() -> bool: - return os.getenv("LITELLM_USE_RUST_OCR", "").strip().lower() in { - "1", - "true", - "yes", - "on", - } - - -_rust_ocr_enabled = _env_enables_rust_ocr() _rust_ocr_impl: RustOcr | None = None _rust_aocr_impl: RustAocr | None = None -def use_litellm_rust( - enabled: bool = True, +def set_rust_ocr( *, ocr: RustOcr | None | _Unset = _UNSET, aocr: RustAocr | None | _Unset = _UNSET, - messages: RustMessages | None | _Unset = _UNSET, - amessages: RustAmessages | None | _Unset = _UNSET, - responses_websocket: Any | None | _Unset = _UNSET, - transcription: Any | None | _Unset = _UNSET, - atranscription: Any | None | _Unset = _UNSET, ) -> None: - global _rust_ocr_enabled, _rust_ocr_impl, _rust_aocr_impl - configuring_ocr: Final = not isinstance(ocr, _Unset) or not isinstance(aocr, _Unset) - configuring_messages: Final = not isinstance(messages, _Unset) or not isinstance(amessages, _Unset) - configuring_responses_websocket: Final = not isinstance(responses_websocket, _Unset) - configuring_transcription: Final = not isinstance(transcription, _Unset) or not isinstance(atranscription, _Unset) - if configuring_ocr or (not configuring_messages and not configuring_responses_websocket): - _rust_ocr_enabled = enabled + global _rust_ocr_impl, _rust_aocr_impl if not isinstance(ocr, _Unset): _rust_ocr_impl = ocr if not isinstance(aocr, _Unset): _rust_aocr_impl = aocr - if configuring_transcription: - from litellm.rust_bridge.transcription import configure_rust_transcription - - configure_rust_transcription( - enabled=enabled, - transcription=transcription, - atranscription=atranscription, - ) - if not configuring_messages and not configuring_responses_websocket: - return - if configuring_messages: - from litellm.rust_bridge.messages import set_rust_messages - - if not isinstance(messages, _Unset) and not isinstance(amessages, _Unset): - set_rust_messages(messages=messages, amessages=amessages) - elif not isinstance(messages, _Unset): - set_rust_messages(messages=messages) - else: - set_rust_messages(amessages=amessages) - if configuring_responses_websocket: - from litellm.rust_bridge.responses_websocket import set_rust_responses_websocket - - set_rust_responses_websocket(connection=responses_websocket) - - -def rust_ocr_enabled() -> bool: - return _rust_ocr_enabled def load_rust_ocr() -> RustOcr | None: diff --git a/litellm/rust_bridge/runtime.py b/litellm/rust_bridge/runtime.py new file mode 100644 index 00000000000..00f06c046a2 --- /dev/null +++ b/litellm/rust_bridge/runtime.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from enum import Enum +from typing import Final, Generic, NoReturn, TypeAlias, TypeVar + +from litellm.exceptions import APIError +from litellm.rust_bridge.bindings import native_exception_types + +NativeT = TypeVar("NativeT") +ResultT = TypeVar("ResultT") + + +class FallbackMode(Enum): + PYTHON = "python" + RUST_REQUIRED = "rust_required" + + +@dataclass(frozen=True, slots=True) +class RustHandled(Generic[ResultT]): + value: ResultT + + +@dataclass(frozen=True, slots=True) +class RustDeclined: + reason: str + + +@dataclass(frozen=True, slots=True) +class RustUnavailable: + pass + + +RustAttempt: TypeAlias = RustHandled[ResultT] | RustDeclined | RustUnavailable + + +@dataclass(frozen=True, slots=True) +class BridgeErrorContext: + route: str + provider: str + model: str + + +def invoke( + *, + native_call: Callable[[], NativeT] | None, + fallback: Callable[[], ResultT], + adapt: Callable[[NativeT], ResultT], + mode: FallbackMode, + context: BridgeErrorContext, +) -> ResultT: + result: Final = attempt(native_call=native_call, adapt=adapt, context=context) + if isinstance(result, RustHandled): + return result.value + if mode is FallbackMode.PYTHON: + return fallback() + _raise_required(result, context) + + +async def ainvoke( + *, + native_call: Callable[[], Awaitable[NativeT]] | None, + fallback: Callable[[], Awaitable[ResultT]], + adapt: Callable[[NativeT], ResultT], + mode: FallbackMode, + context: BridgeErrorContext, +) -> ResultT: + result: Final = await aattempt(native_call=native_call, adapt=adapt, context=context) + if isinstance(result, RustHandled): + return result.value + if mode is FallbackMode.PYTHON: + return await fallback() + _raise_required(result, context) + + +def attempt( + *, + native_call: Callable[[], NativeT] | None, + adapt: Callable[[NativeT], ResultT], + context: BridgeErrorContext, +) -> RustAttempt[ResultT]: + if native_call is None: + return RustUnavailable() + exceptions: Final = native_exception_types() + if exceptions is None: + return RustHandled(adapt(native_call())) + declined, upstream = exceptions + try: + value: Final = native_call() + except declined as error: + return RustDeclined(reason=_decline_reason(error)) + except upstream as error: + _raise_upstream(error, context) + return RustHandled(adapt(value)) + + +async def aattempt( + *, + native_call: Callable[[], Awaitable[NativeT]] | None, + adapt: Callable[[NativeT], ResultT], + context: BridgeErrorContext, +) -> RustAttempt[ResultT]: + if native_call is None: + return RustUnavailable() + exceptions: Final = native_exception_types() + if exceptions is None: + return RustHandled(adapt(await native_call())) + declined, upstream = exceptions + try: + value: Final = await native_call() + except declined as error: + return RustDeclined(reason=_decline_reason(error)) + except upstream as error: + _raise_upstream(error, context) + return RustHandled(adapt(value)) + + +def call(operation: Callable[[], ResultT], context: BridgeErrorContext) -> ResultT: + exceptions: Final = native_exception_types() + if exceptions is None: + return operation() + upstream: Final = exceptions[1] + try: + return operation() + except upstream as error: + _raise_upstream(error, context) + + +async def acall(operation: Callable[[], Awaitable[ResultT]], context: BridgeErrorContext) -> ResultT: + exceptions: Final = native_exception_types() + if exceptions is None: + return await operation() + upstream: Final = exceptions[1] + try: + return await operation() + except upstream as error: + _raise_upstream(error, context) + + +def _decline_reason(error: BaseException) -> str: + reason: Final[object] = error.args[0] if error.args else str(error) + return reason if isinstance(reason, str) else str(reason) + + +def _raise_required( + result: RustDeclined | RustUnavailable, + context: BridgeErrorContext, +) -> NoReturn: + raise RuntimeError(f"Rust {context.route} bridge {_required_reason(result)}") + + +def _required_reason(result: RustDeclined | RustUnavailable) -> str: + match result: + case RustUnavailable(): + return "is unavailable" + case RustDeclined(reason=reason): + return f"declined the request: {reason}" + + +def _raise_upstream(error: BaseException, context: BridgeErrorContext) -> NoReturn: + args: Final[tuple[object, ...]] = error.args + status_value: Final = args[0] if args else 0 + message_value: Final = args[1] if len(args) > 1 else str(error) + status: Final = status_value if isinstance(status_value, int) else 0 + message: Final = message_value if isinstance(message_value, str) else str(message_value) + raise APIError( + status_code=status or 500, + message=f"litellm rust {context.route}: {message}", + llm_provider=context.provider, + model=context.model, + ) from error + + +def identity(value: ResultT) -> ResultT: + return value + + +async def async_none() -> None: + return None diff --git a/litellm/types/llms/gemini_audio_transcription.py b/litellm/types/llms/gemini_audio_transcription.py index cb12e0f45b8..f7e74ba4bf8 100644 --- a/litellm/types/llms/gemini_audio_transcription.py +++ b/litellm/types/llms/gemini_audio_transcription.py @@ -1,7 +1,7 @@ -from typing import Literal, Required +from typing import Literal from pydantic import BaseModel, ConfigDict -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, Required, TypedDict class GeminiTranscriptionAudioInput(TypedDict): diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index 1b8baf2da09..a59fcb1bcb5 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal from urllib.parse import urlsplit import httpx -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, Field from typing_extensions import TypedDict from litellm.types.llms.base import HiddenParams @@ -91,6 +91,33 @@ class MCPPublicServer(BaseModel): mcp_info: dict[str, Any] | None = None +class MCPToolSearchSettings(BaseModel): + """`litellm_settings.mcp_tool_search`: how the native `mcp_tool_search` virtual tool ranks the caller's tools.""" + + model_config = ConfigDict(frozen=True) + + embedding_model: str | None = Field( + default=None, + description="Embedding model from model_list used to rank tools by meaning. Unset keeps keyword matching.", + ) + top_k: int = Field( + default=5, + ge=1, + le=100, + description="Most ranked tools a search returns. A smaller top_k in the tool call wins. Core tools do not count.", + ) + similarity_threshold: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Lowest cosine similarity a tool needs to appear in semantic results (0.0 = no cutoff).", + ) + core_tools: tuple[str, ...] = Field( + default=(), + description="Tool names always returned first when the caller can access them, e.g. `my_server-get_rates`.", + ) + + # OAuth 2.0 token-endpoint client authentication method (RFC 6749 section 2.3.1). MCPTokenEndpointAuthMethod = Literal["client_secret_basic", "client_secret_post"] diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/crowdstrike_aidr.py b/litellm/types/proxy/guardrails/guardrail_hooks/crowdstrike_aidr.py index f47c38af3e3..6beca030a3a 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/crowdstrike_aidr.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/crowdstrike_aidr.py @@ -4,7 +4,18 @@ from .base import GuardrailConfigModel class CrowdStrikeAIDRGuardrailConfigModelOptionalParams(BaseModel): - pass + streaming_end_of_stream_only: bool | None = Field( + default=None, + description="If False (default when unset), post_call scans the accumulated streamed response every " + "streaming_sampling_rate chunks and an in-flight block stops the stream. If True, the guard runs once " + "over the assembled response at end of stream, so flagged content may already have reached the client.", + ) + streaming_sampling_rate: int | None = Field( + default=None, + ge=1, + description="When streaming_end_of_stream_only is False, scan the accumulated streamed response every Nth " + "chunk. Defaults to 5 when unset.", + ) class CrowdStrikeAIDRGuardrailConfigModel(GuardrailConfigModel[CrowdStrikeAIDRGuardrailConfigModelOptionalParams]): diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index f6ee054ceaa..fa73926305b 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -1,7 +1,7 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Any, Literal -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict class PublicModelHubInfo(BaseModel): @@ -43,6 +43,8 @@ class AgentCredentialField(BaseModel): options: list[str] | None = None default_value: str | None = None include_in_litellm_params: bool | None = None + validation_pattern: str | None = None + validation_message: str | None = None class AgentCreateInfo(BaseModel): @@ -71,6 +73,44 @@ class SupportedEndpointsResponse(BaseModel): endpoints: list[SupportedEndpoint] +class AutoRouterPresetTiers(BaseModel): + """Exactly the four built-in tiers the dashboard's preset prefill can apply. + + extra="forbid" on purpose: a tier name this dashboard cannot apply would grey out or crash the + picker, so such a catalog is rejected wholesale and the bundled one serves instead. + """ + + model_config = ConfigDict(extra="forbid") + + SIMPLE: Sequence[str] + MEDIUM: Sequence[str] + COMPLEX: Sequence[str] + REASONING: Sequence[str] + + +class AutoRouterPresetConfig(BaseModel): + """The complexity_router_config a preset prefills. + + Only tiers is validated, because every dashboard consumer dereferences it; everything else + passes through verbatim with unknown fields kept (extra="allow"), so a catalog published after + this proxy shipped still serves its new fields intact. + """ + + model_config = ConfigDict(extra="allow") + + tiers: AutoRouterPresetTiers + + +class AutoRouterPresetRecord(BaseModel): + """One auto-router preset as served to the dashboard's template picker.""" + + model_config = ConfigDict(extra="allow") + + label: str + description: str + complexity_router_config: AutoRouterPresetConfig + + class ComplexityScorerDefaults(BaseModel): """The complexity router's shipped heuristic scorer defaults. diff --git a/litellm/types/rerank.py b/litellm/types/rerank.py index 903781b2ccd..a76e6cf1187 100644 --- a/litellm/types/rerank.py +++ b/litellm/types/rerank.py @@ -4,8 +4,10 @@ https://docs.cohere.com/reference/rerank """ -from pydantic import BaseModel, PrivateAttr -from typing_extensions import Required, TypedDict +from typing import Literal + +from pydantic import BaseModel, ConfigDict, PrivateAttr +from typing_extensions import ReadOnly, Required, TypedDict class RerankRequest(BaseModel): @@ -21,6 +23,18 @@ class RerankRequest(BaseModel): # (e.g. hosted vLLM / Qwen3-Reranker, DeepInfra). Omitted from the outgoing # request when None, so this is fully backward-compatible. instruction: str | None = None + truncate_prompt_tokens: int | None = None + truncation_side: Literal["left", "right"] | None = None + max_tokens_per_query: int | None = None + + +class HostedVLLMRerankTruncationParams(BaseModel): + model_config = ConfigDict(frozen=True) + + truncate_prompt_tokens: int | None = None + truncation_side: Literal["left", "right"] | None = None + max_tokens_per_query: int | None = None + max_tokens_per_doc: int | None = None class OptionalRerankParams(TypedDict, total=False): @@ -32,6 +46,9 @@ class OptionalRerankParams(TypedDict, total=False): max_chunks_per_doc: int | None max_tokens_per_doc: int | None instruction: str | None + truncate_prompt_tokens: ReadOnly[int | None] + truncation_side: ReadOnly[Literal["left", "right"] | None] + max_tokens_per_query: ReadOnly[int | None] class RerankBilledUnits(TypedDict, total=False): diff --git a/litellm/types/router.py b/litellm/types/router.py index e0957383aac..4f4df1a8d2e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -106,6 +106,20 @@ class RetryPolicy(BaseModel): InternalServerErrorRetries: int | None = None +OptionalPreCallChecks = list[ + Literal[ + "prompt_caching", + "router_budget_limiting", + "responses_api_deployment_check", + "deployment_affinity", + "session_affinity", + "forward_client_headers_by_model_group", + "enforce_model_rate_limits", + "encrypted_content_affinity", + ] +] + + class UpdateRouterConfig(BaseModel): """ Set of params that you can modify via `router.update_settings()`. @@ -128,6 +142,7 @@ class UpdateRouterConfig(BaseModel): model_group_alias: dict[str, str | dict] | None = {} enable_tag_filtering: bool | None = None tag_routing_prefix: str | None = None + optional_pre_call_checks: OptionalPreCallChecks | None = None model_config = ConfigDict(protected_namespaces=()) @@ -290,6 +305,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): """ custom_llm_provider: str | None = None + rust: bool | None = None tpm: int | None = None rpm: int | None = None itpm: int | None = None @@ -869,20 +885,6 @@ class FallbackAccessCheck(Protocol): async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: "Router") -> bool: ... -OptionalPreCallChecks = list[ - Literal[ - "prompt_caching", - "router_budget_limiting", - "responses_api_deployment_check", - "deployment_affinity", - "session_affinity", - "forward_client_headers_by_model_group", - "enforce_model_rate_limits", - "encrypted_content_affinity", - ] -] - - class LiteLLM_RouterFileObject(TypedDict, total=False): """ Tracking the litellm params hash, used for mapping the file id to the right model diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5783a39b30c..569fce4f7b8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2839,6 +2839,7 @@ class StandardLoggingRoutingDecisionTierBoundaries(TypedDict): RoutingDecisionCause = Literal[ "heuristic_scorer", + "heuristic_v2", # The scorer found 2+ reasoning markers and forced REASONING regardless of score. # A distinct cause rather than a marker inside `signals`, because it is the fact # that tells a reader the score did NOT choose the tier; encoding it as free text @@ -2851,6 +2852,7 @@ RoutingDecisionCause = Literal[ # scorer, and from "classifier_fallback", which is the scorer running because a call failed: # only this cause means an LLM classifier was configured, reachable, and deliberately skipped. "heuristic_first_short_circuit", + "hybrid_short_circuit", # The operator's classifier plugin (classifier_type 'custom') decided the tier. "classifier_plugin", # The LLM classifier or classifier plugin failed on a router with an operator-defined @@ -2877,6 +2879,10 @@ RoutingDecisionCause = Literal[ # routed model does not accept image input, so the nearest higher capable tier or # default_model served instead. The displaced placement rides in signals. "modality_escalation", + # modality_pin_override replaced a KEPT session-affinity pin for this request only: the turn + # carries an image the pinned model cannot accept. The stored pin is untouched, so the next + # text turn replays it. Distinct from "modality_escalation", which never displaces a pin. + "modality_pin_override", "session_affinity_pin", "session_affinity_escalation", # classification_mode 'user_turn': the request is an agent loop's continuation turn (no new diff --git a/litellm/utils.py b/litellm/utils.py index 252b6756937..ba456fc353b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5106,49 +5106,6 @@ def get_response_string(response_obj: ModelResponse | ModelResponseStream) -> st return "".join(response_parts) -def get_api_key(llm_provider: str, dynamic_api_key: str | None): - api_key = dynamic_api_key or litellm.api_key - # openai - if llm_provider == "openai" or llm_provider == "text-completion-openai": - api_key = api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") - # anthropic - elif llm_provider == "anthropic" or llm_provider == "anthropic_text": - api_key = api_key or litellm.anthropic_key or get_secret("ANTHROPIC_API_KEY") - # ai21 - elif llm_provider == "ai21": - api_key = api_key or litellm.ai21_key or get_secret("AI21_API_KEY") - # aleph_alpha - elif llm_provider == "aleph_alpha": - api_key = api_key or litellm.aleph_alpha_key or get_secret("ALEPH_ALPHA_API_KEY") - # baseten - elif llm_provider == "baseten": - api_key = api_key or litellm.baseten_key or get_secret("BASETEN_API_KEY") - # cohere - elif llm_provider == "cohere" or llm_provider == "cohere_chat": - api_key = api_key or litellm.cohere_key or get_secret("COHERE_API_KEY") - # huggingface - elif llm_provider == "huggingface": - api_key = api_key or litellm.huggingface_key or get_secret("HUGGINGFACE_API_KEY") - # nlp_cloud - elif llm_provider == "nlp_cloud": - api_key = api_key or litellm.nlp_cloud_key or get_secret("NLP_CLOUD_API_KEY") - # replicate - elif llm_provider == "replicate": - api_key = api_key or litellm.replicate_key or get_secret("REPLICATE_API_KEY") - # together_ai - elif llm_provider == "together_ai": - api_key = ( - api_key or litellm.togetherai_api_key or get_secret("TOGETHERAI_API_KEY") or get_secret("TOGETHER_AI_TOKEN") - ) - # nebius - elif llm_provider == "nebius": - api_key = api_key or litellm.nebius_key or get_secret("NEBIUS_API_KEY") - # wandb - elif llm_provider == "wandb": - api_key = api_key or litellm.wandb_key or get_secret("WANDB_API_KEY") - return api_key - - def get_utc_datetime(): import datetime as dt from datetime import datetime diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index 9b0ff71730a..636bdd4b52e 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -7,7 +7,7 @@ import builtins import contextvars from collections.abc import Coroutine, Mapping from functools import partial -from typing import Final +from typing import TYPE_CHECKING, Final import httpx @@ -15,6 +15,11 @@ import litellm from litellm.constants import request_timeout from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.vector_store.transformation import ( + BaseQueryEmbeddingVectorStoreConfig, + VectorStoreEmbeddingExecutor, + vector_store_request_metadata, +) from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( @@ -29,12 +34,25 @@ from litellm.types.vector_stores import ( from litellm.utils import ProviderConfigManager, client from litellm.vector_stores.utils import VectorStoreRequestUtils +if TYPE_CHECKING: + from litellm.router import Router + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() ################################################# +def _direct_vector_store_embedding_executor( + value: object, router: "Router | None", request_kwargs: Mapping[str, object] +) -> VectorStoreEmbeddingExecutor: + if value is not None and not isinstance(value, VectorStoreEmbeddingExecutor): + raise TypeError("Invalid direct vector store embedding executor") + return BaseQueryEmbeddingVectorStoreConfig.query_embedding_executor( + value, router, vector_store_request_metadata(request_kwargs) + ) + + def mock_vector_store_search_response( mock_results: list[VectorStoreSearchResult] | None = None, ): @@ -280,12 +298,18 @@ async def asearch( timeout: float | httpx.Timeout | None = None, # LiteLLM specific params, custom_llm_provider: str | None = None, + router: "Router | None" = None, **kwargs, ) -> VectorStoreSearchResponse: """ Async: Search a vector store for relevant chunks based on a query and file attributes filter. """ - local_vars: Final = locals() + embedding_executor: Final = _direct_vector_store_embedding_executor( + kwargs.pop("_direct_vector_store_embedding_executor", None), router, kwargs + ) + local_vars: Final = { # mutable-ok: exception logging requires a sanitized mutable snapshot + key: value for key, value in locals().items() if key != "embedding_executor" + } try: loop: Final = asyncio.get_event_loop() @@ -308,6 +332,8 @@ async def asearch( extra_body=extra_body, timeout=timeout, custom_llm_provider=custom_llm_provider, + _direct_vector_store_embedding_executor=embedding_executor, + router=router, **kwargs, ) @@ -347,6 +373,7 @@ def search( timeout: float | httpx.Timeout | None = None, # LiteLLM specific params, custom_llm_provider: str | None = None, + router: "Router | None" = None, **kwargs, ) -> VectorStoreSearchResponse | Coroutine[object, object, VectorStoreSearchResponse]: """ @@ -363,12 +390,16 @@ def search( Returns: VectorStoreSearchResponse containing the search results. """ - local_vars: Final = locals() + embedding_executor: Final = _direct_vector_store_embedding_executor( + kwargs.pop("_direct_vector_store_embedding_executor", None), router, kwargs + ) + local_vars: Final = { # mutable-ok: exception logging requires a sanitized mutable snapshot + key: value for key, value in locals().items() if key != "embedding_executor" + } try: litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) _is_async: Final = kwargs.pop("asearch", False) is True - # pull credentials from registry if available if litellm.vector_store_registry is not None and vector_store_id is not None: try: @@ -445,11 +476,13 @@ def search( custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, logging_obj=litellm_logging_obj, + embedding_executor=embedding_executor, extra_headers=extra_headers, extra_body=extra_body, timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), + router=router, ) return response diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a3cfb300ea6..d8a8f84b032 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9643,7 +9643,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2.5-Flash": { "input_cost_per_image_token": 1.75e-06, @@ -9656,7 +9657,8 @@ "supported_endpoints": [ "/v1/images/generations", "/v1/images/edits" - ] + ], + "deprecation_date": "2026-10-01" }, "azure_ai/MAI-Image-2e": { "deprecation_date": "2026-08-15", @@ -10155,7 +10157,9 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "cache_read_input_token_cost": 1.45e-07, + "supports_prompt_caching": true }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", @@ -10169,18 +10173,20 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true - }, - "azure_ai/deepseek-v4-flash-0731": { + "supports_tool_choice": true, "cache_read_input_token_cost": 2.8e-08, + "supports_prompt_caching": true + }, + "azure_ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", - "input_cost_per_token": 1.9e-07, + "input_cost_per_token": 4.4e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5.1e-07, + "output_cost_per_token": 1.32e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", "supports_function_calling": true, "supports_prompt_caching": true, @@ -10400,11 +10406,13 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supports_function_calling": true, "supports_tool_choice": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-07, + "supports_prompt_caching": true }, "azure_ai/kimi-k2.6": { "deprecation_date": "2027-04-16", @@ -10415,7 +10423,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k2-6-in-microsoft-foundry/4513125", + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", "supported_modalities": [ "text", "image" @@ -10426,7 +10434,9 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1.6e-07, + "supports_prompt_caching": true }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, @@ -12110,7 +12120,7 @@ "max_output_tokens": 2048, "max_tokens": 2048, "mode": "chat", - "output_cost_per_token": 2.65e-06, + "output_cost_per_token": 6e-07, "supports_pdf_input": true }, "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": { @@ -23514,6 +23524,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "vertex_ai/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -25351,6 +25418,65 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini/gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "rpm": 2000, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "tpm": 800000, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -25759,6 +25885,63 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "gemini-3.8-flash": { + "prompt_cache_min_tokens": 4096, + "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "input_cost_per_token": 7.5e-07, + "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_flex": 3.75e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 3.75e-06, + "output_cost_per_token": 3.75e-06, + "output_cost_per_token_batches": 1.875e-06, + "output_cost_per_token_flex": 1.875e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_audio_input": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "supports_native_streaming": true, + "input_cost_per_token_priority": 1.35e-06, + "output_cost_per_token_priority": 6.75e-06, + "cache_read_input_token_cost_priority": 1.35e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.014, + "search_context_size_medium": 0.014, + "search_context_size_high": 0.014 + }, + "web_search_billing_unit": "per_query", + "google_maps_grounding_cost_per_query": 0.014 + }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, "input_cost_per_audio_token": 7e-07, @@ -29098,16 +29281,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29119,6 +29305,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29161,16 +29348,19 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, @@ -29182,6 +29372,7 @@ "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, @@ -29225,16 +29416,19 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, "cache_creation_input_token_cost_flex": 1.25e-06, "cache_creation_input_token_cost_priority": 5e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, @@ -29246,6 +29440,7 @@ "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, "output_cost_per_token_above_272k_tokens_flex": 9e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, @@ -29288,16 +29483,19 @@ "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, "cache_creation_input_token_cost_flex": 1.25e-07, "cache_creation_input_token_cost_priority": 5e-07, "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, "input_cost_per_token_above_272k_tokens_flex": 2e-07, + "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, @@ -29309,6 +29507,7 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, "output_cost_per_token_above_272k_tokens_flex": 9e-07, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, @@ -29548,7 +29747,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, @@ -29602,7 +29804,10 @@ "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": false + "supports_minimal_reasoning_effort": false, + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07 }, "gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -29751,7 +29956,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, @@ -29800,7 +30008,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07 }, "gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, @@ -29849,7 +30060,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, @@ -29898,7 +30111,9 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135 }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, @@ -30834,17 +31049,18 @@ }, "gpt-realtime-2": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image": 5e-06, "input_cost_per_token": 4e-06, "litellm_provider": "openai", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, - "output_cost_per_token": 1.6e-05, + "output_cost_per_token": 2.4e-05, "supported_endpoints": [ "/v1/realtime" ], @@ -30908,8 +31124,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -30941,7 +31157,7 @@ "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, "litellm_provider": "openai", - "max_input_tokens": 128000, + "max_input_tokens": 32000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", @@ -32877,6 +33093,88 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "meta/muse-spark-1.3": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "meta", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.25e-06, + "search_context_cost_per_query": { + "search_context_size_high": 0.0025, + "search_context_size_low": 0.0025, + "search_context_size_medium": 0.0025 + }, + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_minimal_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "meta/muse-spark-1.3-contributor": { + "cache_read_input_token_cost": 2e-09, + "input_cost_per_token": 1e-07, + "litellm_provider": "meta", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.0025, + "search_context_size_low": 0.0025, + "search_context_size_medium": 0.0025 + }, + "source": "https://ai.developer.meta.com/docs/pricing-rate-limits", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_minimal_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "meta_llama/Llama-3.3-70B-Instruct": { "litellm_provider": "meta_llama", "max_input_tokens": 128000, @@ -33705,19 +34003,21 @@ "source": "https://mistral.ai/pricing#api-pricing" }, "mistral/magistral-medium-latest": { - "input_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-2506": { "deprecation_date": "2025-11-30", @@ -33736,19 +34036,21 @@ "supports_tool_choice": true }, "mistral/magistral-small-latest": { - "input_cost_per_token": 5e-07, + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", + "output_cost_per_token": 6e-07, + "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/magistral-small-1-2-2509": { "deprecation_date": "2026-07-31", @@ -33880,16 +34182,21 @@ "supports_vision": true }, "mistral/mistral-medium": { - "input_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", - "output_cost_per_token": 8.1e-06, + "output_cost_per_token": 7.5e-06, + "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/mistral-medium-2312": { "deprecation_date": "2025-06-16", @@ -41340,13 +41647,13 @@ "source": "https://docs.together.ai/docs/serverless-models" }, "together_ai/Qwen/Qwen3.8-2.4T-A95B": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, "max_tokens": 1010000, "mode": "chat", - "output_cost_per_token": 6.25e-06, + "output_cost_per_token": 6e-06, "source": "https://docs.together.ai/docs/serverless-models", "supports_prompt_caching": true }, @@ -41956,6 +42263,70 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 1024 }, + "us-gov.anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us-gov.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, @@ -45849,6 +46220,26 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, + "voyage/rerank-3": { + "input_cost_per_token": 5e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, + "voyage/rerank-3-lite": { + "input_cost_per_token": 2e-08, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://docs.voyageai.com/docs/pricing" + }, "voyage/voyage-2": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", @@ -47100,6 +47491,27 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-build-latest": { + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "xai", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://docs.x.ai/developers/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "xai/grok-4.6": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, @@ -57420,6 +57832,34 @@ "supports_tool_choice": true, "supports_vision": false }, + "fireworks_ai/accounts/fireworks/models/glm-5p3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/accounts/fireworks/models/inkling": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 4.05e-06, + "source": "https://fireworks.ai/models/fireworks/inkling", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-8b": { "input_cost_per_token": 1e-07, "output_cost_per_token": 0.0, @@ -57479,5 +57919,542 @@ "supported_endpoints": [ "/v1/audio/transcriptions" ] + }, + "scaleway/glm-5.2": { + "input_cost_per_token": 1.8e-06, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 5.5e-06, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": false + }, + "scaleway/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "scaleway", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_vision": false + }, + "azure_ai/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "deprecation_date": "2026-10-03", + "input_cost_per_token": 9.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-west-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-west-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 7.2e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.88e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_system_messages": true, + "supports_vision": true + }, + "bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 7.8e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "input_cost_per_token": 8.4e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "input_cost_per_token": 1.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock/us-gov-east-1/anthropic.claude-sonnet-5": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 2.4e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock/us-gov-east-1/anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, + "bedrock_output_config_effort_ceiling": "xhigh", + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + "input_cost_per_token": 6e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "prompt_cache_min_tokens": 1024, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_above_272k_tokens": 5.28e-06, + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-06, + "cache_read_input_token_cost": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-07, + "output_cost_per_token": 1.584e-05, + "output_cost_per_token_above_272k_tokens": 2.376e-05 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 2.64e-07, + "input_cost_per_token_above_272k_tokens": 5.28e-07, + "cache_creation_input_token_cost": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-07, + "cache_read_input_token_cost": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-08, + "output_cost_per_token": 1.584e-06, + "output_cost_per_token_above_272k_tokens": 2.376e-06 + }, + "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "bedrock_mantle/us-gov-west-1/xai.grok-4.3": { + "use_openai_responses_path": true, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/", + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.4e-07 + }, + "bedrock_mantle/us-gov-east-1/openai.gpt-5.4": { + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "use_openai_responses_path": true, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "input_cost_per_token": 3.3e-06, + "cache_read_input_token_cost": 3.3e-07, + "output_cost_per_token": 1.98e-05 + }, + "azure/us-gov/gpt-5.1": { + "cache_read_input_token_cost": 1.71875e-07, + "default_reasoning_effort": "none", + "input_cost_per_token": 1.71875e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.375e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us-gov/o3-mini": { + "cache_read_input_token_cost": 7.57e-07, + "input_cost_per_token": 1.513e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 6.05e-06, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "azure/us-gov/text-embedding-3-large": { + "input_cost_per_token": 1.63e-07, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "azure/us-gov/text-embedding-3-small": { + "input_cost_per_token": 2.5e-08, + "litellm_provider": "azure", + "max_input_tokens": 8191, + "max_tokens": 8191, + "mode": "embedding", + "output_cost_per_token": 0.0 + }, + "cloudflare/@cf/openai/whisper": { + "input_cost_per_second": 7.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "cloudflare/@cf/openai/whisper-large-v3-turbo": { + "input_cost_per_second": 8.5e-06, + "litellm_provider": "cloudflare", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/whisper-large-v3-turbo/", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] } } diff --git a/pyproject.toml b/pyproject.toml index 2866e27e84c..d0e5723d1cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,7 +161,7 @@ proxy-runtime = [ "mangum>=0.17.0,<1.0", "azure-ai-contentsafety>=1.0.0,<2.0", "azure-storage-file-datalake>=12.20.0,<13.0", - "pypdf>=6.12.0,<7.0", + "pypdf>=6.16.1,<7.0", "llm-sandbox>=0.3.39,<1.0", "detect-secrets>=1.5.0,<2.0", ] @@ -278,7 +278,10 @@ bindings = "pyo3" features = ["extension-module"] profile = "release" editable-profile = "dev" -include = ["litellm/proxy/_experimental/out/**"] +include = [ + "litellm/proxy/_experimental/out/**", + "litellm/router_strategy/complexity_router/artifacts/*.json", +] exclude = [ "litellm/proxy/enterprise", "litellm/proxy/enterprise/**", @@ -292,7 +295,7 @@ exclude = [ [tool.uv] constraint-dependencies = [ - "tornado>=6.5.6", + "tornado>=6.5.8", "aiohttp>=3.14.2,<4.0", "packaging>=24.0", "soupsieve>=2.8.4", diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 9b1cc977a64..4fcf650a8bc 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -9,7 +9,7 @@ "limit": 809 }, "ANN201": { - "limit": 2001 + "limit": 2000 }, "ANN202": { "limit": 835 @@ -24,7 +24,7 @@ "limit": 133 }, "ANN401": { - "limit": 307 + "limit": 304 }, "ASYNC230": { "limit": 11 @@ -108,7 +108,7 @@ "limit": 3 }, "F401": { - "limit": 13 + "limit": 12 }, "LOG015": { "limit": 5 @@ -147,7 +147,7 @@ "limit": 3 }, "PLR1714": { - "limit": 256 + "limit": 253 }, "PLW0127": { "limit": 57 @@ -156,7 +156,7 @@ "limit": 215 }, "PLW0603": { - "limit": 191 + "limit": 190 }, "PLW1508": { "limit": 190 @@ -177,7 +177,7 @@ "limit": 8 }, "RUF019": { - "limit": 31 + "limit": 27 }, "RUF046": { "limit": 4 @@ -231,7 +231,7 @@ "limit": 5 }, "TID251": { - "limit": 1073 + "limit": 1071 }, "TRY002": { "limit": 524 diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000000..a1598ccbb34 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.98.0" +profile = "minimal" +components = ["rustfmt", "clippy"] diff --git a/tests/code_coverage_tests/check_py310_typing_imports.py b/tests/code_coverage_tests/check_py310_typing_imports.py new file mode 100644 index 00000000000..0cd4d089890 --- /dev/null +++ b/tests/code_coverage_tests/check_py310_typing_imports.py @@ -0,0 +1,150 @@ +import ast +import os +import sys +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +PY311_PLUS_TYPING_NAMES: Final[frozenset[str]] = frozenset( + { + "NotRequired", + "Required", + "Self", + "LiteralString", + "Never", + "assert_never", + "assert_type", + "reveal_type", + "TypeVarTuple", + "Unpack", + "dataclass_transform", + "override", + "TypeAliasType", + "get_original_bases", + "ReadOnly", + "TypeIs", + "NoDefault", + "get_protocol_members", + "is_protocol", + "evaluate_forward_ref", + "TypeForm", + } +) + + +@dataclass(frozen=True, slots=True) +class TypingImportViolation: + file: str + line: int + name: str + + +def _walk_with_ancestors( + node: ast.AST, ancestors: tuple[tuple[ast.AST, str], ...] = () +) -> Iterator[tuple[ast.AST, tuple[tuple[ast.AST, str], ...]]]: + yield node, ancestors + for field_name, field_value in ast.iter_fields(node): + if isinstance(field_value, ast.AST): + yield from _walk_with_ancestors(field_value, (*ancestors, (node, field_name))) + elif isinstance(field_value, list): + for child in field_value: + if isinstance(child, ast.AST): + yield from _walk_with_ancestors(child, (*ancestors, (node, field_name))) + + +def _is_sys_version_info(node: ast.AST) -> bool: + return ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "sys" + and node.attr == "version_info" + ) + + +def _is_version_guarded(ancestors: tuple[tuple[ast.AST, str], ...]) -> bool: + nearest_if: Final[tuple[ast.If, str] | None] = next( + ( + (ancestor, field_name) + for ancestor, field_name in reversed(ancestors) + if isinstance(ancestor, ast.If) + ), + None, + ) + if nearest_if is None: + return False + enclosing_if, branch = nearest_if + test: Final[ast.expr] = enclosing_if.test + if not isinstance(test, ast.Compare) or len(test.ops) != 1 or not _is_sys_version_info(test.left): + return False + operator: Final[ast.cmpop] = test.ops[0] + return (isinstance(operator, (ast.Gt, ast.GtE)) and branch == "body") or ( + isinstance(operator, (ast.Lt, ast.LtE)) and branch == "orelse" + ) + + +def scan_file(file_path: str | os.PathLike[str]) -> tuple[TypingImportViolation, ...]: + path: Final[Path] = Path(file_path) + tree: Final[ast.Module] = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + return tuple( + violation + for node, ancestors in _walk_with_ancestors(tree) + if not _is_version_guarded(ancestors) + for violation in _violations_for_node(node, path) + ) + + +def _violations_for_node( + node: ast.AST, path: Path +) -> tuple[TypingImportViolation, ...]: + if isinstance(node, ast.ImportFrom) and node.module == "typing": + return tuple( + TypingImportViolation(file=str(path), line=node.lineno, name=alias.name) + for alias in node.names + if alias.name in PY311_PLUS_TYPING_NAMES + ) + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "typing" + and node.attr in PY311_PLUS_TYPING_NAMES + ): + return (TypingImportViolation(file=str(path), line=node.lineno, name=node.attr),) + return () + + +def scan_directory(base_dir: str | os.PathLike[str] = ".") -> tuple[TypingImportViolation, ...]: + base_path: Final[Path] = Path(base_dir) + return tuple( + violation + for directory in ( + base_path / "litellm", + base_path / "enterprise", + base_path / "litellm-proxy-extras" / "litellm_proxy_extras", + ) + if directory.exists() + for path in directory.rglob("*.py") + for violation in scan_file(path) + ) + + +def main() -> None: + violations: Final[tuple[TypingImportViolation, ...]] = scan_directory() + if violations: + message: Final[str] = "\n".join( + ( + "Python 3.10-incompatible typing imports found:", + *( + f"{violation.file}:{violation.line}: {violation.name} is unavailable in Python 3.10; " + "import it from typing_extensions instead because litellm supports Python 3.10" + for violation in violations + ), + ) + ) + sys.stdout.write(f"{message}\n") + raise RuntimeError("Import Python 3.10-incompatible typing names from typing_extensions instead") + sys.stdout.write("No Python 3.10-incompatible typing imports found.\n") + + +if __name__ == "__main__": + main() diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 790956156b0..0578dc60119 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -64,6 +64,8 @@ IGNORE_FUNCTIONS = [ "_flatten_form_field", # bounded by the nesting depth of the already-parsed request body (a finite JSON tree, no cycles possible). "_flatten_form_data_field", # bounded by the nesting depth of the already-parsed request body (a finite JSON tree, no cycles possible). "_json_safe", # max depth set (_MAX_DEPTH) plus a seen-ids cycle guard for self-referential input. + "_redact_agent_params_tree", # max depth set (default 10), same shape as _redact_sensitive_litellm_params. + "_restore_redacted_nested_value", # max depth set (default 10), mirrors _redact_agent_params_tree on the write side. ] diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index a5e00799519..0af29f069c6 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -82,6 +82,11 @@ ignored_function_names = [ "_invalidate_access_groups_cache", # Tested indirectly via set_model_list, upsert_model etc. (test files lack "router" in name) "has_buffered_provider_output", # Property, so its reads in test_router.py are never an ast.Call "_resolved_provider", # Tested via get_pattern in test_pattern_match_deployments.py (file lacks "router" in name) + "_request_header", # Tested through Claude Code session routing in test_router.py + "_claude_code_session_router_cache_key", # Tested through Claude Code session routing in test_router.py + "_delete_claude_code_session_router_binding", # Tested through Redis cleanup failure in test_router.py + "_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py + "_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py ] diff --git a/tests/e2e/test_junit_properties.py b/tests/e2e/test_junit_properties.py index c0596177cc1..02c1413c840 100644 --- a/tests/e2e/test_junit_properties.py +++ b/tests/e2e/test_junit_properties.py @@ -24,25 +24,10 @@ from junit_properties import ( ) -class FakeMarker: - def __init__(self, name: str, *args: object) -> None: - self.name = name - self.args = args - - -class FakeItem: - """The three attributes junit_properties reads off a pytest Item.""" - - def __init__( - self, nodeid: str, location: tuple[str, int | None, str], markers: tuple[FakeMarker, ...] = () - ) -> None: - self.nodeid = nodeid - self.location = location - self.user_properties: list[tuple[str, str]] = [] - self._markers = markers - - def iter_markers(self, name: str): - return (marker for marker in self._markers if marker.name == name) +def collected_item(request: pytest.FixtureRequest, name: str) -> pytest.Item: + """The Item pytest collected for test ``name`` in this file: the real nodeid, + location and marker machinery the collection hook reads, as pytest built it.""" + return next(item for item in request.session.items if item.path == request.path and item.name == name) def repo_root() -> Path | None: @@ -109,22 +94,22 @@ class TestSourceFromLocation: class TestResultProperties: - def test_every_test_carries_package_covers_and_source(self) -> None: - item = FakeItem( - "logging/test_x.py::TestFoo::test_bar", - ("logging/test_x.py", 40, "TestFoo.test_bar"), - (FakeMarker("covers", "LOG-1", "LOG-2"),), - ) - assert result_properties(item) == ( - ("package", "logging"), + def test_every_test_carries_package_covers_and_source(self, request: pytest.FixtureRequest) -> None: + """Read off this test's own collected Item, so the nodeid and location are + whatever pytest reports for the launch shape in use, and the marker is added + at run time so the coverage registry's collect-only pass never sees it.""" + test = type(self).test_every_test_carries_package_covers_and_source + request.applymarker(pytest.mark.covers("LOG-1", "LOG-2")) + assert result_properties(collected_item(request, test.__name__)) == ( + ("package", "root"), ("covers", "LOG-1,LOG-2"), - ("source", "tests/e2e/logging/test_x.py:41"), + ("source", f"tests/e2e/test_junit_properties.py:{test.__code__.co_firstlineno}"), ) - def test_attach_is_idempotent(self) -> None: + def test_attach_is_idempotent(self, request: pytest.FixtureRequest) -> None: """Collection can run the hook more than once; a second pass must not double the entries in the report.""" - item = FakeItem("logging/test_x.py::test_bar", ("logging/test_x.py", 40, "test_bar")) + item = collected_item(request, type(self).test_attach_is_idempotent.__name__) attach_result_properties(item) attach_result_properties(item) assert [name for name, _ in item.user_properties] == ["package", "covers", "source"] diff --git a/tests/e2e/ui/constants.ts b/tests/e2e/ui/constants.ts index bb33c90ddf3..3a62252915c 100644 --- a/tests/e2e/ui/constants.ts +++ b/tests/e2e/ui/constants.ts @@ -30,6 +30,7 @@ export const E2E_PROXY_ADMIN_EMAIL = "admin@test.local"; export const E2E_INTERNAL_USER_ID = "e2e-internal-user"; export const E2E_INTERNAL_USER_EMAIL = "internal@test.local"; export const E2E_TEAM_ADMIN_USER_ID = "e2e-team-admin"; +export const E2E_SEEDED_USER_PASSWORD = "E2e-Test-Pass-2026!"; // Key aliases for seeded test keys (match seed.sql) export const E2E_UPDATE_LIMITS_KEY_ALIAS = "e2eUpdateLimitsKey"; diff --git a/tests/e2e/ui/fixtures/seed.sql b/tests/e2e/ui/fixtures/seed.sql index e77b4a16b3d..00ea668ed8f 100644 --- a/tests/e2e/ui/fixtures/seed.sql +++ b/tests/e2e/ui/fixtures/seed.sql @@ -24,18 +24,18 @@ INSERT INTO "LiteLLM_OrganizationTable" ( 'e2e-proxy-admin', 'e2e-proxy-admin' ); --- 4. Users (password hash is scrypt of "test") +-- 4. Users (password hash is scrypt of E2E_SEEDED_USER_PASSWORD from constants.ts) INSERT INTO "LiteLLM_UserTable" ("user_id", "user_email", "user_role", "teams", "password") VALUES - ('e2e-proxy-admin', 'admin@test.local', 'proxy_admin', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org","e2e-team-keygen"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-noteam', 'noteam@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-invitable-by-team-admin', 'invitable-team@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-removable-member', 'removable@test.local', 'internal_user', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'); + ('e2e-proxy-admin', 'admin@test.local', 'proxy_admin', '{"e2e-team-crud"}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', '{}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org","e2e-team-keygen"}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-internal-noteam', 'noteam@test.local', 'internal_user', '{}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-invitable-by-team-admin', 'invitable-team@test.local', 'internal_user', '{}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'), + ('e2e-removable-member', 'removable@test.local', 'internal_user', '{"e2e-team-crud"}', 'scrypt:KdnTJwPb3gswdqSznPE5CC6apeFIMycd6BG7yRWndZa3QZPcVs37y7jvrQCaPUNq'); -- 5. Teams (members_with_roles is required JSON) INSERT INTO "LiteLLM_TeamTable" ( diff --git a/tests/e2e/ui/fixtures/users.ts b/tests/e2e/ui/fixtures/users.ts index 79ee237f334..0457361d9d4 100644 --- a/tests/e2e/ui/fixtures/users.ts +++ b/tests/e2e/ui/fixtures/users.ts @@ -1,6 +1,7 @@ import { ADMIN_STORAGE_PATH, ADMIN_VIEWER_STORAGE_PATH, + E2E_SEEDED_USER_PASSWORD, INTERNAL_USER_STORAGE_PATH, INTERNAL_VIEWER_STORAGE_PATH, TEAM_ADMIN_STORAGE_PATH, @@ -23,22 +24,22 @@ export const users: Record { // Log in via the form as the no-team seeded user. await page.goto("/ui/login"); await page.getByPlaceholder("Enter your username").fill("noteam@test.local"); - await page.getByPlaceholder("Enter your password").fill("test"); + await page.getByPlaceholder("Enter your password").fill(E2E_SEEDED_USER_PASSWORD); await page.getByRole("button", { name: "Login", exact: true }).click(); await expect(page.getByRole("complementary").getByText("Virtual Keys")).toBeVisible({ timeout: 30_000 }); expect(new URL(page.url()).pathname).not.toMatch(/\/connect$/); diff --git a/tests/e2e/ui/tests/logs/logsPagination.spec.ts b/tests/e2e/ui/tests/logs/logsPagination.spec.ts new file mode 100644 index 00000000000..416e1c3171f --- /dev/null +++ b/tests/e2e/ui/tests/logs/logsPagination.spec.ts @@ -0,0 +1,150 @@ +import { test, expect, type APIRequestContext, type Locator, type Page as PlaywrightPage } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; +import { CHAT_MODEL_A, createVirtualKey, sendChatCompletion, waitForSpendLog } from "../../helpers/traffic"; + +/** + * Session-grouped pagination (#38060): a page of N rows must render exactly N session rows, a + * session must never straddle pages, and two callers reusing one session id stay separate rows. + * All traffic is generated per run behind a unique key alias or session id, so concurrent specs + * cannot decide the outcome. + */ + +const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + +/** Every tab stays mounted, so the DOM holds four tables at once; scope to the visible one. */ +const requestLogsRows = (page: PlaywrightPage): Locator => + page.locator("table").filter({ visible: true }).first().locator("tbody tr"); + +const visibleTestId = (page: PlaywrightPage, id: string): Locator => page.getByTestId(id).filter({ visible: true }); + +async function openLogs(page: PlaywrightPage): Promise { + await navigateToPage(page, Page.Logs); + await dismissFeedbackPopup(page); + await expect(visibleTestId(page, "datatable-search")).toBeVisible({ timeout: 20_000 }); +} + +async function openFilterDrawer(page: PlaywrightPage): Promise { + await visibleTestId(page, "datatable-filters-trigger").click(); + const drawer = page.getByRole("dialog", { name: "Filters" }); + await expect(drawer).toBeVisible({ timeout: 10_000 }); + return drawer; +} + +async function applyKeyAliasFilter(page: PlaywrightPage, drawer: Locator, alias: string): Promise { + await drawer.getByRole("combobox", { name: "Search a key alias" }).click(); + await page.keyboard.type(alias); + await page.getByRole("option", { name: alias, exact: true }).first().click(); + await drawer.getByRole("button", { name: "Apply Filters" }).click(); + await expect(drawer).not.toBeVisible({ timeout: 10_000 }); +} + +async function setRowsPerPage(page: PlaywrightPage, size: "25" | "50" | "100"): Promise { + await visibleTestId(page, "pagination-page-size").click(); + await page.getByRole("option", { name: size, exact: true }).click(); +} + +test.describe("Logs page session-grouped pagination", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("a 25-row page renders exactly 25 session rows and no session straddles pages", async ({ page, request }) => { + const suffix = uniqueSuffix(); + const alias = `e2e-logs-pgn-${suffix}`; + const mine = await createVirtualKey(request, { key_alias: alias }); + + const soloIds: string[] = []; + for (let i = 0; i < 26; i++) { + soloIds.push( + await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `logs-pgn-solo-${i}-${suffix}`, + apiKey: mine.key, + }), + ); + } + const sessionA = `sess-pgn-a-${suffix}`; + const sessionB = `sess-pgn-b-${suffix}`; + let lastSessionCallId = ""; + for (let i = 0; i < 7; i++) { + lastSessionCallId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `logs-pgn-a-${i}-${suffix}`, + apiKey: mine.key, + traceId: sessionA, + }); + } + for (let i = 0; i < 3; i++) { + lastSessionCallId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `logs-pgn-b-${i}-${suffix}`, + apiKey: mine.key, + traceId: sessionB, + }); + } + await waitForSpendLog(request, lastSessionCallId); + await waitForSpendLog(request, soloIds[soloIds.length - 1]); + + // 36 calls in 28 session groups: 26 solos plus sessions of 7 and 3. + await openLogs(page); + const drawer = await openFilterDrawer(page); + await applyKeyAliasFilter(page, drawer, alias); + await setRowsPerPage(page, "25"); + + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 28", { timeout: 30_000 }); + await expect(requestLogsRows(page)).toHaveCount(25); + // The sessions are the newest groups, so their single representative rows sit on page 1. + await expect(requestLogsRows(page).filter({ hasText: sessionA })).toHaveCount(1); + await expect(requestLogsRows(page).filter({ hasText: sessionA })).toContainText("7"); + await expect(requestLogsRows(page).filter({ hasText: sessionB })).toHaveCount(1); + + await visibleTestId(page, "pagination-next").click(); + + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 26-28 of 28", { timeout: 30_000 }); + await expect(requestLogsRows(page)).toHaveCount(3); + await expect(requestLogsRows(page).filter({ hasText: sessionA })).toHaveCount(0); + await expect(requestLogsRows(page).filter({ hasText: sessionB })).toHaveCount(0); + }); + + test("two keys reusing one session id stay separate rows", async ({ page, request }) => { + const suffix = uniqueSuffix(); + const mine = await createVirtualKey(request, { key_alias: `e2e-logs-pgn-mine-${suffix}` }); + const theirs = await createVirtualKey(request, { key_alias: `e2e-logs-pgn-theirs-${suffix}` }); + const sharedSession = `sess-pgn-shared-${suffix}`; + + let lastId = ""; + for (let i = 0; i < 2; i++) { + lastId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `logs-pgn-shared-mine-${i}-${suffix}`, + apiKey: mine.key, + traceId: sharedSession, + }); + } + lastId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `logs-pgn-shared-theirs-${suffix}`, + apiKey: theirs.key, + traceId: sharedSession, + }); + await waitForSpendLog(request, lastId); + + await openLogs(page); + const drawer = await openFilterDrawer(page); + await drawer.getByPlaceholder("Enter session ID…").fill(sharedSession); + await drawer.getByRole("button", { name: "Apply Filters" }).click(); + await expect(drawer).not.toBeVisible({ timeout: 10_000 }); + + // One row per caller: reusing a session id must not merge two keys' activity into one row. + await expect(requestLogsRows(page).filter({ hasText: sharedSession })).toHaveCount(2, { timeout: 30_000 }); + + // And each row carries ITS key's totals: two calls badge the first key's row, + // while the other key's single call renders as a plain LLM row. + const mineRow = requestLogsRows(page).filter({ hasText: sharedSession }).filter({ hasText: mine.token }); + const theirsRow = requestLogsRows(page).filter({ hasText: sharedSession }).filter({ hasText: theirs.token }); + await expect(mineRow).toHaveCount(1); + await expect(theirsRow).toHaveCount(1); + await expect(mineRow.getByText("2", { exact: true })).toBeVisible(); + await expect(theirsRow.getByText("LLM", { exact: true })).toBeVisible(); + }); +}); diff --git a/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts b/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts index 5a8bc84cc13..73263c844fa 100644 --- a/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts @@ -10,7 +10,7 @@ test.describe("Second proxy admin", () => { test("an invited admin can log in, mint a key, and call a model with it", async ({ page, browser, request }) => { const suffix = Date.now(); const email = `second-admin-${suffix}@test.local`; - const password = "e2e-second-admin-password"; + const password = "E2e-Second-Admin-Pass-1!"; const auth = { Authorization: `Bearer ${masterKey()}` }; const inviteAdminUser = async (): Promise => { diff --git a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py index 3f9f2bacdd3..98045725177 100644 --- a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py +++ b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py @@ -71,6 +71,48 @@ def setup_vector_store_registry(): ) +@pytest.mark.asyncio +async def test_vector_store_hook_routes_search_through_proxy_router( + setup_vector_store_registry, +): + proxy_router = Mock() + proxy_router.avector_store_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", + search_query="what is litellm?", + data=[ + VectorStoreSearchResult( + score=1.0, + content=[VectorStoreResultContent(text="routed context", type="text")], + ) + ], + ) + ) + logging_obj = Mock() + logging_obj.model_call_details = { + "litellm_params": {"metadata": {"user_api_key_team_id": "team-a"}} + } + + with patch("litellm.proxy.proxy_server.llm_router", proxy_router): + _, messages, _ = await VectorStorePreCallHook().async_get_chat_completion_prompt( + model="chat-model", + messages=[{"role": "user", "content": "what is litellm?"}], + non_default_params={"vector_store_ids": ["T37J8R4WTM"]}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + litellm_logging_obj=logging_obj, + ) + + proxy_router.avector_store_search.assert_awaited_once_with( + vector_store_id="T37J8R4WTM", + query="what is litellm?", + custom_llm_provider="bedrock", + metadata={"user_api_key_team_id": "team-a"}, + ) + assert messages[0]["content"] == "Context:\n\nrouted context\n\n" + + @pytest.mark.asyncio async def test_e2e_bedrock_knowledgebase_retrieval_with_completion( setup_vector_store_registry, @@ -333,6 +375,7 @@ async def test_bedrock_kb_request_body_has_transformed_filters( timeout=None, client=None, _is_async=False, + router: "litellm.Router | None" = None, ): litellm_params_dict = ( litellm_params.model_dump(exclude_none=False) diff --git a/tests/ocr_tests/test_ocr_vertex_ai.py b/tests/ocr_tests/test_ocr_vertex_ai.py index 1ba5b9d0883..1842eb063a5 100644 --- a/tests/ocr_tests/test_ocr_vertex_ai.py +++ b/tests/ocr_tests/test_ocr_vertex_ai.py @@ -5,9 +5,11 @@ Note: Vertex AI OCR automatically converts URLs to base64 data URIs since the Vertex AI endpoint doesn't have internet access. """ -import os import json +import os import tempfile +from typing import Final + import pytest from base_ocr_unit_tests import BaseOCRTest @@ -139,3 +141,19 @@ def test_vertex_ai_ocr_routing(): assert isinstance( deepseek_variant, VertexAIDeepSeekOCRConfig ), "DeepSeek variant should route to VertexAIDeepSeekOCRConfig" + + +@pytest.mark.parametrize("model", ("deepseek-ocr-maas", "deepseek-ai/deepseek-ocr-maas")) +def test_deepseek_request_uses_single_provider_namespace(model: str) -> None: + from litellm.llms.vertex_ai.ocr.deepseek_transformation import ( + VertexAIDeepSeekOCRConfig, + ) + + request: Final = VertexAIDeepSeekOCRConfig().transform_ocr_request( + model=model, + document={"type": "image_url", "image_url": "data:image/png;base64,AA=="}, + optional_params={}, + headers={}, + ) + + assert request.data["model"] == "deepseek-ai/deepseek-ocr-maas" diff --git a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py index 1a7fb1f3e41..abae26e02cd 100644 --- a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py +++ b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py @@ -1,5 +1,5 @@ import httpx -from openai import OpenAI, BadRequestError, APIStatusError +from openai import OpenAI, BadRequestError, NotFoundError, APIStatusError import pytest @@ -105,10 +105,9 @@ def test_streaming_response(): assert len(collected_chunks) > 0 -def test_bad_request_error(): +def test_model_not_found_error(): client = get_test_client() - with pytest.raises(BadRequestError): - # Trigger error with invalid model name + with pytest.raises(NotFoundError): client.responses.create(model="non-existent-model", input="This should fail") diff --git a/tests/proxy_migration_tests/test_prisma_toolchain.py b/tests/proxy_migration_tests/test_prisma_toolchain.py index d12a2c4dd4e..733870f3239 100644 --- a/tests/proxy_migration_tests/test_prisma_toolchain.py +++ b/tests/proxy_migration_tests/test_prisma_toolchain.py @@ -7,26 +7,36 @@ attempt fails identically. These tests pin the two behaviours that keep a container recoverable: an incomplete cache is deleted before Prisma is invoked, and the install gets a budget of its own rather than sharing the one that bounds each migration command. + +``prisma migrate deploy`` gets a budget of its own for the same reason: its +runtime grows with the number of pending migrations, so a fresh database that +replays every migration overran the per-command budget on slow machines and +the proxy gave up after four identical timeouts. """ import ast import json +import logging import os import sys import time +from collections.abc import Callable from pathlib import Path import pytest from litellm_proxy_extras.prisma_toolchain import ( DEFAULT_PRISMA_COMMAND_TIMEOUT, + DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT, PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR, PRISMA_COMMAND_TIMEOUT_ENV_VAR, + PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, ensure_prisma_toolchain, heal_incomplete_nodeenv_cache, node_binary_path, prisma_bootstrap_timeout, prisma_command_timeout, + prisma_migrate_deploy_timeout, ) from litellm_proxy_extras.utils import ProxyExtrasDBManager @@ -42,14 +52,27 @@ import time args = sys.argv[1:] cache_dir = os.environ["PRISMA_NODEENV_CACHE_DIR"] -with pathlib.Path(os.environ["FAKE_PRISMA_LOG"]).open("a") as log: +log_path = pathlib.Path(os.environ["FAKE_PRISMA_LOG"]) +earlier_same_command = sum( + 1 + for line in (log_path.read_text().splitlines() if log_path.exists() else []) + if json.loads(line)["args"][:2] == args[:2] +) +with log_path.open("a") as log: log.write( json.dumps({{"args": args, "cache_dir_present": os.path.isdir(cache_dir)}}) + "\\n" ) time.sleep(float(os.environ.get("FAKE_PRISMA_SLEEP", "0"))) if args[:2] == ["migrate", "deploy"]: + if earlier_same_command == 0: + time.sleep(float(os.environ.get("FAKE_PRISMA_FIRST_DEPLOY_SLEEP", "0"))) + elif os.environ.get("FAKE_PRISMA_LATER_DEPLOY_STDERR"): + print(os.environ["FAKE_PRISMA_LATER_DEPLOY_STDERR"], file=sys.stderr) + sys.exit(1) print("No pending migrations to apply") +if args[:2] == ["db", "push"] and earlier_same_command == 0: + time.sleep(float(os.environ.get("FAKE_PRISMA_FIRST_PUSH_SLEEP", "0"))) sys.exit(0) """ @@ -80,9 +103,14 @@ def toolchain_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}") monkeypatch.delenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, raising=False) monkeypatch.delenv(PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR, raising=False) + monkeypatch.delenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, raising=False) return cache_dir, log_path +def _deploy_calls(log_path: Path) -> list[list[str]]: + return [call["args"] for call in _fake_prisma_calls(log_path) if call["args"][:2] == ["migrate", "deploy"]] + + def _make_incomplete_cache(cache_dir: Path) -> None: (cache_dir / "lib").mkdir(parents=True) (cache_dir / "bin").mkdir() @@ -209,25 +237,111 @@ def test_setup_database_prepares_the_toolchain_before_migrating( assert calls[0]["cache_dir_present"] is False +@pytest.mark.parametrize("use_v2_resolver", [False, True], ids=["v1", "v2"]) +def test_migrate_deploy_is_not_bounded_by_the_per_command_timeout( + toolchain_env: tuple[Path, Path], + monkeypatch: pytest.MonkeyPatch, + use_v2_resolver: bool, +) -> None: + """A fresh database replays every migration, which takes longer than any bookkeeping command.""" + _, log_path = toolchain_env + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, "1") + monkeypatch.setenv("FAKE_PRISMA_FIRST_DEPLOY_SLEEP", "3") + + assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True + assert _deploy_calls(log_path) == [["migrate", "deploy"]] + + +def test_migrate_deploy_stops_at_its_own_timeout( + toolchain_env: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + """The deploy budget still bounds a deploy that hangs, so boot cannot wait forever.""" + _, log_path = toolchain_env + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, "1") + monkeypatch.setenv("FAKE_PRISMA_FIRST_DEPLOY_SLEEP", "60") + monkeypatch.setenv("FAKE_PRISMA_LATER_DEPLOY_STDERR", "Error: P3018 permission denied for schema public") + + started = time.monotonic() + with pytest.raises(RuntimeError, match="insufficient permissions"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + elapsed = time.monotonic() - started + + assert len(_deploy_calls(log_path)) == 2 + assert elapsed < 30 + + +def test_db_push_timeout_hint_names_the_per_command_budget( + toolchain_env: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """``db push`` keeps the per-command budget, so its timeout hint has to name that variable.""" + _, log_path = toolchain_env + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, "1") + monkeypatch.setenv("FAKE_PRISMA_FIRST_PUSH_SLEEP", "3") + + with caplog.at_level(logging.WARNING, logger="litellm_proxy_extras"): + assert ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=False) is True + + assert [call["args"][:2] for call in _fake_prisma_calls(log_path)].count(["db", "push"]) == 2 + assert [record.getMessage() for record in caplog.records if "timed out" in record.getMessage()] == [ + f"Attempt 1 timed out. Raise {PRISMA_COMMAND_TIMEOUT_ENV_VAR} if this database needs longer to apply its schema." + ] + + +@pytest.mark.parametrize( + ("command_timeout", "deploy_timeout", "expected"), + [ + ("900", None, 900.0), + ("12", None, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT), + ("900", "1200", 1200.0), + ("900", "300", 300.0), + ], + ids=["raised_command_budget_carries_over", "lowered_command_budget_does_not", "override_wins_upward", "override_wins_downward"], +) +def test_migrate_deploy_budget_keeps_a_raised_command_budget( + command_timeout: str, deploy_timeout: str | None, expected: float, monkeypatch: pytest.MonkeyPatch +) -> None: + """Deployments that raised the per-command budget to survive a long deploy keep that budget for deploy.""" + monkeypatch.setenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, command_timeout) + if deploy_timeout is None: + monkeypatch.delenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, raising=False) + else: + monkeypatch.setenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, deploy_timeout) + + assert prisma_migrate_deploy_timeout() == expected + + @pytest.mark.parametrize( "raw", ["", "0", "-5", "not-a-number", "nan", "inf", "-inf", "1e400"], ) +@pytest.mark.parametrize( + ("env_var", "read_timeout", "default"), + [ + (PRISMA_COMMAND_TIMEOUT_ENV_VAR, prisma_command_timeout, DEFAULT_PRISMA_COMMAND_TIMEOUT), + (PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, prisma_migrate_deploy_timeout, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT), + ], + ids=["command", "migrate_deploy"], +) def test_unusable_timeout_override_falls_back_to_the_default( - raw: str, monkeypatch: pytest.MonkeyPatch + raw: str, env_var: str, read_timeout: Callable[[], float], default: float, monkeypatch: pytest.MonkeyPatch ) -> None: """A non-finite override would silently disable the timeout it configures.""" - monkeypatch.setenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, raw) + monkeypatch.setenv(env_var, raw) - assert prisma_command_timeout() == DEFAULT_PRISMA_COMMAND_TIMEOUT + assert read_timeout() == default def test_timeout_overrides_are_independent(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv(PRISMA_COMMAND_TIMEOUT_ENV_VAR, "12") monkeypatch.setenv(PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR, "900") + monkeypatch.setenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, "1200") assert prisma_command_timeout() == 12 assert prisma_bootstrap_timeout() == 900 + assert prisma_migrate_deploy_timeout() == 1200 @pytest.mark.parametrize("module", ["utils.py", "replica_identity.py"]) diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 47554913419..54cce9cdd78 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -3076,7 +3076,9 @@ async def test_update_config_success_callback_normalization(): admin_user = UserAPIKeyAuth( user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test" ) - await proxy_server.update_config(config_update, user_api_key_dict=admin_user) + request = MagicMock() + request.json = AsyncMock(return_value={"litellm_settings": {"success_callback": ["SQS", "sQs"]}}) + await proxy_server.update_config(config_update, request=request, user_api_key_dict=admin_user) assert ( "litellm_settings" in upserted diff --git a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py index 3785ccdcfba..096efc33aaf 100644 --- a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py +++ b/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py @@ -1,3 +1,5 @@ +import asyncio +from types import MappingProxyType from unittest.mock import AsyncMock, patch @@ -5,6 +7,7 @@ import pytest import litellm from litellm.caching.caching import DualCache +from litellm.caching.redis_cache import RedisCache from datetime import datetime, timezone from litellm.litellm_core_utils.duration_parser import duration_in_seconds @@ -1332,3 +1335,85 @@ async def test_the_user_scope_has_no_pre_upgrade_counter_to_carry(): await limiter.is_user_within_model_budget( user_id="u1", user_model_max_budget=model_max_budget, model="openai/gpt-4" ) + + +class _SharedFakeRedis(RedisCache): + """Stand-in for the one Redis every replica's DualCache is attached to. + + Only the methods the limiter and DualCache call are implemented, and + ``super().__init__`` is skipped so no connection is opened. + """ + + def __init__(self): + self._store = MappingProxyType({}) + + async def async_set_cache(self, key, value, **kwargs): + self._store = MappingProxyType({**self._store, key: value}) + + async def async_get_cache(self, key, **kwargs): + return self._store.get(key) + + async def async_batch_get_cache(self, key_list, **kwargs): + return {key: self._store.get(key) for key in key_list} + + async def async_increment_pipeline(self, increment_list, **kwargs): + for op in increment_list: + total = self._store.get(op["key"], 0.0) + op["increment_value"] + self._store = MappingProxyType({**self._store, op["key"]: total}) + return [self._store[op["key"]] for op in increment_list] + + +async def _log_spend(limiter, *, key_hash, model_max_budget, response_cost): + await limiter.async_log_success_event( + _success_kwargs( + model_group="gpt-4", + response_cost=response_cost, + key_hash=key_hash, + key_model_max_budget=model_max_budget, + ), + response_obj=None, + start_time=None, + end_time=None, + ) + # The Redis push is scheduled as a task rather than awaited inline. + await asyncio.gather(*(t for t in asyncio.all_tasks() if t is not asyncio.current_task())) + + +@pytest.mark.asyncio +async def test_spend_logged_on_one_replica_is_enforced_and_reported_on_another(): + """ + Each replica increments its own in-memory copy of the per-model counter and + pushes the increment to the shared Redis, so only Redis holds the window's + total. A replica that has served part of the traffic must still enforce and + report the total, not its own share. + + Regression: reads went to the in-memory tier first, so a replica whose local + copy sat under the cap kept admitting requests and /key/info on it reported + that local share, while the shared counter was already over the cap. + """ + shared_redis = _SharedFakeRedis() + replica_a = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache(redis_cache=shared_redis)) + replica_b = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache(redis_cache=shared_redis)) + key_hash = "vk-shared" + model_max_budget = {"gpt-4": {"budget_limit": 1.0, "time_period": "30d"}} + user_api_key = UserAPIKeyAuth(token=key_hash, model_max_budget=model_max_budget) + + await _log_spend(replica_b, key_hash=key_hash, model_max_budget=model_max_budget, response_cost=0.25) + await _log_spend(replica_a, key_hash=key_hash, model_max_budget=model_max_budget, response_cost=0.5) + await _log_spend(replica_a, key_hash=key_hash, model_max_budget=model_max_budget, response_cost=0.5) + + with pytest.raises(litellm.BudgetExceededError): + await replica_b.is_key_within_model_budget(user_api_key, "gpt-4") + + usage_on_b = await build_model_max_budget_usage( + entity_type=Litellm_EntityType.KEY, + entity_id=key_hash, + model_max_budget=model_max_budget, + cache=replica_b.dual_cache, + ) + assert usage_on_b["gpt-4"]["current_spend"] == 1.25 + + # Control: a replica that never served this key reads the same total. + replica_c = _PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache(redis_cache=shared_redis)) + with pytest.raises(litellm.BudgetExceededError): + await replica_c.is_key_within_model_budget(user_api_key, "gpt-4") diff --git a/tests/router_unit_tests/test_router_anthropic_messages_fallback.py b/tests/router_unit_tests/test_router_anthropic_messages_fallback.py index 0c4d1dfc21e..4812d199c06 100644 --- a/tests/router_unit_tests/test_router_anthropic_messages_fallback.py +++ b/tests/router_unit_tests/test_router_anthropic_messages_fallback.py @@ -338,6 +338,112 @@ def test_record_pre_routing_selection_writes_only_the_internal_bucket(): assert kwargs["metadata"] == {"user_id": "u1"} +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_generic_only_row_recovers_safeguard_refusal(stream): + """With no content-policy list configured, a generic fallback row covers safeguard refusals, + so the dashboard's generic fallbacks work without config-only content_policy rows.""" + fake = FakeAnthropicUpstream() + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"fable-tier": ["opus-target"]}]) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", max_tokens=16, stream=stream, messages=[{"role": "user", "content": "hi"}] + ) + body = await _collect(response) if stream else response + + if stream: + assert b'"refusal"' not in body + assert b"text_delta" in body + else: + assert body["stop_reason"] == "end_turn" + assert len(fake.calls) == 2 + assert "claude-opus-5" in fake.calls[1] + + +@pytest.mark.asyncio +async def test_configured_content_policy_list_stays_authoritative_over_generic_rows(): + fake = FakeAnthropicUpstream() + router = Router( + model_list=[FABLE_TIER, OPUS_TARGET], + fallbacks=[{"fable-tier": ["opus-target"]}], + content_policy_fallbacks=[{"unrelated-group": ["opus-target"]}], + ) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", max_tokens=16, messages=[{"role": "user", "content": "hi"}] + ) + + assert response["stop_reason"] == "refusal" + assert len(fake.calls) == 1 + + +def test_refusal_fallback_available_arms_on_generic_rows_only_without_content_policy(): + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"tier-group": ["opus-target"]}]) + stamped = {"litellm_metadata": {PRE_ROUTING_SELECTED_MODEL_KEY: "tier-group"}} + + assert router._refusal_fallback_available("router-group", stamped) is True + assert router._refusal_fallback_available("router-group", {}) is False + assert router._refusal_fallback_available("router-group", {"content_policy_fallbacks": [{"other": ["x"]}]}) is False + + +def test_chat_content_filter_gate_unchanged_by_generic_rows(): + """The generic-row arming is scoped to /v1/messages safeguard refusals; the chat surface's + content_filter gate keeps its long-standing content-policy-only semantics.""" + from litellm.types.utils import Choices, ModelResponse + + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"fable-tier": ["opus-target"]}]) + response = ModelResponse(choices=[Choices(finish_reason="content_filter")]) + + assert router._should_raise_content_policy_error(model="fable-tier", response=response, kwargs={}) is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_disable_fallbacks_returns_the_refusal_instead_of_raising(stream): + """A request that opted out of fallbacks must receive the provider's refusal response, + never a ContentPolicyViolationError the dispatcher refuses to recover.""" + fake = FakeAnthropicUpstream() + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"fable-tier": ["opus-target"]}]) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", + max_tokens=16, + stream=stream, + disable_fallbacks=True, + messages=[{"role": "user", "content": "hi"}], + ) + body = await _collect(response) if stream else response + + if stream: + assert b'"stop_reason": "refusal"' in body + else: + assert body["stop_reason"] == "refusal" + assert len(fake.calls) == 1 + + +@pytest.mark.asyncio +async def test_disable_fallbacks_beats_a_content_policy_row_too(): + fake = FakeAnthropicUpstream() + router = Router( + model_list=[FABLE_TIER, OPUS_TARGET], + content_policy_fallbacks=[{"fable-tier": ["opus-target"]}], + ) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", + max_tokens=16, + disable_fallbacks=True, + messages=[{"role": "user", "content": "hi"}], + ) + + assert response["stop_reason"] == "refusal" + assert len(fake.calls) == 1 + + def test_refusal_gate_keys_on_pre_routing_tier_stamp(): router = _router(content_policy_fallbacks=[{"tier-group": ["opus-target"]}]) diff --git a/tests/router_unit_tests/test_router_embedding_integration.py b/tests/router_unit_tests/test_router_embedding_integration.py index 75dacbaf08e..2cc9914c9b3 100644 --- a/tests/router_unit_tests/test_router_embedding_integration.py +++ b/tests/router_unit_tests/test_router_embedding_integration.py @@ -5,17 +5,218 @@ These tests simulate real-world scenarios where headers and configuration need to be properly propagated through the router to the LLM API. """ -from unittest.mock import MagicMock, patch, AsyncMock +import json +from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +import respx - +import litellm from litellm import Router +from litellm.llms.base_llm.vector_store.transformation import ( + LiteLLMVectorStoreEmbeddingExecutor, + RouterVectorStoreEmbeddingExecutor, +) + +QUERY_VECTOR = [0.5, -0.25, 0.125] +OPENAI_EMBEDDINGS_URL = "https://api.openai.com/v1/embeddings" +STORE_EMBEDDINGS_URL = "https://embedding.example/v1/embeddings" + + +def _mock_embedding_route(respx_mock: respx.MockRouter, url: str) -> respx.Route: + return respx_mock.post(url).mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": QUERY_VECTOR}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + ) + + +def _sent(route: respx.Route, index: int) -> tuple[str, str, list[str]]: + request = route.calls[index].request + body = json.loads(request.read()) + return request.headers["authorization"], body["model"], body["input"] + + +def _alias_router() -> Router: + return Router( + model_list=[ + { + "model_name": "team-alias", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "deployment-key", + }, + } + ] + ) class TestRouterEmbeddingIntegration: """Integration tests for embedding with router configuration.""" + def test_vector_store_request_metadata_prefers_litellm_metadata(self): + assert Router._vector_store_request_metadata( + { + "litellm_metadata": {"user_api_key_team_id": "team-a"}, + "metadata": {"user_api_key_team_id": "team-b"}, + } + ) == {"user_api_key_team_id": "team-a"} + + assert Router._vector_store_request_metadata({"metadata": {"user_api_key_team_id": "team-b"}}) == { + "user_api_key_team_id": "team-b" + } + assert Router._vector_store_request_metadata({}) == {} + + def test_sync_vector_store_wrapper_injects_router_embedding_executor(self): + router = Router(model_list=[]) + original = MagicMock(return_value="searched") + wrapped = router.factory_function(original, call_type="vector_store_search") + + assert ( + wrapped( + vector_store_id="store", + query="query", + custom_llm_provider="valkey", + metadata={"user_api_key_team_id": "team-a"}, + ) + == "searched" + ) + + call_kwargs = original.call_args.kwargs + assert call_kwargs["custom_llm_provider"] == "valkey" + executor = call_kwargs["_direct_vector_store_embedding_executor"] + assert isinstance(executor, RouterVectorStoreEmbeddingExecutor) + assert executor.metadata == {"user_api_key_team_id": "team-a"} + + def test_sync_vector_store_wrapper_preserves_model_routing(self): + router = Router(model_list=[]) + original = MagicMock() + wrapped = router.factory_function(original, call_type="vector_store_search") + + with patch.object(router, "_generic_api_call_with_fallbacks", return_value="routed") as fallback: + assert wrapped(model="vector-alias", vector_store_id="store", query="query") == "routed" + + assert fallback.call_args.kwargs["model"] == "vector-alias" + assert fallback.call_args.kwargs["original_function"] is original + assert isinstance( + fallback.call_args.kwargs["_direct_vector_store_embedding_executor"], + RouterVectorStoreEmbeddingExecutor, + ) + + @pytest.mark.asyncio + async def test_vector_store_embedding_executors_cover_sdk_and_router_paths( + self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + openai_route = _mock_embedding_route(respx_mock, OPENAI_EMBEDDINGS_URL) + store_route = _mock_embedding_route(respx_mock, STORE_EMBEDDINGS_URL) + sdk_executor = LiteLLMVectorStoreEmbeddingExecutor() + + sync_response = sdk_executor.embed("openai/text-embedding-3-small", "sync", {"api_key": "explicit"}) + async_response = await sdk_executor.aembed("openai/text-embedding-3-small", "async", {"api_key": "explicit"}) + + assert sync_response.data[0]["embedding"] == QUERY_VECTOR + assert async_response.data[0]["embedding"] == QUERY_VECTOR + assert _sent(openai_route, 0) == ("Bearer explicit", "text-embedding-3-small", ["sync"]) + assert _sent(openai_route, 1) == ("Bearer explicit", "text-embedding-3-small", ["async"]) + + explicit_config = { + "api_base": "https://embedding.example/v1", + "api_key": "store-key", + "metadata": { + "configured": True, + "user_api_key_team_id": "untrusted-team", + }, + "model": "untrusted-model", + } + mock_router = MagicMock() + mock_router.embedding.return_value = sync_response + router_executor = RouterVectorStoreEmbeddingExecutor( + router=mock_router, + metadata={"user_api_key_team_id": "team-a"}, + ) + assert router_executor.embed("team-alias", "query", explicit_config) is sync_response + mock_router.embedding.assert_called_once_with( + model="team-alias", + input=["query"], + api_base="https://embedding.example/v1", + api_key="store-key", + metadata={"configured": True, "user_api_key_team_id": "team-a"}, + ) + + alias_executor = RouterVectorStoreEmbeddingExecutor( + router=_alias_router(), + metadata={"user_api_key_team_id": "team-a"}, + ) + sync_alias = alias_executor.embed("team-alias", "sync query", explicit_config) + async_alias = await alias_executor.aembed("team-alias", "async query", explicit_config) + + assert sync_alias.data[0]["embedding"] == QUERY_VECTOR + assert async_alias.data[0]["embedding"] == QUERY_VECTOR + assert openai_route.call_count == 2 + assert _sent(store_route, 0) == ("Bearer store-key", "text-embedding-3-small", ["sync query"]) + assert _sent(store_route, 1) == ("Bearer store-key", "text-embedding-3-small", ["async query"]) + + @pytest.mark.asyncio + async def test_router_executor_falls_back_to_sdk_for_models_the_router_does_not_serve( + self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + store_route = _mock_embedding_route(respx_mock, STORE_EMBEDDINGS_URL) + executor = RouterVectorStoreEmbeddingExecutor( + router=_alias_router(), + metadata={"user_api_key_team_id": "team-a"}, + ) + inline_config = {"api_base": "https://embedding.example/v1", "api_key": "store-key"} + + sync_response = executor.embed("openai/text-embedding-3-large", "sync query", inline_config) + async_response = await executor.aembed("openai/text-embedding-3-large", "async query", inline_config) + + assert sync_response.data[0]["embedding"] == QUERY_VECTOR + assert async_response.data[0]["embedding"] == QUERY_VECTOR + assert _sent(store_route, 0) == ("Bearer store-key", "text-embedding-3-large", ["sync query"]) + assert _sent(store_route, 1) == ("Bearer store-key", "text-embedding-3-large", ["async query"]) + + @pytest.mark.asyncio + async def test_router_executor_rejects_unserved_models_without_explicit_config( + self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("OPENAI_API_KEY", "env-key") + openai_route = _mock_embedding_route(respx_mock, OPENAI_EMBEDDINGS_URL) + executor = RouterVectorStoreEmbeddingExecutor( + router=_alias_router(), + metadata={"user_api_key_team_id": "team-a"}, + ) + + with pytest.raises(litellm.BadRequestError): + executor.embed("openai/text-embedding-3-large", "sync query", {}) + with pytest.raises(litellm.BadRequestError): + await executor.aembed("openai/text-embedding-3-large", "async query", {}) + + assert openai_route.call_count == 0 + + def test_router_executor_routes_deployment_model_names_through_the_router( + self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + openai_route = _mock_embedding_route(respx_mock, OPENAI_EMBEDDINGS_URL) + executor = RouterVectorStoreEmbeddingExecutor(router=_alias_router(), metadata={}) + + response = executor.embed("openai/text-embedding-3-small", "query", {}) + + assert response.data[0]["embedding"] == QUERY_VECTOR + assert _sent(openai_route, 0) == ("Bearer deployment-key", "text-embedding-3-small", ["query"]) + def test_embedding_with_deployment_specific_headers(self): """ Test that deployment-specific headers are propagated. @@ -122,9 +323,7 @@ class TestRouterEmbeddingIntegration: router = Router( model_list=model_list, - default_litellm_params={ - "metadata": {"environment": "test", "service": "embedding-service"} - }, + default_litellm_params={"metadata": {"environment": "test", "service": "embedding-service"}}, ) with patch("litellm.embedding") as mock_embedding: @@ -240,9 +439,7 @@ class TestRouterEmbeddingIntegration: # Make multiple calls and verify headers are always present for i in range(5): with patch("litellm.embedding") as mock_embedding: - mock_embedding.return_value = MagicMock( - data=[{"embedding": [0.1, 0.2]}] - ) + mock_embedding.return_value = MagicMock(data=[{"embedding": [0.1, 0.2]}]) router.embedding(model="shared-embedding-model", input=[f"test {i}"]) @@ -327,9 +524,7 @@ class TestRouterEmbeddingIntegration: router = Router( model_list=model_list, - default_litellm_params={ - "headers": {"X-Custom-Azure-Header": "azure-value"} - }, + default_litellm_params={"headers": {"X-Custom-Azure-Header": "azure-value"}}, ) with patch("litellm.embedding") as mock_embedding: diff --git a/tests/router_unit_tests/test_router_endpoints.py b/tests/router_unit_tests/test_router_endpoints.py index d37af5b456a..a06aaa363ad 100644 --- a/tests/router_unit_tests/test_router_endpoints.py +++ b/tests/router_unit_tests/test_router_endpoints.py @@ -1324,6 +1324,98 @@ async def test_init_containers_api_endpoints_managed_id_without_model_id_applies assert call_kw["custom_llm_provider"] == "azure" +@pytest.mark.asyncio +async def test_init_containers_api_endpoints_create_with_model_uses_deployment_credentials(monkeypatch): + """ + ``POST /v1/containers`` carries no container ID, so a ``model`` in the request + body is the only way to pick a deployment. The upstream call must receive that + deployment's ``api_key``/``api_base`` instead of falling back to the global + ``OPENAI_API_KEY`` (which may be unset on the proxy). + """ + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + router = Router( + model_list=[ + { + "model_name": "gpt-5.4", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_key": "sk-model-list-key", + "api_base": "https://custom.openai.example/v1", + }, + } + ] + ) + mock_original_function = AsyncMock(return_value={"id": "cntr_test", "name": "Test Container"}) + + await router._init_containers_api_endpoints( + original_function=mock_original_function, + custom_llm_provider="openai", + name="Test Container", + model="gpt-5.4", + ) + + mock_original_function.assert_called_once() + call_kw = mock_original_function.call_args.kwargs + assert call_kw["api_key"] == "sk-model-list-key" + assert call_kw["api_base"] == "https://custom.openai.example/v1" + assert call_kw["model"] == "openai/gpt-5.4" + assert call_kw["name"] == "Test Container" + + +@pytest.mark.asyncio +async def test_init_containers_api_endpoints_create_without_model_calls_directly(): + """ + Without ``model`` (or with ``model=None`` as the proxy forwards it), create/list + must keep calling the handler directly with global provider credentials. + """ + router = Router(model_list=[]) + router._ageneric_api_call_with_fallbacks = AsyncMock() + mock_original_function = AsyncMock(return_value={"id": "cntr_test"}) + + await router._init_containers_api_endpoints( + original_function=mock_original_function, + custom_llm_provider="openai", + name="Test Container", + model=None, + ) + + router._ageneric_api_call_with_fallbacks.assert_not_called() + mock_original_function.assert_called_once_with(custom_llm_provider="openai", name="Test Container", model=None) + + +@pytest.mark.asyncio +async def test_init_containers_api_endpoints_create_with_unknown_model_passes_through(monkeypatch): + """ + A ``model`` that names no configured deployment must not turn into a 400. The call + falls through to the handler with the caller's model and no injected deployment + credentials, matching the behaviour before model-based routing existed. + """ + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + router = Router( + model_list=[ + { + "model_name": "gpt-5.4", + "litellm_params": {"model": "openai/gpt-5.4", "api_key": "sk-model-list-key"}, + } + ] + ) + mock_original_function = AsyncMock(return_value={"id": "cntr_test"}) + + await router._init_containers_api_endpoints( + original_function=mock_original_function, + custom_llm_provider="openai", + name="Test Container", + model="does-not-exist", + ) + + mock_original_function.assert_called_once() + call_kw = mock_original_function.call_args.kwargs + assert call_kw["model"] == "does-not-exist" + assert call_kw["name"] == "Test Container" + assert "api_key" not in call_kw + assert "api_base" not in call_kw + + def test_router_model_group_encrypted_content_affinity_callback_registration(): from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, diff --git a/tests/rust-python-harness/AGENTS.md b/tests/rust-python-harness/AGENTS.md new file mode 100644 index 00000000000..e9b17027ddc --- /dev/null +++ b/tests/rust-python-harness/AGENTS.md @@ -0,0 +1,44 @@ +# Expected Structure + +```text +tests/rust-python-harness/ +├── __main__.py +│ +├── strategies/ +│ ├── e2e_parity/ +│ │ ├── runner.py +│ │ ├── sdk/ +│ │ │ ├── ocr/ +│ │ │ ├── messages/ +│ │ │ ├── chat_completions/ +│ │ │ └── responses/ +│ │ └── gateway/ +│ │ +│ ├── trace_parity/ +│ │ ├── runner.py +│ │ ├── sdk/ +│ │ └── gateway/ +│ │ +│ └── unit_tests/ +│ ├── runner.py +│ ├── mapping_validator.py +│ ├── python_runner.py +│ └── rust_runner.py +│ +└── shared/ + ├── parity/ + ├── tracing/ + └── reporting/ +``` + +- Run locally only; no CI integration +- `__main__.py` selects strategies and combines their reports; each strategy also runs independently +- `e2e_parity/` compares SDK objects, exceptions, callbacks, and streams, or gateway HTTP responses +- `trace_parity/` compares mapped operations, call counts, and required execution ordering +- E2E and trace runners share orchestration across `sdk/` and `gateway/`; surface-specific execution lives in those folders +- `unit_tests/runner.py` combines mapping validation, Python test runs, and native Rust test runs +- `mapping_validator.py` matches Python/Rust tests by agreed names or annotations and reports missing or ambiguous counterparts +- `python_runner.py` runs existing Python tests with Rust disabled and enabled in separate processes, verifies backend selection, and compares results +- `rust_runner.py` runs Cargo tests; native Rust unit tests stay beside their implementation +- `shared/` contains reusable parity, tracing, and reporting machinery +- Keep fixtures with their owning API and existing Python tests in their current locations diff --git a/tests/rust-python-harness/README.md b/tests/rust-python-harness/README.md new file mode 100644 index 00000000000..e94ac87c3b3 --- /dev/null +++ b/tests/rust-python-harness/README.md @@ -0,0 +1,145 @@ +# Rust ↔ Python SDK parity harness + +This folder is the operator-facing harness for the Rust migration test plan. It runs pytest normally, listens to test events in-process, and redraws a live matrix grouped by testing strategy and SDK-level function. + +The matrix always has these SDK columns: + +- `ocr / aocr` +- `messages / amessages` +- `responses / aresponses` +- `count_tokens` + +The harness has three deliberately broad test-strategy folders: + +| Strategy | Folder | +| --- | --- | +| Public SDK parity over generated and recorded inputs | [`e2e_fuzz_tests/`](e2e_fuzz_tests/) | +| Focused tests of Rust-owned behavior | [`unit_tests_rust/`](unit_tests_rust/) | +| Isolated transform and Python-to-Rust helper coverage | [`validate_sub_methods/`](validate_sub_methods/) | + +## Run it + +From the repository root: + +```bash +poetry run python -m tests.rust-python-harness +``` + +The default runs every configured test once and updates all matching cells in real time. Narrow a run by strategy, SDK function, or both: + +```bash +poetry run python -m tests.rust-python-harness --strategy e2e_fuzz_tests +poetry run python -m tests.rust-python-harness --function messages +poetry run python -m tests.rust-python-harness --strategy validate_sub_methods --function ocr +``` + +For a guided run, use the interactive picker. It asks which strategy rows and SDK +function columns to include, then hands the terminal to the live dashboard. It never +captures keys while tests are running, so Ctrl-C and pytest debugging remain safe. + +```bash +poetry run python -m tests.rust-python-harness --interactive +``` + +Useful operator options: + +```bash +# Inspect coverage and pytest selectors without running anything. +poetry run python -m tests.rust-python-harness --list + +# Stable line-oriented output for CI logs or redirected output. +poetry run python -m tests.rust-python-harness --plain + +# Measure Python reference lines exercised by this parity run and build an HTML heatmap. +poetry run python -m tests.rust-python-harness --coverage + +# Forward pytest options. Use the equals form when the value begins with a dash. +poetry run python -m tests.rust-python-harness --pytest-arg=-x +``` + +The process returns pytest's exit code. A configured selector that collects no test is also a failure. A planned cell has no selector yet and does not fail the run. + +The dashboard adapts to narrow terminals, shows elapsed time and unique-test progress, +and prints the three slowest tests when the run ends. Each failure includes a focused +`poetry run pytest ... -q` command. Redirected output and CI automatically use the +line-oriented plain renderer; `--plain` lets you opt into it locally. + +The final screen includes a confidence score for every SDK section. It is the direct +ratio of required strategy rows with passing evidence, such as `1/3 = 33%`; High means +all required strategies passed, Medium means some passed, and Low means none passed. +This behavioral score is intentionally shown separately from Python and Rust LOC. + +Coverage reports are written outside the three strategy folders at +`target/rust-python-harness/`. Open `python-html/index.html` to inspect executed and +missing Python lines; `python.json` and `python.xml` are available for automation. +Coverage is finalized after pytest exits, because worker processes must flush their +data first. + +## Port coverage and confidence + +Treat these as separate signals instead of one ambiguous coverage percentage: + +| Signal | Tool | What it proves | +| --- | --- | --- | +| Python reference LOC | `coverage.py` / `pytest-cov` via `--coverage` | The mapped Python behavior ran | +| Rust port LOC | `cargo-llvm-cov` | The mapped Rust implementation ran | +| Parity contracts | This harness matrix | Python and Rust had the same observable behavior | + +`validate_sub_methods/` owns the future source-section inventory that maps a stable +Python qualified symbol to its Rust symbol. That inventory is the denominator for +per-function rollups; raw coverage for the entire LiteLLM repository would obscure +the port's real gaps. `unit_tests_rust/` owns direct `cargo-llvm-cov` runs, while +`e2e_fuzz_tests/` owns behavioral parity and fuzz-case counts. Keep Python, Rust, and +parity percentages visible side by side and label section confidence High only when +the mapped implementation exists, every required strategy passes, and both sides meet +their LOC thresholds. Generated Rust LCOV/HTML and the combined index also belong in +`target/rust-python-harness/`, not in a fourth strategy folder. + +## Read the matrix + +| Mark | Meaning | +| --- | --- | +| `✓` | All collected tests passed | +| `✗` | At least one test failed | +| `!` | Test setup or teardown failed | +| `↷` | All collected tests skipped | +| `?` | A configured selector did not collect a test | +| `—` | Strategy is planned but has no test yet | +| `n/a` | Strategy does not apply to this SDK function | +| `◐` | The configured tests cover only part of the TDD's parity contract | + +The initial end-to-end entries deliberately show `◐`: the repository has Rust bridge tests for OCR, Messages, and Responses websocket plumbing, but those are not yet frozen-Python-oracle comparisons. The remaining TDD cells stay visible as planned work instead of disappearing from a green summary. + +## Attach parity tests + +Each of the three folders contains a concise `README.md` and a `strategy.json`. Add a pytest file or node ID to the appropriate SDK function's `selectors` list: + +```json +{ + "coverage": "complete", + "selectors": [ + "tests/rust-python-harness/validate_sub_methods/test_messages.py" + ] +} +``` + +Selectors use the same syntax as pytest. A file selector aggregates every test in the file; a node selector can target one test or parametrized family. The runner deduplicates selectors, so one test may intentionally prove more than one cell without executing twice. + +Use these coverage values: + +- `complete`: implements the full strategy contract for that SDK function. +- `partial`: useful coverage exists, but the TDD contract is not fully proven. +- `planned`: no runnable parity test exists yet. +- `not_applicable`: the strategy cannot apply, such as streaming for OCR. + +Keep comparison mechanics in shared harness modules and provider/function facts in the owning strategy folder. A Python/Rust mismatch is a test failure; do not normalize away observable return types, exception classes, private response fields, chunk ordering, or callback payload differences merely to make a cell green. + +## Architecture + +- `catalog.py` validates and loads every strategy manifest. +- `models.py` owns typed strategy, case, coverage, and run-state models. +- `runner.py` maps live pytest events back to one or more matrix cells. +- `ui.py` renders the interactive Rich dashboard and a dependency-free plain fallback. +- `cli.py` handles filtering and preserves pytest exit semantics. + +The harness is driven from Python, matching the SDK surface and existing test tooling. Rust remains responsible for the implementation under comparison; the harness does not move provider semantics into the PyO3 bridge. diff --git a/tests/rust-python-harness/__init__.py b/tests/rust-python-harness/__init__.py new file mode 100644 index 00000000000..70362674d2b --- /dev/null +++ b/tests/rust-python-harness/__init__.py @@ -0,0 +1,5 @@ +"""Interactive Rust/Python SDK parity test harness.""" + +from .catalog import load_catalog + +__all__ = ["load_catalog"] diff --git a/tests/rust-python-harness/__main__.py b/tests/rust-python-harness/__main__.py new file mode 100644 index 00000000000..bfdcd0c1158 --- /dev/null +++ b/tests/rust-python-harness/__main__.py @@ -0,0 +1,4 @@ +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/rust-python-harness/catalog.py b/tests/rust-python-harness/catalog.py new file mode 100644 index 00000000000..e23b9b125f0 --- /dev/null +++ b/tests/rust-python-harness/catalog.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from .models import Coverage, HarnessCase, SDK_FUNCTIONS, Strategy + +STRATEGIES_ROOT = Path(__file__).parent + + +def _require_string(value: Any, field: str, source: Path) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{source}: {field} must be a non-empty string") + return value + + +def _load_strategy(source: Path) -> Strategy: + with source.open(encoding="utf-8") as stream: + data = json.load(stream) + + strategy_id = _require_string(data.get("id"), "id", source) + label = _require_string(data.get("label"), "label", source) + description = _require_string(data.get("description"), "description", source) + order = data.get("order") + if not isinstance(order, int): + raise ValueError(f"{source}: order must be an integer") + function_data = data.get("functions") + if not isinstance(function_data, dict): + raise ValueError(f"{source}: functions must be an object") + + missing = set(SDK_FUNCTIONS) - set(function_data) + extra = set(function_data) - set(SDK_FUNCTIONS) + if missing or extra: + raise ValueError( + f"{source}: functions must exactly match {SDK_FUNCTIONS}; missing={missing}, extra={extra}" + ) + + cases: list[HarnessCase] = [] + for sdk_function in SDK_FUNCTIONS: + case_data = function_data[sdk_function] + if not isinstance(case_data, dict): + raise ValueError(f"{source}: functions.{sdk_function} must be an object") + try: + coverage = Coverage(case_data.get("coverage")) + except ValueError as exc: + raise ValueError(f"{source}: invalid coverage for {sdk_function}") from exc + selectors = case_data.get("selectors", []) + if not isinstance(selectors, list) or not all( + isinstance(item, str) and item for item in selectors + ): + raise ValueError( + f"{source}: selectors for {sdk_function} must be a list of strings" + ) + if coverage is Coverage.NOT_APPLICABLE and selectors: + raise ValueError( + f"{source}: not_applicable case {sdk_function} cannot have selectors" + ) + cases.append( + HarnessCase( + strategy_id=strategy_id, + strategy_label=label, + sdk_function=sdk_function, + coverage=coverage, + selectors=tuple(selectors), + note=str(case_data.get("note", "")), + ) + ) + + return Strategy( + order=order, + id=strategy_id, + label=label, + description=description, + directory=source.parent, + cases=tuple(cases), + ) + + +def load_catalog(root: Path = STRATEGIES_ROOT) -> tuple[Strategy, ...]: + sources = sorted(root.glob("*/strategy.json")) + if not sources: + raise ValueError(f"No strategy manifests found below {root}") + strategies = tuple( + sorted( + (_load_strategy(source) for source in sources), + key=lambda strategy: strategy.order, + ) + ) + ids = [strategy.id for strategy in strategies] + if len(ids) != len(set(ids)): + raise ValueError(f"Duplicate strategy id in {root}") + return strategies diff --git a/tests/rust-python-harness/cli.py b/tests/rust-python-harness/cli.py new file mode 100644 index 00000000000..f9e97d7ad43 --- /dev/null +++ b/tests/rust-python-harness/cli.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import argparse +import importlib.util +from collections.abc import Sequence +from pathlib import Path + +from .catalog import load_catalog +from .models import HarnessCase, Strategy +from .runner import run_pytest +from .ui import make_dashboard +from .strategies.unit_tests.mapping_validator import FunctionReport, build_function_report + +REPO_ROOT = Path(__file__).resolve().parents[2] +COVERAGE_ROOT = REPO_ROOT / "target" / "rust-python-harness" +SDK_FUNCTION_CHOICES = ("ocr", "messages", "responses", "count_tokens") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="rust-python-harness", + description="Run Rust/Python parity tests with a live strategy-by-SDK-function dashboard.", + ) + parser.add_argument( + "-i", + "--interactive", + action="store_true", + help="pick strategies and SDK functions in a guided terminal menu", + ) + parser.add_argument( + "--list", action="store_true", help="show the catalog without running tests" + ) + parser.add_argument( + "--strategy", + action="append", + default=[], + metavar="ID", + help="run only this strategy", + ) + parser.add_argument( + "--function", + action="append", + default=[], + dest="sdk_functions", + choices=SDK_FUNCTION_CHOICES, + help="run only this SDK function", + ) + parser.add_argument( + "--validate-ledger", + action="store_true", + help=( + "report Python<->Rust test-parity ledger gaps and drift instead of " + "running the dashboard; narrow with --function" + ), + ) + parser.add_argument( + "--plain", + action="store_true", + help="disable the interactive terminal dashboard", + ) + parser.add_argument( + "--coverage", + action="store_true", + help="write Python reference LOC reports (HTML, JSON, and XML)", + ) + parser.add_argument( + "--pytest-arg", + action="append", + default=[], + metavar="ARG", + help="append an argument to pytest (repeatable, for example --pytest-arg=-x)", + ) + return parser + + +def _coverage_pytest_args(output_root: Path = COVERAGE_ROOT) -> tuple[str, ...]: + output_root.mkdir(parents=True, exist_ok=True) + return ( + "--cov=litellm", + "--cov-context=test", + f"--cov-report=json:{output_root / 'python.json'}", + f"--cov-report=xml:{output_root / 'python.xml'}", + f"--cov-report=html:{output_root / 'python-html'}", + ) + + +def _pick_values( + title: str, options: Sequence[tuple[str, str]], input_fn=input +) -> set[str]: + print(f"\n{title} (Enter = all)") + for index, (value, label) in enumerate(options, start=1): + print(f" {index:>2}. {label} [{value}]") + while True: + answer = input_fn("Choose numbers, comma-separated: ").strip() + if not answer: + return set() + try: + indexes = {int(part.strip()) for part in answer.split(",")} + except ValueError: + print("Please enter numbers separated by commas.") + continue + if indexes and all(1 <= index <= len(options) for index in indexes): + return {options[index - 1][0] for index in indexes} + print(f"Choose values from 1 to {len(options)}.") + + +def _interactive_filters(strategies: Sequence[Strategy]) -> tuple[set[str], set[str]]: + strategy_ids = _pick_values( + "Testing strategies", [(strategy.id, strategy.label) for strategy in strategies] + ) + sdk_functions = _pick_values( + "SDK functions", + [(name, name) for name in SDK_FUNCTION_CHOICES], + ) + return strategy_ids, sdk_functions + + +def _select( + strategies: Sequence[Strategy], strategy_ids: set[str], sdk_functions: set[str] +) -> tuple[HarnessCase, ...]: + known_ids = {strategy.id for strategy in strategies} + unknown = strategy_ids - known_ids + if unknown: + raise ValueError(f"Unknown strategy: {', '.join(sorted(unknown))}") + return tuple( + case + for strategy in strategies + if not strategy_ids or strategy.id in strategy_ids + for case in strategy.cases + if not sdk_functions or case.sdk_function in sdk_functions + ) + + +def _print_catalog(strategies: Sequence[Strategy]) -> None: + for strategy in strategies: + print(f"{strategy.id:20} {strategy.label}") + for case in strategy.cases: + selectors = ( + ", ".join(case.selectors) if case.selectors else "no test configured" + ) + print(f" {case.sdk_function:12} {case.coverage.value:14} {selectors}") + + +def _print_function_report(report: FunctionReport) -> None: + print(f"\n{report.sdk_function}") + if report.ledger is None or report.audit is None: + print(" no ledger yet") + return + ledger, audit = report.ledger, report.audit + print( + f" {ledger.mapped_count}/{ledger.total_count} python tests mapped to rust " + f"({ledger.percentage}%)" + ) + print(f" {len(ledger.rust_only_tests)} rust-only tests with no python counterpart") + if audit.is_clean: + print(" ledger is in sync with the live test files") + return + for label, items in ( + ("ledger references a python test that no longer exists", audit.missing_python_tests), + ("python test exists but is not tracked in the ledger", audit.stale_python_tests), + ("ledger references a rust test that no longer exists", audit.missing_rust_tests), + ("rust test exists but is not tracked in the ledger", audit.stale_rust_tests), + ): + for item in items: + print(f" {label}: {item}") + + +def _validate_ledger(sdk_functions: set[str]) -> int: + functions = sdk_functions or set(SDK_FUNCTION_CHOICES) + reports = tuple(build_function_report(function) for function in sorted(functions)) + for report in reports: + _print_function_report(report) + return 0 if all(report.is_clean for report in reports) else 1 + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + if args.coverage and importlib.util.find_spec("pytest_cov") is None: + _parser().error( + "--coverage requires the project's pytest-cov dependency; run with " + "`poetry run python -m tests.rust-python-harness --coverage`" + ) + if args.validate_ledger: + return _validate_ledger(set(args.sdk_functions)) + strategies = load_catalog() + if args.list: + _print_catalog(strategies) + return 0 + + strategy_ids = set(args.strategy) + sdk_functions = set(args.sdk_functions) + if args.interactive: + picked_strategies, picked_functions = _interactive_filters(strategies) + strategy_ids = strategy_ids or picked_strategies + sdk_functions = sdk_functions or picked_functions + + try: + cases = _select(strategies, strategy_ids, sdk_functions) + except ValueError as exc: + _parser().error(str(exc)) + selected_strategy_ids = {case.strategy_id for case in cases} + visible_strategies = tuple( + strategy for strategy in strategies if strategy.id in selected_strategy_ids + ) + dashboard = make_dashboard( + visible_strategies, + plain=args.plain, + confidence_strategies=strategies, + ) + pytest_args = [*args.pytest_arg] + if args.coverage: + pytest_args.extend(_coverage_pytest_args()) + with dashboard: + exit_code, run = run_pytest( + cases=cases, + repo_root=REPO_ROOT, + on_update=dashboard.update, + pytest_args=pytest_args, + ) + dashboard.finish(run, exit_code) + if args.coverage and (COVERAGE_ROOT / "python.json").exists(): + print(f"Python LOC heatmap: {COVERAGE_ROOT / 'python-html' / 'index.html'}") + print(f"Machine-readable coverage: {COVERAGE_ROOT / 'python.json'}") + return exit_code diff --git a/tests/rust-python-harness/e2e_fuzz_tests/README.md b/tests/rust-python-harness/e2e_fuzz_tests/README.md new file mode 100644 index 00000000000..34b12050ff9 --- /dev/null +++ b/tests/rust-python-harness/e2e_fuzz_tests/README.md @@ -0,0 +1,3 @@ +# End-to-end fuzz tests + +Runs the same SDK call through the Python and Rust paths using generated inputs and recorded provider responses. It compares public results, streams, callbacks, and exceptions to catch behavior differences a unit test can miss. diff --git a/tests/rust-python-harness/e2e_fuzz_tests/strategy.json b/tests/rust-python-harness/e2e_fuzz_tests/strategy.json new file mode 100644 index 00000000000..abeea01d9b5 --- /dev/null +++ b/tests/rust-python-harness/e2e_fuzz_tests/strategy.json @@ -0,0 +1,12 @@ +{ + "order": 10, + "id": "e2e_fuzz_tests", + "label": "End-to-end fuzz tests", + "description": "Compare observable Python and Rust SDK behavior over generated and recorded inputs.", + "functions": { + "ocr": {"coverage": "partial", "selectors": ["tests/test_litellm/ocr/test_rust_bridge.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."}, + "messages": {"coverage": "partial", "selectors": ["tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."}, + "responses": {"coverage": "partial", "selectors": ["tests/test_litellm/responses/test_rust_bridge_websocket.py"], "note": "Covers the websocket bridge; full responses parity is still being added."}, + "count_tokens": {"coverage": "planned", "selectors": [], "note": "No Rust count_tokens parity test is present yet."} + } +} diff --git a/tests/rust-python-harness/models.py b/tests/rust-python-harness/models.py new file mode 100644 index 00000000000..21097e0f7d0 --- /dev/null +++ b/tests/rust-python-harness/models.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from time import monotonic +from typing import Iterable + + +class Coverage(str, Enum): + COMPLETE = "complete" + PARTIAL = "partial" + PLANNED = "planned" + NOT_APPLICABLE = "not_applicable" + + +class RunStatus(str, Enum): + NOT_RUN = "not_run" + QUEUED = "queued" + RUNNING = "running" + PASSED = "passed" + FAILED = "failed" + SKIPPED = "skipped" + ERROR = "error" + MISSING = "missing" + PLANNED = "planned" + NOT_APPLICABLE = "not_applicable" + + +class ConfidenceLevel(str, Enum): + HIGH = "HIGH" + MEDIUM = "MEDIUM" + LOW = "LOW" + + +SDK_FUNCTIONS = ("ocr", "messages", "responses", "count_tokens") + + +@dataclass(frozen=True) +class HarnessCase: + strategy_id: str + strategy_label: str + sdk_function: str + coverage: Coverage + selectors: tuple[str, ...] + note: str = "" + + @property + def key(self) -> str: + return f"{self.strategy_id}:{self.sdk_function}" + + +@dataclass(frozen=True) +class Strategy: + order: int + id: str + label: str + description: str + directory: Path + cases: tuple[HarnessCase, ...] + + +@dataclass +class CaseResult: + case: HarnessCase + status: RunStatus = RunStatus.NOT_RUN + collected: set[str] = field(default_factory=set) + completed: set[str] = field(default_factory=set) + passed: int = 0 + failed: int = 0 + skipped: int = 0 + errors: int = 0 + outcomes: dict[str, RunStatus] = field(default_factory=dict) + durations: dict[str, float] = field(default_factory=dict) + + @property + def total(self) -> int: + return len(self.collected) + + @property + def duration(self) -> float: + return sum(self.durations.values()) + + def record(self, nodeid: str, status: RunStatus, duration: float = 0.0) -> None: + """Record a terminal outcome, allowing teardown errors to replace a pass.""" + self.outcomes[nodeid] = status + self.durations[nodeid] = self.durations.get(nodeid, 0.0) + duration + self.completed = set(self.outcomes) + values = tuple(self.outcomes.values()) + self.passed = values.count(RunStatus.PASSED) + self.failed = values.count(RunStatus.FAILED) + self.skipped = values.count(RunStatus.SKIPPED) + self.errors = values.count(RunStatus.ERROR) + self.finalize() + + def set_initial_status(self) -> None: + if self.case.coverage is Coverage.NOT_APPLICABLE: + self.status = RunStatus.NOT_APPLICABLE + elif not self.case.selectors: + self.status = RunStatus.PLANNED + else: + self.status = RunStatus.QUEUED + + def finalize(self) -> None: + if self.status in {RunStatus.NOT_APPLICABLE, RunStatus.PLANNED}: + return + if not self.collected: + self.status = RunStatus.MISSING + elif self.errors: + self.status = RunStatus.ERROR + elif self.failed: + self.status = RunStatus.FAILED + elif self.passed and len(self.completed) == len(self.collected): + self.status = RunStatus.PASSED + elif self.skipped and len(self.completed) == len(self.collected): + self.status = RunStatus.SKIPPED + + +@dataclass +class HarnessRun: + results: dict[str, CaseResult] + current_nodeid: str | None = None + failures: list[tuple[str, str]] = field(default_factory=list) + started_at: float = field(default_factory=monotonic) + finished_at: float | None = None + + @property + def duration(self) -> float: + return (self.finished_at or monotonic()) - self.started_at + + @property + def unique_tests(self) -> int: + return len( + {nodeid for result in self.results.values() for nodeid in result.collected} + ) + + @property + def completed_tests(self) -> int: + return len( + {nodeid for result in self.results.values() for nodeid in result.completed} + ) + + @classmethod + def from_cases(cls, cases: Iterable[HarnessCase]) -> "HarnessRun": + results = {case.key: CaseResult(case=case) for case in cases} + for result in results.values(): + result.set_initial_status() + return cls(results=results) + + +@dataclass(frozen=True) +class SectionConfidence: + sdk_function: str + verified_strategies: int + required_strategies: int + level: ConfidenceLevel + details: tuple[str, ...] + + @property + def percentage(self) -> int: + if not self.required_strategies: + return 0 + return round(100 * self.verified_strategies / self.required_strategies) + + +def section_confidence( + run: HarnessRun, strategies: Iterable[Strategy] +) -> tuple[SectionConfidence, ...]: + strategy_list = tuple(strategies) + scores: list[SectionConfidence] = [] + for sdk_function in SDK_FUNCTIONS: + cases = tuple( + case + for strategy in strategy_list + for case in strategy.cases + if case.sdk_function == sdk_function + and case.coverage is not Coverage.NOT_APPLICABLE + ) + verified = 0 + details: list[str] = [] + for case in cases: + result = run.results.get(case.key) + status = result.status if result is not None else RunStatus.NOT_RUN + if status is RunStatus.PASSED: + verified += 1 + details.append( + f"{STATUS_LABELS[status]} {case.strategy_id} ({case.coverage.value})" + ) + required = len(cases) + if required and verified == required: + level = ConfidenceLevel.HIGH + elif verified: + level = ConfidenceLevel.MEDIUM + else: + level = ConfidenceLevel.LOW + scores.append( + SectionConfidence( + sdk_function=sdk_function, + verified_strategies=verified, + required_strategies=required, + level=level, + details=tuple(details), + ) + ) + return tuple(scores) + + +STATUS_LABELS = { + RunStatus.NOT_RUN: "·", + RunStatus.QUEUED: "○", + RunStatus.RUNNING: "◉", + RunStatus.PASSED: "✓", + RunStatus.FAILED: "✗", + RunStatus.SKIPPED: "↷", + RunStatus.ERROR: "!", + RunStatus.MISSING: "?", + RunStatus.PLANNED: "—", + RunStatus.NOT_APPLICABLE: "n/a", +} diff --git a/tests/rust-python-harness/runner.py b/tests/rust-python-harness/runner.py new file mode 100644 index 00000000000..82393ef234e --- /dev/null +++ b/tests/rust-python-harness/runner.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import os +from collections.abc import Callable, Sequence +from pathlib import Path +from time import monotonic + +import pytest + +from .models import CaseResult, HarnessCase, HarnessRun, RunStatus + +UpdateCallback = Callable[[HarnessRun], None] + + +def selector_matches_node(selector: str, nodeid: str) -> bool: + normalized_selector = selector.replace("\\", "/") + normalized_nodeid = nodeid.replace("\\", "/") + if "::" in normalized_selector: + return normalized_nodeid == normalized_selector or normalized_nodeid.startswith( + f"{normalized_selector}[" + ) + return normalized_nodeid == normalized_selector or normalized_nodeid.startswith( + f"{normalized_selector}::" + ) + + +def selector_path(selector: str) -> Path: + return Path(selector.split("::", 1)[0]) + + +def runnable_selectors( + cases: Sequence[HarnessCase], repo_root: Path +) -> tuple[str, ...]: + selectors = { + selector + for case in cases + for selector in case.selectors + if (repo_root / selector_path(selector)).exists() + } + return tuple(sorted(selectors)) + + +class HarnessPytestPlugin: + def __init__(self, run: HarnessRun, on_update: UpdateCallback) -> None: + self.run = run + self.on_update = on_update + self.node_to_results: dict[str, list[CaseResult]] = {} + + def _notify(self) -> None: + self.on_update(self.run) + + def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: + for item in items: + matched_results: list[CaseResult] = [] + for result in self.run.results.values(): + if any( + selector_matches_node(selector, item.nodeid) + for selector in result.case.selectors + ): + result.collected.add(item.nodeid) + matched_results.append(result) + if matched_results: + self.node_to_results[item.nodeid] = matched_results + for result in self.run.results.values(): + if result.status is RunStatus.QUEUED and not result.collected: + result.status = RunStatus.MISSING + self._notify() + + def pytest_runtest_logstart( + self, nodeid: str, location: tuple[str, int | None, str] + ) -> None: + del location + self.run.current_nodeid = nodeid + for result in self.node_to_results.get(nodeid, []): + if result.status not in {RunStatus.FAILED, RunStatus.ERROR}: + result.status = RunStatus.RUNNING + self._notify() + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + if report.when not in {"setup", "call", "teardown"}: + return + results = self.node_to_results.get(report.nodeid, []) + if not results: + return + + terminal = report.when == "call" or report.failed or report.skipped + if not terminal: + for result in results: + result.durations[report.nodeid] = ( + result.durations.get(report.nodeid, 0.0) + report.duration + ) + return + for result in results: + if report.when == "teardown" and not report.failed: + result.durations[report.nodeid] = ( + result.durations.get(report.nodeid, 0.0) + report.duration + ) + continue + if report.skipped: + status = RunStatus.SKIPPED + elif report.failed and report.when in {"setup", "teardown"}: + status = RunStatus.ERROR + elif report.failed: + status = RunStatus.FAILED + else: + status = RunStatus.PASSED + result.record(report.nodeid, status, report.duration) + if report.failed: + failure = (report.nodeid, str(report.longrepr)) + if failure not in self.run.failures: + self.run.failures.append(failure) + self._notify() + + def pytest_sessionfinish( + self, session: pytest.Session, exitstatus: int | pytest.ExitCode + ) -> None: + del session, exitstatus + self.run.current_nodeid = None + self.run.finished_at = monotonic() + for result in self.run.results.values(): + result.finalize() + self._notify() + + +def run_pytest( + cases: Sequence[HarnessCase], + repo_root: Path, + on_update: UpdateCallback, + pytest_args: Sequence[str] = (), +) -> tuple[int, HarnessRun]: + run = HarnessRun.from_cases(cases) + selectors = runnable_selectors(cases, repo_root) + if not selectors: + for result in run.results.values(): + result.finalize() + run.finished_at = monotonic() + on_update(run) + has_missing_test = any( + result.status is RunStatus.MISSING for result in run.results.values() + ) + exit_code = ( + int(pytest.ExitCode.TESTS_FAILED) + if has_missing_test + else int(pytest.ExitCode.OK) + ) + return exit_code, run + + plugin = HarnessPytestPlugin(run=run, on_update=on_update) + args = [*selectors, "-p", "no:terminal", *pytest_args] + previous_directory = Path.cwd() + try: + os.chdir(repo_root) + exit_code = int(pytest.main(args, plugins=[plugin])) + finally: + os.chdir(previous_directory) + if exit_code == 0 and any( + result.status is RunStatus.MISSING for result in run.results.values() + ): + exit_code = int(pytest.ExitCode.TESTS_FAILED) + return exit_code, run diff --git a/tests/rust-python-harness/shared/__init__.py b/tests/rust-python-harness/shared/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/shared/parity/__init__.py b/tests/rust-python-harness/shared/parity/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/shared/parity/ledger.py b/tests/rust-python-harness/shared/parity/ledger.py new file mode 100644 index 00000000000..40dfed583ae --- /dev/null +++ b/tests/rust-python-harness/shared/parity/ledger.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +@dataclass(frozen=True, slots=True) +class LedgerEntry: + python_file: str + python_test: str + status: str + rust_file: str + rust_test: str + justification: str + reason: str + + +@dataclass(frozen=True, slots=True) +class RustOnlyEntry: + rust_file: str + rust_test: str + reason: str + + +@dataclass(frozen=True, slots=True) +class TestLedger: + sdk_function: str + python_scope: tuple[str, ...] + rust_scope: tuple[str, ...] + entries: tuple[LedgerEntry, ...] + rust_only_tests: tuple[RustOnlyEntry, ...] + + @property + def mapped_count(self) -> int: + return sum(1 for entry in self.entries if entry.status == "mapped") + + @property + def total_count(self) -> int: + return len(self.entries) + + @property + def percentage(self) -> float: + if self.total_count == 0: + return 0.0 + return round(100.0 * self.mapped_count / self.total_count, 1) + + +def _require_string(value: Any, field: str, source: Path) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{source}: {field} must be a non-empty string") + return value + + +def _require_string_list(value: Any, field: str, source: Path) -> tuple[str, ...]: + if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value): + raise ValueError(f"{source}: {field} must be a list of non-empty strings") + return tuple(value) + + +def _load_entry(data: Any, index: int, source: Path) -> LedgerEntry: + if not isinstance(data, dict): + raise ValueError(f"{source}: entries[{index}] must be an object") + python_file = _require_string(data.get("python_file"), f"entries[{index}].python_file", source) + python_test = _require_string(data.get("python_test"), f"entries[{index}].python_test", source) + status = data.get("status") + if status not in ("mapped", "unmapped"): + raise ValueError(f"{source}: entries[{index}].status must be 'mapped' or 'unmapped'") + + if status == "mapped": + rust_file = _require_string(data.get("rust_file"), f"entries[{index}].rust_file", source) + rust_test = _require_string(data.get("rust_test"), f"entries[{index}].rust_test", source) + justification = _require_string( + data.get("justification"), f"entries[{index}].justification", source + ) + return LedgerEntry( + python_file=python_file, + python_test=python_test, + status=status, + rust_file=rust_file, + rust_test=rust_test, + justification=justification, + reason="", + ) + + reason = _require_string(data.get("reason"), f"entries[{index}].reason", source) + return LedgerEntry( + python_file=python_file, + python_test=python_test, + status=status, + rust_file="", + rust_test="", + justification="", + reason=reason, + ) + + +def _load_rust_only_entry(data: Any, index: int, source: Path) -> RustOnlyEntry: + if not isinstance(data, dict): + raise ValueError(f"{source}: rust_only_tests[{index}] must be an object") + return RustOnlyEntry( + rust_file=_require_string(data.get("rust_file"), f"rust_only_tests[{index}].rust_file", source), + rust_test=_require_string(data.get("rust_test"), f"rust_only_tests[{index}].rust_test", source), + reason=_require_string(data.get("reason"), f"rust_only_tests[{index}].reason", source), + ) + + +def load_ledger(path: Path) -> TestLedger: + with path.open(encoding="utf-8") as stream: + data = json.load(stream) + + sdk_function = _require_string(data.get("sdk_function"), "sdk_function", path) + python_scope = _require_string_list(data.get("python_scope"), "python_scope", path) + rust_scope = _require_string_list(data.get("rust_scope"), "rust_scope", path) + + entries_data = data.get("entries") + if not isinstance(entries_data, list): + raise ValueError(f"{path}: entries must be a list") + entries = tuple( + _load_entry(entry, index, path) for index, entry in enumerate(entries_data) + ) + + rust_only_data = data.get("rust_only_tests") + if not isinstance(rust_only_data, list): + raise ValueError(f"{path}: rust_only_tests must be a list") + rust_only_tests = tuple( + _load_rust_only_entry(entry, index, path) for index, entry in enumerate(rust_only_data) + ) + + return TestLedger( + sdk_function=sdk_function, + python_scope=python_scope, + rust_scope=rust_scope, + entries=entries, + rust_only_tests=rust_only_tests, + ) diff --git a/tests/rust-python-harness/strategies/__init__.py b/tests/rust-python-harness/strategies/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/unit_tests/__init__.py b/tests/rust-python-harness/strategies/unit_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json b/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json new file mode 100644 index 00000000000..799a1320463 --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json @@ -0,0 +1,208 @@ +{ + "sdk_function": "ocr", + "python_scope": [ + "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", + "tests/test_litellm/ocr/test_rust_bridge.py", + "tests/test_litellm/ocr/test_ocr_file_input.py", + "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", + "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", + "tests/test_litellm/ocr/test_ocr_native_format.py", + "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", + "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py" + ], + "rust_scope": [ + "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", + "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", + "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", + "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", + "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", + "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs" + ], + "entries": [ + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_should_encode_azure_document_intelligence_model_id", "status": "unmapped", "reason": "model-id URL percent-encoding has no Rust test; Rust only tests pages/features query building"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_should_reject_dot_segment_azure_document_intelligence_model_id", "status": "unmapped", "reason": "model-id dot-segment validation has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_preserves_azure_native_fields", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_response_normalizes_pages", "justification": "both assert page markdown, dimension (inch-to-pixel) normalization, and usage_info.pages_processed from the same Azure succeeded response shape"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_async_transform_ocr_response_preserves_azure_native_fields", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_response_normalizes_pages", "justification": "async twin of the sync case above, same underlying transform is exercised on the Rust side"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_tolerates_missing_native_fields", "status": "unmapped", "reason": "tables/keyValuePairs absence tolerance is not asserted by the Rust response test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_non_succeeded_status_raises", "status": "unmapped", "reason": "no Rust test asserts on a non-succeeded Azure DI status"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_supported_ocr_params_includes_features", "status": "unmapped", "reason": "supported-params list content has no Rust equivalent for Azure"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_native_format_carries_raw_operation", "status": "unmapped", "reason": "native req_format raw-operation passthrough is not tested in Rust"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_async_transform_ocr_response_native_format_carries_raw_operation", "status": "unmapped", "reason": "native req_format raw-operation passthrough is not tested in Rust"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_default_format_omits_raw_operation", "status": "unmapped", "reason": "req_format gating of raw-operation output has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_passes_through_req_format", "status": "unmapped", "reason": "req_format passthrough in map_ocr_params has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_rejects_unknown_req_format_as_bad_request", "status": "unmapped", "reason": "req_format validation error path has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_omits_req_format_query_param", "status": "unmapped", "reason": "no Rust test asserts req_format is excluded from the built URL"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_features", "status": "unmapped", "reason": "features-string normalization in map_ocr_params has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_empty_features_list_omitted", "status": "unmapped", "reason": "empty-features omission has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_invalid_features_raises", "status": "unmapped", "reason": "features validation error path has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_appends_features_query", "status": "unmapped", "reason": "features query-param construction has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_combines_pages_and_features", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_url_normalizes_zero_based_pages", "justification": "both assert 0-based, duplicate page indices are deduped, sorted, and rewritten 1-based into the request URL"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_validate_environment_uses_subscription_key", "status": "unmapped", "reason": "Python-side header derivation from litellm_params; Rust's poll test only checks the header is present, not how it was resolved"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_validate_environment_falls_back_to_entra_token", "status": "unmapped", "reason": "Entra bearer-token fallback logic has no Rust test"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_doc_intelligence_route", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_documentintelligence_and_is_case_insensitive", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_does_not_match_mistral_route", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_does_not_hijack_doc_intelligence", "status": "unmapped", "reason": "api_base resolution from the secret manager runs before the Rust bridge is called, no Rust test exists for it"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_explicit_api_base_is_honoured_for_doc_intelligence", "status": "unmapped", "reason": "api_base precedence resolution is Python-only"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_still_applies_to_mistral_ocr", "status": "unmapped", "reason": "api_base precedence resolution is Python-only"}, + + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_use_litellm_rust_toggles_flag", "status": "unmapped", "reason": "bridge-plumbing: Python-side feature-flag toggle, no Rust equivalent"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_env_var_enables_rust_ocr", "status": "unmapped", "reason": "bridge-plumbing: Python-side env-var flag gating"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_returns_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: dependency-injection test hook, not provider behavior"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_loader_returns_none_when_extension_absent", "status": "unmapped", "reason": "bridge-plumbing: native-extension import/loader fallback"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_loader_caches_absent_extension", "status": "unmapped", "reason": "bridge-plumbing: loader caching behavior"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_available_reflects_loader", "status": "unmapped", "reason": "bridge-plumbing: loader availability check"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_aocr_returns_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: dependency-injection test hook"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_toggle_without_ocr_arg_preserves_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: injected-impl state retention regression"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_explicit_ocr_none_clears_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: injected-impl clearing behavior"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_none_when_extension_absent", "status": "unmapped", "reason": "bridge-plumbing: degrade path when the native extension is missing"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_uses_compiled_extension", "status": "unmapped", "reason": "bridge-plumbing: native module resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_timeout_to_seconds_handles_float_timeout_and_none", "status": "unmapped", "reason": "bridge-plumbing: Python-side timeout normalization helper"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_bridge_wrapper_forwards_prepared_args_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: wrapper argument forwarding, asserted against a fake bridge not the real Rust code"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: async wrapper argument forwarding"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_prepares_request_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: request preparation and response wrapping in Python"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_resolves_key_via_secret_manager_when_missing", "status": "unmapped", "reason": "secret-manager: API key resolution happens in Python before the bridge is invoked"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_prefers_explicit_key_over_resolver", "status": "unmapped", "reason": "secret-manager: key precedence resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_uses_provider_api_key_env_var", "status": "unmapped", "reason": "secret-manager: provider-specific env var name resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_forwards_vertex_routing_metadata", "status": "unmapped", "reason": "secret-manager: vertex routing metadata merge happens in Python"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_manager", "status": "unmapped", "reason": "secret-manager: vertex project/location resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager", "status": "unmapped", "reason": "secret-manager: azure_ai api_base resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint", "status": "unmapped", "reason": "secret-manager: doc-intelligence endpoint resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_runs_pre_call_logging", "status": "unmapped", "reason": "bridge-plumbing: Python logging-object pre_call invocation"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_routes_to_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: routing to a fake bridge, not the real Rust transform"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_routes_azure_ai_to_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: provider-prefix stripping before routing"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_rust_path_converts_file_document_before_bridge", "status": "unmapped", "reason": "file-normalization: raw-bytes-to-data-URI conversion happens in Python before the bridge call"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_exception_type_uses_resolved_provider_context", "status": "unmapped", "reason": "bridge-plumbing: Python exception-type mapping on bridge failure"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_aocr_routes_to_async_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: async routing to a fake bridge"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_aocr_exception_type_uses_resolved_provider_context", "status": "unmapped", "reason": "bridge-plumbing: async exception-type mapping on bridge failure"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_does_not_route_to_rust_when_disabled", "status": "unmapped", "reason": "bridge-plumbing: Python control flow for the toggle-disabled branch, no Rust-owned behavior runs"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_falls_back_to_python_when_bridge_unavailable", "status": "unmapped", "reason": "bridge-plumbing: Python-only fallback when the compiled Rust extension is absent, Rust cannot test its own absence"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_forwards_timeout_to_rust", "status": "unmapped", "reason": "bridge-plumbing: asserts the Python call site forwards a timeout kwarg, Rust receives an already-constructed request"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_passes_default_request_timeout_to_rust", "status": "unmapped", "reason": "bridge-plumbing: asserts the Python call site supplies a default timeout kwarg, no Rust equivalent"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_provider_configs_expose_api_key_env_vars", "status": "unmapped", "reason": "asserts per-provider get_api_key_env_var() strings; the closest Rust test (ocr_dispatch_supports_migrated_providers) asserts provider dispatch/param resolution instead, not API key env var names"}, + + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_pdf_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection is Python-only preprocessing before the bridge call"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_png_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_jpg_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_jpeg_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_gif_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_webp_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_tiff_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_tif_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_bmp_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_be_case_insensitive", "status": "unmapped", "reason": "file-normalization: MIME detection case handling"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_fallback_for_unknown_extension", "status": "unmapped", "reason": "file-normalization: MIME detection fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pdf_pathlib_path_to_document_url", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion happens in Python"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_image_pathlib_path_to_image_url", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_reject_bare_str_path", "status": "unmapped", "reason": "file-normalization: arbitrary-file-read guard on bare str paths"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pathlib_path", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes", "status": "unmapped", "reason": "file-normalization: raw-bytes-to-data-URI conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_explicit_mime_type", "status": "unmapped", "reason": "file-normalization: explicit MIME override on raw bytes"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_image_mime_type", "status": "unmapped", "reason": "file-normalization: explicit MIME override on raw bytes"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object", "status": "unmapped", "reason": "file-normalization: file-like-object conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object_with_name", "status": "unmapped", "reason": "file-normalization: file-like-object name-based MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_missing_file_field", "status": "unmapped", "reason": "file-normalization: missing-field validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_nonexistent_pathlib_path", "status": "unmapped", "reason": "file-normalization: missing-file validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_empty_file", "status": "unmapped", "reason": "file-normalization: empty-file validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_unsupported_type", "status": "unmapped", "reason": "file-normalization: unsupported input type validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_invalid_mime_type", "status": "unmapped", "reason": "file-normalization: MIME-type injection validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_override_mime_type_for_pathlib_path", "status": "unmapped", "reason": "file-normalization: explicit MIME override precedence"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_document_url_for_pdf", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_png", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_jpeg", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_octet_stream", "status": "unmapped", "reason": "file-normalization: filename-based MIME fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_none", "status": "unmapped", "reason": "file-normalization: filename-based MIME fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_fallback_to_octet_stream_for_unknown", "status": "unmapped", "reason": "file-normalization: default MIME fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_preserve_base64_content_correctly", "status": "unmapped", "reason": "file-normalization: binary round-trip through base64"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_from_content_type", "status": "unmapped", "reason": "file-normalization: content-type parameter stripping"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_with_multiple_params", "status": "unmapped", "reason": "file-normalization: content-type parameter stripping"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_reject_file_type_document_in_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body file-type guard, a different mechanism than Rust's URL-fetch SSRF guard"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_accept_document_url_type_in_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body parsing"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_raise_on_invalid_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body parsing error path"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_ignore_document_form_field_injection", "status": "unmapped", "reason": "proxy-layer multipart form-field injection guard, a different mechanism than Rust's URL-fetch SSRF guard"}, + + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_extract_header_in_supported_params", "status": "unmapped", "reason": "Rust's fixed-list test checks the full list as one assertion, not this individual param"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_extract_footer_in_supported_params", "status": "unmapped", "reason": "Rust's fixed-list test checks the full list as one assertion, not this individual param"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_existing_params_still_present", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "supported_params_match_python_mistral_ocr_config", "justification": "both assert the full supported_ocr_params list matches the same fixed set of param names"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_header_passed_through", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_drops_unknown_params", "justification": "both assert extract_header survives map_ocr_params filtering unchanged"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_footer_passed_through", "status": "unmapped", "reason": "Rust's map_ocr_params test does not assert on extract_footer specifically"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_header_and_footer_together", "status": "unmapped", "reason": "combined extract_header+extract_footer passthrough is not asserted together in Rust"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_unknown_param_is_dropped", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_drops_unknown_params", "justification": "both assert an unrecognized param key is dropped while a known one is kept"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestNewSupportedParams::test_new_param_in_supported_list", "status": "unmapped", "reason": "OCR4-specific new params (table_format etc) are not individually verified against the Rust fixed-list test"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestNewParamsMapOcr::test_new_param_passed_through", "status": "unmapped", "reason": "OCR4-specific new params are not individually asserted in the Rust map_ocr_params test"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrRequest::test_param_included_in_request_body", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_builds_mistral_body", "justification": "both assert an optional param value ends up in the built request body alongside model/document"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrRequest::test_multiple_new_params_together", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_builds_mistral_body", "justification": "both assert multiple optional params (table_format/include_image_base64) land correctly in the same request body"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrResponseOcr4Fields::test_blocks_and_confidence_scores_preserved", "status": "unmapped", "reason": "OCR4 blocks/confidence_scores fields are not asserted by the Rust response test"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrResponseOcr4Fields::test_ocr4_fields_survive_model_dump", "status": "unmapped", "reason": "OCR4 tables/hyperlinks/header/footer fields are not asserted by the Rust response test"}, + + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_model_info_ocr4_price", "status": "unmapped", "reason": "cost-calc: pricing/model-info lookup is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr4_cost_scales_with_pages", "status": "unmapped", "reason": "cost-calc: per-page pricing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_pricing_entry", "status": "unmapped", "reason": "cost-calc: cost-map JSON entry validation is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_model_info_price", "status": "unmapped", "reason": "cost-calc: pricing/model-info lookup is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_cost_scales_with_pages", "status": "unmapped", "reason": "cost-calc: per-page pricing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_ocr_and_annotation_pages_at_their_own_rates", "status": "unmapped", "reason": "cost-calc: mixed-rate billing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_only_response", "status": "unmapped", "reason": "cost-calc: annotation-only billing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_pages_when_pages_processed_missing", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_azure_doc_ai_annotation_pages_fall_back_to_ocr_rate", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"}, + + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_serves_default_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_skipped_for_native_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_native_format_rejected_for_provider_without_support_as_bad_request", "status": "unmapped", "reason": "provider-support validation for req_format happens in Python"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_unknown_format_rejected_for_provider_without_support_as_bad_request", "status": "unmapped", "reason": "req_format validation error path is Python-only"}, + + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestHandlerDiscovery::test_handler_discovered_for_ocr", "status": "unmapped", "reason": "guardrail-translation handler discovery is a Python proxy-layer concern"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestHandlerDiscovery::test_handler_discovered_for_aocr", "status": "unmapped", "reason": "guardrail-translation handler discovery is a Python proxy-layer concern"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_document_url", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_image_url", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_no_document", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_invalid_document", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_input_blocking_guardrail", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_single_page", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_multiple_pages", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_empty_pages", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_page_with_empty_markdown", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_preserves_page_metadata", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_output_blocking_guardrail", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestPIIMaskingScenario::test_pii_masking_in_ocr_pages", "status": "unmapped", "reason": "PII redaction in the translation handler has no Rust equivalent"}, + + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_read_req_format_from_header", "status": "unmapped", "reason": "proxy-layer header parsing has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_prefer_body_req_format_over_header", "status": "unmapped", "reason": "proxy-layer body-vs-header precedence has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_omit_req_format_when_header_absent", "status": "unmapped", "reason": "proxy-layer parsing has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_reject_unknown_req_format", "status": "unmapped", "reason": "proxy-layer validation has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_return_native_payload_with_litellm_response_headers", "status": "unmapped", "reason": "proxy-layer response construction has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_return_normalized_response_when_no_native_payload", "status": "unmapped", "reason": "proxy-layer response construction has no Rust equivalent"} + ], + "rust_only_tests": [ + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "rust_custom_logger_reads_failure_payload_for_non_ocr_call_type", "reason": "exercises the non-OCR (acompletion) call-type branch of the logger; the OCR branch is covered separately by rust_custom_logger_reads_success_payload_for_ocr"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "no_callback_fast_path_dispatches_nothing", "reason": "Rust-only fast-path optimization test for when zero callbacks are registered; Python has no equivalent no-op dispatch path"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "with_standard_logging_payload_keeps_top_level_fields_in_sync", "reason": "Rust-internal builder-method invariant, Python has no equivalent internal builder"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "blocks_private_and_metadata_ips", "reason": "SSRF IP-blocking helper has no Python unit test; Python relies on the proxy-layer JSON/form guards instead"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "convert_document_url_rejects_loopback_fetch", "reason": "URL-fetch SSRF protection is Rust-gateway-only"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "convert_document_url_leaves_data_uri_untouched", "reason": "URL-fetch SSRF protection is Rust-gateway-only"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_passes_short_strings_through", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_caps_long_payloads", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_does_not_split_multibyte_chars", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_dispatch_supports_migrated_providers", "reason": "Rust-internal provider-config dispatch table has no equivalent Python unit test"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "string_headers_accepts_string_values", "reason": "Rust-gateway header-coercion helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "auth_header_detection_is_case_insensitive", "reason": "Rust-gateway header-detection helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_runs_pre_during_and_success_hooks", "reason": "full gateway-level guardrail-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_runs_failure_hook_on_provider_error", "reason": "full gateway-level failure-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_pre_call_block_skips_provider_socket", "reason": "full gateway-level pre-call-block-plus-socket-skip test with no Python equivalent at this integration scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_does_not_duplicate_authorization_header_when_header_is_supplied", "reason": "outgoing HTTP header dedup at the Rust gateway has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "document_intelligence_poll_uses_resolved_subscription_key", "reason": "full Azure DI poll-loop integration test with no Python equivalent at this scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "string_headers_rejects_non_string_values", "reason": "Rust-gateway header-coercion error path has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "azure_ai_reuses_mistral_body_transform", "reason": "Rust-internal delegation-to-Mistral-transform implementation detail, no Python test asserts this delegation"}, + {"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_request_uses_base64_source_for_data_uri", "reason": "no Python test asserts on the base64Source request body shape"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_mistral_url_uses_project_location_and_model", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_mistral_reuses_mistral_body_transform", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_deepseek_request_uses_ocr_endpoint_shape", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_deepseek_response_wraps_markdown_content", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_rejects_non_object_document", "reason": "non-object document rejection has no dedicated Python unit test"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_response_normalizes_mistral_json", "reason": "Python's response tests target OCR4-specific fields only, none asserts the same base normalization this Rust test checks"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "complete_url_defaults_and_dedupes_v1", "reason": "URL-building/defaulting for Mistral has no Python unit test"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "resolve_api_key_prefers_param_then_env", "reason": "API key resolution precedence at the Rust provider-config layer has no Python unit test"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "resolve_api_key_errors_when_absent", "reason": "API key resolution error path at the Rust provider-config layer has no Python unit test"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "rust_custom_logger_reads_success_payload_for_ocr", "reason": "Rust-internal custom-logger dispatch for OCR payloads has no Python unit test at this layer"} + ] +} diff --git a/tests/rust-python-harness/strategies/unit_tests/mapping_validator.py b/tests/rust-python-harness/strategies/unit_tests/mapping_validator.py new file mode 100644 index 00000000000..b8355e04266 --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/mapping_validator.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from ...shared.parity.ledger import TestLedger, load_ledger +from .python_runner import enumerate_python_tests +from .rust_runner import enumerate_rust_tests + +REPO_ROOT = Path(__file__).resolve().parents[4] +LEDGER_ROOT = Path(__file__).parent / "ledgers" + + +def ledger_path_for(sdk_function: str) -> Path: + return LEDGER_ROOT / sdk_function / f"{sdk_function}_test_ledger.json" + + +@dataclass(frozen=True, slots=True) +class AuditReport: + missing_python_tests: tuple[str, ...] + stale_python_tests: tuple[str, ...] + missing_rust_tests: tuple[str, ...] + stale_rust_tests: tuple[str, ...] + + @property + def is_clean(self) -> bool: + return not ( + self.missing_python_tests + or self.stale_python_tests + or self.missing_rust_tests + or self.stale_rust_tests + ) + + +def _ledger_python_tests_by_file(ledger: TestLedger) -> dict[str, set[str]]: + grouping: dict[str, set[str]] = {path: set() for path in ledger.python_scope} + for entry in ledger.entries: + grouping.setdefault(entry.python_file, set()).add(entry.python_test) + return grouping + + +def _ledger_rust_tests_by_file(ledger: TestLedger) -> dict[str, set[str]]: + grouping: dict[str, set[str]] = {path: set() for path in ledger.rust_scope} + for entry in ledger.entries: + if entry.status == "mapped": + grouping.setdefault(entry.rust_file, set()).add(entry.rust_test) + for rust_only in ledger.rust_only_tests: + grouping.setdefault(rust_only.rust_file, set()).add(rust_only.rust_test) + return grouping + + +def audit_ledger(ledger: TestLedger, repo_root: Path = REPO_ROOT) -> AuditReport: + missing_python: list[str] = [] + stale_python: list[str] = [] + for python_file, ledger_tests in _ledger_python_tests_by_file(ledger).items(): + actual_tests = enumerate_python_tests(repo_root, python_file) + for missing in sorted(ledger_tests - actual_tests): + missing_python.append(f"{python_file}:{missing}") + for stale in sorted(actual_tests - ledger_tests): + stale_python.append(f"{python_file}:{stale}") + + missing_rust: list[str] = [] + stale_rust: list[str] = [] + for rust_file, ledger_tests in _ledger_rust_tests_by_file(ledger).items(): + actual_tests = enumerate_rust_tests(repo_root, rust_file) + for missing in sorted(ledger_tests - actual_tests): + missing_rust.append(f"{rust_file}:{missing}") + for stale in sorted(actual_tests - ledger_tests): + stale_rust.append(f"{rust_file}:{stale}") + + return AuditReport( + missing_python_tests=tuple(missing_python), + stale_python_tests=tuple(stale_python), + missing_rust_tests=tuple(missing_rust), + stale_rust_tests=tuple(stale_rust), + ) + + +@dataclass(frozen=True, slots=True) +class FunctionReport: + sdk_function: str + ledger: TestLedger | None + audit: AuditReport | None + + @property + def has_ledger(self) -> bool: + return self.ledger is not None + + @property + def is_clean(self) -> bool: + return self.audit is None or self.audit.is_clean + + +def build_function_report(sdk_function: str, repo_root: Path = REPO_ROOT) -> FunctionReport: + path = ledger_path_for(sdk_function) + if not path.exists(): + return FunctionReport(sdk_function=sdk_function, ledger=None, audit=None) + ledger = load_ledger(path) + return FunctionReport( + sdk_function=sdk_function, ledger=ledger, audit=audit_ledger(ledger, repo_root) + ) diff --git a/tests/rust-python-harness/strategies/unit_tests/python_runner.py b/tests/rust-python-harness/strategies/unit_tests/python_runner.py new file mode 100644 index 00000000000..a7528d27756 --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/python_runner.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import ast +from pathlib import Path + + +def enumerate_python_tests(repo_root: Path, relative_path: str) -> frozenset[str]: + source = (repo_root / relative_path).read_text(encoding="utf-8") + tree = ast.parse(source, filename=relative_path) + + module_level: list[str] = [] + for node in ast.iter_child_nodes(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test_"): + module_level.append(node.name) + elif isinstance(node, ast.ClassDef): + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) and child.name.startswith( + "test_" + ): + module_level.append(f"{node.name}::{child.name}") + + return frozenset(module_level) diff --git a/tests/rust-python-harness/strategies/unit_tests/rust_runner.py b/tests/rust-python-harness/strategies/unit_tests/rust_runner.py new file mode 100644 index 00000000000..6b855adbc4b --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/rust_runner.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +import re +from pathlib import Path + +_RUST_TEST_PATTERN = re.compile( + r"#\[(?:test|tokio::test)\][^\n]*\n(?:[^\n]*\n)*?\s*(?:async\s+)?fn\s+(\w+)\s*\(" +) + + +def enumerate_rust_tests(repo_root: Path, relative_path: str) -> frozenset[str]: + source = (repo_root / relative_path).read_text(encoding="utf-8") + return frozenset(match.group(1) for match in _RUST_TEST_PATTERN.finditer(source)) diff --git a/tests/rust-python-harness/ui.py b/tests/rust-python-harness/ui.py new file mode 100644 index 00000000000..57fedd17fa6 --- /dev/null +++ b/tests/rust-python-harness/ui.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +import os +import shlex +import sys +from collections.abc import Sequence +from contextlib import AbstractContextManager +from pathlib import Path +from typing import Any + +from .models import ( + Coverage, + HarnessRun, + RunStatus, + SDK_FUNCTIONS, + Strategy, + section_confidence, +) + +STATUS_GLYPHS = { + RunStatus.NOT_RUN: "·", + RunStatus.QUEUED: "○", + RunStatus.RUNNING: "◉", + RunStatus.PASSED: "✓", + RunStatus.FAILED: "✗", + RunStatus.SKIPPED: "↷", + RunStatus.ERROR: "!", + RunStatus.MISSING: "?", + RunStatus.PLANNED: "—", + RunStatus.NOT_APPLICABLE: "n/a", +} + +STATUS_STYLES = { + RunStatus.QUEUED: "dim", + RunStatus.RUNNING: "bold cyan", + RunStatus.PASSED: "bold green", + RunStatus.FAILED: "bold red", + RunStatus.SKIPPED: "yellow", + RunStatus.ERROR: "bold red", + RunStatus.MISSING: "magenta", + RunStatus.PLANNED: "dim", + RunStatus.NOT_APPLICABLE: "dim", +} + + +def _format_duration(seconds: float) -> str: + if seconds < 1: + return f"{seconds * 1000:.0f}ms" + if seconds < 60: + return f"{seconds:.1f}s" + return f"{int(seconds // 60)}m {seconds % 60:.0f}s" + + +def _rerun_command(nodeid: str) -> str: + return f"poetry run pytest {shlex.quote(nodeid)} -q" + + +def _summary(run: HarnessRun) -> tuple[int, int, int, int]: + outcomes: dict[str, RunStatus] = {} + for result in run.results.values(): + outcomes.update(result.outcomes) + return ( + list(outcomes.values()).count(RunStatus.PASSED), + list(outcomes.values()).count(RunStatus.FAILED), + list(outcomes.values()).count(RunStatus.ERROR), + list(outcomes.values()).count(RunStatus.SKIPPED), + ) + + +def _cell_text(run: HarnessRun, strategy_id: str, sdk_function: str) -> tuple[str, str]: + result = run.results.get(f"{strategy_id}:{sdk_function}") + if result is None: + return "", "" + counts = "" + if result.total: + counts = f" {len(result.completed)}/{result.total}" + coverage = " ◐" if result.case.coverage is Coverage.PARTIAL else "" + return f"{STATUS_GLYPHS[result.status]}{counts}{coverage}", STATUS_STYLES.get( + result.status, "" + ) + + +class RichDashboard(AbstractContextManager["RichDashboard"]): + def __init__( + self, + strategies: Sequence[Strategy], + confidence_strategies: Sequence[Strategy], + ) -> None: + from rich.console import Console + from rich.live import Live + + self.strategies = strategies + self.confidence_strategies = confidence_strategies + self.console = Console() + self.live: Any = Live( + console=self.console, refresh_per_second=12, transient=False + ) + + def _table(self, run: HarnessRun) -> Any: + from rich import box + from rich.table import Table + from rich.text import Text + + narrow = self.console.width < 96 + if narrow: + table = Table(box=box.SIMPLE_HEAVY, expand=True, show_header=False) + table.add_column("Strategy", ratio=3) + table.add_column("Results", ratio=5) + for strategy in self.strategies: + values = [] + for sdk_function in SDK_FUNCTIONS: + value, style = _cell_text(run, strategy.id, sdk_function) + if value: + values.append( + Text.assemble((f"{sdk_function} ", "dim"), (value, style)) + ) + table.add_row(strategy.label, Text(" ").join(values)) + return table + + table = Table(box=box.ROUNDED, expand=True, title="Strategy × SDK function") + table.add_column("Strategy", ratio=3) + for label in ("ocr/aocr", "messages", "responses", "count_tokens"): + table.add_column(label, justify="center", ratio=1) + for strategy in self.strategies: + cells = [] + for sdk_function in SDK_FUNCTIONS: + value, style = _cell_text(run, strategy.id, sdk_function) + cells.append(Text(value, style=style)) + table.add_row(strategy.label, *cells) + return table + + def __enter__(self) -> "RichDashboard": + self.live.__enter__() + return self + + def __exit__(self, *args: object) -> None: + self.live.__exit__(*args) + + def update(self, run: HarnessRun) -> None: + from rich.markup import escape + from rich.panel import Panel + + active = run.current_nodeid or "Waiting for test events…" + if len(active) > max(40, self.console.width - 16): + active = f"…{active[-(self.console.width - 17):]}" + passed, failed, errors, skipped = _summary(run) + progress = ( + f"[bold]{run.completed_tests}/{run.unique_tests}[/bold] tests " + f"[green]{passed} passed[/green] [red]{failed + errors} failed[/red] " + f"[yellow]{skipped} skipped[/yellow] [dim]{_format_duration(run.duration)}[/dim]" + ) + legend = "✓ pass ✗ fail ! error ↷ skip\n? configured test missing — planned ◐ partial coverage" + self.live.update( + Panel( + self._table(run), + title="⚡ Rust ↔ Python parity lab", + subtitle=f"{progress}\n[dim]{escape(active)}[/dim]\n{legend}", + border_style="cyan", + ) + ) + + def finish(self, run: HarnessRun, exit_code: int) -> None: + self.update(run) + if run.failures: + from rich.markup import escape + from rich.panel import Panel + + for nodeid, detail in run.failures[:5]: + rerun = _rerun_command(nodeid) + self.console.print( + Panel( + f"{escape(detail)}\n\n[bold]Rerun just this test[/bold]\n" + f"[cyan]{escape(rerun)}[/cyan]", + title=f"✗ {escape(nodeid)}", + border_style="red", + ) + ) + durations: dict[str, float] = {} + for result in run.results.values(): + for nodeid, duration in result.durations.items(): + durations[nodeid] = max(duration, durations.get(nodeid, 0.0)) + if durations: + slow = sorted(durations.items(), key=lambda item: item[1], reverse=True)[:3] + self.console.print( + "[bold]Slowest tests[/bold] " + + " • ".join( + f"{Path(nodeid).name} [dim]{_format_duration(duration)}[/dim]" + for nodeid, duration in slow + ) + ) + from rich import box + from rich.table import Table + + confidence_table = Table( + title="Port confidence by SDK section", box=box.ROUNDED, expand=True + ) + confidence_table.add_column("SDK section") + confidence_table.add_column("Score", justify="right") + confidence_table.add_column("Confidence") + confidence_table.add_column("Strategy evidence", ratio=4) + confidence_styles = {"HIGH": "green", "MEDIUM": "yellow", "LOW": "red"} + for score in section_confidence(run, self.confidence_strategies): + confidence_table.add_row( + score.sdk_function, + f"{score.verified_strategies}/{score.required_strategies} {score.percentage}%", + f"[{confidence_styles[score.level.value]}]{score.level.value}[/]", + " ".join(score.details), + ) + self.console.print(confidence_table) + self.console.print( + "[dim]Score = required strategies with passing evidence. " + "LOC coverage remains a separate report.[/dim]" + ) + style = "green" if exit_code == 0 else "red" + self.console.print( + f"[{style}]Harness finished in {_format_duration(run.duration)} " + f"(exit {exit_code})[/{style}]" + ) + + +class PlainDashboard(AbstractContextManager["PlainDashboard"]): + def __init__( + self, + strategies: Sequence[Strategy], + confidence_strategies: Sequence[Strategy], + ) -> None: + self.strategies = strategies + self.confidence_strategies = confidence_strategies + self._seen: dict[str, tuple[RunStatus, int]] = {} + + def __enter__(self) -> "PlainDashboard": + print("Rust <-> Python SDK parity harness", flush=True) + return self + + def __exit__(self, *args: object) -> None: + return None + + def update(self, run: HarnessRun) -> None: + for key, result in run.results.items(): + state = (result.status, len(result.completed)) + if self._seen.get(key) != state: + self._seen[key] = state + progress = ( + f" {len(result.completed)}/{result.total}" if result.total else "" + ) + print( + f"{STATUS_GLYPHS[result.status]} {key}: {result.status.value}{progress}", + flush=True, + ) + + def finish(self, run: HarnessRun, exit_code: int) -> None: + self.update(run) + passed, failed, errors, skipped = _summary(run) + print( + f"Summary: {passed} passed, {failed} failed, {errors} errors, " + f"{skipped} skipped in {_format_duration(run.duration)}", + flush=True, + ) + for nodeid, _ in run.failures[:5]: + print(f"Rerun: {_rerun_command(nodeid)}", flush=True) + print("Port confidence by SDK section", flush=True) + for score in section_confidence(run, self.confidence_strategies): + print( + f" {score.sdk_function:12} " + f"{score.verified_strategies}/{score.required_strategies} " + f"{score.percentage:3}% {score.level.value:6} " + f"{' | '.join(score.details)}", + flush=True, + ) + print( + " Score = required strategies with passing evidence; LOC is reported separately.", + flush=True, + ) + print(f"Harness finished with exit code {exit_code}", flush=True) + + +def make_dashboard( + strategies: Sequence[Strategy], + plain: bool = False, + confidence_strategies: Sequence[Strategy] | None = None, +) -> RichDashboard | PlainDashboard: + confidence_strategies = confidence_strategies or strategies + interactive_terminal = ( + sys.stdout.isatty() + and not os.environ.get("CI") + and os.environ.get("TERM") != "dumb" + ) + if not plain and interactive_terminal: + try: + import rich # noqa: F401 + + return RichDashboard(strategies, confidence_strategies) + except ImportError: + pass + return PlainDashboard(strategies, confidence_strategies) diff --git a/tests/rust-python-harness/unit_tests_rust/README.md b/tests/rust-python-harness/unit_tests_rust/README.md new file mode 100644 index 00000000000..12c7eb0089c --- /dev/null +++ b/tests/rust-python-harness/unit_tests_rust/README.md @@ -0,0 +1,3 @@ +# Rust unit tests + +Holds focused Cargo tests for Rust-owned parsing, transforms, errors, and streaming behavior. These tests make failures fast to diagnose before the Python bridge or full SDK path is involved. diff --git a/tests/rust-python-harness/unit_tests_rust/strategy.json b/tests/rust-python-harness/unit_tests_rust/strategy.json new file mode 100644 index 00000000000..89e897c872d --- /dev/null +++ b/tests/rust-python-harness/unit_tests_rust/strategy.json @@ -0,0 +1,12 @@ +{ + "order": 20, + "id": "unit_tests_rust", + "label": "Rust unit tests", + "description": "Exercise Rust-owned behavior directly with focused unit tests.", + "functions": { + "ocr": {"coverage": "planned", "selectors": []}, + "messages": {"coverage": "planned", "selectors": []}, + "responses": {"coverage": "planned", "selectors": []}, + "count_tokens": {"coverage": "planned", "selectors": []} + } +} diff --git a/tests/rust-python-harness/validate_sub_methods/README.md b/tests/rust-python-harness/validate_sub_methods/README.md new file mode 100644 index 00000000000..24894366f23 --- /dev/null +++ b/tests/rust-python-harness/validate_sub_methods/README.md @@ -0,0 +1,3 @@ +# Validate sub-methods + +Checks each request, response, stream, and error-mapping sub-method independently across Python and Rust. It also validates that traced Python helpers have an explicit Rust implementation and parity test. diff --git a/tests/rust-python-harness/validate_sub_methods/strategy.json b/tests/rust-python-harness/validate_sub_methods/strategy.json new file mode 100644 index 00000000000..6e6381678e0 --- /dev/null +++ b/tests/rust-python-harness/validate_sub_methods/strategy.json @@ -0,0 +1,12 @@ +{ + "order": 30, + "id": "validate_sub_methods", + "label": "Validate sub-methods", + "description": "Compare isolated transforms and verify Python-to-Rust helper coverage.", + "functions": { + "ocr": {"coverage": "planned", "selectors": []}, + "messages": {"coverage": "planned", "selectors": []}, + "responses": {"coverage": "planned", "selectors": []}, + "count_tokens": {"coverage": "planned", "selectors": []} + } +} diff --git a/tests/sdk_function_trace/README.md b/tests/sdk_function_trace/README.md new file mode 100644 index 00000000000..d3a3b654aea --- /dev/null +++ b/tests/sdk_function_trace/README.md @@ -0,0 +1,30 @@ +# SDK function tracing + +The compare runner executes the same SDK calls through the Python engine and the Rust native bridge against a local HTTP provider fixture, then prints their pipeline trees side by side. Matching calls align on the same row in green; Python-only calls are blue, Rust-only calls yellow, and reordered calls red. Gaps preserve execution order and each column retains its own nesting. A comparison column labels every row even without color. Colors are enabled in terminals unless `NO_COLOR` is set. A difference summary follows (shared step order, python-only steps, rust-only steps). Each invocation must issue exactly one HTTP request. It requires the LiteLLM Python dependencies and the native extension built with tracing support + +From the repository root, using the project's Python environment: + +```bash +uv run python -m tests.sdk_function_trace.compare +uv run python -m tests.sdk_function_trace.compare --route ocr +uv run python -m tests.sdk_function_trace.compare --route ocr --sync +uv run python -m tests.sdk_function_trace.compare --route all --both --check +``` + +Calls default to async; use `--sync` for synchronous calls or `--both` for the complete matrix. Python sync Messages raises `not implemented for sync calls`; only that exact failure is marked `SKIP`, and the runner still executes Rust sync Messages and subsequent routes. Bedrock transcription has no independent Python provider implementation: its Python trace covers SDK dispatch into Rust + +Both engines are projected onto a shared per-route step table (`steps.py`): canonical names such as `transform_ocr_request` map Python functions (`MistralOCRConfig.transform_ocr_request`) and Rust spans (`transform_ocr_request`) to the same label. Only the first occurrence of each step is kept. Python indentation uses each event's actual frame ancestors and the nearest already displayed ancestor, so returned helpers and coroutine resumptions do not create false parents. Rust indentation uses instrumented span ancestry. Unmatched Rust span names pass through unchanged. `--full` prints every captured runtime event; validation still uses projected steps + +Every report checks required stage presence and dependency order. Provider lookup must precede request transformation, which must precede HTTP, followed by response transformation. The handler must precede HTTP; parameter mapping and supported-parameter checks must precede request transformation. Environment validation and URL construction, where mapped, must precede HTTP. Python transcription is checked only through native dispatch. `--check` also requires identical canonical step sequences for comparable routes and exits nonzero for missing, extra, or reordered steps, or an unexpected call failure, after finishing all selected cases + +Individual stage checks are separate from cross-language `step parity`. Passing stage checks cannot override a failing step comparison. Bedrock transcription and Python sync Messages report `UNAVAILABLE` for cross-language parity because they lack an independent Python execution to compare. Absolute nesting depth is not a cross-language gate: async Python Messages dispatches its handler onto another thread. See `route-comparison.md` for the audited matrix and remaining contract limitations + +The Python runner uses the existing `profile_python` / `sys.setprofile` collector, selecting executed code under the installed `litellm` source directory instead of maintaining a function-name allowlist. It prints source locations and qualified function names, including repeated calls. Coroutine resumptions are counted once per invocation. It profiles the current thread and threads created during the call, including the fresh async executor. Existing worker threads are not retroactively profiled; background Python calls may appear, and indentation follows selected Python stack ancestors within each thread + +The Rust runner calls the compiled PyO3 SDK entrypoints with `trace=True`. The existing `FunctionTrace` subscriber collects `#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]` spans for the route entrypoint, preparation, provider lookup, HTTP handler, and selected provider transformations. The shared `http_request` helper instruments the existing Rust send operation without changing clients, timeouts, signing, or error mapping. Function names come from the actual functions. `WithSubscriber` attaches the collector to each future across async polls. Arguments and provider payloads are not recorded in trace events. Uninstrumented functions do not appear; this is scoped instrumentation, not an exhaustive native call graph + +Tracing is opt-in: native calls without `trace=True` keep their original response shape. Traced calls return `{"response": ..., "trace": [{"function": ..., "depth": ...}]}`. The runners print only trace events. Missing native support or empty traces fail instead of falling back to source searching. The old `--repo`, `--signatures`, and `--calls` options are removed + +`profile_python(functions)` still supports direct function references for focused parity checks. `assert_function_trace_parity` compares selected Python events with Rust events supplied by an executable scenario. Successful stage checks prove the declared pipeline ran in a valid dependency order for this fixture; they do not assert identical function contracts, request bodies, responses, streaming behavior, or live-provider correctness + +Build the extension with `maturin develop` in the project's virtual environment. Then run either command above to get the executed function order diff --git a/tests/sdk_function_trace/__init__.py b/tests/sdk_function_trace/__init__.py new file mode 100644 index 00000000000..da62b8041f6 --- /dev/null +++ b/tests/sdk_function_trace/__init__.py @@ -0,0 +1,13 @@ +from tests.sdk_function_trace.harness import ( + TraceScenario, + TraceStep, + assert_function_trace_parity, +) +from tests.sdk_function_trace.profiler import FunctionTraceEvent + +__all__ = [ + "FunctionTraceEvent", + "TraceScenario", + "TraceStep", + "assert_function_trace_parity", +] diff --git a/tests/sdk_function_trace/compare.py b/tests/sdk_function_trace/compare.py new file mode 100644 index 00000000000..941c1b6e067 --- /dev/null +++ b/tests/sdk_function_trace/compare.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import argparse +import os +import sys +from typing import Final + +from tests.sdk_function_trace.fixtures import ROUTES +from tests.sdk_function_trace.report import compare, render + + +def _run(route: str, asynchronous: bool, *, full: bool, colorize: bool) -> bool: + comparison: Final = compare(route, asynchronous=asynchronous) + sys.stdout.write(render(comparison, full=full, colorize=colorize)) + return comparison.passed + + +def main() -> None: + parser: Final = argparse.ArgumentParser(description="Compare Python and Rust SDK pipeline steps per route") + parser.add_argument("--route", choices=("all", *ROUTES), default="all") + mode: Final = parser.add_mutually_exclusive_group() + mode.add_argument("--async", dest="asynchronous", action="store_true", default=True) + mode.add_argument("--sync", dest="asynchronous", action="store_false") + mode.add_argument("--both", action="store_true", help="run async and sync for every selected route") + parser.add_argument( + "--check", action="store_true", help="exit nonzero for missing, extra, or reordered comparable steps" + ) + parser.add_argument( + "--full", action="store_true", help="print every captured runtime event instead of pipeline steps" + ) + args: Final = parser.parse_args() + os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") + colorize: Final = sys.stdout.isatty() and "NO_COLOR" not in os.environ + results: Final = tuple( + _run(selected, selected_mode, full=args.full, colorize=colorize) + for selected in ROUTES + if args.route in ("all", selected) + for selected_mode in ((True, False) if args.both else (args.asynchronous,)) + ) + if args.check and not all(results): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/sdk_function_trace/fixtures.py b/tests/sdk_function_trace/fixtures.py new file mode 100644 index 00000000000..47bbe839627 --- /dev/null +++ b/tests/sdk_function_trace/fixtures.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import base64 +import io +import json +import wave +from collections.abc import Callable +from dataclasses import dataclass +from typing import Final, Protocol, cast + +from tests.sdk_function_trace.mock_provider import MockProviderResponse +from tests.sdk_function_trace.steps import Engine + +ANTHROPIC_MODEL: Final = "claude-sonnet-5" +OCR_MODEL: Final = "mistral-ocr-latest" +AUDIO_MODEL: Final = "mistral.voxtral-mini-3b-2507" + + +class SdkCall(Protocol): + def __call__(self, **kwargs: object) -> object: ... + + +@dataclass(frozen=True, slots=True) +class Fixture: + kwargs: dict[str, object] + provider_response: MockProviderResponse + + +@dataclass(frozen=True, slots=True) +class RouteSpec: + label: str + python_entrypoints: tuple[str, str] + rust_entrypoints: tuple[str, str] + fixture: Callable[[Engine], Fixture] + + +@dataclass(frozen=True, slots=True) +class Invocation: + function: SdkCall + kwargs: dict[str, object] + provider_response: MockProviderResponse + label: str + + +def audio_bytes() -> bytes: + with io.BytesIO() as buffer: + with wave.open(buffer, "wb") as audio: + audio.setnchannels(1) + audio.setsampwidth(2) + audio.setframerate(16000) + audio.writeframes(b"\x00\x00" * 1600) + return buffer.getvalue() + + +def _anthropic_message_response() -> MockProviderResponse: + body: Final = { + "id": "msg_trace", + "type": "message", + "role": "assistant", + "model": ANTHROPIC_MODEL, + "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 2, "output_tokens": 3}, + } + return MockProviderResponse(200, (("content-type", "application/json"),), json.dumps(body).encode()) + + +def _conversation() -> dict[str, object]: + return {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 16} + + +def _ocr_fixture(engine: Engine) -> Fixture: + return Fixture( + kwargs={ + "model": f"mistral/{OCR_MODEL}", + "document": {"type": "document_url", "document_url": "https://example.com/document.pdf"}, + **({"optional_params": {"pages": [0]}} if engine == "rust" else {"pages": [0]}), + }, + provider_response=MockProviderResponse( + 200, + (("content-type", "application/json"),), + json.dumps( + { + "pages": [{"index": 0, "markdown": "hello"}], + "model": OCR_MODEL, + "usage_info": {"pages_processed": 1}, + } + ).encode(), + ), + ) + + +def _chat_completions_fixture(engine: Engine) -> Fixture: + conversation: Final = _conversation() + payload: Final = ( + {"messages": conversation["messages"], "optional_params": {"max_tokens": 16}} + if engine == "rust" + else conversation + ) + return Fixture( + kwargs={"model": f"anthropic/{ANTHROPIC_MODEL}", **payload}, + provider_response=_anthropic_message_response(), + ) + + +def _messages_fixture(engine: Engine) -> Fixture: + conversation: Final = _conversation() + payload: Final = {"body": {**conversation, "model": ANTHROPIC_MODEL}} if engine == "rust" else conversation + return Fixture( + kwargs={"model": f"anthropic/{ANTHROPIC_MODEL}", **payload}, + provider_response=_anthropic_message_response(), + ) + + +def _transcription_fixture(engine: Engine) -> Fixture: + credentials: Final = { + "aws_access_key_id": "test-access", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-east-1", + } + payload: Final = ( + { + "audio": {"data": base64.b64encode(audio_bytes()).decode(), "format": "wav"}, + "optional_params": credentials, + } + if engine == "rust" + else {"file": ("sample.wav", audio_bytes(), "audio/wav"), **credentials} + ) + return Fixture( + kwargs={"model": f"bedrock/{AUDIO_MODEL}", **payload}, + provider_response=MockProviderResponse( + 200, + (("content-type", "application/json"),), + json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 2, "outputTokens": 3, "totalTokens": 5}, + } + ).encode(), + ), + ) + + +ROUTE_SPECS: Final[dict[str, RouteSpec]] = { + "chat_completions": RouteSpec( + label="anthropic", + python_entrypoints=("completion", "acompletion"), + rust_entrypoints=("chat_completions", "achat_completions"), + fixture=_chat_completions_fixture, + ), + "audio_transcription": RouteSpec( + label="bedrock (Rust-only provider; Python trace covers SDK dispatch)", + python_entrypoints=("transcription", "atranscription"), + rust_entrypoints=("transcription", "atranscription"), + fixture=_transcription_fixture, + ), + "messages": RouteSpec( + label="anthropic", + python_entrypoints=("create", "acreate"), + rust_entrypoints=("messages", "amessages"), + fixture=_messages_fixture, + ), + "ocr": RouteSpec( + label="mistral", + python_entrypoints=("ocr", "aocr"), + rust_entrypoints=("ocr", "aocr"), + fixture=_ocr_fixture, + ), +} + +ROUTES: Final = tuple(ROUTE_SPECS) + + +def sdk_invocation(route: str, *, engine: Engine, asynchronous: bool) -> Invocation: + import litellm + from litellm.anthropic_interface import messages as sdk_messages + from litellm.rust_bridge import get_native_bridge + + rust: Final = engine == "rust" + bridge: Final = get_native_bridge() if rust else None + if rust and bridge is None: + raise RuntimeError("Build the native extension first: maturin develop") + spec: Final = ROUTE_SPECS.get(route) + if spec is None: + raise ValueError(f"Unknown route: {route}") + fixture: Final = spec.fixture(engine) + owner: Final = bridge if rust else (sdk_messages if route == "messages" else litellm) + entrypoint: Final = (spec.rust_entrypoints if rust else spec.python_entrypoints)[int(asynchronous)] + return Invocation( + function=cast(SdkCall, getattr(owner, entrypoint)), + kwargs={ + **fixture.kwargs, + "api_key": "test-key", + **({"trace": True, "timeout_seconds": 5} if rust else {"timeout": 5}), + }, + provider_response=fixture.provider_response, + label=spec.label, + ) diff --git a/tests/sdk_function_trace/harness.py b/tests/sdk_function_trace/harness.py new file mode 100644 index 00000000000..8f707402449 --- /dev/null +++ b/tests/sdk_function_trace/harness.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from types import FunctionType +from typing import Final, cast + +from tests.sdk_function_trace.profiler import FunctionTraceEvent, profile_python + + +@dataclass(frozen=True, slots=True) +class TraceStep: + function: FunctionType + depth: int + + +@dataclass(frozen=True, slots=True) +class TraceScenario: + steps: tuple[TraceStep, ...] + invoke_python: Callable[[], object] + invoke_rust: Callable[[], Sequence[FunctionTraceEvent]] + + +def assert_function_trace_parity(scenario: TraceScenario) -> None: + expected: Final = tuple( + FunctionTraceEvent(function=step.function.__name__, depth=step.depth) for step in scenario.steps + ) + functions: Final = cast(tuple[FunctionType, ...], tuple(step.function for step in scenario.steps)) + with profile_python(functions) as profiler: + scenario.invoke_python() + python_trace: Final = tuple(profiler.events) + rust_trace: Final = tuple(scenario.invoke_rust()) + + if python_trace != expected: + raise AssertionError(f"Python function trace differs: {python_trace!r} != {expected!r}") + if rust_trace != expected: + raise AssertionError(f"Rust function trace differs: {rust_trace!r} != {expected!r}") + if python_trace != rust_trace: + raise AssertionError(f"Python and Rust function traces differ: {python_trace!r} != {rust_trace!r}") diff --git a/tests/sdk_function_trace/mock_provider.py b/tests/sdk_function_trace/mock_provider.py new file mode 100644 index 00000000000..37eca665586 --- /dev/null +++ b/tests/sdk_function_trace/mock_provider.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Lock, Thread +from typing import Final, cast + + +@dataclass(frozen=True, slots=True) +class MockProviderResponse: + status_code: int + headers: tuple[tuple[str, str], ...] + body: bytes + + +class _MockProviderServer(ThreadingHTTPServer): + def __init__(self, response: MockProviderResponse) -> None: + super().__init__(("127.0.0.1", 0), _MockProviderHandler) + self.response: Final = response + self._request_count = 0 + self._request_count_lock: Final = Lock() + + def record_request(self) -> None: + with self._request_count_lock: + self._request_count += 1 + + @property + def request_count(self) -> int: + with self._request_count_lock: + return self._request_count + + +class _MockProviderHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + content_length: Final = int(self.headers.get("content-length", "0")) + self.rfile.read(content_length) + server: Final = cast(_MockProviderServer, self.server) + server.record_request() + self.send_response(server.response.status_code) + for name, value in server.response.headers: + self.send_header(name, value) + self.send_header("content-length", str(len(server.response.body))) + self.end_headers() + self.wfile.write(server.response.body) + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 # matches BaseHTTPRequestHandler + pass + + +@contextmanager +def mock_provider(response: MockProviderResponse) -> Generator[str]: + server: Final = _MockProviderServer(response) + thread: Final = Thread(target=server.serve_forever, daemon=True) + thread.start() + host, port = cast(tuple[str, int], server.server_address) + try: + yield f"http://{host}:{port}" + finally: + server.shutdown() + server.server_close() + thread.join() + if server.request_count != 1: + raise AssertionError(f"expected one provider request, received {server.request_count}") diff --git a/tests/sdk_function_trace/ocr-comparison.md b/tests/sdk_function_trace/ocr-comparison.md new file mode 100644 index 00000000000..d252480e218 --- /dev/null +++ b/tests/sdk_function_trace/ocr-comparison.md @@ -0,0 +1,59 @@ +# OCR Python and Rust comparison + +Audited implementation revision: `edcba483b2`. The implementations do not match in function contracts, call structure, or all tested response behavior. This audit changes the source listing coverage, not OCR runtime behavior + +Run both source listings from the repository root: + +```bash +python3 tests/sdk_function_trace/list_python_steps.py --route ocr --signatures --calls +uv run tests/sdk_function_trace/list_rust_steps.py --route ocr --signatures --calls +``` + +Both cover Mistral, Azure AI Mistral, Azure Document Intelligence, Vertex Mistral, and Vertex DeepSeek. Listings show declarations and source call sites, not executed traces + +## Function contracts + +Comparing Python `BaseOCRConfig` with Rust `OcrProviderConfig`, omitting `self` and language-specific ownership details: + +| Python | Rust | Difference | +| --- | --- | --- | +| `get_supported_ocr_params(model)` | `supported_ocr_params()` | Name and model argument | +| `get_api_key_env_var()` | No corresponding method | Missing contract | +| `map_ocr_params(non_default_params, optional_params, model)` | `map_ocr_params(non_default_params)` | Missing accumulator and model | +| `validate_environment(headers, model, api_key, api_base, litellm_params, **kwargs)` | Separate auth/key/header helpers | Different contract | +| `get_complete_url(api_base, model, optional_params, litellm_params, **kwargs)` | `complete_url(api_base, model, optional_params, env_lookup)` | Name and context | +| `transform_ocr_request(model, document, optional_params, headers, **kwargs)` | `transform_ocr_request(model, document, optional_params)` | Missing headers and extra context | +| `async_transform_ocr_request(...)` | No corresponding method | Missing async override | +| `transform_ocr_response(model, raw_response, logging_obj, **kwargs)` | `transform_ocr_response(model, response_json)` | Missing HTTP metadata, logging and extra context | +| `async_transform_ocr_response(...)` | No corresponding method | Missing async override | +| `get_error_class(error_message, status_code, headers)` | Central Rust error mapping | Different contract | + +Python's default mapper returns the supplied `optional_params`; Rust's filters `non_default_params`. Provider overrides must also be compared + +Python maps parameters during SDK preparation, before HTTP-handler environment validation and URL construction. Rust resolves auth and URL before mapping parameters in `prepare_provider_request`. Python has async provider transforms; both native entrypoints execute the same Rust async route using synchronous transform hooks, with polling and document downloading in gateway helpers + +The native bindings also accept `optional_params` and `timeout_seconds`, while the Python SDK accepts `**kwargs` and `timeout`. Public SDK calls with Rust enabled still execute Python preparation before entering Rust, so matching SDK responses would not prove matching standalone Rust steps + +## Runtime results + +Built the native extension from the audited source using `cargo build -p litellm-python-bridge --features extension-module --offline`. Supplied that build's functions through `use_litellm_rust` dependency injection. Ran public `litellm.ocr` and `litellm.aocr` with Rust disabled and enabled against identical local HTTP response fixtures, requiring one request per invocation + +Successful `model_dump()` results and failure exception classes were compared. These checks cover Mistral response outcomes only, not request equality, error messages, live providers, or every execution branch + +| Mistral response fixture | Sync | Async | Observation | +| --- | --- | --- | --- | +| Valid page/model/usage | Match | Match | Same normalized response | +| Model omitted | Match | Match | Both use the requested model | +| `model: null` | Different | Different | Python rejects; Rust uses the requested model | +| `pages: null` | Different | Different | Python rejects; Rust returns an empty array | +| Invalid page element | Match | Match | Both reject during response validation | + +Six of ten fixture/mode comparisons match, four differ. Rust's Mistral response transform conflates missing values with explicit nulls through `as_array`/`as_str` fallbacks. Python preserves explicit nulls into response validation, which rejects them + +## Other provider gaps found in source + +Azure Document Intelligence's Python configuration supports `pages`, `features`, and `req_format`; Rust lists only `pages`. Python normalizes parameters before URL construction; Rust normalizes pages during URL construction + +Python preserves Azure `content`, `tables`, and `keyValuePairs`, and supports retaining the native operation payload. Rust's `OcrResponseData` has no corresponding fields, and its Azure transform does not preserve those values + +Azure and Vertex async document transforms and Azure polling also use different helper contracts. Their runtime equivalence was not tested in this audit diff --git a/tests/sdk_function_trace/profiler.py b/tests/sdk_function_trace/profiler.py new file mode 100644 index 00000000000..c71c74ab0d3 --- /dev/null +++ b/tests/sdk_function_trace/profiler.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import sys +import threading +from collections.abc import Generator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from types import CodeType, FrameType, FunctionType +from typing import Final + + +@dataclass(frozen=True, slots=True) +class FunctionTraceEvent: + function: str + depth: int + ancestors: tuple[str, ...] | None = None + + +class PythonProfiler: + def __init__(self, functions: Sequence[FunctionType], source_root: Path | None = None) -> None: + self._source_root: Final = str(source_root.resolve()) + "/" if source_root is not None else None + self._names_by_code: Final = {function.__code__: function.__name__ for function in functions} + self._seen_frames: Final[set[FrameType]] = set() + self.events: Final[list[FunctionTraceEvent]] = [] + + def __call__(self, frame: FrameType, event: str, _arg: object) -> None: + if event != "call" or frame in self._seen_frames: + return + function_name: Final = self.function_name(frame.f_code) + if function_name is None: + return + ancestors: Final = tuple( + name for ancestor in _frame_ancestors(frame) if (name := self.function_name(ancestor.f_code)) is not None + ) + self._seen_frames.add(frame) + self.events.append( + FunctionTraceEvent( + function=function_name, + depth=len(ancestors), + ancestors=ancestors if self._source_root is not None else None, + ) + ) + + def function_name(self, code: CodeType) -> str | None: + if self._source_root is None: + return self._names_by_code.get(code) + if not code.co_filename.startswith(self._source_root): + return None + relative: Final = code.co_filename.removeprefix(self._source_root) + return f"{relative}:{code.co_firstlineno} {getattr(code, 'co_qualname', code.co_name)}" + + +def _frame_ancestors(frame: FrameType) -> Generator[FrameType]: + ancestor: Final = frame.f_back + if ancestor is not None: + yield ancestor + yield from _frame_ancestors(ancestor) + + +@contextmanager +def profile_python( + functions: Sequence[FunctionType] = (), *, source_root: Path | None = None, threads: bool = False +) -> Generator[PythonProfiler]: + profiler: Final = PythonProfiler(functions, source_root) + previous_thread: Final = threading.getprofile() + if threads: + threading.setprofile(profiler) + previous: Final = sys.getprofile() + sys.setprofile(profiler) + try: + yield profiler + finally: + sys.setprofile(previous) + if threads: + threading.setprofile(previous_thread) diff --git a/tests/sdk_function_trace/report.py b/tests/sdk_function_trace/report.py new file mode 100644 index 00000000000..9b654e571f8 --- /dev/null +++ b/tests/sdk_function_trace/report.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final + +from tests.sdk_function_trace.fixtures import ROUTE_SPECS +from tests.sdk_function_trace.profiler import FunctionTraceEvent +from tests.sdk_function_trace.runtime import ( + TraceDiff, + TraceFailed, + TraceOk, + TraceRun, + TraceSkipped, + attempt_trace, + trace_diff, +) +from tests.sdk_function_trace.steps import Engine, pipeline_issues, pipeline_steps +from tests.sdk_function_trace.table import format_trace_table + +_PYTHON_ONLY_COLOR: Final = "\033[34m" +_RUST_ONLY_COLOR: Final = "\033[33m" +_RESET: Final = "\033[0m" + +_ENGINE_COLOR: Final[dict[Engine, str]] = {"python": _PYTHON_ONLY_COLOR, "rust": _RUST_ONLY_COLOR} + + +@dataclass(frozen=True, slots=True) +class EngineReport: + engine: Engine + run: TraceRun + events: tuple[FunctionTraceEvent, ...] + steps: tuple[FunctionTraceEvent, ...] + issues: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class Comparison: + route: str + label: str + asynchronous: bool + engines: tuple[EngineReport, ...] + diff: TraceDiff + + @property + def comparable(self) -> bool: + return self.route != "audio_transcription" and all(isinstance(report.run, TraceOk) for report in self.engines) + + @property + def passed(self) -> bool: + return ( + (not self.comparable or self.diff.matches) + and not any(report.issues for report in self.engines) + and all(not isinstance(report.run, TraceFailed) for report in self.engines) + ) + + +def _events(run: TraceRun) -> tuple[FunctionTraceEvent, ...]: + match run: + case TraceOk(events=events): + return events + case TraceSkipped() | TraceFailed(): + return () + + +def _engine_report(route: str, engine: Engine, run: TraceRun) -> EngineReport: + events: Final = _events(run) + steps: Final = pipeline_steps(route, engine, events) + issues: Final = pipeline_issues(route, engine, steps) if isinstance(run, TraceOk) else () + return EngineReport(engine=engine, run=run, events=events, steps=steps, issues=issues) + + +def compare(route: str, *, asynchronous: bool) -> Comparison: + runs: Final = { + engine: attempt_trace(route, engine=engine, asynchronous=asynchronous) for engine in ("python", "rust") + } + engines: Final = tuple(_engine_report(route, engine, run) for engine, run in runs.items()) + return Comparison( + route=route, + label=ROUTE_SPECS[route].label, + asynchronous=asynchronous, + engines=engines, + diff=trace_diff(engines[0].steps, engines[1].steps), + ) + + +def _tree_line(event: FunctionTraceEvent, only: frozenset[str], marker: str, color: str, *, colorize: bool) -> str: + line: Final = f"{' ' * event.depth}{event.function}" + (f" {marker}" if event.function in only else "") + return f"{color}{line}{_RESET}\n" if colorize and event.function in only else f"{line}\n" + + +def _tree_lines( + events: tuple[FunctionTraceEvent, ...], + only: frozenset[str], + marker: str, + color: str, + *, + colorize: bool, +) -> tuple[str, ...]: + return tuple(_tree_line(event, only, marker, color, colorize=colorize) for event in events) + + +def _engine_lines( + report: EngineReport, diff: TraceDiff, *, comparable: bool, full: bool, colorize: bool +) -> tuple[str, ...]: + match report.run: + case TraceSkipped(reason=reason): + return (f"{report.engine}: SKIP ({reason})\n\n",) + case TraceFailed(reason=reason): + return (f"{report.engine}: FAIL ({reason})\n\n",) + case TraceOk(): + shown: Final = report.events if full else report.steps + only: Final = ( + () if full or not comparable else (diff.python_only if report.engine == "python" else diff.rust_only) + ) + return ( + f"{report.engine} ({len(shown)} steps)\n\n", + *_tree_lines( + shown, + frozenset(only), + f"<- {report.engine} only", + _ENGINE_COLOR[report.engine], + colorize=colorize, + ), + "\n", + ) + + +def _parity_lines(comparison: Comparison) -> tuple[str, ...]: + if not comparison.comparable: + if comparison.route == "audio_transcription": + return ("step parity: UNAVAILABLE (Bedrock transcription has no independent Python implementation)\n",) + return ("step parity: UNAVAILABLE (both engines must complete)\n",) + diff: Final = comparison.diff + order: Final = "the same" if diff.shared_order_matches else "a different" + return ( + "diff\n\n", + f"shared steps appear in {order} order\n", + f"python-only: {', '.join(diff.python_only) or 'none'}\n", + f"rust-only: {', '.join(diff.rust_only) or 'none'}\n\n", + f"step parity: {'PASS' if diff.matches else 'FAIL'}\n", + ) + + +def _stage_lines(comparison: Comparison) -> tuple[str, ...]: + return tuple( + f"{report.engine} " + f"{'SDK dispatch only' if comparison.route == 'audio_transcription' and report.engine == 'python' else 'pipeline'}: " + f"{'FAIL: ' + '; '.join(report.issues) if report.issues else 'PASS'}\n" + for report in comparison.engines + if isinstance(report.run, TraceOk) + ) + + +def render(comparison: Comparison, *, full: bool, colorize: bool) -> str: + mode: Final = "async" if comparison.asynchronous else "sync" + traces: Final = ( + (format_trace_table(comparison.engines[0].steps, comparison.engines[1].steps, colorize=colorize) + "\n\n",) + if not full and all(isinstance(report.run, TraceOk) for report in comparison.engines) + else tuple( + line + for report in comparison.engines + for line in _engine_lines( + report, comparison.diff, comparable=comparison.comparable, full=full, colorize=colorize + ) + ) + ) + return "".join( + ( + f"route: {comparison.route} provider: {comparison.label} mode: {mode}\n\n", + *traces, + *_parity_lines(comparison), + *_stage_lines(comparison), + "Each successful invocation issued exactly one local provider request\n\n", + ) + ) diff --git a/tests/sdk_function_trace/route-comparison.md b/tests/sdk_function_trace/route-comparison.md new file mode 100644 index 00000000000..009d3544d05 --- /dev/null +++ b/tests/sdk_function_trace/route-comparison.md @@ -0,0 +1,26 @@ +# SDK route trace audit + +Run the four native HTTP route families in both modes from the repository root: + +```bash +uv run python -m tests.sdk_function_trace.compare --route all --both --check +``` + +The local fixture matrix on 2026-09-02 completed 15 successful engine invocations and one expected skip. Every successful invocation issued exactly one local HTTP request. All five comparable route/mode pairs have identical canonical steps in the same order, with no Python-only or Rust-only steps + +| Route | Python async | Python sync | Rust async | Rust sync | +| --- | --- | --- | --- | --- | +| Chat completions, Anthropic | Pass | Pass | Pass | Pass | +| Messages, Anthropic | Pass | Unsupported, skipped | Pass | Pass | +| OCR, Mistral | Pass | Pass | Pass | Pass | +| Audio transcription, Bedrock | Dispatch only | Dispatch only | Pass | Pass | + +The same canonical step sequence ran in sync and async for each engine with both modes available. Bedrock transcription's Python SDK delegates to Rust, so its two successful calls do not establish independent provider parity. Realtime and Responses WebSockets are outside this HTTP fixture runner + +Chat and OCR also have identical projected nesting in both modes. Async Messages has the same helper nesting beneath its handler, but Python starts that handler on a worker thread, so it appears as a second root. The comparison preserves this physical thread boundary and checks step order independently of absolute depth + +Rust now resolves chat providers and supported parameters before entering its handler. Chat and Messages validate the environment and transform requests inside their handlers. Messages builds the final URL after transformation. OCR resolves its config and maps supported parameters during preparation, then validates credentials, builds the URL, and transforms the request inside its handler. Its during-call guardrails still run before HTTP, within the provider-call lifecycle phase + +The environment hooks execute credential and header validation. Chat's supported-parameter hooks return OpenAI names paired with provider names and feed the existing request acceptance checks. The direct Rust API still accepts provider-mapped parameters, and its supported subset is smaller than Python's. Matching the pipeline does not establish identical parameter contracts + +`--check` now fails if either comparable engine has missing, extra, or reordered canonical steps, even if its individual stage checks pass. Bedrock transcription and sync Messages report `UNAVAILABLE` for cross-language parity; native execution is still checked. Passing establishes step coverage and order for one non-streaming fixture per route, not complete request, response, error, or provider parity. The previously recorded OCR response gaps remain in `ocr-comparison.md` diff --git a/tests/sdk_function_trace/runtime.py b/tests/sdk_function_trace/runtime.py new file mode 100644 index 00000000000..d5bf15694bc --- /dev/null +++ b/tests/sdk_function_trace/runtime.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import asyncio +import os +from collections.abc import Awaitable, Generator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final, cast +from unittest.mock import patch + +from pydantic import BaseModel, ConfigDict + +from tests.sdk_function_trace.fixtures import Invocation, sdk_invocation +from tests.sdk_function_trace.mock_provider import mock_provider +from tests.sdk_function_trace.profiler import FunctionTraceEvent, profile_python +from tests.sdk_function_trace.steps import Engine + + +class TraceEventPayload(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + function: str + depth: int + + +class TraceResponsePayload(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + response: object + trace: tuple[TraceEventPayload, ...] | list[TraceEventPayload] + + +@contextmanager +def _python_engine() -> Generator[None]: + from litellm.rust_bridge import ocr as ocr_bridge + + previous_ocr: Final = ocr_bridge.rust_ocr_enabled() + with patch.dict(os.environ, {"LITELLM_RUST": "false"}): + ocr_bridge.use_litellm_rust(False) + try: + yield + finally: + ocr_bridge.use_litellm_rust(previous_ocr) + + +def _invoke(case: Invocation, api_base: str, *, asynchronous: bool) -> object: + async def invoke_async() -> object: + return await cast("Awaitable[object]", case.function(**case.kwargs, api_base=api_base)) + + if asynchronous: + return asyncio.run(invoke_async()) + return case.function(**case.kwargs, api_base=api_base) + + +def collect(case: Invocation, api_base: str, *, engine: Engine, asynchronous: bool) -> tuple[FunctionTraceEvent, ...]: + import litellm + + if engine == "rust": + payload: Final = TraceResponsePayload.model_validate(_invoke(case, api_base, asynchronous=asynchronous)) + return tuple(FunctionTraceEvent(event.function, event.depth) for event in payload.trace) + with profile_python(source_root=Path(litellm.__file__).parent, threads=True) as profiler: + _invoke(case, api_base, asynchronous=asynchronous) + return tuple(profiler.events) + + +def run_trace(route: str, *, engine: Engine, asynchronous: bool = False) -> tuple[FunctionTraceEvent, ...]: + case: Final = sdk_invocation(route, engine=engine, asynchronous=asynchronous) + with _python_engine(), mock_provider(case.provider_response) as api_base: + events: Final = collect(case, api_base, engine=engine, asynchronous=asynchronous) + if not events: + raise RuntimeError(f"No runtime events for {route}; rebuild the native extension with tracing support") + return events + + +@dataclass(frozen=True, slots=True) +class TraceOk: + events: tuple[FunctionTraceEvent, ...] + + +@dataclass(frozen=True, slots=True) +class TraceSkipped: + reason: str + + +@dataclass(frozen=True, slots=True) +class TraceFailed: + reason: str + + +TraceRun = TraceOk | TraceSkipped | TraceFailed + + +def attempt_trace(route: str, *, engine: Engine, asynchronous: bool) -> TraceRun: + try: + return TraceOk(run_trace(route, engine=engine, asynchronous=asynchronous)) + except Exception as error: + reason: Final = f"{type(error).__name__}: {error}" + if ( + route == "messages" + and engine == "python" + and not asynchronous + and isinstance(error, ValueError) + and str(error) == "anthropic_messages_handler is not implemented for sync calls" + ): + return TraceSkipped(reason) + return TraceFailed(reason) + + +@dataclass(frozen=True, slots=True) +class TraceDiff: + python_only: tuple[str, ...] + rust_only: tuple[str, ...] + shared_order_matches: bool + + @property + def matches(self) -> bool: + return not self.python_only and not self.rust_only and self.shared_order_matches + + +def trace_diff(python: tuple[FunctionTraceEvent, ...], rust: tuple[FunctionTraceEvent, ...]) -> TraceDiff: + python_names: Final = {event.function for event in python} + rust_names: Final = {event.function for event in rust} + shared_python: Final = tuple(event.function for event in python if event.function in rust_names) + shared_rust: Final = tuple(event.function for event in rust if event.function in python_names) + return TraceDiff( + python_only=tuple(event.function for event in python if event.function not in rust_names), + rust_only=tuple(event.function for event in rust if event.function not in python_names), + shared_order_matches=bool(shared_python) and shared_python == shared_rust, + ) diff --git a/tests/sdk_function_trace/steps.py b/tests/sdk_function_trace/steps.py new file mode 100644 index 00000000000..bb50d4ebe57 --- /dev/null +++ b/tests/sdk_function_trace/steps.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import re +from collections.abc import Sequence +from dataclasses import dataclass +from functools import reduce +from typing import Final, Literal + +from tests.sdk_function_trace.profiler import FunctionTraceEvent + +Engine = Literal["python", "rust"] + + +@dataclass(frozen=True, slots=True) +class Step: + name: str + python: re.Pattern[str] | None + rust: str | None + + +def _step(name: str, python: str | None = None, rust: str | None = None) -> Step: + return Step(name, re.compile(python) if python is not None else None, rust) + + +_POST: Final = r"AsyncHTTPHandler\.post$|HTTPHandler\.post$" + +STEPS: Final[dict[str, tuple[Step, ...]]] = { + "ocr": ( + _step("ocr", r"ocr/main\.py:\d+ a?ocr$", "ocr"), + _step("prepare_ocr_call", r"ocr/main\.py:\d+ _prepare_ocr_request$", "prepare_ocr_call"), + _step("get_provider_ocr_config", r"ProviderConfigManager\.get_provider_ocr_config$", "ocr_provider_config"), + _step("supported_ocr_params", r"get_supported_ocr_params$", "supported_ocr_params"), + _step("map_ocr_params", r"(? tuple[str, ...]: + names: Final = tuple(event.function for event in events) + required: Final = tuple(step.name for step in STEPS[route] if getattr(step, engine) is not None) + missing: Final = tuple(f"missing {name}" for name in required if name not in names) + provider: Final = next(name for name in required if name.startswith("get_provider_")) + handler: Final = next(name for name in required if name.startswith("execute_")) + dispatch_only: Final = route == "audio_transcription" and engine == "python" + request: Final = next( + (name for name in required if name.startswith("transform_") and name.endswith("request")), handler + ) + response: Final = next( + (name for name in required if name.startswith("transform_") and name.endswith("response")), handler + ) + phases: Final = ( + (route, "map_transcription_params", provider, handler) + if dispatch_only + else (route, provider, request, "http_request", response) + ) + extra_edges: Final = ( + () + if dispatch_only + else ( + (handler, "http_request"), + *((name, request) for name in required if name.startswith(("map_", "supported_"))), + *((name, "http_request") for name in ("validate_environment", "complete_url") if name in required), + ) + ) + edges: Final = (*zip(phases, phases[1:]), *extra_edges) + return missing + tuple( + f"{before} must precede {after}" + for before, after in edges + if before in names and after in names and names.index(before) >= names.index(after) + ) + + +def _canonical_name(route: str, engine: Engine, function: str) -> str | None: + for step in STEPS[route]: + if engine == "python": + if step.python is not None and step.python.search(function): + return step.name + elif step.rust is not None and function == step.rust: + return step.name + return function if engine == "rust" else None + + +@dataclass(frozen=True, slots=True) +class _Projection: + shown: tuple[FunctionTraceEvent, ...] = () + stack: tuple[tuple[int, int], ...] = () + seen: frozenset[str] = frozenset() + + +def _project(route: str, engine: Engine, state: _Projection, event: FunctionTraceEvent) -> _Projection: + stack: Final = tuple(pair for pair in state.stack if event.depth > pair[0]) + name: Final = _canonical_name(route, engine, event.function) + if name is None or name in state.seen: + return _Projection(state.shown, stack, state.seen) + depth: Final = ( + next( + ( + kept.depth + 1 + for ancestor in event.ancestors + for kept in state.shown + if kept.function == _canonical_name(route, engine, ancestor) + ), + 0, + ) + if event.ancestors is not None + else stack[-1][1] + 1 + if stack + else 0 + ) + return _Projection( + state.shown + (FunctionTraceEvent(function=name, depth=depth),), + stack + ((event.depth, depth),), + state.seen | {name}, + ) + + +def pipeline_steps(route: str, engine: Engine, events: Sequence[FunctionTraceEvent]) -> tuple[FunctionTraceEvent, ...]: + projection: Final = reduce(lambda state, event: _project(route, engine, state, event), events, _Projection()) + return projection.shown diff --git a/tests/sdk_function_trace/table.py b/tests/sdk_function_trace/table.py new file mode 100644 index 00000000000..2124d7e3faf --- /dev/null +++ b/tests/sdk_function_trace/table.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from collections.abc import Iterator +from difflib import SequenceMatcher +from typing import Final + +from tests.sdk_function_trace.profiler import FunctionTraceEvent + + +def _aligned_rows( + python: tuple[FunctionTraceEvent, ...], rust: tuple[FunctionTraceEvent, ...] +) -> Iterator[tuple[FunctionTraceEvent | None, FunctionTraceEvent | None]]: + matcher: Final = SequenceMatcher( + a=tuple(event.function for event in python), + b=tuple(event.function for event in rust), + autojunk=False, + ) + for tag, python_start, python_end, rust_start, rust_end in matcher.get_opcodes(): + if tag == "equal": + yield from zip(python[python_start:python_end], rust[rust_start:rust_end]) + else: + yield from ((event, None) for event in python[python_start:python_end]) + yield from ((None, event) for event in rust[rust_start:rust_end]) + + +def _label(event: FunctionTraceEvent | None) -> str: + return f"{' ' * event.depth}{event.function}" if event is not None else "" + + +def _status( + python: FunctionTraceEvent | None, + rust: FunctionTraceEvent | None, + python_names: frozenset[str], + rust_names: frozenset[str], +) -> tuple[str, str]: + if python is not None and rust is not None: + return "match", "\033[32m" + if python is not None: + return ("reordered", "\033[31m") if python.function in rust_names else ("python only", "\033[34m") + if rust is not None: + return ("reordered", "\033[31m") if rust.function in python_names else ("rust only", "\033[33m") + return "", "" + + +def format_trace_table( + python: tuple[FunctionTraceEvent, ...], + rust: tuple[FunctionTraceEvent, ...], + *, + colorize: bool, +) -> str: + python_header: Final = f"python ({len(python)} steps)" + rust_header: Final = f"rust ({len(rust)} steps)" + python_width: Final = max(len(python_header), *(len(_label(event)) for event in python), 0) + rust_width: Final = max(len(rust_header), *(len(_label(event)) for event in rust), 0) + python_names: Final = frozenset(event.function for event in python) + rust_names: Final = frozenset(event.function for event in rust) + border: Final = f"+-{'-' * python_width}-+-{'-' * rust_width}-+-------------+" + rows: Final = tuple( + f"{color}{line}\033[0m" if colorize else line + for left, right in _aligned_rows(python, rust) + for status, color in (_status(left, right, python_names, rust_names),) + for line in (f"| {_label(left):<{python_width}} | {_label(right):<{rust_width}} | {status:<11} |",) + ) + return "\n".join( + ( + border, + f"| {python_header:<{python_width}} | {rust_header:<{rust_width}} | {'comparison':<11} |", + border, + *rows, + border, + ) + ) diff --git a/tests/sdk_function_trace/test_mock_provider.py b/tests/sdk_function_trace/test_mock_provider.py new file mode 100644 index 00000000000..88d7d5392d0 --- /dev/null +++ b/tests/sdk_function_trace/test_mock_provider.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from contextlib import ExitStack +from typing import Final +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +import pytest + +from tests.sdk_function_trace.mock_provider import MockProviderResponse, mock_provider + + +def test_mock_provider_preserves_error_response() -> None: + response: Final = MockProviderResponse(429, (("retry-after", "2"),), b'{"error":"rate limited"}') + with mock_provider(response) as api_base: + with pytest.raises(HTTPError) as error: + urlopen(Request(api_base, data=b"{}"), timeout=5) + with error.value as received: + assert received.code == 429 + assert received.headers["retry-after"] == "2" + assert received.read() == response.body + + +@pytest.mark.parametrize("request_count", [0, 2]) +def test_mock_provider_rejects_missing_or_duplicate_requests(request_count: int) -> None: + response: Final = MockProviderResponse(200, (), b"{}") + with ExitStack() as stack: + api_base: Final = stack.enter_context(mock_provider(response)) + for _ in range(request_count): + with urlopen(Request(api_base, data=b"{}"), timeout=5) as received: + assert received.read() == response.body + with pytest.raises(AssertionError, match=f"expected one provider request, received {request_count}"): + stack.close() diff --git a/tests/sdk_function_trace/test_profiler.py b/tests/sdk_function_trace/test_profiler.py new file mode 100644 index 00000000000..10a266fb1e8 --- /dev/null +++ b/tests/sdk_function_trace/test_profiler.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from types import FunctionType +from typing import Final, cast + +import pytest + +from tests.sdk_function_trace import ( + FunctionTraceEvent, + TraceScenario, + TraceStep, + assert_function_trace_parity, +) +from tests.sdk_function_trace.profiler import profile_python + + +class First: + @staticmethod + def run() -> None: + return None + + +class Second: + @staticmethod + def run() -> None: + return None + + +def test_profiler_matches_code_objects_and_keeps_repeated_calls() -> None: + with profile_python((First.run,)) as profiler: + Second.run() + First.run() + First.run() + + assert profiler.events == [ + FunctionTraceEvent(function="run", depth=0), + FunctionTraceEvent(function="run", depth=0), + ] + + +def test_profiler_records_selected_function_nesting_depth() -> None: + class Nested: + @staticmethod + def run() -> None: + First.run() + + with profile_python((Nested.run, First.run)) as profiler: + Nested.run() + + assert profiler.events == [ + FunctionTraceEvent(function="run", depth=0), + FunctionTraceEvent(function="run", depth=1), + ] + + +def test_profiler_restores_previous_profiler_after_failure() -> None: + previous: Final = sys.getprofile() + + with profile_python((First.run,)) as outer: + with pytest.raises(RuntimeError, match="stop"): + with profile_python((Second.run,)): + raise RuntimeError("stop") + assert sys.getprofile() is outer + First.run() + + assert sys.getprofile() is previous + assert outer.events == [FunctionTraceEvent(function="run", depth=0)] + + +def test_profiler_does_not_count_coroutine_resumption_as_another_call() -> None: + async def suspended() -> None: + await asyncio.sleep(0) + First.run() + await asyncio.sleep(0) + + with profile_python((suspended, First.run)) as profiler: + asyncio.run(suspended()) + + assert profiler.events == [ + FunctionTraceEvent(function="suspended", depth=0), + FunctionTraceEvent(function="run", depth=1), + ] + + +def test_source_profiler_records_real_frame_ancestry() -> None: + def outer() -> None: + First.run() + + with profile_python(source_root=Path(__file__).parent) as profiler: + outer() + Second.run() + + outer_event, first_event, second_event = ( + event for event in profiler.events if event.function.startswith("test_profiler.py:") + ) + assert first_event.ancestors is not None + assert outer_event.function in first_event.ancestors + assert second_event.ancestors is not None + assert outer_event.function not in second_event.ancestors + + +@pytest.mark.parametrize( + "rust_trace", + [ + (), + (FunctionTraceEvent(function="renamed", depth=0),), + (FunctionTraceEvent(function="run", depth=1),), + (FunctionTraceEvent(function="run", depth=0),) * 2, + ], + ids=["missing", "renamed", "wrong-depth", "extra-call"], +) +def test_harness_rejects_rust_function_trace_drift(rust_trace: tuple[FunctionTraceEvent, ...]) -> None: + with pytest.raises(AssertionError, match="Rust function trace differs"): + assert_function_trace_parity( + TraceScenario( + steps=(TraceStep(cast(FunctionType, First.run), depth=0),), + invoke_python=First.run, + invoke_rust=lambda: rust_trace, + ) + ) + + +def test_harness_rejects_python_function_trace_drift() -> None: + with pytest.raises(AssertionError, match="Python function trace differs"): + assert_function_trace_parity( + TraceScenario( + steps=(TraceStep(cast(FunctionType, First.run), depth=0),), + invoke_python=Second.run, + invoke_rust=lambda: (FunctionTraceEvent(function="run", depth=0),), + ) + ) + + +def test_harness_accepts_matching_traces() -> None: + assert_function_trace_parity( + TraceScenario( + steps=(TraceStep(cast(FunctionType, First.run), depth=0),), + invoke_python=First.run, + invoke_rust=lambda: (FunctionTraceEvent(function="run", depth=0),), + ) + ) + + +def test_harness_rejects_reordered_calls() -> None: + def begin() -> None: + return None + + def finish() -> None: + return None + + with pytest.raises(AssertionError, match="Rust function trace differs"): + assert_function_trace_parity( + TraceScenario( + steps=( + TraceStep(cast(FunctionType, begin), depth=0), + TraceStep(cast(FunctionType, finish), depth=0), + ), + invoke_python=lambda: (begin(), finish()), + invoke_rust=lambda: ( + FunctionTraceEvent(function="finish", depth=0), + FunctionTraceEvent(function="begin", depth=0), + ), + ) + ) diff --git a/tests/sdk_function_trace/test_runtime.py b/tests/sdk_function_trace/test_runtime.py new file mode 100644 index 00000000000..015cba55083 --- /dev/null +++ b/tests/sdk_function_trace/test_runtime.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import Final + +import pytest + +from tests.sdk_function_trace.runtime import ( + TraceFailed, + TraceSkipped, + attempt_trace, + run_trace, + trace_diff, +) +from tests.sdk_function_trace.steps import pipeline_issues, pipeline_steps + + +def test_sync_messages_records_the_known_python_limitation() -> None: + result: Final = attempt_trace("messages", engine="python", asynchronous=False) + + assert isinstance(result, TraceSkipped) + assert result.reason == "ValueError: anthropic_messages_handler is not implemented for sync calls" + + +def test_unexpected_call_failure_is_not_skipped() -> None: + result: Final = attempt_trace("unknown", engine="python", asynchronous=False) + + assert isinstance(result, TraceFailed) + assert result.reason == "ValueError: Unknown route: unknown" + + +@pytest.mark.parametrize( + ("route", "asynchronous"), + (("chat_completions", False), ("chat_completions", True), ("messages", True), ("ocr", False), ("ocr", True)), +) +def test_compiled_routes_match_python_steps(route: str, asynchronous: bool) -> None: + from litellm.rust_bridge import get_native_bridge + + if get_native_bridge() is None: + pytest.skip("build the native bridge to run executed route parity") + python: Final = pipeline_steps(route, "python", run_trace(route, engine="python", asynchronous=asynchronous)) + rust: Final = pipeline_steps(route, "rust", run_trace(route, engine="rust", asynchronous=asynchronous)) + + assert pipeline_issues(route, "python", python) == () + assert pipeline_issues(route, "rust", rust) == () + assert trace_diff(python, rust).matches + if route != "messages": + assert python == rust diff --git a/tests/sdk_function_trace/test_steps.py b/tests/sdk_function_trace/test_steps.py new file mode 100644 index 00000000000..b5432951187 --- /dev/null +++ b/tests/sdk_function_trace/test_steps.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +from typing import Final + +import pytest + +from tests.sdk_function_trace.profiler import FunctionTraceEvent +from tests.sdk_function_trace.runtime import trace_diff +from tests.sdk_function_trace.steps import pipeline_issues, pipeline_steps + + +def test_python_ocr_projection_keeps_pipeline_and_drops_noise() -> None: + events: Final = ( + FunctionTraceEvent("utils.py:1747 client..wrapper_async", 0), + FunctionTraceEvent("ocr/main.py:331 aocr", 1), + FunctionTraceEvent("ocr/main.py:70 _prepare_ocr_request", 2), + FunctionTraceEvent("litellm_core_utils/get_llm_provider_logic.py:142 get_llm_provider", 3), + FunctionTraceEvent("utils.py:9303 ProviderConfigManager.get_provider_ocr_config", 3), + FunctionTraceEvent("llms/mistral/ocr/transformation.py:34 MistralOCRConfig.get_supported_ocr_params", 4), + FunctionTraceEvent("llms/mistral/ocr/transformation.py:72 MistralOCRConfig.map_ocr_params", 4), + FunctionTraceEvent("llms/mistral/ocr/transformation.py:34 MistralOCRConfig.get_supported_ocr_params", 5), + FunctionTraceEvent("llms/custom_httpx/llm_http_handler.py:1705 BaseLLMHTTPHandler.async_ocr", 2), + FunctionTraceEvent("llms/mistral/ocr/transformation.py:94 MistralOCRConfig.validate_environment", 4), + FunctionTraceEvent("llms/mistral/ocr/transformation.py:124 MistralOCRConfig.get_complete_url", 4), + FunctionTraceEvent("llms/base_llm/ocr/transformation.py:209 BaseOCRConfig.async_transform_ocr_request", 5), + FunctionTraceEvent("llms/mistral/ocr/transformation.py:149 MistralOCRConfig.transform_ocr_request", 6), + FunctionTraceEvent("llms/custom_httpx/http_handler.py:654 AsyncHTTPHandler.post", 6), + FunctionTraceEvent("llms/base_llm/ocr/transformation.py:255 BaseOCRConfig.async_transform_ocr_response", 4), + FunctionTraceEvent("llms/mistral/ocr/transformation.py:200 MistralOCRConfig.transform_ocr_response", 5), + FunctionTraceEvent("cost_calculator.py:1874 ocr_cost", 6), + ) + + assert pipeline_steps("ocr", "python", events) == ( + FunctionTraceEvent("ocr", 0), + FunctionTraceEvent("prepare_ocr_call", 1), + FunctionTraceEvent("get_provider_ocr_config", 2), + FunctionTraceEvent("supported_ocr_params", 3), + FunctionTraceEvent("map_ocr_params", 3), + FunctionTraceEvent("execute_ocr_provider_call", 1), + FunctionTraceEvent("validate_environment", 2), + FunctionTraceEvent("complete_url", 2), + FunctionTraceEvent("transform_ocr_request", 3), + FunctionTraceEvent("http_request", 3), + FunctionTraceEvent("transform_ocr_response", 2), + ) + + +def test_rust_ocr_projection_reuses_step_names_and_keeps_unknown_spans() -> None: + events: Final = ( + FunctionTraceEvent("ocr", 0), + FunctionTraceEvent("prepare_ocr_call", 1), + FunctionTraceEvent("map_ocr_params", 2), + FunctionTraceEvent("supported_ocr_params", 3), + FunctionTraceEvent("map_ocr_params", 2), + FunctionTraceEvent("transform_ocr_request", 2), + FunctionTraceEvent("execute_ocr_provider_call", 2), + FunctionTraceEvent("transform_ocr_response", 3), + FunctionTraceEvent("new_uninstrumented_span", 3), + ) + + assert pipeline_steps("ocr", "rust", events) == ( + FunctionTraceEvent("ocr", 0), + FunctionTraceEvent("prepare_ocr_call", 1), + FunctionTraceEvent("map_ocr_params", 2), + FunctionTraceEvent("supported_ocr_params", 3), + FunctionTraceEvent("transform_ocr_request", 2), + FunctionTraceEvent("execute_ocr_provider_call", 2), + FunctionTraceEvent("transform_ocr_response", 3), + FunctionTraceEvent("new_uninstrumented_span", 3), + ) + + +def test_projection_resets_depth_on_thread_root() -> None: + events: Final = ( + FunctionTraceEvent("main.py:387 acompletion", 1), + FunctionTraceEvent("llms/anthropic/chat/handler.py:255 AnthropicChatCompletion.acompletion_function", 2), + FunctionTraceEvent( + "llms/anthropic/experimental_pass_through/messages/handler.py:416 anthropic_messages_handler", 0 + ), + FunctionTraceEvent( + "llms/anthropic/experimental_pass_through/messages/transformation.py:575" + " AnthropicMessagesConfig.transform_anthropic_messages_request", + 4, + ), + ) + + assert pipeline_steps("chat_completions", "python", events) == ( + FunctionTraceEvent("chat_completions", 0), + FunctionTraceEvent("execute_chat_completions_provider_call", 1), + ) + assert pipeline_steps("messages", "python", events) == ( + FunctionTraceEvent("execute_messages_provider_call", 0), + FunctionTraceEvent("transform_request", 1), + ) + + +@pytest.mark.parametrize("function", ("completion", "completion_function", "acompletion_function")) +def test_chat_projection_includes_sync_and_async_handlers(function: str) -> None: + events: Final = (FunctionTraceEvent(f"llms/anthropic/chat/handler.py:100 AnthropicChatCompletion.{function}", 0),) + + assert pipeline_steps("chat_completions", "python", events) == ( + FunctionTraceEvent("execute_chat_completions_provider_call", 0), + ) + + +def test_trace_diff_reports_no_difference_for_identical_steps() -> None: + steps: Final = ( + FunctionTraceEvent("ocr", 0), + FunctionTraceEvent("transform_ocr_request", 1), + ) + + diff: Final = trace_diff(steps, steps) + + assert diff.python_only == () + assert diff.rust_only == () + assert diff.shared_order_matches + assert diff.matches + + +def test_trace_diff_reports_exclusive_steps_and_reordered_shared_steps() -> None: + python: Final = ( + FunctionTraceEvent("ocr", 0), + FunctionTraceEvent("supported_ocr_params", 1), + FunctionTraceEvent("map_ocr_params", 1), + FunctionTraceEvent("http_request", 2), + ) + rust: Final = ( + FunctionTraceEvent("ocr", 0), + FunctionTraceEvent("map_ocr_params", 1), + FunctionTraceEvent("supported_ocr_params", 2), + FunctionTraceEvent("transform_ocr_response", 2), + ) + + diff: Final = trace_diff(python, rust) + + assert diff.python_only == ("http_request",) + assert diff.rust_only == ("transform_ocr_response",) + assert not diff.shared_order_matches + assert not diff.matches + + +def test_trace_diff_does_not_claim_empty_or_disjoint_traces_match() -> None: + assert not trace_diff((), ()).shared_order_matches + assert not trace_diff((FunctionTraceEvent("ocr", 0),), (FunctionTraceEvent("messages", 0),)).shared_order_matches + + +def test_projection_uses_actual_ancestors_after_coroutine_resumption() -> None: + entrypoint: Final = "main.py:387 acompletion" + handler: Final = "llms/anthropic/chat/handler.py:255 AnthropicChatCompletion.acompletion_function" + events: Final = ( + FunctionTraceEvent(entrypoint, 0, ()), + FunctionTraceEvent(handler, 1, (entrypoint,)), + FunctionTraceEvent("utils.py:100 unrelated_worker", 0, ()), + FunctionTraceEvent("llms/anthropic/chat/transformation.py:100 transform_response", 1, (handler,)), + ) + + assert pipeline_steps("chat_completions", "python", events) == ( + FunctionTraceEvent("chat_completions", 0), + FunctionTraceEvent("execute_chat_completions_provider_call", 1), + FunctionTraceEvent("transform_response", 2), + ) + + +def test_projection_does_not_nest_siblings_under_a_returned_config_lookup() -> None: + events: Final = ( + FunctionTraceEvent("main.py:387 completion", 0), + FunctionTraceEvent("utils.py:100 ProviderConfigManager.get_provider_chat_config", 1), + FunctionTraceEvent("utils.py:200 unrelated_helper", 1), + FunctionTraceEvent("llms/anthropic/chat/transformation.py:100 transform_request", 2), + ) + + assert pipeline_steps("chat_completions", "python", events) == ( + FunctionTraceEvent("chat_completions", 0), + FunctionTraceEvent("get_provider_chat_config", 1), + FunctionTraceEvent("transform_request", 1), + ) + + +CHAT_RUST_STEPS: Final = ( + "chat_completions", + "get_provider_chat_config", + "supported_openai_params", + "execute_chat_completions_provider_call", + "validate_environment", + "transform_request", + "http_request", + "transform_response", +) + + +@pytest.mark.parametrize("missing", CHAT_RUST_STEPS) +def test_pipeline_check_rejects_missing_stages(missing: str) -> None: + steps: Final = tuple(FunctionTraceEvent(name, 0) for name in CHAT_RUST_STEPS if name != missing) + + assert f"missing {missing}" in pipeline_issues("chat_completions", "rust", steps) + + +def test_pipeline_check_rejects_http_before_request_transformation() -> None: + steps: Final = tuple( + FunctionTraceEvent(name, 0) + for name in ( + "chat_completions", + "get_provider_chat_config", + "supported_openai_params", + "execute_chat_completions_provider_call", + "validate_environment", + "http_request", + "transform_request", + "transform_response", + ) + ) + + assert "transform_request must precede http_request" in pipeline_issues("chat_completions", "rust", steps) + + +def test_step_parity_rejects_different_handler_boundaries_even_with_valid_stages() -> None: + rust: Final = tuple(FunctionTraceEvent(name, 0) for name in CHAT_RUST_STEPS) + python: Final = tuple( + FunctionTraceEvent(name, 0) + for name in ( + "chat_completions", + "get_provider_chat_config", + "supported_openai_params", + "validate_environment", + "transform_request", + "execute_chat_completions_provider_call", + "http_request", + "transform_response", + ) + ) + + assert not trace_diff(python, rust).shared_order_matches + assert not trace_diff(python, rust).matches + assert pipeline_issues("chat_completions", "python", python) == () + assert pipeline_issues("chat_completions", "rust", rust) == () + + +def test_step_parity_rejects_an_exclusive_helper_with_matching_shared_order() -> None: + rust: Final = tuple(FunctionTraceEvent(name, 0) for name in CHAT_RUST_STEPS) + python: Final = (*rust, FunctionTraceEvent("unmatched_helper", 0)) + diff: Final = trace_diff(python, rust) + + assert diff.shared_order_matches + assert not diff.matches diff --git a/tests/sdk_function_trace/test_table.py b/tests/sdk_function_trace/test_table.py new file mode 100644 index 00000000000..c2341a391a9 --- /dev/null +++ b/tests/sdk_function_trace/test_table.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import re +from typing import Final + +from tests.sdk_function_trace.profiler import FunctionTraceEvent +from tests.sdk_function_trace.table import format_trace_table + + +def test_table_aligns_matches_after_missing_steps_and_preserves_indentation() -> None: + python: Final = ( + FunctionTraceEvent("ocr", 0), + FunctionTraceEvent("python_helper", 1), + FunctionTraceEvent("http_request", 2), + ) + rust: Final = ( + FunctionTraceEvent("ocr", 0), + FunctionTraceEvent("rust_helper", 1), + FunctionTraceEvent("http_request", 1), + ) + output: Final = format_trace_table(python, rust, colorize=False) + rows: Final = tuple(line.split("|")[1:-1] for line in output.splitlines() if line.startswith("|")) + + assert tuple(tuple(cell.strip() for cell in row) for row in rows) == ( + ("python (3 steps)", "rust (3 steps)", "comparison"), + ("ocr", "ocr", "match"), + ("python_helper", "", "python only"), + ("", "rust_helper", "rust only"), + ("http_request", "http_request", "match"), + ) + assert rows[-1][0].startswith(" http_request") + assert rows[-1][1].startswith(" http_request") + assert len({len(line) for line in output.splitlines()}) == 1 + assert "\033[" not in output + + +def test_table_marks_reordered_calls_and_keeps_both_execution_orders() -> None: + python: Final = tuple(FunctionTraceEvent(name, 0) for name in ("ocr", "map", "validate", "http")) + rust: Final = tuple(FunctionTraceEvent(name, 0) for name in ("ocr", "validate", "map", "http")) + output: Final = format_trace_table(python, rust, colorize=True) + plain: Final = re.sub(r"\033\[[0-9;]*m", "", output) + rows: Final = tuple(line.split("|")[1:-1] for line in plain.splitlines() if line.startswith("|"))[1:] + + assert tuple(row[0].strip() for row in rows if row[0].strip()) == tuple(event.function for event in python) + assert tuple(row[1].strip() for row in rows if row[1].strip()) == tuple(event.function for event in rust) + assert plain.count("reordered") == 2 + assert output.count("\033[31m") == 2 + assert "only" not in output + + +def test_table_colors_match_and_exclusive_rows_without_changing_alignment() -> None: + python: Final = (FunctionTraceEvent("ocr", 0), FunctionTraceEvent("python_helper", 1)) + rust: Final = (FunctionTraceEvent("ocr", 0), FunctionTraceEvent("rust_helper", 1)) + colored: Final = format_trace_table(python, rust, colorize=True) + + assert re.sub(r"\033\[[0-9;]*m", "", colored) == format_trace_table(python, rust, colorize=False) + assert next(line for line in colored.splitlines() if "match" in line).startswith("\033[32m") + assert next(line for line in colored.splitlines() if "python only" in line).startswith("\033[34m") + assert next(line for line in colored.splitlines() if "rust only" in line).startswith("\033[33m") + + +def test_table_handles_empty_traces() -> None: + output: Final = format_trace_table((), (), colorize=False) + + assert "python (0 steps)" in output + assert "rust (0 steps)" in output + assert "match" not in output diff --git a/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py index 5503a5668bf..a8fe464ec32 100644 --- a/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py +++ b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py @@ -11,7 +11,9 @@ Verifies that: import json +import httpx import pytest +import respx from unittest.mock import AsyncMock, MagicMock, patch @@ -295,6 +297,195 @@ class TestTransformation: assert headers["Authorization"].startswith("AWS4-HMAC-SHA256") +SESSION_HEADER = "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id" +CONTEXT_ID = "conversation-alpha-0001-0000000000000000" +KEY_HASH = "hashed-key-of-caller-one" + + +def _params_with_context(context_id: object) -> dict: + return {"message": {**SAMPLE_PARAMS["message"], "contextId": context_id}} + + +def _scoped(context_id: str, key_hash: str) -> str: + import hashlib + + return f"{hashlib.sha256(key_hash.encode()).hexdigest()[:16]}-{context_id}" + + +def _session_header(params: dict, litellm_params: dict) -> str: + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=params, + litellm_params=litellm_params, + ) + return headers[SESSION_HEADER] + + +@pytest.fixture +def httpx_transport(monkeypatch): + import litellm + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +class TestRequestScopedRuntimeSession: + """message.contextId selects the AgentCore runtime session, scoped to the calling key.""" + + def test_context_id_scoped_to_calling_key(self): + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, + ) + + litellm_params = {**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: KEY_HASH} + assert _session_header(_params_with_context(CONTEXT_ID), litellm_params) == _scoped(CONTEXT_ID, KEY_HASH) + + def test_context_id_used_verbatim_without_principal(self): + assert _session_header(_params_with_context(CONTEXT_ID), SAMPLE_LITELLM_PARAMS) == CONTEXT_ID + + def test_same_context_id_reuses_session_and_other_context_isolated(self): + first = _session_header(_params_with_context(CONTEXT_ID), SAMPLE_LITELLM_PARAMS) + second = _session_header(_params_with_context(CONTEXT_ID), SAMPLE_LITELLM_PARAMS) + other = _session_header( + _params_with_context("conversation-beta-00002-0000000000000000"), + SAMPLE_LITELLM_PARAMS, + ) + assert first == second + assert other != first + + def test_same_context_id_from_different_keys_is_isolated(self): + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, + ) + + params = _params_with_context(CONTEXT_ID) + caller_one = _session_header(params, {**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: KEY_HASH}) + caller_two = _session_header( + params, {**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: "hashed-key-of-caller-two"} + ) + assert caller_one != caller_two + assert caller_one.endswith(f"-{CONTEXT_ID}") + assert caller_two.endswith(f"-{CONTEXT_ID}") + + def test_context_id_takes_precedence_over_configured_session(self): + litellm_params = {**SAMPLE_LITELLM_PARAMS, "runtimeSessionId": "a" * 40} + assert _session_header(_params_with_context(CONTEXT_ID), litellm_params) == CONTEXT_ID + + def test_configured_session_is_fallback_without_context_id(self): + litellm_params = {**SAMPLE_LITELLM_PARAMS, "runtimeSessionId": "a" * 40} + assert _session_header(SAMPLE_PARAMS, litellm_params) == "a" * 40 + assert _session_header(_params_with_context(""), litellm_params) == "a" * 40 + + def test_no_context_id_and_no_config_generates_new_session_per_request(self): + first = _session_header(SAMPLE_PARAMS, SAMPLE_LITELLM_PARAMS) + second = _session_header(SAMPLE_PARAMS, SAMPLE_LITELLM_PARAMS) + assert first != second + assert 33 <= len(first) <= 256 + + @pytest.mark.parametrize( + "context_id", + [ + "short-context-id", + "x" * 257, + ], + ) + def test_invalid_context_id_rejected_with_clear_error(self, context_id): + import litellm + + with pytest.raises(litellm.BadRequestError, match="Invalid AgentCore runtime session id") as exc_info: + _session_header(_params_with_context(context_id), SAMPLE_LITELLM_PARAMS) + assert exc_info.value.status_code == 400 + assert "33-256" in str(exc_info.value) + + def test_scoped_context_id_shorter_than_33_rejected(self): + import litellm + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, + ) + + litellm_params = {**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: KEY_HASH} + with pytest.raises(litellm.BadRequestError, match=_scoped("c" * 15, KEY_HASH)): + _session_header(_params_with_context("c" * 15), litellm_params) + assert _session_header(_params_with_context("c" * 16), litellm_params) == _scoped("c" * 16, KEY_HASH) + + def test_invalid_configured_session_rejected(self): + import litellm + + litellm_params = {**SAMPLE_LITELLM_PARAMS, "runtimeSessionId": "too-short"} + with pytest.raises(litellm.BadRequestError, match="Invalid AgentCore runtime session id"): + _session_header(SAMPLE_PARAMS, litellm_params) + + def test_non_string_context_id_falls_back(self): + litellm_params = {**SAMPLE_LITELLM_PARAMS, "runtimeSessionId": "a" * 40} + assert _session_header(_params_with_context(12345), litellm_params) == "a" * 40 + + def test_spoofed_session_header_does_not_override_context_id(self): + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=_params_with_context(CONTEXT_ID), + litellm_params=SAMPLE_LITELLM_PARAMS, + agent_extra_headers={SESSION_HEADER: "s" * 40}, + ) + assert headers[SESSION_HEADER] == CONTEXT_ID + + @pytest.mark.asyncio + async def test_context_id_session_header_on_outbound_non_streaming_post(self, httpx_transport): + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, + ) + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + with respx.mock(assert_all_called=True) as router: + route = router.post(url__regex=r".*/invocations.*").mock( + return_value=httpx.Response(200, json={"jsonrpc": "2.0", "id": "req-001", "result": {}}) + ) + await BedrockAgentCoreA2AConfig().handle_non_streaming( + request_id="req-001", + params=_params_with_context(CONTEXT_ID), + litellm_params={**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: KEY_HASH}, + ) + + assert route.calls.last.request.headers[SESSION_HEADER] == _scoped(CONTEXT_ID, KEY_HASH) + + @pytest.mark.asyncio + async def test_context_id_session_header_on_outbound_streaming_post(self, httpx_transport): + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, + ) + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + with respx.mock(assert_all_called=True) as router: + route = router.post(url__regex=r".*/invocations.*").mock( + return_value=httpx.Response(200, json={"jsonrpc": "2.0", "id": "req-001", "result": {}}) + ) + events = [ + event + async for event in BedrockAgentCoreA2AConfig().handle_streaming( + request_id="req-001", + params=_params_with_context(CONTEXT_ID), + litellm_params={**SAMPLE_LITELLM_PARAMS, A2A_USER_API_KEY_HASH_PARAM: KEY_HASH}, + ) + ] + + assert events == [{"jsonrpc": "2.0", "id": "req-001", "result": {}}] + assert route.calls.last.request.headers[SESSION_HEADER] == _scoped(CONTEXT_ID, KEY_HASH) + + class TestNonStreaming: """Test end-to-end non-streaming flow.""" diff --git a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py index fbd7e36e298..293f75b7592 100644 --- a/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py +++ b/tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py @@ -8,6 +8,7 @@ import pytest import litellm from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.rust_bridge import configuration from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -109,10 +110,12 @@ class RaisingAsyncMessages: @pytest.fixture(autouse=True) def _reset_rust_flag(): - litellm.use_litellm_rust(False, messages=None, amessages=None) + rust_messages.set_rust_messages(messages=None, amessages=None) + configuration.reset_rust_configuration() rust_bridge_loader._cached_bridge = rust_bridge_loader._BRIDGE_SENTINEL yield - litellm.use_litellm_rust(False, messages=None, amessages=None) + rust_messages.set_rust_messages(messages=None, amessages=None) + configuration.reset_rust_configuration() rust_bridge_loader._cached_bridge = rust_bridge_loader._BRIDGE_SENTINEL @@ -122,17 +125,6 @@ def test_load_rust_messages_returns_injected_impl(): assert rust_messages.load_rust_messages() is bridge -def test_configuring_messages_does_not_enable_ocr(): - from litellm.rust_bridge.ocr import rust_ocr_enabled - - litellm.use_litellm_rust(False) - assert rust_ocr_enabled() is False - - litellm.use_litellm_rust(True, messages=RecordingMessages()) - - assert rust_ocr_enabled() is False - - def test_bare_use_litellm_rust_still_toggles_ocr(): from litellm.rust_bridge.ocr import rust_ocr_enabled @@ -264,7 +256,7 @@ async def test_gate_falls_back_to_python_when_bridge_raises(): @pytest.mark.asyncio async def test_gate_skips_rust_when_flag_absent(): bridge = ExplodingAsyncMessages() - litellm.use_litellm_rust(True, amessages=bridge) + rust_messages.set_rust_messages(amessages=bridge) response = await _gate(litellm_params=GenericLiteLLMParams(api_key="sk-azure")) @@ -272,6 +264,18 @@ async def test_gate_skips_rust_when_flag_absent(): assert bridge.calls == 0 +@pytest.mark.asyncio +async def test_gate_uses_process_enable_without_request_override(): + bridge = RecordingAsyncMessages() + rust_messages.set_rust_messages(amessages=bridge) + litellm.use_litellm_rust(True) + + response = await _gate(litellm_params=GenericLiteLLMParams(api_key="sk-azure")) + + assert response is not None + assert bridge.calls[0]["custom_llm_provider"] == "azure_ai" + + @pytest.mark.asyncio async def test_gate_skips_rust_when_flag_false(): bridge = ExplodingAsyncMessages() @@ -305,7 +309,7 @@ async def test_gate_invokes_rust_for_native_anthropic_provider(): @pytest.mark.asyncio async def test_gate_invokes_rust_when_env_var_set(monkeypatch): bridge = RecordingAsyncMessages() - litellm.use_litellm_rust(True, amessages=bridge) + rust_messages.set_rust_messages(amessages=bridge) monkeypatch.setenv("LITELLM_RUST", "1") response = await _gate( @@ -320,7 +324,7 @@ async def test_gate_invokes_rust_when_env_var_set(monkeypatch): @pytest.mark.asyncio async def test_gate_env_var_falsey_does_not_enable(monkeypatch): bridge = ExplodingAsyncMessages() - litellm.use_litellm_rust(True, amessages=bridge) + rust_messages.set_rust_messages(amessages=bridge) monkeypatch.setenv("LITELLM_RUST", "0") response = await _gate( diff --git a/tests/test_litellm/containers/test_container_api.py b/tests/test_litellm/containers/test_container_api.py index 885c4cd294a..16a3844431e 100644 --- a/tests/test_litellm/containers/test_container_api.py +++ b/tests/test_litellm/containers/test_container_api.py @@ -152,6 +152,42 @@ class TestContainerAPI: assert response.id == "cntr_async_123" assert response.name == "Async Test Container" + @pytest.mark.asyncio + async def test_acreate_container_encodes_router_model_id(self): + """ + The async handler returns a coroutine, so the managed-ID encoding must run + after it resolves. Otherwise follow-up calls (retrieve/delete/files) lose the + deployment and fall back to global provider credentials. + """ + upstream_response = ContainerObject( + id="cntr_upstream_123", + object="container", + created_at=1747857508, + status="running", + expires_after={"anchor": "last_active_at", "minutes": 20}, + last_active_at=1747857508, + name="Routed Container", + ) + + async def _resolve_upstream(): + return upstream_response + + with patch.object( # test-quality-ok: create_container exposes no client seam, only the handler + base_llm_http_handler, + "container_create_handler", + side_effect=lambda **kwargs: _resolve_upstream() if kwargs["_is_async"] else upstream_response, + ): + response = await acreate_container( + name="Routed Container", + custom_llm_provider="openai", + litellm_metadata={"model_info": {"id": "deployment-abc"}}, + ) + + decoded = ResponsesAPIRequestUtils._decode_container_id(response.id) + assert decoded["model_id"] == "deployment-abc" + assert decoded["custom_llm_provider"] == "openai" + assert decoded["response_id"] == "cntr_upstream_123" + @pytest.mark.asyncio async def test_alist_containers_basic(self): """Test basic async container listing functionality.""" diff --git a/tests/test_litellm/integrations/azure_storage/test_azure_storage.py b/tests/test_litellm/integrations/azure_storage/test_azure_storage.py index 16c518ff412..a96eae0f9c3 100644 --- a/tests/test_litellm/integrations/azure_storage/test_azure_storage.py +++ b/tests/test_litellm/integrations/azure_storage/test_azure_storage.py @@ -1,10 +1,15 @@ +import asyncio import sys +import threading from unittest.mock import AsyncMock, MagicMock, patch import pytest - -from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger +from litellm.integrations.azure_storage.azure_storage import ( + AzureBlobStorageLogger, + _cached_credential_chain_token_provider, +) +from litellm.types.secret_managers.get_azure_ad_token_provider import AzureCredentialType from litellm.types.utils import StandardLoggingPayload @@ -25,6 +30,26 @@ def mock_gov_env_vars(mock_env_vars, monkeypatch): monkeypatch.setenv("AZURE_STORAGE_ENDPOINT_SUFFIX", "core.usgovcloudapi.net") +@pytest.fixture +def workload_identity_env_vars(monkeypatch): + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_NAME", "test-account") + monkeypatch.setenv("AZURE_STORAGE_FILE_SYSTEM", "test-container") + for unset in ( + "AZURE_STORAGE_TENANT_ID", + "AZURE_STORAGE_CLIENT_ID", + "AZURE_STORAGE_CLIENT_SECRET", + "AZURE_STORAGE_ACCOUNT_KEY", + "AZURE_STORAGE_ENDPOINT_SUFFIX", + "AZURE_CLIENT_SECRET", + "AZURE_CREDENTIAL", + "AZURE_SCOPE", + ): + monkeypatch.delenv(unset, raising=False) + monkeypatch.setenv("AZURE_CLIENT_ID", "workload-identity-client-id") + monkeypatch.setenv("AZURE_TENANT_ID", "workload-identity-tenant-id") + monkeypatch.setenv("AZURE_FEDERATED_TOKEN_FILE", "/var/run/secrets/azure/tokens/azure-identity-token") + + @pytest.mark.asyncio async def test_async_upload_payload_to_azure_blob_storage(mock_env_vars): """ @@ -32,17 +57,12 @@ async def test_async_upload_payload_to_azure_blob_storage(mock_env_vars): a payload to Azure Blob Storage using the 3-step process (create, append, flush). """ with ( - patch( - "litellm.integrations.azure_storage.azure_storage.get_async_httpx_client" - ) as mock_get_client, - patch( - "litellm.llms.azure.common_utils.get_azure_ad_token_from_entra_id" - ) as mock_get_token, + patch("litellm.integrations.azure_storage.azure_storage.get_async_httpx_client") as mock_get_client, + patch("litellm.integrations.azure_storage.azure_storage.get_azure_ad_token_from_entra_id") as mock_get_token, ): # Create mock HTTP client mock_http_client = AsyncMock() - mock_response = AsyncMock() - mock_response.raise_for_status = AsyncMock() + mock_response = MagicMock() mock_http_client.put.return_value = mock_response mock_http_client.patch.return_value = mock_response mock_get_client.return_value = mock_http_client @@ -79,9 +99,7 @@ async def test_async_upload_payload_to_azure_blob_storage(mock_env_vars): put_call_args = mock_http_client.put.call_args assert put_call_args[0][0] == f"{expected_base_url}?resource=file" assert put_call_args[1]["headers"]["x-ms-version"] is not None - assert ( - put_call_args[1]["headers"]["Authorization"] == "Bearer mock-azure-ad-token" - ) + assert put_call_args[1]["headers"]["Authorization"] == "Bearer mock-azure-ad-token" # Step 2: Append data assert mock_http_client.patch.call_count == 2 # Called for append and flush @@ -89,9 +107,7 @@ async def test_async_upload_payload_to_azure_blob_storage(mock_env_vars): assert append_call[0][0] == f"{expected_base_url}?action=append&position=0" assert append_call[1]["headers"]["x-ms-version"] is not None assert append_call[1]["headers"]["Content-Type"] == "application/json" - assert ( - append_call[1]["headers"]["Authorization"] == "Bearer mock-azure-ad-token" - ) + assert append_call[1]["headers"]["Authorization"] == "Bearer mock-azure-ad-token" assert "test-log-id-123" in append_call[1]["data"] # Step 3: Flush data @@ -110,9 +126,7 @@ async def test_async_upload_payload_uses_configured_endpoint_suffix(mock_gov_env AZURE_STORAGE_ENDPOINT_SUFFIX must reach the Entra-ID REST upload path so a sovereign-cloud account is addressed instead of the commercial dfs host. """ - with patch( - "litellm.integrations.azure_storage.azure_storage.get_async_httpx_client" - ) as mock_get_client: + with patch("litellm.integrations.azure_storage.azure_storage.get_async_httpx_client") as mock_get_client: mock_http_client = AsyncMock() mock_response = MagicMock() mock_http_client.put.return_value = mock_response @@ -127,17 +141,10 @@ async def test_async_upload_payload_uses_configured_endpoint_suffix(mock_gov_env await logger.async_upload_payload_to_azure_blob_storage(test_payload) - expected_base_url = ( - "https://test-account.dfs.core.usgovcloudapi.net/test-container/gov-log-id.json" - ) + expected_base_url = "https://test-account.dfs.core.usgovcloudapi.net/test-container/gov-log-id.json" assert mock_http_client.put.call_args[0][0] == f"{expected_base_url}?resource=file" - assert ( - mock_http_client.patch.call_args_list[0][0][0] - == f"{expected_base_url}?action=append&position=0" - ) - assert mock_http_client.patch.call_args_list[1][0][0].startswith( - f"{expected_base_url}?action=flush" - ) + assert mock_http_client.patch.call_args_list[0][0][0] == f"{expected_base_url}?action=append&position=0" + assert mock_http_client.patch.call_args_list[1][0][0].startswith(f"{expected_base_url}?action=flush") @pytest.mark.asyncio @@ -148,9 +155,7 @@ async def test_service_client_uses_configured_endpoint_suffix(mock_gov_env_vars) """ fake_aio_module = MagicMock() - with patch.dict( - sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module} - ): + with patch.dict(sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module}): logger = AzureBlobStorageLogger() await logger.get_service_client() @@ -160,14 +165,180 @@ async def test_service_client_uses_configured_endpoint_suffix(mock_gov_env_vars) ) +@pytest.mark.asyncio +async def test_upload_authenticates_through_the_credential_chain_under_workload_identity( + workload_identity_env_vars, +): + build_provider = MagicMock(return_value=lambda: "workload-identity-token") + with patch( # test-quality-ok: REST client is created inside the method; assert emitted request headers + "litellm.integrations.azure_storage.azure_storage.get_async_httpx_client" + ) as mock_get_client: + mock_http_client = AsyncMock() + mock_http_client.put.return_value = MagicMock() + mock_http_client.patch.return_value = MagicMock() + mock_get_client.return_value = mock_http_client + + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=build_provider) + await logger.async_upload_payload_to_azure_blob_storage({"id": "wif-log-id"}) + + build_provider.assert_called_once_with() + assert logger.azure_auth_token == "workload-identity-token" + sent_headers = [mock_http_client.put.call_args[1]["headers"]] + [ + call[1]["headers"] for call in mock_http_client.patch.call_args_list + ] + assert len(sent_headers) == 3 + assert all(headers["Authorization"] == "Bearer workload-identity-token" for headers in sent_headers) + + +def test_default_chain_provider_is_storage_scoped_and_built_once_per_process(): + _cached_credential_chain_token_provider.cache_clear() + with ( + patch( # test-quality-ok: assert the default factory's fixed scope and credential type without constructing Azure SDK credentials + "litellm.integrations.azure_storage.azure_storage.get_azure_ad_token_provider", + return_value=lambda: "chain-token", + ) as mock_builder + ): + first = _cached_credential_chain_token_provider() + second = _cached_credential_chain_token_provider() + _cached_credential_chain_token_provider.cache_clear() + + assert first is second + assert first() == "chain-token" + mock_builder.assert_called_once_with( + azure_scope="https://storage.azure.com/.default", + azure_credential=AzureCredentialType.DefaultAzureCredential, + ) + + +@pytest.mark.asyncio +async def test_chain_tokens_are_read_from_the_provider_on_every_refresh( + workload_identity_env_vars, +): + provider = MagicMock(side_effect=["chain-token-1", "chain-token-2"]) + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=MagicMock(return_value=provider)) + await logger.set_valid_azure_ad_token() + first_token = logger.azure_auth_token + await logger.set_valid_azure_ad_token() + + assert first_token == "chain-token-1" + assert logger.azure_auth_token == "chain-token-2" + assert provider.call_count == 2 + + +@pytest.mark.asyncio +async def test_chain_token_read_yields_to_the_event_loop(workload_identity_env_vars): + """ + The chain walk is blocking I/O (IMDS probe, CLI subprocess), so reading the provider + inline would stall every request on the worker. Prove other coroutines run during the read. + """ + loop_was_free = threading.Event() + + def provider() -> str: + if not loop_was_free.wait(timeout=5): + raise TimeoutError("the event loop never ran the observer while the token was being read") + return "chain-token" + + async def observer(): + loop_was_free.set() + + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=MagicMock(return_value=provider)) + observer_task = asyncio.create_task(observer()) + await logger.set_valid_azure_ad_token() + await observer_task + + assert logger.azure_auth_token == "chain-token" + + +@pytest.mark.asyncio +async def test_empty_string_service_principal_vars_still_use_the_credential_chain( + workload_identity_env_vars, monkeypatch +): + for name in ("AZURE_STORAGE_TENANT_ID", "AZURE_STORAGE_CLIENT_ID", "AZURE_STORAGE_CLIENT_SECRET"): + monkeypatch.setenv(name, "") + + logger = AzureBlobStorageLogger( + build_credential_chain_token_provider=MagicMock(return_value=lambda: "workload-identity-token") + ) + await logger.set_valid_azure_ad_token() + + assert logger.azure_auth_token == "workload-identity-token" + + +@pytest.mark.asyncio +async def test_client_secret_auth_still_uses_the_storage_scoped_service_principal(mock_env_vars): + build_provider = MagicMock() + with ( + patch( # test-quality-ok: assert the storage scope passed to the shared token factory without making an external auth call + "litellm.integrations.azure_storage.azure_storage.get_azure_ad_token_from_entra_id", + return_value=lambda: "client-secret-token", + ) as mock_entra_id + ): + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=build_provider) + await logger.set_valid_azure_ad_token() + + assert logger.azure_auth_token == "client-secret-token" + build_provider.assert_not_called() + assert mock_entra_id.call_args.kwargs == { + "tenant_id": "test-tenant-id", + "client_id": "test-client-id", + "client_secret": "test-client-secret", + "scope": "https://storage.azure.com/.default", + } + + +@pytest.mark.parametrize( + "missing_var", + ["AZURE_STORAGE_TENANT_ID", "AZURE_STORAGE_CLIENT_ID", "AZURE_STORAGE_CLIENT_SECRET"], +) +@pytest.mark.asyncio +async def test_partially_configured_service_principal_still_names_the_missing_variable( + mock_env_vars, monkeypatch, missing_var +): + monkeypatch.delenv(missing_var) + + build_provider = MagicMock() + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=build_provider) + with pytest.raises(ValueError, match=f"Missing required environment variable: {missing_var}"): + await logger.set_valid_azure_ad_token() + + build_provider.assert_not_called() + + +@pytest.mark.asyncio +async def test_account_key_auth_never_requests_a_token(workload_identity_env_vars, monkeypatch): + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_KEY", "dGVzdC1rZXk=") + + file_client = MagicMock() + file_client.create_file = AsyncMock() + file_client.append_data = AsyncMock() + file_client.flush_data = AsyncMock() + directory_client = MagicMock() + directory_client.exists = AsyncMock(return_value=True) + directory_client.get_file_client = MagicMock(return_value=file_client) + file_system_client = MagicMock() + file_system_client.get_directory_client = MagicMock(return_value=directory_client) + service_client = MagicMock() + service_client.get_file_system_client = MagicMock(return_value=file_system_client) + fake_aio_module = MagicMock() + fake_aio_module.DataLakeServiceClient = MagicMock(return_value=service_client) + + build_provider = MagicMock() + with patch.dict(sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module}): + logger = AzureBlobStorageLogger(build_credential_chain_token_provider=build_provider) + await logger.async_upload_payload_to_azure_blob_storage({"id": "account-key-log-id"}) + + build_provider.assert_not_called() + assert logger.azure_auth_token is None + file_client.flush_data.assert_awaited_once() + assert fake_aio_module.DataLakeServiceClient.call_args.kwargs["credential"] == "dGVzdC1rZXk=" + + @pytest.mark.asyncio async def test_service_client_defaults_to_commercial_endpoint(mock_env_vars): """Unset AZURE_STORAGE_ENDPOINT_SUFFIX keeps the pre-existing commercial host""" fake_aio_module = MagicMock() - with patch.dict( - sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module} - ): + with patch.dict(sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module}): logger = AzureBlobStorageLogger() await logger.get_service_client() diff --git a/tests/test_litellm/integrations/otel/test_langfuse_logger.py b/tests/test_litellm/integrations/otel/test_langfuse_logger.py new file mode 100644 index 00000000000..8f93a9a564f --- /dev/null +++ b/tests/test_litellm/integrations/otel/test_langfuse_logger.py @@ -0,0 +1,359 @@ +"""Tests for ``LangfuseOpenTelemetryV2``: the root observation's input and output are stamped from the +request-task hooks, while the root span is still recording, so Langfuse can show them on the trace.""" + +import asyncio +import json +from collections.abc import AsyncIterator, Sequence +from typing import Final + +import pytest + +pytest.importorskip("opentelemetry") + +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 + +import litellm # noqa: E402 +from litellm.caching.dual_cache import DualCache # noqa: E402 +from litellm.integrations.otel.logger import build_otel_v2_logger # noqa: E402 +from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402 +from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402 +from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402 +from litellm.integrations.otel.plumbing import providers # noqa: E402 +from litellm.integrations.otel.plumbing.context import set_request_root_span # noqa: E402 +from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 # noqa: E402 +from litellm.proxy._types import UserAPIKeyAuth # noqa: E402 +from litellm.proxy.utils import ProxyLogging # noqa: E402 +from litellm.types.llms.openai import ( # noqa: E402 + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) +from litellm.types.utils import ( # noqa: E402 + Choices, + Delta, + Embedding, + EmbeddingResponse, + Message, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) + +INPUT_ATTR: Final = "langfuse.observation.input" +OUTPUT_ATTR: Final = "langfuse.observation.output" +CHAT_DATA: Final = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "ping"}]} + + +@pytest.fixture(autouse=True) +def _reset_request_root_span(): + otel_context._request_root_span.set(None) + yield + otel_context._request_root_span.set(None) + + +def _logger(*, capture: str = "span_only", mappers: Sequence[str] = ("genai", "langfuse")): + cfg = OpenTelemetryV2Config(exporter="in_memory", mapper_names=list(mappers), capture_message_content=capture) + exporter = InMemorySpanExporter() + tracer_provider = providers.build_tracer_provider(cfg, exporter=exporter) + return build_otel_v2_logger(config=cfg, tracer_provider=tracer_provider), exporter + + +def _start_root(logger): + root = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + set_request_root_span(root) + return root + + +def _root_attrs(exporter): + by_name = {span.name: span for span in exporter.get_finished_spans()} + return dict(by_name[LITELLM_PROXY_REQUEST_SPAN_NAME].attributes or {}) + + +def _run_request(logger, data: dict, call_type: str, response: object): + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), data, call_type)) + asyncio.run(logger.async_post_call_success_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), response=response)) + root.end() + + +async def _relay(logger, chunks: Sequence[object], data: dict) -> list[object]: + async def source() -> AsyncIterator[object]: + for chunk in chunks: + yield chunk + + return [chunk async for chunk in logger.async_post_call_streaming_iterator_hook(UserAPIKeyAuth(), source(), data)] + + +def _run_stream(logger, data: dict, chunks: Sequence[object]) -> list[object]: + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), data, "acompletion")) + relayed = asyncio.run(_relay(logger, chunks, data)) + root.end() + return relayed + + +def _chat_chunk(content: str | None, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-1", + created=1, + model="gpt-5.4-mini", + choices=[StreamingChoices(index=0, delta=Delta(content=content), finish_reason=finish_reason)], + ) + + +def _responses_api_response() -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_1", + created_at=1, + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "pong", "annotations": []}], + } + ], + ) + + +def _anthropic_sse_frames() -> tuple[bytes, ...]: + events = ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "po"}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ng"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 2}}, + {"type": "message_stop"}, + ) + return tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events) + + +def test_chat_request_stamps_root_observation_input_and_output(): + logger, exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + + _run_request(logger, CHAT_DATA, "acompletion", response) + + attrs = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [{"role": "user", "content": "ping"}] + output = json.loads(attrs[OUTPUT_ATTR]) + assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] + + +def test_responses_request_folds_instructions_into_input_and_stamps_output_items(): + logger, exporter = _logger() + data = {"model": "gpt-5.4-mini", "instructions": "be terse", "input": "ping"} + + _run_request(logger, data, "aresponses", _responses_api_response()) + + attrs = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "ping"}, + ] + output = json.loads(attrs[OUTPUT_ATTR]) + assert output[0]["role"] == "assistant" + assert output[0]["content"][0]["text"] == "pong" + + +def test_anthropic_messages_request_folds_system_into_input_and_stamps_content_blocks(): + logger, exporter = _logger() + data = {"model": "claude-sonnet-4-5", "system": "be terse", "messages": [{"role": "user", "content": "ping"}]} + response = {"type": "message", "role": "assistant", "content": [{"type": "text", "text": "pong"}]} + + _run_request(logger, data, "aanthropic_messages", response) + + attrs = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "ping"}, + ] + assert json.loads(attrs[OUTPUT_ATTR]) == [{"role": "assistant", "content": [{"type": "text", "text": "pong"}]}] + + +def test_chat_stream_relays_chunks_untouched_and_stamps_assembled_output(): + logger, exporter = _logger() + chunks = (_chat_chunk("po"), _chat_chunk("ng"), _chat_chunk(None, finish_reason="stop")) + + relayed = _run_stream(logger, CHAT_DATA, chunks) + + assert [id(chunk) for chunk in relayed] == [id(chunk) for chunk in chunks] + output = json.loads(_root_attrs(exporter)[OUTPUT_ATTR]) + assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] + + +def test_responses_stream_stamps_output_from_the_completed_event(): + logger, exporter = _logger() + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_responses_api_response() + ) + chunks = ({"type": "response.created"}, {"type": "response.output_text.delta", "delta": "pong"}, completed) + + relayed = _run_stream(logger, {"model": "gpt-5.4-mini", "input": "ping"}, chunks) + + assert relayed == list(chunks) + output = json.loads(_root_attrs(exporter)[OUTPUT_ATTR]) + assert output[0]["content"][0]["text"] == "pong" + + +def test_anthropic_sse_stream_stamps_output_from_the_assembled_frames(): + logger, exporter = _logger() + frames = _anthropic_sse_frames() + + relayed = _run_stream( + logger, {"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "ping"}]}, frames + ) + + assert relayed == list(frames) + output = json.loads(_root_attrs(exporter)[OUTPUT_ATTR]) + assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] + + +def test_root_observation_io_survives_the_root_ending_before_the_success_callback(): + logger, exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) + logger.log_pre_api_call( + model="gpt-5.4-mini", + messages=[], + kwargs={"litellm_call_id": "call_1", "litellm_params": {"metadata": {}}}, + ) + asyncio.run( + logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + root.end() + + payload = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-5.4-mini", + "messages": CHAT_DATA["messages"], + "response": response.model_dump(), + "status": "success", + "litellm_call_id": "call_1", + "metadata": {}, + "hidden_params": {}, + } + asyncio.run( + logger.async_log_success_event( + {"standard_logging_object": payload, "litellm_params": {"metadata": {}}}, response, None, None + ) + ) + + attrs = _root_attrs(exporter) + assert INPUT_ATTR in attrs and OUTPUT_ATTR in attrs + generation = next(span for span in exporter.get_finished_spans() if span.name != LITELLM_PROXY_REQUEST_SPAN_NAME) + assert OUTPUT_ATTR in dict(generation.attributes or {}) + + +def test_root_input_is_the_request_as_the_pre_call_chain_left_it(): + logger, exporter = _logger() + raw = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "my ssn is 123-45-6789"}]} + masked = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "my ssn is [REDACTED]"}]} + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="noted"))]) + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), raw, "acompletion")) + asyncio.run(logger.async_post_call_success_hook(data=masked, user_api_key_dict=UserAPIKeyAuth(), response=response)) + root.end() + + assert json.loads(_root_attrs(exporter)[INPUT_ATTR]) == masked["messages"] + + +def test_root_already_ended_is_left_alone(): + logger, exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + root = _start_root(logger) + root.end() + + asyncio.run( + logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs + + +def test_responses_without_a_message_body_stamp_neither_input_nor_output(): + logger, exporter = _logger() + embedding = EmbeddingResponse(model="e", data=[Embedding(embedding=[0.1], index=0, object="embedding")]) + + _run_request(logger, {"model": "e", "input": "ping"}, "aembedding", embedding) + + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs + + +def test_unrenderable_output_never_raises_into_the_request(): + logger, exporter = _logger() + + _run_request(logger, CHAT_DATA, "acompletion", object()) + + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs + + +@pytest.mark.parametrize( + ("capture", "mappers"), + [("no_content", ("genai", "langfuse")), ("span_only", ("genai",))], +) +def test_factory_keeps_the_base_logger_unless_langfuse_content_capture_is_on(capture, mappers): + logger, exporter = _logger(capture=capture, mappers=mappers) + + _run_request(logger, CHAT_DATA, "acompletion", ModelResponse()) + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs + + +@pytest.mark.parametrize( + ("capture", "mappers", "relays_streams"), + [ + ("span_only", ("genai", "langfuse"), True), + ("no_content", ("genai", "langfuse"), False), + ("span_only", ("genai",), False), + ], +) +def test_only_langfuse_content_capture_takes_proxy_streams_off_the_fast_path( + monkeypatch, capture, mappers, relays_streams +): + logger, _ = _logger(capture=capture, mappers=mappers) + monkeypatch.setattr(litellm, "callbacks", [logger]) + + assert ProxyLogging._callback_capabilities().has_iterator_override is relays_streams + + +def test_langfuse_otel_preset_builds_a_logger_that_stamps_the_root(monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk") + monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") + monkeypatch.setenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "span_only") + is_otel_v2_enabled.cache_clear() + + loggers: list = [] + try: + built = _maybe_construct_otel_v2("langfuse_otel", loggers) + assert built is not None + assert _maybe_construct_otel_v2("langfuse_otel", loggers) is built + root = _start_root(built) + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + asyncio.run( + built.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + attrs = dict(root.attributes or {}) + assert INPUT_ATTR in attrs and OUTPUT_ATTR in attrs + finally: + is_otel_v2_enabled.cache_clear() diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index e995cbae782..de8b654987b 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -2858,6 +2858,27 @@ class TestRecordGatewayInjection: AnthropicCacheControlHook.record_gateway_injection(kwargs, 0) assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT + def test_an_every_deployment_mark_survives_a_later_per_deployment_stamp(self): + """A per-leg stamp like the Bedrock converse tool_config one describes one leg of + a payload every leg sends, so narrowing an every-deployment mark to that leg's + deployment would uncredit whichever leg gets billed after a failover.""" + kwargs: dict = {"litellm_metadata": {self.KEY: ""}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 1) + assert kwargs["litellm_metadata"][self.KEY] == "" + + def test_a_pre_choice_pass_stamps_the_sentinel_over_a_provisional_deployment(self): + """The router's prompt-management factory stamps a provisional deployment's + model_info into kwargs before the prompt pass runs, and any other deployment can + end up billed, so the pass declares every-deployment scope explicitly.""" + kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 1, injected_for_every_deployment=True) + assert kwargs["litellm_metadata"][self.KEY] == "" + + def test_a_per_deployment_mark_still_follows_the_latest_leg(self): + kwargs: dict = {"litellm_metadata": {self.KEY: "dep-old"}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 1) + assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT + def test_v1_messages_auto_injection_stamps_the_marker(self, monkeypatch): monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}} diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index d978eb48c12..7d70b9a8862 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -2237,3 +2237,202 @@ class TestRecordsOwnGuardrailInformation: ) assert _guardrail_entries(request_data) == [] + + +class _ApplyOnlyObserver(CustomGuardrail): + """Overrides only apply_guardrail, like panw_prisma_airs; inherits async_logging_hook.""" + + def __init__(self, block: bool = False): + from litellm.types.guardrails import GuardrailEventHooks + + super().__init__(guardrail_name="apply-only-observer", event_hook=GuardrailEventHooks.logging_only) + self.block = block + self.calls: list = [] + + @log_guardrail_information + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + from fastapi import HTTPException + + self.calls.append((input_type, list(inputs.get("texts") or []))) + if self.block: + raise HTTPException(status_code=400, detail={"error": "flagged"}) + return GenericGuardrailAPIInputs(texts=["[MASKED]" for _ in inputs.get("texts") or []]) + + +def _logged_call(messages: list | str) -> tuple[dict, object]: + from litellm.types.utils import Choices, Message, ModelResponse + + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="general kenobi"))]) + kwargs = { + "model": "gpt-5.4-mini", + "messages": messages, + "litellm_call_id": "call-1", + "litellm_params": {"metadata": {"user_api_key_user_id": "u1"}}, + "optional_params": {}, + "standard_logging_object": {"guardrail_information": None}, + } + return kwargs, response + + +class TestLoggingOnlyApplyGuardrail: + """LIT-4876 regression: a guardrail in mode logging_only that implements only + apply_guardrail must still run against the logged request and response and + record guardrail_information, instead of inheriting the CustomLogger no-op.""" + + @pytest.mark.asyncio + async def test_runs_apply_guardrail_observe_only_and_records_verdict(self): + guardrail = _ApplyOnlyObserver() + messages = [{"role": "user", "content": "hello there"}] + kwargs, response = _logged_call(messages) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])] + assert out_kwargs["messages"] == [{"role": "user", "content": "hello there"}] + assert out_response.choices[0].message.content == "general kenobi" + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_name"] for e in entries] == ["apply-only-observer", "apply-only-observer"] + assert {e["guardrail_mode"] for e in entries} == {"logging_only"} + assert {e["guardrail_status"] for e in entries} == {"success"} + assert "standard_logging_guardrail_information" not in kwargs["litellm_params"]["metadata"] + assert kwargs["standard_logging_object"] == {"guardrail_information": None} + + @pytest.mark.asyncio + async def test_appends_to_pre_call_verdicts_without_duplicating_them(self): + guardrail = _ApplyOnlyObserver() + kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) + pre_call_entry = {"guardrail_name": "pii-blocker", "guardrail_mode": "pre_call", "guardrail_status": "success"} + kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] = [pre_call_entry] + kwargs["standard_logging_object"]["guardrail_information"] = [pre_call_entry] + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_name"] for e in entries] == ["pii-blocker", "apply-only-observer", "apply-only-observer"] + assert kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] == [pre_call_entry] + + @pytest.mark.asyncio + async def test_request_copy_failure_is_swallowed(self): + import threading + + guardrail = _ApplyOnlyObserver() + kwargs, response = _logged_call([{"role": "user", "content": "hello there", "lock": threading.Lock()}]) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [] + assert out_kwargs is kwargs + assert out_response is response + + @pytest.mark.asyncio + async def test_block_verdict_is_recorded_without_raising(self): + guardrail = _ApplyOnlyObserver(block=True) + kwargs, response = _logged_call([{"role": "user", "content": "flagged content"}]) + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [("request", ["flagged content"])] + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["guardrail_intervened"] + + @pytest.mark.asyncio + async def test_call_type_without_translation_is_skipped(self): + guardrail = _ApplyOnlyObserver() + kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.amoderation.value) + + assert guardrail.calls == [] + assert out_kwargs["standard_logging_object"]["guardrail_information"] is None + + @pytest.mark.asyncio + async def test_aembedding_scans_logged_input(self): + from litellm.types.utils import EmbeddingResponse + + guardrail = _ApplyOnlyObserver() + kwargs, _ = _logged_call("hello there") + response = EmbeddingResponse(data=[{"embedding": [0.1], "index": 0, "object": "embedding"}]) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.aembedding.value) + + assert guardrail.calls == [("request", ["hello there"])] + assert out_kwargs["messages"] == "hello there" + assert out_response is response + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["success"] + + @pytest.mark.asyncio + async def test_native_lifecycle_hook_guardrail_is_left_alone(self): + class _NativeHooks(_ApplyOnlyObserver): + use_native_lifecycle_hooks = True + + guardrail = _NativeHooks() + kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) + + out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [] + assert out_kwargs is kwargs + assert out_response is response + + @pytest.mark.asyncio + async def test_aresponses_scans_logged_messages_when_input_is_cleared(self): + from litellm.types.llms.openai import ResponsesAPIResponse + + guardrail = _ApplyOnlyObserver() + kwargs, _ = _logged_call([{"role": "user", "content": "hello there"}]) + kwargs["input"] = None + response = ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="gpt-5.4-mini", + object="response", + status="completed", + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "general kenobi"}], + } + ], + ) + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.aresponses.value) + + assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])] + entries = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["success", "success"] + + @pytest.mark.asyncio + async def test_async_success_handler_records_verdict_in_standard_logging_object(self): + import datetime as dt + + from litellm.litellm_core_utils.litellm_logging import Logging + + guardrail = _ApplyOnlyObserver() + guardrail.default_on = True + messages = [{"role": "user", "content": "hello there"}] + _, response = _logged_call(messages) + logging_obj = Logging( + model="gpt-5.4-mini", + messages=messages, + stream=False, + call_type=CallTypes.acompletion.value, + start_time=dt.datetime.now(), + litellm_call_id="call-1", + function_id="fn-1", + dynamic_async_success_callbacks=[guardrail], + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {}}, optional_params={}, model="gpt-5.4-mini", custom_llm_provider="openai" + ) + + await logging_obj.async_success_handler( + result=response, start_time=dt.datetime.now(), end_time=dt.datetime.now() + ) + + assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])] + entries = logging_obj.model_call_details["standard_logging_object"]["guardrail_information"] + assert [e["guardrail_status"] for e in entries] == ["success", "success"] diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 0e1c832ebf5..b7f0ca1efe1 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1522,7 +1522,7 @@ def test_gpt_5_6_alias_prices_match_sol(local_model_cost_map): sol = litellm.model_cost["gpt-5.6-sol"] cost_fields = sorted(field for field in sol if "cost" in field) - assert len(cost_fields) == 23 + assert len(cost_fields) == 27 for field in cost_fields: assert alias.get(field) == sol.get(field), field @@ -4039,8 +4039,8 @@ def test_fast_service_tier_matches_priority_above_the_context_threshold(_local_m ) assert fast == priority - assert fast[0] == pytest.approx(300_000 * 8e-06, rel=1e-9) - assert fast[1] == pytest.approx(1_000 * 3e-05, rel=1e-9) + assert fast[0] == pytest.approx(300_000 * 1.6e-05, rel=1e-9) + assert fast[1] == pytest.approx(1_000 * 6e-05, rel=1e-9) def test_priority_reasoning_tokens_bill_at_the_priority_output_rate(_local_model_cost_map): @@ -4200,6 +4200,86 @@ def test_generic_cost_per_token_gemini_37_flash(_local_model_cost_map): assert completion_cost == pytest.approx(0.001875) +GEMINI_38_FLASH_LAUNCH_PRICING = [ + ("gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08), + ("gemini/gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08), + ("vertex_ai/gemini-3.8-flash", 7.5e-07, 3.75e-06, 7.5e-08), +] + + +@pytest.mark.parametrize("model,input_cost,output_cost,cache_read_cost", GEMINI_38_FLASH_LAUNCH_PRICING) +def test_gemini_38_flash_launch_pricing(model, input_cost, output_cost, cache_read_cost, _local_model_cost_map): + model_cost_map = litellm.model_cost[model] + assert model_cost_map["input_cost_per_token"] == input_cost + assert model_cost_map["output_cost_per_token"] == output_cost + assert model_cost_map["output_cost_per_reasoning_token"] == output_cost + assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost + assert model_cost_map["mode"] == "chat" + assert model_cost_map["supports_reasoning"] is True + assert model_cost_map["supports_function_calling"] is True + assert model_cost_map["max_input_tokens"] == 1048576 + + +GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH = ( + "input_cost_per_token", + "output_cost_per_token", + "output_cost_per_reasoning_token", + "cache_read_input_token_cost", + "input_cost_per_token_batches", + "output_cost_per_token_batches", + "input_cost_per_token_flex", + "output_cost_per_token_flex", + "cache_read_input_token_cost_flex", + "input_cost_per_token_priority", + "output_cost_per_token_priority", + "cache_read_input_token_cost_priority", + "search_context_cost_per_query", + "google_maps_grounding_cost_per_query", + "prompt_cache_min_tokens", + "max_input_tokens", + "max_output_tokens", + "supports_reasoning", + "supports_function_calling", + "supports_prompt_caching", + "supports_vision", + "supports_pdf_input", + "supports_audio_input", + "supports_video_input", + "supports_response_schema", + "supports_tool_choice", + "supports_web_search", + "supports_url_context", +) + + +@pytest.mark.parametrize("prefix", ["", "gemini/", "vertex_ai/"]) +def test_gemini_38_flash_matches_37_flash_promotional_pricing(prefix, _local_model_cost_map): + new_model = litellm.model_cost[f"{prefix}gemini-3.8-flash"] + old_model = litellm.model_cost[f"{prefix}gemini-3.7-flash"] + for field in GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH: + assert new_model[field] == old_model[field], field + + +def test_generic_cost_per_token_gemini_38_flash(_local_model_cost_map): + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=200, + text_tokens=300, + ), + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=1000), + ) + prompt_cost, completion_cost = generic_cost_per_token( + model="gemini-3.8-flash", + usage=usage, + custom_llm_provider="gemini", + ) + assert prompt_cost == pytest.approx(0.00075) + assert completion_cost == pytest.approx(0.001875) + + def test_grok_46_launch_pricing(_local_model_cost_map): model_cost_map = litellm.model_cost["xai/grok-4.6"] assert model_cost_map["input_cost_per_token"] == 2e-06 diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index 15b7ae9d07a..1778eca25ef 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -1013,6 +1013,48 @@ def test_an_unmapped_exception_with_no_model_or_provider_is_a_connection_error(q assert "boom" in raised.value.message +def _raise_and_map( + model: str | None, original_exception: Exception, custom_llm_provider: str | None +) -> None: + """Calls exception_type() from inside the except block, as litellm/main.py does, + so traceback.format_exc() has a real stack.""" + try: + raise original_exception + except type(original_exception) as caught: + exception_type( + model=model, + original_exception=caught, + custom_llm_provider=custom_llm_provider, + ) + + +def test_an_unmapped_exception_message_keeps_traceback_for_sdk_callers(quiet_exception_mapping): + """Direct SDK callers debug unmapped provider exceptions with this traceback; + only the proxy's response boundary strips it.""" + with pytest.raises(litellm.APIConnectionError) as raised: + _raise_and_map( + model="MiniMax-M2.5", + original_exception=RuntimeError("socket hung up"), + custom_llm_provider="minimax", + ) + + assert "Traceback (most recent call last)" in raised.value.message + assert "test_exception_mapping_utils.py" in raised.value.message + + +def test_an_unmapped_exception_with_no_model_or_provider_message_keeps_traceback( + quiet_exception_mapping, +): + with pytest.raises(litellm.APIConnectionError) as raised: + _raise_and_map( + model=None, + original_exception=ValueError("boom"), + custom_llm_provider=None, + ) + + assert "Traceback (most recent call last)" in raised.value.message + + CONTEXT_WINDOW_MESSAGE = "This model's maximum context length is 4096 tokens." CONTENT_POLICY_MESSAGE = ( '{"error": {"type": "invalid_request_error", "code": "content_policy_violation"}}' diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 366f61ded49..f1de7390b5b 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6002,7 +6002,9 @@ async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_o """The savings gate reads litellm_gateway_injected_cache from the request's metadata bucket. Recording lives in the shared prompt-hook wrappers, so chat, /v1/responses, router prompt deployments, and proxy prompt templates all mark - injected requests the same way; a hook that injects nothing leaves no marker.""" + injected requests the same way; a hook that injects nothing leaves no marker. + A pass that runs before deployment choice declares it and gets the every-deployment + sentinel, which a later per-deployment pass never narrows.""" from litellm.integrations.custom_prompt_management import CustomPromptManagement class _InjectingHook(CustomPromptManagement): @@ -6086,6 +6088,28 @@ async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_o ) assert "litellm_gateway_injected_cache" not in untouched["metadata"] + pre_choice = {"metadata": {}, "model_info": {"id": "dep-of-this-attempt"}} + logging_obj.get_chat_completion_prompt( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + non_default_params={}, + prompt_variables=None, + prompt_management_logger=_InjectingHook(), + request_kwargs=pre_choice, + injected_for_every_deployment=True, + ) + assert pre_choice["metadata"]["litellm_gateway_injected_cache"] == "" + + await logging_obj.async_get_chat_completion_prompt( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "a fresh turn"}], + non_default_params={}, + prompt_variables=None, + prompt_management_logger=_InjectingHook(), + request_kwargs=pre_choice, + ) + assert pre_choice["metadata"]["litellm_gateway_injected_cache"] == "" + def test_get_standard_logging_object_payload_reads_overhead_from_logging_obj_for_dict_results(logging_obj): """LIT-5466: /v1/messages returns a plain dict with no _hidden_params, so the overhead diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 8ac050a04f9..bacbcbf132b 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -592,6 +592,59 @@ def test_stream_chunk_builder_litellm_usage_chunks(): assert usage.total_tokens == 77 +def test_calculate_usage_honors_openai_sdk_completion_usage_chunks(): + from openai.types.completion_usage import CompletionUsage + + content_chunk = ModelResponseStream( + id="chatcmpl-sdk-usage-1", + created=1745513206, + model="mantle-claude", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta( + provider_specific_fields=None, + content="ok", + role=None, + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields=None, + stream_options={"include_usage": True}, + ) + usage_chunk = ModelResponseStream( + id="chatcmpl-sdk-usage-1", + created=1745513207, + model="mantle-claude", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[], + provider_specific_fields=None, + stream_options={"include_usage": True}, + ) + usage_chunk.usage = CompletionUsage( + prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704 + ) + assert type(usage_chunk.usage) is CompletionUsage + + chunks = [content_chunk, usage_chunk] + usage = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, model="mantle-claude", completion_output="" + ) + + assert usage.prompt_tokens == 20 + assert usage.completion_tokens == 60 + assert usage.total_tokens == 80 + assert getattr(usage, "cost", None) == pytest.approx(0.000704) + + def test_get_model_from_chunks_azure_model_router(): """ Test that _get_model_from_chunks finds the actual model from Azure Model Router chunks. diff --git a/tests/test_litellm/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py b/tests/test_litellm/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py new file mode 100644 index 00000000000..dd7e8fadcd8 --- /dev/null +++ b/tests/test_litellm/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py @@ -0,0 +1,35 @@ +"""Tests for litellm/llms/a2a/chat/guardrail_translation/handler.py.""" + +import json + +from litellm.llms.a2a.chat.guardrail_translation.handler import A2AGuardrailHandler +from litellm.llms.base_llm.guardrail_translation.base_translation import StreamingScanKey + + +def _text_event(text: str) -> str: + return json.dumps( + { + "jsonrpc": "2.0", + "id": "req-1", + "result": {"kind": "message", "role": "agent", "parts": [{"kind": "text", "text": text}]}, + } + ) + + +def _status_event() -> str: + return json.dumps({"jsonrpc": "2.0", "id": "req-1", "result": {"kind": "status-update", "status": {}}}) + + +class TestA2AGuardrailHandlerStreamingScanKey: + def test_key_joins_the_text_of_every_message_event(self): + key = A2AGuardrailHandler().get_streaming_scan_key([_text_event("hello "), _text_event("world")]) + assert key == StreamingScanKey(texts=("hello world",)) + + def test_events_without_text_leave_the_key_unchanged(self): + handler = A2AGuardrailHandler() + events = [_text_event("hello")] + assert handler.get_streaming_scan_key(events + [_status_event()]) == handler.get_streaming_scan_key(events) + + def test_unparseable_items_are_ignored(self): + key = A2AGuardrailHandler().get_streaming_scan_key([_text_event("hi"), "not json", b"bytes"]) + assert key.texts == ("hi",) diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 0fe7730e91e..3044a321aa6 100644 --- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -13,6 +13,7 @@ import pytest from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.base_llm.guardrail_translation.base_translation import StreamingScanKey from litellm.llms.anthropic.chat.guardrail_translation.handler import ( AnthropicMessagesHandler, ) @@ -1991,3 +1992,56 @@ class TestStructuredWriteBackKeepsToolResults: } later_blocks = [b for m in messages[tool_use_index + 1 :] for b in self._blocks(m)] assert {"type": "text", "text": "Now fetch the page."} in later_blocks + + +class TestAnthropicMessagesHandlerStreamingScanKey: + """get_streaming_scan_key mirrors what process_output_streaming_response would scan""" + + @staticmethod + def _sse(event_type, data): + return f"event: {event_type}\ndata: {json.dumps(data)}\n\n".encode() + + def _text_delta(self, text): + return self._sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}, + ) + + def test_key_is_empty_before_any_text_arrives(self): + head = self._sse("message_start", {"type": "message_start", "message": {"stop_reason": None}}) + key = AnthropicMessagesHandler().get_streaming_scan_key([head]) + assert key == StreamingScanKey(texts=("",)) + + def test_key_accumulates_text_deltas(self): + key = AnthropicMessagesHandler().get_streaming_scan_key([self._text_delta("hello "), self._text_delta("world")]) + assert key.texts == ("hello world",) + assert key.stream_ended is False + + def _stop(self, stop_reason): + return self._sse( + "message_delta", + {"type": "message_delta", "delta": {"stop_reason": stop_reason, "stop_sequence": None}, "usage": {}}, + ) + + def test_stop_without_tool_use_scans_the_same_payload(self): + handler = AnthropicMessagesHandler() + open_key = handler.get_streaming_scan_key([self._text_delta("hi")]) + ended_key = handler.get_streaming_scan_key([self._text_delta("hi"), self._stop("end_turn")]) + assert ended_key.stream_ended is True + assert ended_key == open_key + + def test_tool_use_blocks_enter_the_key_once_the_stream_has_ended(self): + handler = AnthropicMessagesHandler() + tool_use = self._sse( + "content_block_start", + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {}}, + }, + ) + open_key = handler.get_streaming_scan_key([self._text_delta("hi"), tool_use]) + ended_key = handler.get_streaming_scan_key([self._text_delta("hi"), tool_use, self._stop("tool_use")]) + assert open_key == StreamingScanKey(texts=("hi",)) + assert len(ended_key.tool_calls) == 1 and "get_weather" in ended_key.tool_calls[0] + assert ended_key != open_key diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 0f9f8259bef..8ea8db5fb65 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3127,6 +3127,31 @@ def test_reasoning_effort_accepts_dict_shape_for_non_adaptive_model( ) +@pytest.mark.parametrize( + "model,budget_tokens,expected", + [ + ("claude-opus-4-8", 4096, ({"type": "adaptive"}, {"effort": "high"})), + ("claude-opus-4-7", 24000, ({"type": "adaptive"}, {"effort": "xhigh"})), + ("claude-opus-4-6", 4096, ({"type": "enabled", "budget_tokens": 4096}, None)), + ("claude-sonnet-4-5-20250929", 4096, ({"type": "enabled", "budget_tokens": 4096}, None)), + ], +) +def test_legacy_thinking_translated_to_adaptive_on_adaptive_only_models(model, budget_tokens, expected): + """Adaptive-only models reject thinking={type: enabled} with a 400, so the + legacy shape must be upgraded to adaptive + output_config.effort on + /chat/completions too, while models that accept it keep the caller's budget.""" + config = AnthropicConfig() + + result = config.map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": budget_tokens}, "max_tokens": 64000}, + optional_params={}, + model=model, + drop_params=False, + ) + + assert (result["thinking"], result.get("output_config")) == expected + + @pytest.mark.parametrize( "bad_value", [ diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_stop_reason.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_stop_reason.py new file mode 100644 index 00000000000..4b95b36fec3 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_stop_reason.py @@ -0,0 +1,79 @@ +""" +Regression tests for issue #34692. + +ollama_chat streams tool_calls in a mid-stream chunk while its final +(``done: true``) chunk carries only ``done_reason: "stop"``. The provider +iterator must remember the earlier tool_calls and stamp +``finish_reason="tool_calls"`` on the final chunk, so the Anthropic +``/v1/messages`` bridge emits ``stop_reason: "tool_use"``. Before the fix the +bridge emitted ``stop_reason: "end_turn"`` and Anthropic tool-runners +(Claude Code, ``messages.stream``) silently dropped the tool call. +""" + +import pytest + +from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( + AnthropicStreamWrapper, +) +from litellm.llms.ollama.chat.transformation import ( + OllamaChatCompletionResponseIterator, +) +from litellm.types.utils import ModelResponseStream + +_OLLAMA_TOOL_CHUNK = { + "model": "qwen3:8b", + "message": { + "role": "assistant", + "content": "", + "tool_calls": [{"function": {"name": "get_weather", "arguments": {"city": "San Francisco"}}}], + }, + "done": False, +} +_OLLAMA_DONE_CHUNK = { + "model": "qwen3:8b", + "message": {"role": "assistant", "content": ""}, + "done": True, + "done_reason": "stop", + "prompt_eval_count": 100, + "eval_count": 20, +} + + +def _ollama_streamed_chunks() -> list[ModelResponseStream]: + iterator = OllamaChatCompletionResponseIterator(streaming_response=iter([]), sync_stream=True) + return [iterator.chunk_parser(_OLLAMA_TOOL_CHUNK), iterator.chunk_parser(_OLLAMA_DONE_CHUNK)] + + +class _AsyncStream: + def __init__(self, items: list[ModelResponseStream]): + self._it = iter(items) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._it) + except StopIteration: + raise StopAsyncIteration + + +def _assert_tool_use_stop_reason(events: list[dict]) -> None: + block_types = [e["content_block"]["type"] for e in events if e.get("type") == "content_block_start"] + assert "tool_use" in block_types, f"no tool_use content block opened: {events}" + message_deltas = [e for e in events if e.get("type") == "message_delta"] + assert message_deltas, f"no message_delta emitted: {events}" + assert message_deltas[-1]["delta"]["stop_reason"] == "tool_use", ( + f"expected stop_reason 'tool_use', got: {message_deltas[-1]}" + ) + + +def test_ollama_mid_stream_tool_call_yields_tool_use_stop_reason_sync(): + wrapper = AnthropicStreamWrapper(completion_stream=iter(_ollama_streamed_chunks()), model="qwen3:8b") + _assert_tool_use_stop_reason(list(wrapper)) + + +@pytest.mark.asyncio +async def test_ollama_mid_stream_tool_call_yields_tool_use_stop_reason_async(): + wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(_ollama_streamed_chunks()), model="qwen3:8b") + _assert_tool_use_stop_reason([event async for event in wrapper]) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index ad4c3d6bfbb..e819433c269 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -296,21 +296,15 @@ async def test_bedrock_converse_budget_tokens_preserved(): mock_acompletion.assert_called_once() call_kwargs = mock_acompletion.call_args.kwargs - print( - "acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str) - ) + print("acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str)) # Verify thinking parameter is passed through with budget_tokens preserved thinking_param = call_kwargs.get("thinking") - assert ( - thinking_param is not None - ), "thinking parameter should be passed to acompletion" - assert ( - thinking_param.get("type") == "enabled" - ), "thinking.type should be 'enabled'" - assert ( - thinking_param.get("budget_tokens") == 1024 - ), f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" + assert thinking_param is not None, "thinking parameter should be passed to acompletion" + assert thinking_param.get("type") == "enabled", "thinking.type should be 'enabled'" + assert thinking_param.get("budget_tokens") == 1024, ( + f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" + ) def test_openai_model_with_thinking_converts_to_reasoning(): @@ -342,23 +336,18 @@ def test_openai_model_with_thinking_converts_to_reasoning(): call_kwargs = mock_responses.call_args.kwargs # Verify reasoning is set (converted from thinking) - assert ( - "reasoning" in call_kwargs - ), "reasoning should be passed to litellm.responses" + assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses" # budget_tokens=1024 -> effort="low" (at the LOW budget threshold) # reasoning_auto_summary is False by default, so no summary key expected_reasoning = {"effort": "low"} assert call_kwargs["reasoning"] == expected_reasoning, ( - f"reasoning should be {expected_reasoning} for budget_tokens=1024, " - f"got {call_kwargs.get('reasoning')}" + f"reasoning should be {expected_reasoning} for budget_tokens=1024, got {call_kwargs.get('reasoning')}" ) assert "summary" not in call_kwargs["reasoning"] # Verify thinking is NOT passed directly to the Responses API - assert ( - "thinking" not in call_kwargs - ), "thinking should NOT be passed directly to litellm.responses" + assert "thinking" not in call_kwargs, "thinking should NOT be passed directly to litellm.responses" class TestThinkingParameterTransformation: @@ -411,9 +400,7 @@ class TestThinkingParameterTransformation: thinking=thinking, model="openai/gpt-5.2", ) - assert result == { - "reasoning_effort": {"effort": "high", "summary": "detailed"} - } + assert result == {"reasoning_effort": {"effort": "high", "summary": "detailed"}} finally: litellm.reasoning_auto_summary = original @@ -611,9 +598,9 @@ class TestThinkingSummaryPreservation: mock_responses.assert_called_once() call_kwargs = mock_responses.call_args.kwargs reasoning = call_kwargs["reasoning"] - assert ( - reasoning["summary"] == "concise" - ), f"Expected summary='concise', got summary='{reasoning.get('summary')}'" + assert reasoning["summary"] == "concise", ( + f"Expected summary='concise', got summary='{reasoning.get('summary')}'" + ) def test_responses_adapter_preserves_summary(self): """translate_thinking_to_reasoning should include summary when user provides it.""" @@ -622,9 +609,7 @@ class TestThinkingSummaryPreservation: ) thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"} - result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( - thinking - ) + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) assert result == {"effort": "high", "summary": "concise"} def test_responses_adapter_no_summary_by_default(self): @@ -638,11 +623,7 @@ class TestThinkingSummaryPreservation: try: litellm.reasoning_auto_summary = False thinking = {"type": "enabled", "budget_tokens": 5000} - result = ( - LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( - thinking - ) - ) + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) assert result == {"effort": "high"} assert result is not None and "summary" not in result finally: @@ -659,9 +640,7 @@ class TestThinkingSummaryPreservation: thinking=thinking, model="openai/gpt-5.2", ) - assert result == { - "reasoning_effort": {"effort": "high", "summary": "concise"} - } + assert result == {"reasoning_effort": {"effort": "high", "summary": "concise"}} def test_translate_thinking_for_model_disabled_stays_plain_string_when_auto_summary_enabled(self): """Disabled thinking must stay a plain string even when reasoning_auto_summary is on.""" @@ -807,9 +786,7 @@ def test_presanitized_flag_not_leaked_to_provider_params(): def fake_base_handler(*args, **kwargs): captured.update(kwargs) - captured["optional"] = kwargs.get( - "anthropic_messages_optional_request_params", {} - ) + captured["optional"] = kwargs.get("anthropic_messages_optional_request_params", {}) return "stub" with patch.object( @@ -974,6 +951,38 @@ def test_gate_passthrough_skipped_when_only_chat_completions_supported(monkeypat assert "config" not in captured +@pytest.mark.parametrize( + "model_info, expected_ttl_support", + [ + ({"supported_endpoints": ["/v1/messages"]}, False), + ({"supported_endpoints": ["/v1/messages"], "cache_control_ttl": True}, True), + ({"supported_endpoints": ["/v1/messages"], "cache_control_ttl": "yes"}, False), + ], +) +def test_gate_passthrough_forwards_cache_control_ttl_only_when_deployment_opts_in( + monkeypatch, model_info, expected_ttl_support +): + """The passthrough config strips cache_control.ttl unless the deployment sets + model_info.cache_control_ttl to exactly true.""" + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + anthropic_messages_handler, + ) + + captured, _ = _gate_stubs(monkeypatch) + + result = anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "Hello"}], + model="openai/some-model", + api_key="sk-test", + api_base="https://host/v1", + model_info=model_info, + ) + + assert result == "native-passthrough" + assert captured["config"].supports_cache_control_ttl() is expected_ttl_support + + def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag(): """Regional and provider-prefixed Claude 4.8+/5 entries carry ``supports_mid_conversation_system``, but the bare first-party keys @@ -987,9 +996,7 @@ def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_sys import litellm - cost_map_path = os.path.join( - os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json" - ) + cost_map_path = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json") with open(cost_map_path) as f: cost_map = json.load(f) rules = cost_map["fallback_generalizations"]["rules"] @@ -1028,9 +1035,7 @@ def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_sys ("perplexity/sonar", "sonar", "https://api.perplexity.ai/chat/completions"), ], ) -async def test_messages_strips_provider_prefix_exactly_once( - requested_model, expected_wire_model, expected_url -): +async def test_messages_strips_provider_prefix_exactly_once(requested_model, expected_wire_model, expected_url): """ BerriAI/litellm#37716: only the leading provider segment may be stripped on the way upstream. diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_transformation.py b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_transformation.py index 9dac914ca4d..9e2bfb08852 100644 --- a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_transformation.py +++ b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_transformation.py @@ -362,8 +362,8 @@ class TestAzureAnthropicConfig: ) assert "xhigh" in str(exc_info.value) - def test_extra_body_promotion_does_not_clobber_top_level(self): - """Top-level ``optional_params`` wins over duplicates in ``extra_body``.""" + def test_extra_body_promotion_overrides_mapped_top_level(self): + """The caller's ``extra_body`` wins over a mapped top-level duplicate, like the native ``anthropic`` passthrough.""" config = AzureAnthropicConfig() messages = [{"role": "user", "content": "Hello"}] @@ -383,7 +383,31 @@ class TestAzureAnthropicConfig: headers=headers, ) - assert result["output_config"] == {"effort": "low"} + assert result["output_config"] == {"effort": "high"} + + def test_legacy_thinking_upgrade_keeps_caller_effort_from_extra_body(self, local_model_cost_map): + config = AzureAnthropicConfig() + + mapped = config.map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 1024}, "max_tokens": 100}, + optional_params={}, + model="claude-opus-4-8", + drop_params=False, + ) + assert mapped["thinking"] == {"type": "adaptive"} + assert mapped["output_config"] == {"effort": "low"} + + result = config.transform_request( + model="claude-opus-4-8", + messages=[{"role": "user", "content": "Hello"}], + optional_params={**mapped, "extra_body": {"output_config": {"effort": "high"}}}, + litellm_params={"api_key": "test-key"}, + headers={"api-key": "test-key", "anthropic-version": "2023-06-01"}, + ) + + assert result["thinking"] == {"type": "adaptive"} + assert result["output_config"] == {"effort": "high"} + assert "extra_body" not in result def test_context_management_mixed_edits_beta_headers(self): """Test that context_management with both compact and other edits adds both beta headers""" diff --git a/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py b/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py index b924ea8f93f..222ead92b54 100644 --- a/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py +++ b/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py @@ -27,6 +27,20 @@ def mock_gov_env_vars(mock_env_vars, monkeypatch): monkeypatch.setenv("AZURE_STORAGE_ENDPOINT_SUFFIX", GOV_SUFFIX) +@pytest.fixture +def credential_chain_env_vars(monkeypatch): + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_NAME", "test-account") + monkeypatch.setenv("AZURE_STORAGE_FILE_SYSTEM", "test-container") + for name in ( + "AZURE_STORAGE_TENANT_ID", + "AZURE_STORAGE_CLIENT_ID", + "AZURE_STORAGE_CLIENT_SECRET", + "AZURE_STORAGE_ACCOUNT_KEY", + "AZURE_STORAGE_ENDPOINT_SUFFIX", + ): + monkeypatch.delenv(name, raising=False) + + def _make_backend() -> AzureBlobStorageBackend: backend = AzureBlobStorageBackend() backend.azure_auth_token = "mock-azure-ad-token" @@ -42,6 +56,29 @@ def _mock_upload_client() -> AsyncMock: return client +@pytest.mark.asyncio +async def test_upload_file_with_credential_chain(credential_chain_env_vars): + client = _mock_upload_client() + build_provider = MagicMock(return_value=lambda: "workload-identity-token") + + with patch( # test-quality-ok: the backend creates its REST client internally; assert the emitted authorization header + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=client + ): + backend = AzureBlobStorageBackend(build_credential_chain_token_provider=build_provider) + storage_url = await backend.upload_file( + file_content=b"hello", + filename="report.json", + content_type="application/json", + path_prefix="logs", + file_naming_strategy="original_filename", + ) + + build_provider.assert_called_once_with() + assert storage_url == "https://test-account.blob.core.windows.net/test-container/logs/report.json" + assert client.put.call_args[1]["headers"]["Authorization"] == "Bearer workload-identity-token" + assert client.patch.call_count == 2 + + @pytest.mark.parametrize( "env_fixture, expected_suffix", [("mock_env_vars", "core.windows.net"), ("mock_gov_env_vars", GOV_SUFFIX)], @@ -125,10 +162,7 @@ async def test_download_file_accepts_url_persisted_before_the_suffix_was_set(moc ) assert content == b"file-bytes" - assert ( - client.get.call_args[0][0] - == f"https://test-account.blob.{GOV_SUFFIX}/test-container/logs/report.json" - ) + assert client.get.call_args[0][0] == f"https://test-account.blob.{GOV_SUFFIX}/test-container/logs/report.json" @pytest.mark.parametrize( @@ -178,10 +212,51 @@ async def test_download_file_drops_query_string_from_the_stored_url(mock_env_var "https://test-account.blob.core.windows.net/test-container/logs/report.json?sig=redacted&se=2026" ) - assert ( - client.get.call_args[0][0] - == "https://test-account.blob.core.windows.net/test-container/logs/report.json" - ) + assert client.get.call_args[0][0] == "https://test-account.blob.core.windows.net/test-container/logs/report.json" + + +@pytest.mark.parametrize( + "malicious_filename", + [ + "report.jsonl/../../etc/cron.d/evil", + "a.b/../../../root/.ssh/authorized_keys", + ], +) +@pytest.mark.parametrize("strategy", ["uuid", "timestamp"]) +@pytest.mark.asyncio +async def test_generate_file_name_strips_path_traversal_from_extension(mock_env_vars, malicious_filename, strategy): + """ + original_filename.split(".")[-1] does not parse path structure, so a filename whose + last "." is followed by a directory traversal sequence used to put that sequence + straight into the blob path built from this name. The mutant this pins is reverting + _safe_extension() back to that bare split. + """ + backend = _make_backend() + generated = backend._generate_file_name(malicious_filename, strategy) + assert "/" not in generated + assert ".." not in generated + + +@pytest.mark.asyncio +async def test_generate_file_name_uuid_strategy_preserves_ordinary_extension(mock_env_vars): + backend = _make_backend() + generated = backend._generate_file_name("data.jsonl", "uuid") + assert generated.endswith(".jsonl") + + +@pytest.mark.asyncio +async def test_generate_file_name_original_filename_strategy_strips_directory_components(mock_env_vars): + """The blob name must never carry a directory the caller supplied, traversal or not.""" + backend = _make_backend() + generated = backend._generate_file_name("../../etc/passwd", "original_filename") + assert generated == "passwd" + + +@pytest.mark.asyncio +async def test_generate_file_name_null_byte_filename_falls_back_to_safe_default(mock_env_vars): + backend = _make_backend() + generated = backend._generate_file_name("report.pdf\x00.exe", "uuid") + assert "\x00" not in generated @pytest.mark.parametrize( diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 41d82e4f960..0d7573a2536 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -671,3 +671,46 @@ def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice assert "output_format" not in result assert "tools" in result assert "tool_choice" not in result + + +@pytest.mark.parametrize("model", ["us.anthropic.claude-sonnet-5", "us.anthropic.claude-fable-5-1"]) +def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_thinking(local_model_cost_map, model): + result = AmazonAnthropicClaudeConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "schema": {"type": "object", "properties": {"result": {"type": "string"}}}, + }, + }, + "thinking": {"type": "enabled", "budget_tokens": 4096}, + "max_tokens": 8192, + }, + optional_params={}, + model=model, + drop_params=False, + ) + + assert "tools" in result + assert result["thinking"] == {"type": "adaptive"} + assert result["output_config"] == {"effort": "high"} + + +def test_bedrock_chat_invoke_response_format_stub_still_upgrades_legacy_thinking(local_model_cost_map): + """Regression: the tool-based ``response_format`` path swaps in a Claude 3 stub + model before the shared Anthropic mapping, which hid the adaptive-only model + from the legacy ``thinking`` upgrade and left ``type=enabled`` on the wire.""" + result = AmazonAnthropicClaudeConfig().map_openai_params( + non_default_params={ + "response_format": {"type": "json_object"}, + "thinking": {"type": "enabled", "budget_tokens": 4096}, + "max_tokens": 8192, + }, + optional_params={}, + model="us.anthropic.claude-fable-5-1", + drop_params=False, + ) + + assert result["thinking"] == {"type": "adaptive"} + assert result["output_config"] == {"effort": "high"} diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 70f3153ed7e..cb05cdb9451 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -979,6 +979,34 @@ def test_config_blocks_do_not_leak_into_inference_config(): assert data["serviceTier"] == {"type": "priority"} +@pytest.mark.parametrize( + "model", + [ + "anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8", + "amazon.nova-pro-v1:0", + "us.meta.llama4-maverick-17b-instruct-v1:0", + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abcdef123456", + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.amazon.nova-pro-v1:0", + ], +) +def test_client_metadata_stripped_from_converse_request(model): + data = AmazonConverseConfig()._transform_request_helper( + model=model, + system_content_blocks=[], + optional_params={ + "maxTokens": 16, + "anthropic_beta": ["computer-use-2025-01-24"], + "client_metadata": {"originator": "codex_cli_rs"}, + }, + messages=None, + ) + + fields = data["additionalModelRequestFields"] + assert "client_metadata" not in fields + assert fields["anthropic_beta"] == ["computer-use-2025-01-24"] + + def test_parallel_tool_calls_config_kept_for_sonnet_5(monkeypatch): old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") old_cost = litellm.model_cost @@ -6347,6 +6375,82 @@ def test_adaptive_thinking_passes_through_on_46_plus_converse(model): assert optional_params.get("thinking") == {"type": "adaptive"} +@pytest.mark.parametrize( + "model,budget_tokens,expected_effort", + [ + ("anthropic.claude-opus-4-8", 4096, "high"), + ("us.anthropic.claude-opus-4-8", 2000, "low"), + ("global.anthropic.claude-opus-4-8", 12000, "xhigh"), + ("us.anthropic.claude-opus-4-7", 3000, "medium"), + ("anthropic.claude-fable-5", 4096, "high"), + ], +) +def test_legacy_thinking_translated_to_adaptive_on_adaptive_only_converse(model, budget_tokens, expected_effort): + """Adaptive-only models (4.7+, 5 families) reject thinking={type: enabled} + with a 400 on Bedrock Converse, so the legacy shape from callers like Claude + Code must be upgraded to thinking={type: adaptive} + output_config.effort + derived from budget_tokens, matching the /v1/messages passthrough.""" + config = AmazonConverseConfig() + + optional_params = config.map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": budget_tokens}, "max_tokens": 64000}, + optional_params={}, + model=model, + drop_params=False, + ) + request = config.transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request["additionalModelRequestFields"]["thinking"] == {"type": "adaptive"} + assert request["additionalModelRequestFields"]["output_config"] == {"effort": expected_effort} + + +def test_legacy_thinking_translation_keeps_caller_output_config_effort_converse(): + config = AmazonConverseConfig() + + optional_params = config.map_openai_params( + non_default_params={ + "output_config": {"effort": "low"}, + "thinking": {"type": "enabled", "budget_tokens": 12000}, + "max_tokens": 64000, + }, + optional_params={}, + model="anthropic.claude-opus-4-8", + drop_params=False, + ) + + assert optional_params["thinking"] == {"type": "adaptive"} + assert optional_params["output_config"] == {"effort": "low"} + + +@pytest.mark.parametrize( + "model", + [ + "us.anthropic.claude-opus-4-6", + "anthropic.claude-3-5-sonnet-20241022-v2:0", + ], +) +def test_legacy_thinking_forwarded_verbatim_when_model_accepts_it_converse(model): + """The 4.6 family and pre-adaptive models accept thinking={type: enabled} + natively, so the caller's budget_tokens cap must keep applying.""" + config = AmazonConverseConfig() + + optional_params = config.map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 4096}, "max_tokens": 8192}, + optional_params={}, + model=model, + drop_params=False, + ) + + assert optional_params["thinking"] == {"type": "enabled", "budget_tokens": 4096} + assert "output_config" not in optional_params + + def test_adaptive_thinking_dropped_when_max_tokens_too_small_converse(): """When max_tokens can't fit even the minimum thinking budget, the raw adaptive block must be dropped entirely rather than translated, so the diff --git a/tests/test_litellm/llms/bedrock/test_mantle.py b/tests/test_litellm/llms/bedrock/test_mantle.py index d34517f61f6..09be2118001 100644 --- a/tests/test_litellm/llms/bedrock/test_mantle.py +++ b/tests/test_litellm/llms/bedrock/test_mantle.py @@ -128,6 +128,12 @@ def test_mantle_messages_url_construction(): _VPC_ENDPOINT = "https://vpce-0a1b2c3d.bedrock-mantle.us-gov-west-1.vpce.amazonaws.com" +@pytest.fixture(autouse=True) +def no_ambient_mantle_api_base(monkeypatch): + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + + + def test_mantle_chat_url_honors_api_base_host(): config = AmazonMantleConfig() url = config.get_complete_url( @@ -193,6 +199,48 @@ def test_mantle_messages_url_honors_aws_bedrock_runtime_endpoint(): assert url == f"{_VPC_ENDPOINT}/anthropic/v1/messages" +_ENV_ENDPOINT = "https://bedrock-mantle.us-east-1.api.aws.internal.example.com" + + +@pytest.mark.parametrize("config_cls", [AmazonMantleConfig, AmazonMantleMessagesConfig]) +@pytest.mark.parametrize( + "env_value", + [_ENV_ENDPOINT, f"{_ENV_ENDPOINT}/", f"{_ENV_ENDPOINT}/v1", f"{_ENV_ENDPOINT}/openai/v1"], +) +def test_mantle_url_honors_bedrock_mantle_api_base_env(monkeypatch, config_cls, env_value): + monkeypatch.setenv("BEDROCK_MANTLE_API_BASE", env_value) + url = config_cls().get_complete_url( + api_base=None, + api_key=None, + model="mantle/anthropic.claude-mythos-preview", + optional_params={"aws_region_name": "us-east-1"}, + litellm_params={}, + ) + assert url == f"{_ENV_ENDPOINT}/anthropic/v1/messages" + + +@pytest.mark.parametrize("config_cls", [AmazonMantleConfig, AmazonMantleMessagesConfig]) +@pytest.mark.parametrize( + ("api_base", "optional_params"), + [ + (_VPC_ENDPOINT, {"aws_region_name": "us-gov-west-1"}), + (None, {"aws_region_name": "us-gov-west-1", "aws_bedrock_runtime_endpoint": _VPC_ENDPOINT}), + ], +) +def test_mantle_url_explicit_endpoint_beats_bedrock_mantle_api_base_env( + monkeypatch, config_cls, api_base, optional_params +): + monkeypatch.setenv("BEDROCK_MANTLE_API_BASE", _ENV_ENDPOINT) + url = config_cls().get_complete_url( + api_base=api_base, + api_key=None, + model="mantle/anthropic.claude-mythos-preview", + optional_params=optional_params, + litellm_params={}, + ) + assert url == f"{_VPC_ENDPOINT}/anthropic/v1/messages" + + def test_mantle_transform_request_strips_prefix_and_adds_model(): config = AmazonMantleConfig() request = config.transform_request( diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index cd775abf136..56b111f294e 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -486,6 +486,71 @@ class TestBedrockMantleChatAuth: assert "/us-east-2/bedrock/aws4_request" in authorization assert requests[0]["url"].startswith("https://bedrock-mantle.us-east-2.api.aws") + def test_completion_per_request_role_reaches_signer_and_not_the_body(self, monkeypatch): + from unittest.mock import MagicMock, Mock + + from botocore.credentials import Credentials + + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + from litellm.llms.custom_httpx.http_handler import HTTPHandler + from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler + from litellm.types.utils import ModelResponse + + for var in ("BEDROCK_MANTLE_API_KEY", "AWS_BEARER_TOKEN_BEDROCK", "BEDROCK_MANTLE_API_BASE"): + monkeypatch.delenv(var, raising=False) + + signer = BaseAWSLLM() + signer.get_credentials = MagicMock( + return_value=Credentials( + access_key="ASIAEXAMPLE", + secret_key="YXNzdW1lZC1yb2xlLXNlY3JldC1hc3N1bWVk", + token="assumed-session-token", + ) + ) + url = "https://bedrock-mantle.us-east-1.api.aws/openai/v1/chat/completions" + client = HTTPHandler(client=httpx.Client()) + client.post = Mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1733529600, + "model": "google.gemma-4-31b", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + request=httpx.Request("POST", url), + ) + ) + + BaseLLMHTTPHandler().completion( + model="google.gemma-4-31b", + messages=[{"role": "user", "content": "hello"}], + api_base=None, + custom_llm_provider="bedrock_mantle", + model_response=ModelResponse(), + encoding=None, + logging_obj=Mock(), + optional_params={}, + timeout=10, + litellm_params={ + "aws_role_name": "arn:aws:iam::000000000000:role/attributed-role", + "aws_session_name": "user-123", + "aws_region_name": "us-east-1", + }, + acompletion=False, + client=client, + provider_config=BedrockMantleChatConfig(aws_signer=signer), + ) + + credential_kwargs = signer.get_credentials.call_args.kwargs + assert credential_kwargs["aws_role_name"] == "arn:aws:iam::000000000000:role/attributed-role" + assert credential_kwargs["aws_session_name"] == "user-123" + sent = client.post.call_args.kwargs + assert sent["headers"]["Authorization"].startswith("AWS4-HMAC-SHA256") + assert not [key for key in json.loads(sent["data"]) if key.startswith("aws_")] + class TestBedrockMantleProjectHeader: def test_validate_environment_sets_openai_project_header(self): diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index 16d57437043..9e64bfafa54 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -1314,3 +1314,236 @@ async def test_finalizer_on_live_loop_disposes_foreign_loop_session_without_sche assert AsyncHTTPHandler._finalizer_close_tasks == baseline_tasks assert session.closed + + +@pytest.fixture +def forward_proxy_server(): + """Plain HTTP forward proxy that records the absolute URIs it is asked to fetch.""" + from http.server import BaseHTTPRequestHandler, HTTPServer + from socketserver import ThreadingMixIn + + seen_uris: list[str] = [] + + class RecordingProxyHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self): + seen_uris.append(self.path) + self.send_response(200) + self.send_header("Content-Length", "9") + self.end_headers() + self.wfile.write(b"via-proxy") + + def log_message(self, format, *args): + pass + + class ThreadedServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + + server = ThreadedServer(("127.0.0.1", 0), RecordingProxyHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", seen_uris + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +# `.invalid` never resolves (RFC 6761), so the only way this request can succeed is through the proxy +_PROXY_ONLY_UPSTREAM_URL = "http://upstream.invalid/v1/models" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("disable_aiohttp_transport", [True, False]) +@pytest.mark.parametrize("force_ipv4", [True, False]) +async def test_async_handler_honours_proxy_env_for_every_transport( + forward_proxy_server, monkeypatch: pytest.MonkeyPatch, disable_aiohttp_transport: bool, force_ipv4: bool +): + proxy_url, seen_uris = forward_proxy_server + monkeypatch.setenv("HTTP_PROXY", proxy_url) + monkeypatch.delenv("NO_PROXY", raising=False) + monkeypatch.delenv("no_proxy", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", disable_aiohttp_transport) + monkeypatch.setattr(litellm, "force_ipv4", force_ipv4) + + handler = AsyncHTTPHandler() + try: + response = await handler.get(_PROXY_ONLY_UPSTREAM_URL) + finally: + await handler.close() + + assert response.text == "via-proxy" + assert seen_uris == [_PROXY_ONLY_UPSTREAM_URL] + + +@pytest.mark.parametrize("force_ipv4", [True, False]) +def test_sync_handler_honours_proxy_env(forward_proxy_server, monkeypatch: pytest.MonkeyPatch, force_ipv4: bool): + proxy_url, seen_uris = forward_proxy_server + monkeypatch.setenv("HTTP_PROXY", proxy_url) + monkeypatch.delenv("NO_PROXY", raising=False) + monkeypatch.delenv("no_proxy", raising=False) + monkeypatch.setattr(litellm, "force_ipv4", force_ipv4) + + handler = HTTPHandler() + try: + response = handler.get(_PROXY_ONLY_UPSTREAM_URL) + finally: + handler.close() + + assert response.text == "via-proxy" + assert seen_uris == [_PROXY_ONLY_UPSTREAM_URL] + + +@pytest.mark.asyncio +async def test_force_ipv4_httpx_transport_honours_no_proxy(keepalive_server, monkeypatch: pytest.MonkeyPatch): + """NO_PROXY hosts must still go direct when the proxy mounts are supplied by litellm instead of httpx.""" + monkeypatch.setenv("HTTP_PROXY", "http://proxy.invalid:3128") + monkeypatch.setenv("NO_PROXY", "127.0.0.1") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "force_ipv4", True) + + handler = AsyncHTTPHandler() + try: + response = await handler.get(keepalive_server) + finally: + await handler.close() + + assert response.text == "ok" + + +@pytest.fixture +def private_ca_tls_upstream(tmp_path: pathlib.Path): + """HTTPS server behind a CONNECT proxy, both on localhost; the server's cert is signed by a test-only CA.""" + import datetime + import select + import socket + from http.server import BaseHTTPRequestHandler, HTTPServer + from socketserver import ThreadingMixIn + + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import NameOID + + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "upstream.invalid")]) + now = datetime.datetime.now(datetime.timezone.utc) + cert = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(minutes=1)) + .not_valid_after(now + datetime.timedelta(hours=1)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("upstream.invalid")]), critical=False) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .sign(key, hashes.SHA256()) + ) + ca_pem = tmp_path / "ca.pem" + ca_pem.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_pem = tmp_path / "key.pem" + key_pem.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ) + ) + + class OkTlsHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self): + self.send_response(200) + self.send_header("Content-Length", "6") + self.end_headers() + self.wfile.write(b"ok-tls") + + def log_message(self, format, *args): + pass + + class ThreadedServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + + tls_server = ThreadedServer(("127.0.0.1", 0), OkTlsHandler) + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_ctx.load_cert_chain(str(ca_pem), str(key_pem)) + tls_server.socket = server_ctx.wrap_socket(tls_server.socket, server_side=True) + tls_port = tls_server.server_port + + class ConnectProxyHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_CONNECT(self): + upstream = socket.create_connection(("127.0.0.1", tls_port)) + self.send_response(200, "Connection established") + self.end_headers() + sockets = [self.connection, upstream] + while True: + readable, _, _ = select.select(sockets, [], [], 5) + if not readable: + break + for src in readable: + data = src.recv(65536) + if not data: + upstream.close() + return + (upstream if src is self.connection else self.connection).sendall(data) + + def log_message(self, format, *args): + pass + + proxy_server = ThreadedServer(("127.0.0.1", 0), ConnectProxyHandler) + threads = [ + threading.Thread(target=tls_server.serve_forever, daemon=True), + threading.Thread(target=proxy_server.serve_forever, daemon=True), + ] + for thread in threads: + thread.start() + try: + yield f"http://127.0.0.1:{proxy_server.server_port}", str(ca_pem) + finally: + for server in (proxy_server, tls_server): + server.shutdown() + server.server_close() + for thread in threads: + thread.join(timeout=5) + + +@pytest.mark.asyncio +async def test_force_ipv4_https_proxy_mount_uses_handler_ca_bundle( + private_ca_tls_upstream, monkeypatch: pytest.MonkeyPatch +): + proxy_url, ca_pem = private_ca_tls_upstream + monkeypatch.setenv("HTTPS_PROXY", proxy_url) + monkeypatch.delenv("NO_PROXY", raising=False) + monkeypatch.delenv("no_proxy", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "force_ipv4", True) + + handler = AsyncHTTPHandler(ssl_verify=ca_pem) + try: + response = await handler.get("https://upstream.invalid/v1/models") + finally: + await handler.close() + + assert response.text == "ok-tls" + + +def test_sync_force_ipv4_https_proxy_mount_uses_handler_ca_bundle( + private_ca_tls_upstream, monkeypatch: pytest.MonkeyPatch +): + proxy_url, ca_pem = private_ca_tls_upstream + monkeypatch.setenv("HTTPS_PROXY", proxy_url) + monkeypatch.delenv("NO_PROXY", raising=False) + monkeypatch.delenv("no_proxy", raising=False) + monkeypatch.setattr(litellm, "force_ipv4", True) + + handler = HTTPHandler(ssl_verify=ca_pem) + try: + response = handler.get("https://upstream.invalid/v1/models") + finally: + handler.close() + + assert response.text == "ok-tls" diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 26f841c1146..1d583c16ad7 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -2295,6 +2295,25 @@ async def test_anthropic_invalid_thinking_signature_retry_resigns_bedrock_reques assert retry_authorization != first_attempt_headers["Authorization"] +def test_aws_signing_overrides_only_fills_missing_credentials(): + from litellm.llms.custom_httpx.llm_http_handler import _aws_signing_overrides + + overrides = _aws_signing_overrides( + {"temperature": 0.2, "aws_region_name": "us-west-2"}, + { + "aws_role_name": "arn:aws:iam::000000000000:role/attributed", + "aws_session_name": "user-123", + "aws_region_name": "us-east-1", + "api_key": "not-an-aws-param", + }, + ) + + assert dict(overrides) == { + "aws_role_name": "arn:aws:iam::000000000000:role/attributed", + "aws_session_name": "user-123", + } + + class TestServerFulfilledToolsInRequest: """_server_fulfilled_tools_in_request gates the buffered (non-leaking) streaming mode for server-fulfilled tools like headroom_retrieve.""" diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py index 41fb2589655..71661cc532b 100644 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -422,6 +422,27 @@ def test_databricks_config_probes_capabilities_under_databricks_namespace(): assert DatabricksConfig().custom_llm_provider == "databricks" +@pytest.mark.parametrize( + "model, expected_thinking, expected_output_config", + [ + ("databricks-claude-opus-4-8", {"type": "adaptive"}, {"effort": "high"}), + ("databricks-claude-opus-4-6", {"type": "enabled", "budget_tokens": 4096}, None), + ], + ids=["adaptive_only_upgrades_to_adaptive", "legacy_capable_forwards_verbatim"], +) +def test_map_openai_params_upgrades_legacy_thinking_on_adaptive_only_claude( + model, expected_thinking, expected_output_config +): + mapped = DatabricksConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 4096}}, + optional_params={}, + model=model, + drop_params=False, + ) + assert mapped["thinking"] == expected_thinking + assert mapped.get("output_config") == expected_output_config + + def _streaming_chunk(usage=None, choices=None): base = { "id": "chatcmpl-test", diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index d7cc89868af..ec8725db5f7 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -8,6 +8,7 @@ import litellm from litellm import get_model_info, supports_reasoning, supports_vision from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -235,6 +236,21 @@ def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id(): ) +def test_get_fireworks_session_id_ignores_proxy_generated_session_id(): + """general_settings.missing_session_id: generate stamps a fresh id per request; sending it + as x-session-affinity would pin every request to a different node.""" + assert ( + get_fireworks_session_id( + { + "litellm_session_id": "generated-1", + "litellm_trace_id": "generated-1", + "metadata": {"session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True}, + } + ) + is None + ) + + def test_handle_message_content_with_tool_calls(): config = FireworksAIConfig() message = Message( diff --git a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py index e6e6aa946d5..9a62fcf6f0f 100644 --- a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py @@ -1,8 +1,14 @@ +import json import os import sys +from typing import Final +from unittest.mock import MagicMock, patch +import httpx import pytest +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig from litellm.rerank_api.rerank_utils import get_optional_rerank_params from litellm.types.rerank import ( @@ -87,9 +93,7 @@ class TestHostedVLLMRerankTransform: assert "instruction" not in body def test_map_cohere_rerank_params_raises_on_max_chunks_per_doc(self): - with pytest.raises( - ValueError, match="Hosted VLLM does not support max_chunks_per_doc" - ): + with pytest.raises(ValueError, match="Hosted VLLM does not support max_chunks_per_doc"): self.config.map_cohere_rerank_params( non_default_params=None, model=self.model, @@ -104,12 +108,10 @@ class TestHostedVLLMRerankTransform: url = self.config.get_complete_url(base, self.model) assert url == "https://api.example.com/rerank" # Already ends with /rerank - url2 = self.config.get_complete_url( - "https://api.example.com/rerank", self.model - ) + url2 = self.config.get_complete_url("https://api.example.com/rerank", self.model) assert url2 == "https://api.example.com/rerank" # Raises if api_base is None - with pytest.raises(ValueError, match='api_base must be provided for Hosted VLLM rerank'): + with pytest.raises(ValueError, match="api_base must be provided for Hosted VLLM rerank"): self.config.get_complete_url(None, self.model) def test_transform_response(self): @@ -173,3 +175,121 @@ class TestGetOptionalRerankParamsInstruction: documents=["doc1", "doc2"], ) assert "instruction" not in params + + +class TestHostedVLLMRerankTruncationParams: + def setup_method(self): + self.config = HostedVLLMRerankConfig() + self.model = "hosted-vllm-model" + + def test_map_cohere_rerank_params_forwards_vllm_truncation_params(self): + params: Final = self.config.map_cohere_rerank_params( + non_default_params={ + "truncate_prompt_tokens": 512, + "truncation_side": "left", + "max_tokens_per_query": 64, + "metadata": {"user_api_key": "sk-test"}, + }, + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + max_tokens_per_doc=128, + ) + assert params["truncate_prompt_tokens"] == 512 + assert params["truncation_side"] == "left" + assert params["max_tokens_per_query"] == 64 + assert params["max_tokens_per_doc"] == 128 + assert "metadata" not in params + + @pytest.mark.parametrize( + "bad_params", + [{"truncation_side": "middle"}, {"truncate_prompt_tokens": "lots"}, {"max_tokens_per_query": -1.5}], + ) + def test_map_cohere_rerank_params_rejects_invalid_truncation_params_as_400(self, bad_params: dict[str, object]): + with pytest.raises(litellm.UnsupportedParamsError) as raised: + self.config.map_cohere_rerank_params( + non_default_params=dict(bad_params), + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + ) + assert raised.value.status_code == 400 + assert next(iter(bad_params)) in str(raised.value) + + def test_map_cohere_rerank_params_omits_truncation_params_when_absent(self): + params: Final = self.config.map_cohere_rerank_params( + non_default_params={"metadata": {"user_api_key": "sk-test"}}, + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + ) + body: Final = self.config.transform_rerank_request(model=self.model, optional_rerank_params=params, headers={}) + truncation_keys: Final = { + "truncate_prompt_tokens", + "truncation_side", + "max_tokens_per_query", + "max_tokens_per_doc", + } + assert not truncation_keys & body.keys() + assert body == { + "model": self.model, + "query": "test query", + "documents": ["doc1", "doc2"], + "return_documents": True, + } + + def test_transform_request_forwards_truncation_params(self): + body: Final = self.config.transform_rerank_request( + model=self.model, + optional_rerank_params={ + "query": "test query", + "documents": ["doc1", "doc2"], + "truncate_prompt_tokens": 512, + "truncation_side": "left", + "max_tokens_per_query": 64, + "max_tokens_per_doc": 128, + }, + headers={}, + ) + assert body["truncate_prompt_tokens"] == 512 + assert body["truncation_side"] == "left" + assert body["max_tokens_per_query"] == 64 + assert body["max_tokens_per_doc"] == 128 + + def test_transform_request_omits_truncation_params_when_absent(self): + body: Final = self.config.transform_rerank_request( + model=self.model, + optional_rerank_params={"query": "test query", "documents": ["doc1", "doc2"]}, + headers={}, + ) + assert "truncate_prompt_tokens" not in body + assert "truncation_side" not in body + assert "max_tokens_per_query" not in body + assert "max_tokens_per_doc" not in body + + def test_rerank_sends_truncate_prompt_tokens_to_vllm(self): + client: Final = HTTPHandler() + mock_response: Final = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "id": "score-1", + "results": [{"index": 0, "relevance_score": 0.5}], + "usage": {"total_tokens": 512}, + } + with patch.object(client, "post", return_value=mock_response) as mock_post: + litellm.rerank( + model="hosted_vllm/BAAI/bge-reranker-base", + api_base="http://vllm.local:8000", + query="List all the unique case ids", + documents=["a document longer than the reranker context window"], + truncate_prompt_tokens=512, + truncation_side="left", + client=client, + ) + sent_body: Final = json.loads(mock_post.call_args.kwargs["data"]) + assert mock_post.call_args.kwargs["url"] == "http://vllm.local:8000/rerank" + assert sent_body["truncate_prompt_tokens"] == 512 + assert sent_body["truncation_side"] == "left" diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index 8f3dbf7b0d9..25f9645faa0 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -615,6 +615,46 @@ class TestOllamaFinishReasonLength: result.choices[0].finish_reason == "stop" ), f"Expected 'stop' for natural finish, got '{result.choices[0].finish_reason}'" + def test_finish_reason_tool_calls_streamed_before_done_chunk(self): + """Streaming: tool_calls arriving mid-stream (not on the done chunk) must + still produce finish_reason='tool_calls' on the final chunk. + + Regression test for https://github.com/BerriAI/litellm/issues/34692: + Ollama emits tool_calls in an earlier chunk and the done chunk carries + none, which left finish_reason at 'stop' and made the Anthropic + /v1/messages bridge emit stop_reason 'end_turn' instead of 'tool_use'. + """ + iterator = OllamaChatCompletionResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + tool_chunk = { + "model": "qwen3:8b", + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + {"function": {"name": "get_weather", "arguments": {"city": "San Francisco"}}} + ], + }, + "done": False, + } + done_chunk = { + "model": "qwen3:8b", + "message": {"role": "assistant", "content": ""}, + "done": True, + "done_reason": "stop", + } + + tool_result = iterator.chunk_parser(tool_chunk) + assert tool_result.choices[0].delta.tool_calls is not None + + done_result = iterator.chunk_parser(done_chunk) + assert ( + done_result.choices[0].finish_reason == "tool_calls" + ), f"Expected 'tool_calls' when tool_calls were streamed earlier, got '{done_result.choices[0].finish_reason}'" + class TestOllamaReasoningContentStreaming: """Test that reasoning_content is properly extracted from all thinking chunks.""" diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 7dd6065063a..cebab2512d0 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -12,6 +12,7 @@ import pytest from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.base_llm.guardrail_translation.base_translation import StreamingScanKey from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, ) @@ -1643,3 +1644,74 @@ class TestCheckStreamingHasEnded: ) ] assert handler._check_streaming_has_ended(chunks) is True + + +class TestStreamingScanKey: + """get_streaming_scan_key identifies what a sampled round would scan so the + unified hook can skip rounds that would re-scan already-cleared text""" + + @staticmethod + def _chunk(content, finish_reason=None, index=0): + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + return ModelResponseStream( + choices=[StreamingChoices(index=index, delta=Delta(content=content), finish_reason=finish_reason)] + ) + + def test_key_carries_accumulated_text_and_open_stream(self): + handler = OpenAIChatCompletionsHandler() + key = handler.get_streaming_scan_key([self._chunk("hel"), self._chunk("lo")]) + assert key == StreamingScanKey(texts=("hello",)) + + def test_chunks_without_text_leave_the_key_unchanged(self): + handler = OpenAIChatCompletionsHandler() + before = handler.get_streaming_scan_key([self._chunk("hel"), self._chunk("lo")]) + after = handler.get_streaming_scan_key([self._chunk("hel"), self._chunk("lo"), self._chunk(None)]) + assert after == before + + def test_finish_chunk_without_tool_calls_scans_the_same_payload(self): + handler = OpenAIChatCompletionsHandler() + open_key = handler.get_streaming_scan_key([self._chunk("hi")]) + ended_key = handler.get_streaming_scan_key([self._chunk("hi"), self._chunk(None, finish_reason="stop")]) + assert open_key.stream_ended is False + assert ended_key.stream_ended is True + assert ended_key == open_key + + def test_tool_calls_only_enter_the_key_once_the_stream_has_ended(self): + from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponseStream, + StreamingChoices, + ) + + handler = OpenAIChatCompletionsHandler() + tool_call = ChatCompletionDeltaToolCall( + id="call_1", index=0, type="function", function=Function(name="get_weather", arguments='{"city": "Paris"}') + ) + tool_chunk = ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content=None, tool_calls=[tool_call]), finish_reason=None)] + ) + open_key = handler.get_streaming_scan_key([self._chunk("hi"), tool_chunk]) + ended_key = handler.get_streaming_scan_key( + [self._chunk("hi"), tool_chunk, self._chunk(None, finish_reason="stop")] + ) + assert open_key == StreamingScanKey(texts=("hi",)) + assert ended_key.texts == ("hi",) + assert len(ended_key.tool_calls) == 1 and "get_weather" in ended_key.tool_calls[0] + assert ended_key != open_key + + def test_text_after_the_first_choice_finishes_still_changes_the_key(self): + handler = OpenAIChatCompletionsHandler() + first_done = [self._chunk("a", index=0), self._chunk("b", finish_reason="stop", index=0)] + key_at_first_finish = handler.get_streaming_scan_key(first_done) + key_after_more_text = handler.get_streaming_scan_key(first_done + [self._chunk("y", index=1)]) + assert key_at_first_finish.stream_ended is True + assert key_after_more_text.stream_ended is True + assert key_after_more_text != key_at_first_finish + + def test_non_stream_items_are_ignored(self): + handler = OpenAIChatCompletionsHandler() + key = handler.get_streaming_scan_key([self._chunk("hi"), b"data: [DONE]"]) + assert key.texts == ("hi",) diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index 315b6948bd8..295121167d6 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -5,6 +5,8 @@ Tests the handler's ability to process input/output for the Responses API with guardrail transformations. """ +import copy +from collections.abc import Callable from typing import Any, List, Literal, Optional, Tuple from unittest.mock import AsyncMock, MagicMock @@ -19,6 +21,10 @@ from litellm.llms import get_guardrail_translation_mapping from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, ) +from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools +from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, +) from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.responses.main import GenericResponseOutputItem, OutputText from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs @@ -1287,14 +1293,14 @@ class TestOpenAIResponsesHandlerToolInjection: """A tool a guardrail injects must survive the write-back to Responses format.""" def test_merge_keeps_guardrail_appended_tool(self): - """_merge_tools_after_guardrail must not drop the extra appended tool.""" - handler = OpenAIResponsesHandler() + """merge_guardrailed_tools must not drop the extra appended tool.""" original = [{"type": "function", "name": "a"}] - remapped = [ - {"type": "function", "name": "a"}, - {"type": "function", "name": "b"}, + groups = [form.chat_tools for form in LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(original)] + guardrailed = [ + *groups[0], + {"type": "function", "function": {"name": "b", "description": "", "parameters": {"type": "object"}}}, ] - merged = handler._merge_tools_after_guardrail(original, remapped) + merged = merge_guardrailed_tools(original, groups, guardrailed) assert [t["name"] for t in merged] == ["a", "b"] @pytest.mark.asyncio @@ -1323,6 +1329,194 @@ class TestOpenAIResponsesHandlerToolInjection: assert "injected_tool" in names +class ToolEditingGuardrail(CustomGuardrail): + """Guardrail that rewrites the flattened chat tools it was handed through ``edit``""" + + def __init__(self, edit: Callable[[list[dict]], list[dict]], **kwargs): + super().__init__(**kwargs) + self.edit = edit + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Any | None = None, + ) -> GenericGuardrailAPIInputs: + inputs["tools"] = self.edit(list(inputs.get("tools") or [])) + return inputs + + +def _codex_request(input_value): + """A Responses API request shaped like what the Codex CLI sends when an MCP server is configured""" + return { + "model": "gpt-5.3-codex", + "input": input_value, + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Weather lookup", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + "strict": False, + }, + { + "type": "namespace", + "name": "mcp__confluence", + "description": "Confluence tools", + "tools": [ + { + "type": "function", + "name": "confluence_get_page", + "description": "Get a page", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + "strict": False, + }, + { + "type": "function", + "name": "confluence_search", + "description": "Search pages", + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, + "strict": False, + }, + ], + }, + { + "type": "custom", + "name": "apply_patch", + "description": "Apply a patch", + "format": {"type": "grammar", "syntax": "lark", "definition": 'start: "x"'}, + }, + {"type": "web_search"}, + ], + } + + +def _tool_named(tools, name): + return next(tool for tool in tools if tool.get("name") == name) + + +class TestOpenAIResponsesHandlerNamespaceTools: + """Codex sends MCP tools as ``namespace`` tools; a guardrail must never flatten them (GH #39183)""" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "input_value", + ["hi", [{"role": "user", "content": "hi", "type": "message"}]], + ids=["string_input", "list_input"], + ) + async def test_pass_through_guardrail_leaves_tools_untouched(self, input_value): + data = _codex_request(input_value) + expected_tools = copy.deepcopy(data["tools"]) + + result = await OpenAIResponsesHandler().process_input_messages( + data, MockPassThroughGuardrail(guardrail_name="test") + ) + + assert result["tools"] == expected_tools + + @pytest.mark.asyncio + async def test_appending_guardrail_keeps_namespace_and_adds_tool(self): + data = _codex_request("hi") + expected_tools = copy.deepcopy(data["tools"]) + + result = await OpenAIResponsesHandler().process_input_messages( + data, ToolAppendingGuardrail(guardrail_name="test") + ) + + assert result["tools"][:-1] == expected_tools + assert result["tools"][-1]["type"] == "function" + assert result["tools"][-1]["name"] == "injected_tool" + + @pytest.mark.asyncio + async def test_dropping_one_member_prunes_only_that_member(self): + data = _codex_request("hi") + expected_tools = copy.deepcopy(data["tools"]) + guardrail = ToolEditingGuardrail( + edit=lambda tools: [t for t in tools if t["function"]["name"] != "mcp__confluence__confluence_search"], + guardrail_name="test", + ) + + result = await OpenAIResponsesHandler().process_input_messages(data, guardrail) + + namespace = _tool_named(result["tools"], "mcp__confluence") + assert [member["name"] for member in namespace["tools"]] == ["confluence_get_page"] + assert namespace["tools"][0] == expected_tools[1]["tools"][0] + assert [t for t in result["tools"] if t is not namespace] == [expected_tools[0], *expected_tools[2:]] + + @pytest.mark.asyncio + async def test_editing_a_member_lands_on_that_member_without_the_namespace_prefix(self): + data = _codex_request("hi") + expected_tools = copy.deepcopy(data["tools"]) + + def redact_search(tools): + for tool in tools: + if tool["function"]["name"] == "mcp__confluence__confluence_search": + tool["function"]["description"] = "Confluence tools\n\nREDACTED" + return tools + + result = await OpenAIResponsesHandler().process_input_messages( + data, ToolEditingGuardrail(edit=redact_search, guardrail_name="test") + ) + + namespace = _tool_named(result["tools"], "mcp__confluence") + assert namespace["tools"][0] == expected_tools[1]["tools"][0] + assert namespace["tools"][1] == {**expected_tools[1]["tools"][1], "description": "REDACTED"} + assert {k: v for k, v in namespace.items() if k != "tools"} == { + k: v for k, v in expected_tools[1].items() if k != "tools" + } + + @pytest.mark.asyncio + async def test_dropping_every_member_drops_the_namespace(self): + data = _codex_request("hi") + expected_tools = copy.deepcopy(data["tools"]) + guardrail = ToolEditingGuardrail( + edit=lambda tools: [t for t in tools if not t["function"]["name"].startswith("mcp__confluence__")], + guardrail_name="test", + ) + + result = await OpenAIResponsesHandler().process_input_messages(data, guardrail) + + assert result["tools"] == [expected_tools[0], *expected_tools[2:]] + + @pytest.mark.asyncio + async def test_edited_top_level_function_is_rewritten_in_place(self): + data = _codex_request("hi") + expected_tools = copy.deepcopy(data["tools"]) + + def rename_weather(tools): + for tool in tools: + if tool["function"]["name"] == "get_weather": + tool["function"]["description"] = "Weather lookup (guarded)" + return tools + + result = await OpenAIResponsesHandler().process_input_messages( + data, ToolEditingGuardrail(edit=rename_weather, guardrail_name="test") + ) + + assert result["tools"][0] == {**expected_tools[0], "description": "Weather lookup (guarded)"} + assert result["tools"][1:] == expected_tools[1:] + + +class TestOpenAIResponsesHandlerMalformedTools: + @pytest.mark.asyncio + async def test_request_tools_that_are_not_a_list_never_reach_the_guardrail(self): + handler = OpenAIResponsesHandler() + seen: list[list[dict]] = [] + + def record(tools): + seen.append(tools) + return tools + + guardrail = ToolEditingGuardrail(edit=record, guardrail_name="test") + data = {"input": "hi", "tools": {"type": "function", "name": "get_weather"}} + + result = await handler.process_input_messages(data, guardrail) + + assert seen == [[]] + assert result["input"] == "hi" + + class TestBuildBlockSseChunks: """build_block_sse_chunks turns a streaming ModifyResponseException into 200 SSE events""" @@ -1537,3 +1731,93 @@ class TestBuildBlockSseChunks: dones = [payload for payload in payloads if payload["type"] == "response.output_item.done"] assert len(dones) == 1 assert dones[0]["item"]["content"][0]["text"] == "Blocked by policy." + + +class TestOpenAIResponsesHandlerStreamingScanKey: + """get_streaming_scan_key mirrors what process_output_streaming_response would scan""" + + @staticmethod + def _delta(sequence_number, text): + return { + "type": "response.output_text.delta", + "sequence_number": sequence_number, + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": text, + } + + def test_no_events_yields_no_key(self): + assert OpenAIResponsesHandler().get_streaming_scan_key([]) is None + + def test_key_accumulates_deltas_while_the_stream_is_open(self): + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamingScanKey + + key = OpenAIResponsesHandler().get_streaming_scan_key([self._delta(0, "hel"), self._delta(1, "lo")]) + assert key == StreamingScanKey(texts=("hello",)) + + def test_typed_delta_events_accumulate_like_dicts(self): + from litellm.types.llms.openai import OutputTextDeltaEvent + + events = [ + OutputTextDeltaEvent( + type="response.output_text.delta", + item_id="msg_1", + output_index=0, + content_index=0, + delta=text, + sequence_number=i, + ) + for i, text in enumerate(("hel", "lo")) + ] + key = OpenAIResponsesHandler().get_streaming_scan_key(events) + assert key.texts == ("hello",) + assert key.stream_ended is False + + def test_events_without_text_leave_the_key_unchanged(self): + handler = OpenAIResponsesHandler() + events = [self._delta(0, "hi")] + quiet = events + [{"type": "response.in_progress", "sequence_number": 1}] + assert handler.get_streaming_scan_key(quiet) == handler.get_streaming_scan_key(events) + + @staticmethod + def _completed(sequence_number, output): + return {"type": "response.completed", "sequence_number": sequence_number, "response": {"output": output}} + + def test_completed_event_keys_on_the_final_output_text(self): + handler = OpenAIResponsesHandler() + message = {"type": "message", "content": [{"type": "output_text", "text": "hi"}]} + open_key = handler.get_streaming_scan_key([self._delta(0, "hi")]) + ended_key = handler.get_streaming_scan_key([self._delta(0, "hi"), self._completed(1, [message])]) + assert ended_key.stream_ended is True + assert ended_key == open_key + + def test_completed_event_with_a_function_call_changes_the_key(self): + handler = OpenAIResponsesHandler() + message = {"type": "message", "content": [{"type": "output_text", "text": "hi"}]} + function_call = {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": "{}"} + open_key = handler.get_streaming_scan_key([self._delta(0, "hi")]) + ended_key = handler.get_streaming_scan_key([self._delta(0, "hi"), self._completed(1, [message, function_call])]) + assert ended_key.texts == ("hi",) + assert len(ended_key.tool_calls) == 1 and "get_weather" in ended_key.tool_calls[0] + assert ended_key != open_key + + def test_completed_event_reads_every_output_text_part(self): + from litellm.types.responses.main import GenericResponseOutputItem, OutputText + + item = GenericResponseOutputItem( + type="message", + id="msg_1", + status="completed", + role="assistant", + content=[ + OutputText(type="output_text", text="one", annotations=[]), + OutputText(type="output_text", text="two", annotations=[]), + ], + ) + key = OpenAIResponsesHandler().get_streaming_scan_key([self._completed(0, [item])]) + assert key.texts == ("one", "two") + + def test_output_item_done_round_is_never_deduped(self): + done = {"type": "response.output_item.done", "sequence_number": 1, "item": {"type": "function_call"}} + assert OpenAIResponsesHandler().get_streaming_scan_key([self._delta(0, "hi"), done]) is None diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py new file mode 100644 index 00000000000..9c236d81f51 --- /dev/null +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py @@ -0,0 +1,198 @@ +""" +Unit tests for merge_guardrailed_tools, which writes guardrail-returned chat tools back onto the +Responses API request tools they were flattened from +""" + +import copy + +from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools +from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, +) +from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import GuardrailToolParam + + +def _groups(tools): + return [form.chat_tools for form in LiteLLMCompletionResponsesConfig.responses_tools_to_chat_forms(tools)] + + +def _flat(groups): + return [chat_tool for group in groups for chat_tool in group] + + +def _function(name, description=""): + return {"type": "function", "name": name, "description": description, "parameters": {"type": "object"}} + + +def test_unchanged_tools_come_back_as_the_original_objects(): + original = [ + _function("a"), + {"type": "namespace", "name": "ns", "description": "NS", "tools": [_function("x"), _function("y")]}, + {"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp"}, + {"type": "web_search"}, + ] + groups = _groups(original) + + merged = merge_guardrailed_tools(original, groups, _flat(groups)) + + assert list(merged) == original + assert all(merged_tool is original_tool for merged_tool, original_tool in zip(merged, original)) + + +def test_guardrail_reordering_unchanged_tools_keeps_request_order(): + original = [_function("a"), {"type": "namespace", "name": "ns", "tools": [_function("x")]}, {"type": "web_search"}] + groups = _groups(original) + + merged = merge_guardrailed_tools(original, groups, list(reversed(_flat(groups)))) + + assert list(merged) == original + + +def test_duplicate_function_names_are_matched_by_ordinal(): + original = [_function("dup", "first"), _function("dup", "second")] + groups = _groups(original) + + merged = merge_guardrailed_tools(original, groups, _flat(groups)[:1]) + + assert list(merged) == [original[0]] + + +def test_interleaved_duplicate_names_keep_their_own_ordinals(): + original = [ + _function("dup", "a"), + _function("other", "x"), + _function("dup", "b"), + _function("dup", "c"), + _function("other", "y"), + ] + groups = _groups(original) + flat = _flat(groups) + edited = {**flat[3], "function": {**flat[3]["function"], "description": "changed"}} + + merged = merge_guardrailed_tools(original, groups, [*flat[:3], edited, flat[4]]) + + assert list(merged) == [*original[:3], {**_function("dup", "changed"), "strict": False}, original[4]] + assert all(merged[position] is original[position] for position in (0, 1, 2, 4)) + + +def test_edited_mcp_tool_is_rewritten(): + original = [{"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp"}] + groups = _groups(original) + edited = [{**groups[0][0], "allowed_tools": ["read_wiki_structure"]}] + + merged = merge_guardrailed_tools(original, groups, edited) + + assert list(merged) == edited + + +def test_injected_tool_lands_after_the_request_tools_when_request_had_none(): + injected = {"type": "function", "function": {"name": "b", "description": "d", "parameters": {"type": "object"}}} + + merged = merge_guardrailed_tools([], [], [injected]) + + assert list(merged) == [ + {"type": "function", "name": "b", "description": "d", "parameters": {"type": "object"}, "strict": False} + ] + + +def test_empty_guardrail_output_keeps_only_tools_never_sent_to_the_guardrail(): + original = [_function("a"), {"type": "web_search"}, {"type": "namespace", "name": "ns", "tools": [_function("x")]}] + + merged = merge_guardrailed_tools(original, _groups(original), []) + + assert list(merged) == [{"type": "web_search"}] + + +def test_member_edit_strips_only_the_namespace_description_prefix(): + original = [{"type": "namespace", "name": "ns", "description": "NS", "tools": [_function("x", "X doc")]}] + groups = _groups(original) + assert groups[0][0]["function"]["description"] == "NS\n\nX doc" + edited = [{**groups[0][0], "function": {**groups[0][0]["function"], "description": "NS\n\nX doc (guarded)"}}] + + merged = merge_guardrailed_tools(original, groups, edited) + + assert list(merged) == [ + {"type": "namespace", "name": "ns", "description": "NS", "tools": [_function("x", "X doc (guarded)")]} + ] + + +def test_namespace_keeps_a_non_function_member_when_a_function_member_is_edited(): + custom_member = {"type": "custom", "name": "grep", "description": "Grep", "format": {"type": "text"}} + original = [ + {"type": "namespace", "name": "ns", "description": "NS", "tools": [_function("read", "Read"), custom_member]} + ] + groups = _groups(original) + edited = copy.deepcopy(_flat(groups)) + edited[0]["function"]["description"] = "NS\n\nEDITED" + + merged = merge_guardrailed_tools(original, groups, edited) + + assert len(merged) == 1 + assert [member["name"] for member in merged[0]["tools"]] == ["read", "grep"] + assert merged[0]["tools"][0]["description"] == "EDITED" + assert merged[0]["tools"][1] == custom_member + + +def test_namespace_keeps_its_non_function_members_when_every_function_member_is_dropped(): + custom_member = {"type": "custom", "name": "grep", "description": "Grep", "format": {"type": "text"}} + original = [ + {"type": "namespace", "name": "ns", "description": "NS", "tools": [_function("read"), custom_member]}, + _function("a"), + ] + groups = _groups(original) + + merged = merge_guardrailed_tools(original, groups, [groups[1][0]]) + + assert list(merged) == [{"type": "namespace", "name": "ns", "description": "NS", "tools": [custom_member]}, _function("a")] + + +def test_member_extras_edited_by_the_guardrail_land_on_that_member(): + original = [{"type": "namespace", "name": "ns", "tools": [_function("read")]}] + groups = _groups(original) + edited = copy.deepcopy(_flat(groups)) + edited[0]["cache_control"] = {"type": "ephemeral"} + + merged = merge_guardrailed_tools(original, groups, edited) + + assert merged[0]["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert merged[0]["tools"][0]["name"] == "read" + + +def test_guardrail_output_is_read_once(): + original = [_function("a"), {"type": "namespace", "name": "ns", "tools": [_function("x")]}] + groups = _groups(original) + + merged = merge_guardrailed_tools(original, groups, (chat_tool for chat_tool in _flat(groups))) + + assert list(merged) == original + + +def test_pydantic_guardrail_tools_round_trip_like_dicts(): + original = [_function("a"), {"type": "namespace", "name": "ns", "tools": [_function("x", "X doc")]}] + groups = _groups(original) + models = [GuardrailToolParam.model_validate(chat_tool) for chat_tool in _flat(groups)] + + merged = merge_guardrailed_tools(original, groups, models) + + assert list(merged) == original + assert all(merged_tool is original_tool for merged_tool, original_tool in zip(merged, original)) + + +def test_pydantic_guardrail_edit_lands_on_the_member(): + original = [{"type": "namespace", "name": "ns", "tools": [_function("x", "X doc")]}] + groups = _groups(original) + edited = copy.deepcopy(_flat(groups)) + edited[0]["function"]["description"] = "EDITED" + + merged = merge_guardrailed_tools(original, groups, [GuardrailToolParam.model_validate(edited[0])]) + + assert list(merged) == [{"type": "namespace", "name": "ns", "tools": [_function("x", "EDITED")]}] + + +def test_non_object_guardrail_items_are_dropped(): + original = [_function("a")] + groups = _groups(original) + + merged = merge_guardrailed_tools(original, groups, [*_flat(groups), "junk", None]) + + assert list(merged) == original diff --git a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py index 9a6a039a470..67a56fdcd79 100644 --- a/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py @@ -318,3 +318,203 @@ def test_json_provider_messages_config_probes_capabilities_under_provider_slug() ) assert JSONProviderAnthropicMessagesConfig(provider).custom_llm_provider == "exampleprovider" assert OpenAILikeAnthropicMessagesConfig().custom_llm_provider == "anthropic" + + +def _cache_control_request_params() -> tuple[list, dict]: + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "write a regex for a US phone number", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + } + ] + optional_params = { + "max_tokens": 256, + "system": [ + { + "type": "text", + "text": "You are Claude Code.", + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + } + ], + "tools": [ + { + "name": "lookup", + "input_schema": {"type": "object"}, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + } + return messages, optional_params + + +def test_request_strips_cache_control_ttl_everywhere(config): + """Regression: Claude Code always sends ``cache_control: {type: ephemeral, + ttl: 1h}``, and strict non-Anthropic /v1/messages validators 400 the whole + request on the ttl extension (``cache_control.ttl: 1h is not supported``).""" + messages, optional_params = _cache_control_request_params() + + payload = config.transform_anthropic_messages_request( + model="some-model", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert payload["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + assert payload["system"][0]["cache_control"] == {"type": "ephemeral"} + assert payload["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert messages[0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + +def test_request_defaults_missing_cache_control_type_and_drops_non_dict(config): + payload = config.transform_anthropic_messages_request( + model="some-model", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "a", "cache_control": {"ttl": "1h"}}, + {"type": "text", "text": "b", "cache_control": None}, + ], + } + ], + anthropic_messages_optional_request_params={"max_tokens": 64}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + blocks = payload["messages"][0]["content"] + assert blocks[0]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in blocks[1] + + +def test_native_anthropic_config_keeps_cache_control_ttl(): + """Anthropic itself accepts ttl, so the normalization must stay scoped to + the OpenAI-like passthrough and never reach the native Anthropic path.""" + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) + + messages, optional_params = _cache_control_request_params() + payload = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-20250514", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert payload["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + assert payload["system"][0]["cache_control"] == {"type": "ephemeral", "ttl": "5m"} + + +def test_deployment_opt_in_keeps_cache_control_ttl(): + config = OpenAILikeAnthropicMessagesConfig(cache_control_ttl=True) + payload = config.transform_anthropic_messages_request( + model="some-model", + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + } + ], + anthropic_messages_optional_request_params={"max_tokens": 16}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert payload["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + +def test_json_provider_constraint_opts_into_cache_control_ttl(): + from litellm.llms.openai_like.json_loader import SimpleProviderConfig + from litellm.llms.openai_like.messages.transformation import ( + JSONProviderAnthropicMessagesConfig, + ) + + base_data = {"base_url": "https://api.example.com/v1", "api_key_env": "EXAMPLE_API_KEY"} + strict = JSONProviderAnthropicMessagesConfig(SimpleProviderConfig(slug="strictprov", data=base_data)) + lenient = JSONProviderAnthropicMessagesConfig( + SimpleProviderConfig(slug="lenientprov", data={**base_data, "constraints": {"cache_control_ttl": True}}) + ) + + def transform(provider_config): + messages, optional_params = _cache_control_request_params() + return provider_config.transform_anthropic_messages_request( + model="some-model", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert transform(strict)["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + assert transform(lenient)["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} + + +def test_request_strips_ttl_only_where_the_messages_api_defines_cache_control(config): + """Regression: the sanitizer must only touch ``cache_control`` where the + Messages API defines it (request, system, tools, content blocks, tool_result + content), never application data such as ``tool_use.input`` or a tool's + ``input_schema`` that happens to contain a ``cache_control`` key.""" + tool_input = {"cache_control": {"type": "ephemeral", "ttl": "1h"}, "query": "x"} + input_schema = { + "type": "object", + "properties": {"cache_control": {"type": "string", "ttl": "1h"}}, + } + messages = [ + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "lookup", "input": tool_input}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + "content": [ + {"type": "text", "text": "result", "cache_control": {"type": "ephemeral", "ttl": "1h"}} + ], + }, + {"type": "text", "text": "plain string content stays", "cache_control": {"ttl": "1h"}}, + ], + }, + {"role": "user", "content": "a plain string message"}, + ] + optional_params = { + "max_tokens": 64, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + "tools": [ + { + "name": "lookup", + "input_schema": input_schema, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + } + + payload = config.transform_anthropic_messages_request( + model="some-model", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert payload["cache_control"] == {"type": "ephemeral"} + assert payload["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert payload["tools"][0]["input_schema"] == input_schema + assert payload["messages"][0]["content"][0]["input"] == tool_input + tool_result = payload["messages"][1]["content"][0] + assert tool_result["cache_control"] == {"type": "ephemeral"} + assert tool_result["content"][0]["cache_control"] == {"type": "ephemeral"} + assert payload["messages"][1]["content"][1]["cache_control"] == {"type": "ephemeral"} + assert payload["messages"][2] == {"role": "user", "content": "a plain string message"} diff --git a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py index 7085e45cdc3..4b58d220623 100644 --- a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py +++ b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, Mock +from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest @@ -9,6 +9,18 @@ from litellm.llms.s3_vectors.vector_stores.transformation import ( from litellm.types.vector_stores import VectorStoreSearchResponse +def _mock_router(model_names, sync=False): + """Router mock serving the given embedding model names.""" + router = MagicMock() + router.get_model_list.return_value = [{"model_name": name} for name in model_names] + embedding_response = Mock(data=[{"embedding": [0.1, 0.2, 0.3]}]) + if sync: + router.embedding = MagicMock(return_value=embedding_response) + else: + router.aembedding = AsyncMock(return_value=embedding_response) + return router + + class TestS3VectorsVectorStoreConfig: def test_init(self): """Test that S3VectorsVectorStoreConfig initializes correctly""" @@ -28,19 +40,174 @@ class TestS3VectorsVectorStoreConfig: url = config.get_complete_url(None, litellm_params) assert url == "https://s3vectors.us-west-2.api.aws" - def test_get_complete_url_missing_region(self): - """Test that missing region raises error""" + def test_get_complete_url_missing_region(self, monkeypatch): + """Missing region falls back to the default region (parity with ingestion)""" + monkeypatch.delenv("AWS_REGION_NAME", raising=False) + monkeypatch.delenv("AWS_REGION", raising=False) config = S3VectorsVectorStoreConfig() - litellm_params = {} - with pytest.raises(ValueError, match="aws_region_name is required"): - config.get_complete_url(None, litellm_params) + url = config.get_complete_url(None, {}) + assert url == "https://s3vectors.us-west-2.api.aws" + + def test_get_complete_url_uses_env_region(self, monkeypatch): + """Missing region param resolves from AWS_REGION_NAME env var""" + monkeypatch.setenv("AWS_REGION_NAME", "eu-west-1") + monkeypatch.delenv("AWS_REGION", raising=False) + config = S3VectorsVectorStoreConfig() + url = config.get_complete_url(None, {}) + assert url == "https://s3vectors.eu-west-1.api.aws" + + def test_get_complete_url_invalid_region_format(self): + """Invalid region format raises""" + config = S3VectorsVectorStoreConfig() + with pytest.raises(ValueError, match="Invalid AWS region format"): + config.get_complete_url(None, {"aws_region_name": "Bad_Region!"}) - @pytest.mark.skip(reason="Requires embedding API call, tested in integration tests") def test_transform_search_request(self): - """Test search request transformation""" - # This test requires making an actual embedding API call - # It's better tested in integration tests - pass + """Full request-body transformation with a router-injected embedding""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["text-embedding-3-small"], sync=True) + + url, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={"max_num_results": 7}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + router=router, + ) + + assert url == "https://s3vectors.us-west-2.api.aws/QueryVectors" + assert request_body == { + "vectorBucketName": "test-bucket", + "indexName": "test-index", + "queryVector": {"float32": [0.1, 0.2, 0.3]}, + "topK": 7, + "returnDistance": True, + "returnMetadata": True, + } + assert mock_logging_obj.model_call_details["query"] == "test query" + + @pytest.mark.asyncio + async def test_atransform_search_uses_router_for_virtual_model(self): + """Regression: router-served embedding models must resolve via the router, + not a bare litellm.aembedding call (which has no deployment credentials).""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["my-embedding-model"]) + + with patch("litellm.aembedding", new=AsyncMock()) as mock_bare_aembedding: # test-quality-ok: guards that the bare-embedding path is not taken; dispatch seam is the behavior under test + url, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "my-embedding-model"}, + extra_body=None, + router=router, + ) + + router.aembedding.assert_awaited_once_with(model="my-embedding-model", input=["test query"]) + mock_bare_aembedding.assert_not_awaited() + assert request_body["queryVector"]["float32"] == [0.1, 0.2, 0.3] + assert request_body["topK"] == 5 # default + + @pytest.mark.asyncio + async def test_atransform_search_falls_back_when_router_does_not_serve_model(self): + """Router present but embedding_model is not a router deployment -> + bare litellm.aembedding keeps working (provider-prefixed + env creds stores).""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["some-other-model"]) + + mock_bare = AsyncMock(return_value=Mock(data=[{"embedding": [0.4, 0.5]}])) + with patch("litellm.aembedding", new=mock_bare): # test-quality-ok: stubs the bare-embedding fallback whose request body the test asserts on + _, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "azure/text-embedding-3-small"}, + extra_body=None, + router=router, + ) + + mock_bare.assert_awaited_once_with(model="azure/text-embedding-3-small", input=["test query"]) + router.aembedding.assert_not_awaited() + assert request_body["queryVector"]["float32"] == [0.4, 0.5] + + @pytest.mark.asyncio + async def test_atransform_search_without_router_uses_bare_embedding(self): + """Backward compat: no router -> bare litellm.aembedding as before""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + + mock_bare = AsyncMock(return_value=Mock(data=[{"embedding": [0.6, 0.7]}])) + with patch("litellm.aembedding", new=mock_bare): # test-quality-ok: stubs the bare-embedding fallback whose request body the test asserts on + _, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + ) + + mock_bare.assert_awaited_once_with(model="text-embedding-3-small", input=["test query"]) + assert request_body["queryVector"]["float32"] == [0.6, 0.7] + + def test_transform_search_uses_router_for_virtual_model_sync(self): + """Sync twin: router-served embedding model resolves via router.embedding""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["my-embedding-model"], sync=True) + + with patch("litellm.embedding", new=MagicMock()) as mock_bare_embedding: # test-quality-ok: guards that the bare-embedding path is not taken; dispatch seam is the behavior under test + _, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "my-embedding-model"}, + extra_body=None, + router=router, + ) + + router.embedding.assert_called_once_with(model="my-embedding-model", input=["test query"]) + mock_bare_embedding.assert_not_called() + assert request_body["queryVector"]["float32"] == [0.1, 0.2, 0.3] + + def test_transform_search_without_router_uses_bare_embedding_sync(self): + """Sync twin: no router -> bare litellm.embedding as before""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + + mock_bare = MagicMock(return_value=Mock(data=[{"embedding": [0.8, 0.9]}])) + with patch("litellm.embedding", new=mock_bare): # test-quality-ok: stubs the bare-embedding fallback whose request body the test asserts on + _, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + ) + + mock_bare.assert_called_once_with(model="text-embedding-3-small", input=["test query"]) + assert request_body["queryVector"]["float32"] == [0.8, 0.9] def test_transform_search_request_invalid_vector_store_id(self): """Test that invalid vector_store_id format raises error""" diff --git a/tests/test_litellm/llms/valkey/vector_stores/test_valkey_transformation.py b/tests/test_litellm/llms/valkey/vector_stores/test_valkey_transformation.py index a2ee2c2bdb1..aa114f128c5 100644 --- a/tests/test_litellm/llms/valkey/vector_stores/test_valkey_transformation.py +++ b/tests/test_litellm/llms/valkey/vector_stores/test_valkey_transformation.py @@ -67,20 +67,52 @@ class FakeAsyncEmbeddingFn(FakeEmbeddingFn): return SimpleNamespace(data=[{"embedding": self.embedding}]) +class FakeEmbeddingExecutor: + def __init__(self, embedding): + self.embedding = embedding + self.captured = None + + def embed(self, model, query, configuration): + self.captured = (model, query, configuration) + return SimpleNamespace(data=[{"embedding": self.embedding}]) + + async def aembed(self, model, query, configuration): + self.captured = (model, query, configuration) + return SimpleNamespace(data=[{"embedding": self.embedding}]) + + def _doc(doc_id, distance, **fields): return SimpleNamespace(id=doc_id, vector_distance=str(distance), **fields) -def _search(config, client=None, query="what is litellm", optional_params=None, litellm_params=None): +def _search(config, client=None, query="what is litellm", optional_params=None, litellm_params=None, executor=None): return config.execute_search_vector_store_request( vector_store_id="my_index", query=query, vector_store_search_optional_params=optional_params or {}, litellm_logging_obj=MagicMock(), litellm_params={"litellm_embedding_model": "openai/text-embedding-3-small", **(litellm_params or {})}, + embedding_executor=executor, ) +def test_sync_search_uses_request_embedding_executor_without_overwriting_explicit_config(): + executor = FakeEmbeddingExecutor([0.1, 0.2]) + config = ValkeyVectorStoreConfig(sync_client=FakeRedis()) + embedding_config = {"api_key": "store-specific-key", "aws_region_name": "us-west-2"} + + _search( + config, + litellm_params={ + "litellm_embedding_model": "team-embedding-alias", + "litellm_embedding_config": embedding_config, + }, + executor=executor, + ) + + assert executor.captured == ("team-embedding-alias", "what is litellm", embedding_config) + + def test_sync_search_builds_knn_query_with_packed_vector(): embedding_fn = FakeEmbeddingFn([0.1, 0.2, 0.3]) client = FakeRedis() diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 8c1de12e7d9..4679b978f78 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -1096,10 +1096,13 @@ def test_natively_signed_parallel_turn_never_carries_a_placeholder(model): "gemini-3.5-flash", "gemini-3.6-flash", "gemini-3.7-flash", + "gemini-3.8-flash", "vertex_ai/gemini-3.5-flash", "vertex_ai/gemini-3.7-flash", + "vertex_ai/gemini-3.8-flash", "gemini/gemini-3.5-flash", "gemini/gemini-3.7-flash", + "gemini/gemini-3.8-flash", ], ) def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model): diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index bd07bec900f..d2788408e09 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1185,6 +1185,18 @@ def test_vertex_ai_map_thinking_param_with_budget_tokens_0(): } +def test_vertex_ai_map_thinking_param_without_budget_tokens_for_gemini_3(): + v = VertexGeminiConfig() + result = v.map_openai_params( + non_default_params={"thinking": {"type": "enabled"}}, + optional_params={}, + model="gemini-3.5-flash", + drop_params=False, + ) + + assert result["thinkingConfig"] == {"includeThoughts": True} + + def test_vertex_ai_map_tools(): v = VertexGeminiConfig() optional_params = {} diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 9419f88a981..a57672cfbfb 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -752,3 +752,26 @@ def test_vertex_ai_fable_5_1_response_format_uses_native_output_format(local_mod assert "output_format" in result_params assert "tool_choice" not in result_params assert "tools" not in result_params + + +def test_vertex_ai_anthropic_tool_based_response_format_still_upgrades_legacy_thinking(local_model_cost_map): + result_params = VertexAIAnthropicConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "schema": {"type": "object", "properties": {"result": {"type": "string"}}}, + }, + }, + "thinking": {"type": "enabled", "budget_tokens": 4096}, + "max_tokens": 8192, + }, + optional_params={}, + model="claude-opus-4-8", + drop_params=False, + ) + + assert "tools" in result_params + assert result_params["thinking"] == {"type": "adaptive"} + assert result_params["output_config"] == {"effort": "high"} diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index acad249a2bb..0764aec7185 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -1,7 +1,7 @@ """Tests for the optional Rust-backed OCR path.""" -import importlib import builtins +import importlib import types from typing import Any @@ -10,6 +10,7 @@ import pytest import litellm from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.rust_bridge import configuration # `litellm/__init__.py` does `from .ocr.main import *`, which binds the `ocr` # function onto `litellm.ocr` and shadows the submodule, so import the modules @@ -214,10 +215,12 @@ def build_prepared_request( @pytest.fixture(autouse=True) def _reset_rust_flag(): """Keep the global toggle isolated between tests.""" - rust_bridge.use_litellm_rust(False, ocr=None, aocr=None) + rust_bridge.set_rust_ocr(ocr=None, aocr=None) + configuration.reset_rust_configuration() rust_bridge_loader._cached_bridge = rust_bridge_loader._BRIDGE_SENTINEL yield - rust_bridge.use_litellm_rust(False, ocr=None, aocr=None) + rust_bridge.set_rust_ocr(ocr=None, aocr=None) + configuration.reset_rust_configuration() rust_bridge_loader._cached_bridge = rust_bridge_loader._BRIDGE_SENTINEL @@ -247,7 +250,14 @@ def test_use_litellm_rust_toggles_flag(): def test_env_var_enables_rust_ocr(monkeypatch): monkeypatch.setenv("LITELLM_USE_RUST_OCR", "1") - assert rust_bridge._env_enables_rust_ocr() is True + with pytest.warns(DeprecationWarning, match="LITELLM_USE_RUST_OCR is deprecated"): + assert rust_bridge.rust_ocr_enabled() is True + + +def test_explicit_false_overrides_process_enable(): + litellm.use_litellm_rust(True) + + assert ocr_main._rust_ocr_enabled(build_prepared_request(litellm_params={"rust": False})) is False def test_load_rust_ocr_returns_injected_impl(): @@ -471,9 +481,7 @@ def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing(): ocr_main._run_rust_ocr( prepared_request=build_prepared_request(api_key=None, timeout=None), - resolve_api_key=lambda name: ( - "sk-from-vault" if name == "MISTRAL_API_KEY" else None - ), + resolve_api_key=lambda name: "sk-from-vault" if name == "MISTRAL_API_KEY" else None, ) assert bridge.calls[0]["api_key"] == "sk-from-vault" @@ -580,9 +588,7 @@ def test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager(): api_base=None, timeout=None, ), - resolve_api_key=lambda name: ( - "https://azure.example.com" if name == "AZURE_AI_API_BASE" else None - ), + resolve_api_key=lambda name: "https://azure.example.com" if name == "AZURE_AI_API_BASE" else None, ) assert bridge.calls[0]["api_base"] == "https://azure.example.com" @@ -600,9 +606,7 @@ def test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint(): timeout=None, ), resolve_api_key=lambda name: ( - "https://document-intelligence.example.com" - if name == "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT" - else None + "https://document-intelligence.example.com" if name == "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT" else None ), ) @@ -815,9 +819,6 @@ def test_ocr_provider_configs_expose_api_key_env_vars(): assert BaseOCRConfig().get_api_key_env_var() is None assert MistralOCRConfig().get_api_key_env_var() == "MISTRAL_API_KEY" assert AzureAIOCRConfig().get_api_key_env_var() == "AZURE_AI_API_KEY" - assert ( - AzureDocumentIntelligenceOCRConfig().get_api_key_env_var() - == "AZURE_DOCUMENT_INTELLIGENCE_API_KEY" - ) + assert AzureDocumentIntelligenceOCRConfig().get_api_key_env_var() == "AZURE_DOCUMENT_INTELLIGENCE_API_KEY" assert VertexAIOCRConfig().get_api_key_env_var() == "VERTEX_AI_API_KEY" assert VertexAIDeepSeekOCRConfig().get_api_key_env_var() == "VERTEX_AI_API_KEY" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 0144fbb17dd..c8ea4867f2c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -6005,6 +6005,44 @@ class TestMCPDcrBridgeDelegateAdmission: await MCPRequestHandler.process_mcp_request(scope) assert exc_info.value.status_code == 503 + assert exc_info.value.detail == ( + "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." + ) + + async def test_user_subject_envelope_permanent_db_fault_is_503_not_worded_as_transient(self): + """A query engine fault that never heals (a missing engine binary) still fails admission with 503, + but the detail must not call the database "temporarily unreachable" or ask the client to retry: the + DCR client would loop on a retry that can never succeed. The fault reaches the handler wrapped in + get_user_object's bare ValueError, so the wording has to be picked off the wrapped cause.""" + from prisma.engine.errors import BinaryNotFoundError + + envelope = self._mint_bridge_envelope(user_id="sso-user-7") + scope = { + "type": "http", + "method": "POST", + "path": "/mcp/bridge_delegate_server", + "headers": [(b"authorization", f"Bearer {envelope}".encode("latin-1"))], + } + with ( + patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling admission tests + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + ) as mock_mgr, + patch( # test-quality-ok: the envelope opener reads master_key off the proxy module, no injection seam + "litellm.proxy.proxy_server.master_key", self._MASTER_KEY + ), + self._patch_user_reload( + side_effect=self._wrapped_user_lookup_error(BinaryNotFoundError("query engine binary not found")) + ), + ): + mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server() + with pytest.raises(HTTPException) as exc_info: + await MCPRequestHandler.process_mcp_request(scope) + + assert exc_info.value.status_code == 503 + assert "temporarily unreachable" not in exc_info.value.detail + assert "retry shortly" not in exc_info.value.detail.lower() + assert "BinaryNotFoundError" in exc_info.value.detail + assert "will not clear by retrying" in exc_info.value.detail async def test_user_subject_envelope_scim_deactivated_user_fails_closed_401(self): """SCIM-deactivating the envelope's user revokes it immediately: the reloaded user carries diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_endpoint.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_endpoint.py index 5f277db2f72..db3a1a386a3 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_endpoint.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_token_endpoint.py @@ -7,6 +7,7 @@ cache's hit/single-flight behavior. Each assertion fails under a real mutation o """ import asyncio +import gc import json from unittest.mock import AsyncMock, MagicMock, patch @@ -359,6 +360,110 @@ async def test_cache_invalidate_only_evicts_the_named_key(): assert calls == 2 +@pytest.mark.asyncio +async def test_cache_invalidate_mid_compute_is_not_overwritten_by_that_compute(): + """A bearer minted before an invalidation must never be served after it. + + The compute is suspended at the token endpoint when the invalidation lands, so its write is + the one that would resurrect the evicted bearer for the rest of its TTL. The caller it was + minted for still gets it; the *cache* is what the invalidation is about. + """ + cache = ExchangedTokenCache() + mint_started, release_mint = asyncio.Event(), asyncio.Event() + + async def slow_mint(): + mint_started.set() + await release_mint.wait() + return _ok_token("bearer-minted-before-invalidation") + + async def re_mint(): + return _ok_token("bearer-minted-after-invalidation") + + in_flight = asyncio.create_task(cache.get_or_compute("slot", slow_mint, fingerprint="fp")) + await mint_started.wait() + + assert not in_flight.done() + cache.invalidate("slot") + release_mint.set() + + raced = await in_flight + assert isinstance(raced, Ok) and raced.ok == "bearer-minted-before-invalidation" + + after = await cache.get_or_compute("slot", re_mint, fingerprint="fp") + assert isinstance(after, Ok) and after.ok == "bearer-minted-after-invalidation" + + +@pytest.mark.asyncio +async def test_cache_invalidate_mid_compute_survives_garbage_collection(): + """The record of an invalidation must outlive a collection cycle taken mid-compute. + + Per-key state is held weakly so idle keys do not accumulate. If the state a compute checks + before writing were collectible while that compute is suspended, the check would read as + "nothing was invalidated" and the stale write would land; the running compute has to pin it. + """ + cache = ExchangedTokenCache() + mint_started, release_mint = asyncio.Event(), asyncio.Event() + + async def slow_mint(): + mint_started.set() + await release_mint.wait() + return _ok_token("bearer-minted-before-invalidation") + + async def re_mint(): + return _ok_token("bearer-minted-after-invalidation") + + in_flight = asyncio.create_task(cache.get_or_compute("slot", slow_mint, fingerprint="fp")) + await mint_started.wait() + + assert not in_flight.done() + cache.invalidate("slot") + gc.collect() + release_mint.set() + await in_flight + + after = await cache.get_or_compute("slot", re_mint, fingerprint="fp") + assert isinstance(after, Ok) and after.ok == "bearer-minted-after-invalidation" + + +@pytest.mark.asyncio +async def test_cache_stores_a_compute_that_started_after_the_invalidation(): + """Only the mint that predates the invalidation loses its write. + + A caller queued behind the single-flight lock computes after the eviction, so its token is + fresh and must be cached; otherwise the fix would trade one stale bearer for re-minting on + every subsequent resolution. + """ + cache = ExchangedTokenCache() + mint_started, release_mint = asyncio.Event(), asyncio.Event() + + async def slow_mint(): + mint_started.set() + await release_mint.wait() + return _ok_token("bearer-minted-before-invalidation") + + async def re_mint(): + return _ok_token("bearer-minted-after-invalidation") + + async def must_not_run(): + pytest.fail("the mint that followed the invalidation should have been cached") + + in_flight = asyncio.create_task(cache.get_or_compute("slot", slow_mint, fingerprint="fp")) + await mint_started.wait() + queued = asyncio.create_task(cache.get_or_compute("slot", re_mint, fingerprint="fp")) + await asyncio.sleep(0) + + assert not queued.done() + cache.invalidate("slot") + release_mint.set() + + raced, fresh = await asyncio.gather(in_flight, queued) + assert isinstance(raced, Ok) and raced.ok == "bearer-minted-before-invalidation" + assert isinstance(fresh, Ok) and fresh.ok == "bearer-minted-after-invalidation" + + served = await cache.get_or_compute("slot", must_not_run, fingerprint="fp") + assert isinstance(served, Ok) and served.ok == "bearer-minted-after-invalidation" + + @pytest.mark.asyncio async def test_cache_does_not_store_a_failed_compute(): cache = ExchangedTokenCache() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 598e9276423..588eba4adb7 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -6432,6 +6432,23 @@ async def test_bridge_mint_db_outage_is_503_before_upstream(): response, post = await _prepare_only_bridge_exchange("unavailable") assert response.status_code == 503 assert json.loads(response.body)["error"] == "temporarily_unavailable" + assert "retry shortly" in json.loads(response.body)["error_description"] + post.assert_not_called() + + +@pytest.mark.asyncio +async def test_bridge_mint_permanent_db_fault_is_503_without_retry_advice(): + """A query engine fault that never heals is still a 503 (the gateway is at fault, not the client), but + the description must not tell the client the database is temporarily unreachable and to retry: that + sends an operator to wait out an outage that is not one. The code stays temporarily_unavailable, the + only RFC 6749 error a client treats as a server-side 503.""" + response, post = await _prepare_only_bridge_exchange("faulted") + assert response.status_code == 503 + body = json.loads(response.body) + assert body["error"] == "temporarily_unavailable" + assert "temporarily unreachable" not in body["error_description"] + assert "retry shortly" not in body["error_description"] + assert "not a transient outage" in body["error_description"] post.assert_not_called() @@ -7143,6 +7160,56 @@ async def test_resolve_active_litellm_key_db_outage_is_unavailable(proxy_globals assert await _resolve_active_litellm_key(request) == "unavailable" +@pytest.mark.asyncio +async def test_resolve_active_litellm_key_permanent_engine_fault_is_faulted(proxy_globals): + """A query engine that is missing or version-skewed cannot resolve any key until the deployment is + repaired, so the resolver reports "faulted" (still statused 503 by the mint) rather than "unavailable", + whose wording promises the outage is transient and asks the client to retry.""" + from prisma.engine.errors import BinaryNotFoundError + + from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + _resolve_active_litellm_key, + ) + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + class _FaultedPrisma: + async def get_data(self, token, table_name, parent_otel_span=None, proxy_logging_obj=None): + raise BinaryNotFoundError("query engine binary not found") + + proxy_globals.user_api_key_cache = UserApiKeyCache() + proxy_globals.prisma_client = _FaultedPrisma() + + request = _token_request({"x-litellm-api-key": "sk-during-engine-fault"}) + assert await _resolve_active_litellm_key(request) == "faulted" + + +@pytest.mark.asyncio +async def test_resolve_active_litellm_key_transport_error_over_permanent_fault_is_faulted(proxy_globals): + """A reconnect that dies on a missing engine binary raises the transport error last, with the + BinaryNotFoundError as __context__. The binary is what blocks recovery, so the key read is "faulted", + not the "unavailable" that the outer ConnectError alone would suggest.""" + import httpx + from prisma.engine.errors import BinaryNotFoundError + + from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + _resolve_active_litellm_key, + ) + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + class _ReconnectFailedPrisma: + async def get_data(self, token, table_name, parent_otel_span=None, proxy_logging_obj=None): + try: + raise BinaryNotFoundError("query engine binary not found") + except BinaryNotFoundError: + raise httpx.ConnectError("All connection attempts failed") + + proxy_globals.user_api_key_cache = UserApiKeyCache() + proxy_globals.prisma_client = _ReconnectFailedPrisma() + + request = _token_request({"x-litellm-api-key": "sk-during-failed-reconnect"}) + assert await _resolve_active_litellm_key(request) == "faulted" + + @pytest.mark.asyncio async def test_resolve_active_litellm_key_no_database_is_unresolvable(proxy_globals): """With no database connection configured the gateway cannot verify the presented key at all, so @@ -7214,6 +7281,26 @@ async def test_reload_active_user_by_id_db_outage_is_unavailable(proxy_globals): assert await _reload_active_user_by_id("sso-user-7") == "unavailable" +@pytest.mark.asyncio +async def test_reload_active_user_by_id_permanent_engine_fault_is_faulted(proxy_globals): + """A permanent query engine fault while re-validating the user on refresh is "faulted", not + "unavailable": both are 503s, but only the transient one may tell the client to retry. get_user_object + wraps the fault in a bare ValueError, so the classification has to read the wrapped cause.""" + from prisma.engine.errors import MismatchedVersionsError + + from litellm.proxy._experimental.mcp_server.bridge_token_flow import _reload_active_user_by_id + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + proxy_globals.user_api_key_cache = UserApiKeyCache() + proxy_globals.prisma_client = object() + + with patch( # test-quality-ok: get_user_object is the DB seam that wraps the fault; same patch as the outage sibling + "litellm.proxy.auth.auth_checks.get_user_object", + new=AsyncMock(side_effect=_wrapped_user_lookup_error(MismatchedVersionsError(expected="1", got="2"))), + ): + assert await _reload_active_user_by_id("sso-user-7") == "faulted" + + @pytest.mark.asyncio async def test_token_endpoint_uses_client_secret_basic_when_configured(): """LIT-4091: a server with token_endpoint_auth_method=client_secret_basic must send the diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py index 32a3f70c357..1670370f082 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py @@ -426,6 +426,7 @@ async def test_token_rejects_expired_code_and_missing_configuration(): [ ("no_active_key", 400, "invalid_grant"), ("unavailable", 503, "temporarily_unavailable"), + ("faulted", 503, "temporarily_unavailable"), ("unresolvable", 500, "server_error"), ], ) @@ -460,6 +461,25 @@ async def test_token_gates_on_live_user_revalidation(failure, expected_status, e assert json.loads(response.body)["error"] == expected_error +def test_permanent_db_fault_503_does_not_promise_a_retry_will_help(): + """Both DB failures are 503 temporarily_unavailable (the only OAuth error a client reads as a + server-side outage), so the description is the one place the two are told apart: a transient outage + says retry, a fault that never heals must say retrying will not help and point at the deployment.""" + from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( + _consent_lookup_failure_response, + _mint_failure_response, + _reload_failure_response, + ) + + for render in (_reload_failure_response, _consent_lookup_failure_response, _mint_failure_response): + transient = json.loads(render("unavailable").body)["error_description"] + faulted = json.loads(render("faulted").body)["error_description"] + assert transient == "the gateway database is unavailable; retry" + assert "retry" not in faulted.replace("retrying will not help", "") + assert "not a transient outage" in faulted + assert "retrying will not help" in faulted + + @pytest.mark.asyncio async def test_flow_is_single_use_shared_cache_rejects_second_complete(): """A double-submit of the finish step mints only ONE code: the second complete over the @@ -1250,6 +1270,7 @@ async def test_native_authorize_refuses_a_hosted_redirect_for_the_proxy_api(): "failure, status, error", [ ("unavailable", 503, "temporarily_unavailable"), + ("faulted", 503, "temporarily_unavailable"), ("unresolvable", 500, "server_error"), ("no_active_key", 403, "access_denied"), ], @@ -1424,6 +1445,7 @@ async def test_native_code_without_a_minter_is_refused_server_side(): ("team_required", 400, "invalid_grant"), ("no_active_key", 400, "invalid_grant"), ("unavailable", 503, "temporarily_unavailable"), + ("faulted", 503, "temporarily_unavailable"), ("unresolvable", 500, "server_error"), ], ) @@ -1805,5 +1827,12 @@ async def test_introspect_fails_closed_on_dead_user_and_503s_on_outage(): status, body = await _introspect(minted.token.get_secret_value(), reload_user=_reload_user_outage) assert (status, body["error"]) == (503, "temporarily_unavailable") + async def _reload_user_faulted(user_id: str): + return "faulted" + + status, body = await _introspect(minted.token.get_secret_value(), reload_user=_reload_user_faulted) + assert (status, body["error"]) == (503, "temporarily_unavailable") + assert "not a transient outage" in body["error_description"] + status, body = await _introspect(minted.token.get_secret_value(), master_key=None) assert (status, body["error"]) == (500, "server_error") diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 82f74cda835..3f6d8f8837c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -8198,6 +8198,79 @@ class TestPreemptive401ModeAware: await self._run(delegate, self.LITELLM_KEY_HEADERS, has_stored_token=False) +def _make_obo_server(alias: str) -> MCPServer: + return MCPServer( + server_id=f"id-{alias}", + name=alias, + alias=alias, + server_name=alias, + url=f"https://{alias}.test/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2_token_exchange, + token_exchange_endpoint="https://idp.test/token", + client_id="cid", + client_secret="csecret", + mcp_info={"server_name": alias}, + ) + + +class TestOboPreflightScopedToAllowedServers: + """The connect-time OBO exchange is an outbound IdP call whose result is cached, so it must + only run for a server the caller's key resolves to through the allowed set, not for any + server the requested path happens to name.""" + + SUBJECT_HEADERS = {"Authorization": "Bearer upstream-subject-token"} + + async def _run(self, requested: MCPServer, allowed: list[MCPServer], user_api_key_auth: UserAPIKeyAuth | None): + from litellm.proxy._experimental.mcp_server import server as server_module + + allowed_lookup = AsyncMock(return_value=allowed) + preflight = AsyncMock() + with ( + patch.object( # test-quality-ok: route handler reads the module-level manager, no injection seam + server_module.global_mcp_server_manager, "get_mcp_server_by_name", return_value=requested + ), + patch.object( # test-quality-ok: the exchanger is the observable; a real one would call an IdP + server_module.global_mcp_server_manager, "preflight_token_exchange", preflight + ), + patch.object( # test-quality-ok: allowed-set resolution needs the DB; the test controls its answer + server_module, "_get_allowed_mcp_servers", allowed_lookup + ), + ): + await server_module._raise_preemptive_401_for_unauthenticated_servers( + scope={"type": "http", "method": "POST", "path": f"/mcp/{requested.alias}", "headers": []}, + mcp_servers=[requested.alias], + oauth2_headers=self.SUBJECT_HEADERS, + mcp_server_auth_headers=None, + user_api_key_auth=user_api_key_auth, + client_ip="10.0.0.7", + ) + return allowed_lookup, preflight + + @pytest.mark.asyncio + async def test_unentitled_key_never_reaches_the_exchanger(self): + requested = _make_obo_server("obo_tools") + key = UserAPIKeyAuth(api_key="sk-plain-only") + + allowed_lookup, preflight = await self._run( + requested, allowed=[_make_obo_server("plain_tools")], user_api_key_auth=key + ) + + preflight.assert_not_awaited() + allowed_lookup.assert_awaited_once_with( + user_api_key_auth=key, mcp_servers=[requested.alias], client_ip="10.0.0.7" + ) + + @pytest.mark.asyncio + async def test_entitled_key_still_exchanges_at_connect(self): + requested = _make_obo_server("obo_tools") + key = UserAPIKeyAuth(api_key="sk-obo") + + _, preflight = await self._run(requested, allowed=[requested], user_api_key_auth=key) + + preflight.assert_awaited_once_with(server=requested, oauth2_headers=self.SUBJECT_HEADERS, user_api_key_auth=key) + + @pytest.mark.asyncio async def test_post_mcp_call_guardrails_return_the_rewritten_result(): """The result a post_mcp_call guardrail rewrote must be what the caller sends back.""" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py index 16221f44efe..239f89ebd90 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py @@ -10,33 +10,36 @@ Covers: """ import json +from collections.abc import Sequence from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest +from mcp.types import Tool +import litellm from litellm.models.object_permission import LiteLLM_ObjectPermissionTable from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.proxy._experimental.mcp_server.tool_search import ( AGENT_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME, MCP_TOOL_SEARCH_TOOL_NAME, + SemanticToolRanker, + ToolSearchResult, coerce_top_k, get_virtual_tool_definitions, + search_mcp_tools, search_tools, ) from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.common_utils.semantic_text_index import EmbeddingFailed, SemanticTextIndex, Vector +from litellm.types.mcp import MCPToolSearchSettings -def _make_tools(specs: list[tuple[str, str]]) -> list[dict[str, Any]]: - return [ - { - "name": name, - "description": desc, - "inputSchema": {"type": "object", "properties": {}}, - } - for name, desc in specs - ] +def _make_tools(specs: list[tuple[str, str]]) -> tuple[Tool, ...]: + return tuple( + Tool(name=name, description=desc, inputSchema={"type": "object", "properties": {}}) for name, desc in specs + ) def _make_perm(**kwargs: Any) -> LiteLLM_ObjectPermissionTable: @@ -54,6 +57,160 @@ SAMPLE_TOOLS = _make_tools( ) +FX_TOOL = Tool( + name="treasury-get_rates", + description="Get foreign exchange rates for a currency pair", + inputSchema={"type": "object", "properties": {}}, +) +WEATHER_TOOL = Tool( + name="weather-forecast", + description="Get the weather forecast for a city", + inputSchema={"type": "object", "properties": {}}, +) +CALENDAR_TOOL = Tool( + name="calendar-create_event", + description="Create a calendar event", + inputSchema={"type": "object", "properties": {}}, +) +CATALOG = (FX_TOOL, WEATHER_TOOL, CALENDAR_TOOL) + +# A stand-in embedding space: "FX" sits next to the foreign-exchange tool and far from the rest. +FAKE_VECTORS: dict[str, Vector] = { + "FX": (1.0, 0.0), + f"{FX_TOOL.name}\n{FX_TOOL.description}": (0.9, 0.1), + f"{WEATHER_TOOL.name}\n{WEATHER_TOOL.description}": (0.3, 1.0), + f"{CALENDAR_TOOL.name}\n{CALENDAR_TOOL.description}": (0.0, 1.0), +} + + +class RecordingEmbedder: + def __init__(self) -> None: + self.calls: list[tuple[str, ...]] = [] + + async def __call__(self, texts: Sequence[str]) -> Sequence[Vector]: + self.calls.append(tuple(texts)) + return tuple(FAKE_VECTORS[text] for text in texts) + + +def _ranker(embedder: RecordingEmbedder | None = None) -> SemanticToolRanker: + return SemanticToolRanker(embed=embedder or RecordingEmbedder(), embedding_model="emb", index=SemanticTextIndex()) + + +def _names(results: Sequence[ToolSearchResult] | EmbeddingFailed) -> list[str]: + assert not isinstance(results, EmbeddingFailed) + return [tool["name"] for tool in results] + + +class TestSearchMcpTools: + @pytest.mark.asyncio + async def test_semantic_mode_finds_foreign_exchange_tool_for_fx(self) -> None: + keyword_only = await search_mcp_tools("FX", CATALOG, 5, MCPToolSearchSettings(), ranker=None) + assert _names(keyword_only) == [] + + results = await search_mcp_tools("FX", CATALOG, 5, MCPToolSearchSettings(embedding_model="emb"), _ranker()) + assert _names(results) == [FX_TOOL.name, WEATHER_TOOL.name, CALENDAR_TOOL.name] + assert not isinstance(results, EmbeddingFailed) + assert results[0]["score"] > results[1]["score"] > results[2]["score"] + assert results[0]["inputSchema"] == FX_TOOL.inputSchema + + @pytest.mark.asyncio + async def test_similarity_threshold_drops_weak_matches(self) -> None: + settings = MCPToolSearchSettings(embedding_model="emb", similarity_threshold=0.5) + results = await search_mcp_tools("FX", CATALOG, 5, settings, _ranker()) + assert _names(results) == [FX_TOOL.name] + + @pytest.mark.asyncio + async def test_request_top_k_limits_semantic_results(self) -> None: + results = await search_mcp_tools("FX", CATALOG, 2, MCPToolSearchSettings(embedding_model="emb"), _ranker()) + assert _names(results) == [FX_TOOL.name, WEATHER_TOOL.name] + + @pytest.mark.asyncio + async def test_configured_top_k_caps_request_top_k(self) -> None: + settings = MCPToolSearchSettings(embedding_model="emb", top_k=1) + assert _names(await search_mcp_tools("FX", CATALOG, 50, settings, _ranker())) == [FX_TOOL.name] + assert _names(await search_mcp_tools("weather", CATALOG, 50, MCPToolSearchSettings(top_k=1), None)) == [ + WEATHER_TOOL.name + ] + + @pytest.mark.asyncio + async def test_core_tools_lead_and_do_not_consume_top_k(self) -> None: + settings = MCPToolSearchSettings(embedding_model="emb", top_k=1, core_tools=(CALENDAR_TOOL.name,)) + results = await search_mcp_tools("FX", CATALOG, 1, settings, _ranker()) + assert _names(results) == [CALENDAR_TOOL.name, FX_TOOL.name] + assert not isinstance(results, EmbeddingFailed) + assert "score" not in results[0] + + @pytest.mark.asyncio + async def test_core_tools_apply_in_keyword_mode_too(self) -> None: + settings = MCPToolSearchSettings(core_tools=(CALENDAR_TOOL.name,)) + assert _names(await search_mcp_tools("weather", CATALOG, 5, settings, None)) == [ + CALENDAR_TOOL.name, + WEATHER_TOOL.name, + ] + + @pytest.mark.asyncio + async def test_core_tools_outside_the_callers_catalog_are_not_returned(self) -> None: + settings = MCPToolSearchSettings(embedding_model="emb", core_tools=("payroll-run", CALENDAR_TOOL.name)) + results = await search_mcp_tools("FX", (FX_TOOL, WEATHER_TOOL), 5, settings, _ranker()) + assert _names(results) == [FX_TOOL.name, WEATHER_TOOL.name] + + @pytest.mark.asyncio + async def test_core_tools_are_listed_once_and_never_embedded(self) -> None: + embedder = RecordingEmbedder() + settings = MCPToolSearchSettings(embedding_model="emb", core_tools=(FX_TOOL.name, FX_TOOL.name)) + results = await search_mcp_tools("FX", CATALOG, 5, settings, _ranker(embedder)) + assert _names(results) == [FX_TOOL.name, WEATHER_TOOL.name, CALENDAR_TOOL.name] + assert all(FX_TOOL.description not in text for call in embedder.calls for text in call) + + @pytest.mark.asyncio + async def test_empty_query_returns_only_core_tools_without_embedding(self) -> None: + embedder = RecordingEmbedder() + settings = MCPToolSearchSettings(embedding_model="emb", core_tools=(CALENDAR_TOOL.name,)) + assert _names(await search_mcp_tools("", CATALOG, 5, settings, _ranker(embedder))) == [CALENDAR_TOOL.name] + assert embedder.calls == [] + + @pytest.mark.asyncio + async def test_repeat_searches_only_embed_the_query(self) -> None: + embedder = RecordingEmbedder() + ranker = _ranker(embedder) + settings = MCPToolSearchSettings(embedding_model="emb") + await search_mcp_tools("FX", CATALOG, 5, settings, ranker) + await search_mcp_tools("FX", CATALOG, 5, settings, ranker) + assert [len(call) for call in embedder.calls] == [4, 1] + + @pytest.mark.asyncio + async def test_embedding_failure_is_reported_not_raised(self) -> None: + async def failing(texts: Sequence[str]) -> Sequence[Vector]: + raise ValueError("embedding model is down") + + ranker = SemanticToolRanker(embed=failing, embedding_model="emb", index=SemanticTextIndex()) + result = await search_mcp_tools("FX", CATALOG, 5, MCPToolSearchSettings(embedding_model="emb"), ranker) + assert isinstance(result, EmbeddingFailed) + assert "embedding model is down" in result.reason + + +class TestMcpToolSearchSettings: + def test_rejects_out_of_range_values(self) -> None: + from pydantic import ValidationError + + with pytest.raises(ValidationError): + MCPToolSearchSettings(top_k=0) + with pytest.raises(ValidationError): + MCPToolSearchSettings(similarity_threshold=1.5) + + def test_yaml_shape_round_trips(self) -> None: + settings = MCPToolSearchSettings.model_validate( + {"embedding_model": "emb", "top_k": 3, "similarity_threshold": 0.2, "core_tools": ["a", "b"]} + ) + assert settings.core_tools == ("a", "b") + assert settings.model_dump() == { + "embedding_model": "emb", + "top_k": 3, + "similarity_threshold": 0.2, + "core_tools": ("a", "b"), + } + + class TestCoerceTopK: def test_int_passthrough(self) -> None: assert coerce_top_k(3) == 3 @@ -92,10 +249,10 @@ class TestSearchTools: assert len(results) <= 2 def test_empty_query_returns_empty(self) -> None: - assert search_tools("", SAMPLE_TOOLS) == [] + assert search_tools("", SAMPLE_TOOLS) == () def test_no_match_returns_empty(self) -> None: - assert search_tools("xyzzy_nonexistent_zzz", SAMPLE_TOOLS) == [] + assert search_tools("xyzzy_nonexistent_zzz", SAMPLE_TOOLS) == () def test_matches_description_not_just_name(self) -> None: results = search_tools("channel", SAMPLE_TOOLS) @@ -603,6 +760,63 @@ class TestCallToolRestApiVirtualTools: assert result.isError is True assert result.content[0].text == "set agent_search_embedding_model" + def _semantic_request(self, query: str = "FX") -> MagicMock: + return self._make_request({"name": MCP_TOOL_SEARCH_TOOL_NAME, "arguments": {"query": query}}) + + @pytest.mark.asyncio + async def test_mcp_tool_search_ranks_the_callers_catalog_with_the_configured_embedding_model( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "mcp_tool_search", {"embedding_model": "emb", "similarity_threshold": 0.5}) + user_api_key_dict = UserAPIKeyAuth( + api_key="k", team_id="team-1", object_permission=_make_perm(mcp_tool_search_enabled=True) + ) + + async def fake_aembedding(model: str, input: list[str], metadata: dict[str, Any]) -> MagicMock: + assert model == "emb" + assert metadata["user_api_key"] == "k" + assert metadata["user_api_key_team_id"] == "team-1" + response = MagicMock() + response.model_dump.return_value = {"data": [{"embedding": list(FAKE_VECTORS[t])} for t in input]} + return response + + router = MagicMock() + router.aembedding = AsyncMock(side_effect=fake_aembedding) + with ( + patch( # test-quality-ok: the proxy's router is a module global; the handler reaches it the way production does + "litellm.proxy.proxy_server.llm_router", router + ), + patch( # test-quality-ok: the authorized catalog is the seam every virtual tool shares; the ranking under test stays real + "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + new_callable=AsyncMock, + return_value=AggregateToolListing(tools=list(CATALOG), outcomes={}), + ) as mock_list, + ): + result = await self._get_call_fn()(request=self._semantic_request(), user_api_key_dict=user_api_key_dict) + + assert mock_list.await_args.kwargs["user_api_key_auth"] is user_api_key_dict + assert result.isError is False + assert [t["name"] for t in json.loads(result.content[0].text)] == [FX_TOOL.name] + + @pytest.mark.asyncio + async def test_mcp_tool_search_reports_missing_router_as_tool_error(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "mcp_tool_search", {"embedding_model": "emb"}) + user_api_key_dict = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + with patch( # test-quality-ok: the proxy's router is a module global; the handler reaches it the way production does + "litellm.proxy.proxy_server.llm_router", None + ): + result = await self._get_call_fn()(request=self._semantic_request(), user_api_key_dict=user_api_key_dict) + assert result.isError is True + assert "mcp_tool_search.embedding_model" in result.content[0].text + + @pytest.mark.asyncio + async def test_mcp_tool_search_reports_invalid_settings_as_tool_error(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "mcp_tool_search", {"top_k": 0}) + user_api_key_dict = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + result = await self._get_call_fn()(request=self._semantic_request(), user_api_key_dict=user_api_key_dict) + assert result.isError is True + assert "top_k" in result.content[0].text + @pytest.mark.asyncio async def test_agent_search_requires_flag_enabled(self) -> None: from fastapi import HTTPException diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 0480bbc40a7..d441c05090b 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -1,4 +1,5 @@ import asyncio +import inspect import json import sys from datetime import datetime @@ -13,6 +14,7 @@ import pytest from fastapi import HTTPException from starlette.requests import Request +from litellm.constants import MCP_TOOL_LISTING_TIMEOUT from litellm.proxy._experimental.mcp_server import rest_endpoints from litellm.proxy._experimental.mcp_server.auth import ( user_api_key_auth_mcp as auth_mcp, @@ -109,6 +111,71 @@ class TestExecuteWithMcpClient: assert result["status"] == "error" assert "stack_trace" not in result + @pytest.mark.asyncio + async def test_timeout_caps_hanging_operation_and_names_url(self, monkeypatch): + async def fake_create_client(*args, **kwargs): + return object() + + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "_create_mcp_client", + fake_create_client, + ) + + async def hanging_operation(client): + await asyncio.Event().wait() + + payload = NewMCPServerRequest( + server_name="example", + url="https://mcp.example.com/mcp/", + auth_type=MCPAuth.none, + ) + + result = await asyncio.wait_for( + rest_endpoints._execute_with_mcp_client(payload, hanging_operation, timeout_seconds=0.05), + timeout=5, + ) + + assert result["error"] is True + assert "https://mcp.example.com/mcp/" in result["message"] + + @pytest.mark.asyncio + async def test_timeout_covers_client_creation(self, monkeypatch): + async def hanging_create_client(*args, **kwargs): + await asyncio.Event().wait() + + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "_create_mcp_client", + hanging_create_client, + ) + + async def unreached_operation(client): + return {"status": "ok"} + + payload = NewMCPServerRequest( + server_name="example", + url="https://mcp.example.com/mcp/", + auth_type=MCPAuth.none, + ) + + result = await asyncio.wait_for( + rest_endpoints._execute_with_mcp_client(payload, unreached_operation, timeout_seconds=0.05), + timeout=5, + ) + + assert result["error"] is True + assert "https://mcp.example.com/mcp/" in result["message"] + + def test_timeout_defaults_to_tool_listing_timeout(self): + default = inspect.signature(rest_endpoints._execute_with_mcp_client).parameters["timeout_seconds"].default + assert default == MCP_TOOL_LISTING_TIMEOUT + + def test_connection_error_message_timeout_names_url_and_budget(self): + message = rest_endpoints._connection_error_message(TimeoutError(), "https://api.example.com/mcp/", 30.0) + assert "https://api.example.com/mcp/" in message + assert "30s" in message + @pytest.mark.asyncio async def test_forwards_static_headers(self, monkeypatch): """Ensure static_headers are forwarded to the MCP client during test calls. @@ -3168,17 +3235,21 @@ class TestConnectionErrorMessage: secret = "Bearer sk-super-secret-token" exc = httpx.LocalProtocolError(f"Illegal header value b' {secret}'") - message = rest_endpoints._connection_error_message(exc) + message = rest_endpoints._connection_error_message(exc, "https://example.com", 30.0) assert "header" in message.lower() assert secret not in message def test_connect_error_points_at_reachability(self): - message = rest_endpoints._connection_error_message(httpx.ConnectError("All connection attempts failed")) + message = rest_endpoints._connection_error_message( + httpx.ConnectError("All connection attempts failed"), "https://example.com", 30.0 + ) assert "unreachable" in message.lower() def test_timeout_error_message(self): - message = rest_endpoints._connection_error_message(httpx.ConnectTimeout("timed out")) + message = rest_endpoints._connection_error_message( + httpx.ConnectTimeout("timed out"), "https://example.com", 30.0 + ) assert "unreachable" in message.lower() def test_http_status_error_includes_status_code(self): @@ -3188,11 +3259,11 @@ class TestConnectionErrorMessage: request=httpx.Request("POST", "http://x/"), response=response, ) - message = rest_endpoints._connection_error_message(exc) + message = rest_endpoints._connection_error_message(exc, "https://example.com", 30.0) assert "503" in message def test_unknown_error_falls_back_to_generic(self): - message = rest_endpoints._connection_error_message(RuntimeError("weird")) + message = rest_endpoints._connection_error_message(RuntimeError("weird"), "https://example.com", 30.0) assert "weird" not in message assert "proxy logs" in message.lower() diff --git a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py index 2ff38af80b1..43034f889f6 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py @@ -918,6 +918,62 @@ async def test_task_method_failure_hook_uses_enriched_request_data(): assert failure_data.get("agent_id") == "test-agent" +@pytest.mark.asyncio +async def test_agentcore_invalid_context_id_returns_jsonrpc_invalid_params_400(): + from litellm.proxy._types import UserAPIKeyAuth + + agent = _make_agent_mock() + agent.litellm_params = { + "custom_llm_provider": "bedrock", + "model": "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/demo", + "api_key": "test-jwt-token", + } + mock_request = _make_request_mock( + "message/send", + { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "Hello"}], + "messageId": "msg-1", + "contextId": "too-short", + } + }, + ) + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1") + + mock_proxy_logging = MagicMock() + mock_proxy_logging.pre_call_hook = AsyncMock( + side_effect=lambda user_api_key_dict, data, call_type: data + ) + mock_proxy_logging.post_call_failure_hook = AsyncMock(return_value=None) + + with ExitStack() as stack: + for p in _base_patches(agent): + stack.enter_context(p) + stack.enter_context( + patch( # test-quality-ok: same proxy_logging_obj injection the sibling failure-hook test uses; no HTTP call is made because the request is rejected before signing + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging + ) + ) + + from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a + + response = await invoke_agent_a2a( + agent_id="test-agent", + request=mock_request, + fastapi_response=MagicMock(), + user_api_key_dict=user_api_key_dict, + ) + + body = json.loads(response.body.decode()) + assert response.status_code == 400 + assert body["id"] == "req-1" + assert body["error"]["code"] == -32602 + assert "Invalid AgentCore runtime session id" in body["error"]["message"] + assert "Internal error" not in body["error"]["message"] + mock_proxy_logging.post_call_failure_hook.assert_awaited_once() + + @pytest.mark.asyncio async def test_get_extended_agent_card_rewrites_url(): from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py index 7ce62fdf648..231626c7eb5 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py @@ -8,7 +8,18 @@ from unittest.mock import AsyncMock, MagicMock import pytest -from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry, GrantMigrationResult +from litellm.constants import REDACTED_BY_LITELM_STRING +from litellm.proxy.agent_endpoints.agent_registry import ( + AgentRegistry, + GrantMigrationResult, + _restore_redacted_litellm_params, + redact_sensitive_agent_litellm_params, +) + +# Obviously-fake stand-ins for a real AWS credential pair (LIT-6736 regression +# fixtures) -- never a real key shape, and must never appear in any response. +SENTINEL_AWS_ACCESS_KEY_ID: Final = "AKIATESTSENTINEL0000" +SENTINEL_AWS_SECRET_ACCESS_KEY: Final = "test-sentinel-do-not-use-secret-value" def _sample_agent_card_params() -> dict: @@ -49,6 +60,7 @@ async def test_update_agent_in_db_clears_static_headers_and_extra_headers_when_o mock_update = AsyncMock(return_value=updated_agent) mock_prisma.db.litellm_agentstable.update = mock_update + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock(return_value=None) # Agent config WITHOUT static_headers or extra_headers (omitted) agent_config = { @@ -95,6 +107,7 @@ async def test_update_agent_in_db_preserves_explicit_static_headers_and_extra_he mock_update = AsyncMock(return_value=updated_agent) mock_prisma.db.litellm_agentstable.update = mock_update + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock(return_value=None) agent_config = { "agent_name": "Updated Agent", @@ -436,6 +449,9 @@ async def test_update_agent_in_db_raises_when_row_deleted_mid_update(): guard the code dereferences None and reports an opaque AttributeError instead of the id.""" registry: Final = AgentRegistry() mock_prisma: Final = MagicMock() + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=SimpleNamespace(litellm_params={}, object_permission_id=None) + ) mock_prisma.db.litellm_agentstable.update = AsyncMock(return_value=None) with pytest.raises(Exception, match="Error updating agent in DB") as exc_info: @@ -485,3 +501,492 @@ async def test_delete_agent_from_db_raises_when_row_already_gone(): await registry.delete_agent_from_db(agent_id="agent-123", prisma_client=mock_prisma) assert str(exc_info.value) == "Error deleting agent from DB: Agent not found, passed agent_id=agent-123" + + +# ---------- LIT-6736: agent litellm_params secret redaction ---------- + + +def test_redact_sensitive_agent_litellm_params_masks_secrets_keeps_the_rest(): + """The sentinel secret must never appear in the redacted output; non-secret + keys (model reference, is_public) must survive untouched.""" + redacted = redact_sensitive_agent_litellm_params( + { + "aws_access_key_id": SENTINEL_AWS_ACCESS_KEY_ID, + "aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, + "model": "bedrock/agentcore/my-agent", + "is_public": True, + } + ) + + assert SENTINEL_AWS_ACCESS_KEY_ID not in json.dumps(redacted) + assert SENTINEL_AWS_SECRET_ACCESS_KEY not in json.dumps(redacted) + assert redacted["aws_access_key_id"] == REDACTED_BY_LITELM_STRING + assert redacted["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert redacted["model"] == "bedrock/agentcore/my-agent" + assert redacted["is_public"] is True + + +def test_redact_sensitive_agent_litellm_params_recurses_into_nested_dicts(): + """A secret nested one level down (e.g. a per-provider sub-config) must + also be redacted, not just top-level keys.""" + redacted = redact_sensitive_agent_litellm_params( + {"provider_config": {"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "region": "us-east-1"}} + ) + + assert SENTINEL_AWS_SECRET_ACCESS_KEY not in json.dumps(redacted) + assert redacted["provider_config"]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert redacted["provider_config"]["region"] == "us-east-1" + + +def test_redact_sensitive_agent_litellm_params_handles_none_and_json_string(): + assert redact_sensitive_agent_litellm_params(None) is None + + serialized = json.dumps({"api_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "model": "gpt-4"}) + redacted = redact_sensitive_agent_litellm_params(serialized) + + assert SENTINEL_AWS_SECRET_ACCESS_KEY not in redacted + assert json.loads(redacted)["api_key"] == REDACTED_BY_LITELM_STRING + assert json.loads(redacted)["model"] == "gpt-4" + + +def test_redact_sensitive_agent_litellm_params_recurses_into_lists_of_dicts(): + """A secret nested inside a list of provider sub-configs (a shape a + non-sensitively-named key can legitimately hold) must also be redacted, + not silently returned as-is.""" + redacted = redact_sensitive_agent_litellm_params( + { + "provider_configs": [ + {"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "region": "us-east-1"}, + {"aws_secret_access_key": "other-" + SENTINEL_AWS_SECRET_ACCESS_KEY, "region": "us-west-2"}, + ] + } + ) + + assert SENTINEL_AWS_SECRET_ACCESS_KEY not in json.dumps(redacted) + assert redacted["provider_configs"][0]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert redacted["provider_configs"][0]["region"] == "us-east-1" + assert redacted["provider_configs"][1]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert redacted["provider_configs"][1]["region"] == "us-west-2" + + +def test_redact_sensitive_agent_litellm_params_redacts_secrets_inside_model_list(): + """The exact shape flagged in review: litellm_params.model_list, where each + entry carries its own nested litellm_params with a provider credential.""" + redacted = redact_sensitive_agent_litellm_params( + { + "model_list": [ + { + "model_name": "gpt-4", + "litellm_params": {"api_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "model": "gpt-4"}, + }, + { + "model_name": "claude", + "litellm_params": { + "aws_secret_access_key": "other-" + SENTINEL_AWS_SECRET_ACCESS_KEY, + "model": "bedrock/claude", + }, + }, + ] + } + ) + + assert SENTINEL_AWS_SECRET_ACCESS_KEY not in json.dumps(redacted) + assert redacted["model_list"][0]["litellm_params"]["api_key"] == REDACTED_BY_LITELM_STRING + assert redacted["model_list"][0]["litellm_params"]["model"] == "gpt-4" + assert redacted["model_list"][1]["litellm_params"]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert redacted["model_list"][1]["litellm_params"]["model"] == "bedrock/claude" + + +def test_restore_redacted_litellm_params_preserves_secret_inside_model_list(): + """The write-side counterpart: a caller editing a model_list entry's own + non-secret field (renaming it) while leaving that same entry's nested + secret masked must not corrupt the stored per-deployment credential. + List entries correspond by position (see the module docstring on + ``_restore_redacted_nested_value``), so this -- the common "edit this + entry, keep its secret" pattern -- must keep working.""" + existing = { + "agent_name": "my-agent", + "model_list": [ + { + "model_name": "gpt-4", + "litellm_params": {"api_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "model": "gpt-4"}, + }, + ], + } + incoming = { + "agent_name": "my-agent-renamed", + "model_list": [ + { + "model_name": "gpt-4-renamed", + "litellm_params": {"api_key": REDACTED_BY_LITELM_STRING, "model": "gpt-4"}, + }, + ], + } + + restored = _restore_redacted_litellm_params(incoming, existing) + + assert SENTINEL_AWS_SECRET_ACCESS_KEY == restored["model_list"][0]["litellm_params"]["api_key"] + assert restored["model_list"][0]["model_name"] == "gpt-4-renamed" + assert restored["agent_name"] == "my-agent-renamed" + + +def test_restore_redacted_litellm_params_matches_list_entries_by_position(): + """Documents the accepted trade-off: a list has no stable per-element + identity in a plain ``dict[str, object]`` schema, so restoration matches + entries by index, the same correspondence every other part of this merge + (and the endpoints' full-replace-on-PUT semantics) already assumes. If a + caller both reorders the list AND echoes back a masked marker in the same + request, a credential can end up attached to a different logical entry. + That is a known, narrow limitation -- not a leak between different + agents or tenants, since it only reshuffles one agent's own stored + values -- and this test pins the current, deliberate behavior rather + than asserting it away.""" + existing = { + "model_list": [ + {"model_name": "gpt-4", "litellm_params": {"api_key": SENTINEL_AWS_SECRET_ACCESS_KEY}}, + {"model_name": "claude", "litellm_params": {"api_key": "other-" + SENTINEL_AWS_SECRET_ACCESS_KEY}}, + ], + } + incoming = { + "model_list": [ + # Same index (0) now holds what used to be at index 1's entry. + {"model_name": "claude", "litellm_params": {"api_key": REDACTED_BY_LITELM_STRING}}, + ], + } + + restored = _restore_redacted_litellm_params(incoming, existing) + + assert restored["model_list"][0]["litellm_params"]["api_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY + + +def test_restore_redacted_litellm_params_recovers_a_whole_subtree_collapsed_by_the_depth_cap(): + """Past the read-side recursion depth cap, a whole nested subtree is + collapsed to the flat REDACTED_BY_LITELM marker rather than a dict/list. + If the caller echoes that flat marker back unchanged, the whole + subtree -- not just the literal marker string -- must be restored.""" + existing_subtree = {"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "region": "us-east-1"} + incoming = {"provider_config": REDACTED_BY_LITELM_STRING} + existing = {"provider_config": existing_subtree} + + restored = _restore_redacted_litellm_params(incoming, existing) + + assert restored["provider_config"] == existing_subtree + + +def test_redact_sensitive_agent_litellm_params_does_not_reinterpret_plain_string_values_as_json(): + """A plain non-JSON string value (most string leaves) must pass through + unchanged rather than failing to parse and getting redacted.""" + redacted = redact_sensitive_agent_litellm_params({"model": "bedrock/agentcore/my-agent", "is_public": True}) + + assert redacted["model"] == "bedrock/agentcore/my-agent" + assert redacted["is_public"] is True + + +@pytest.mark.asyncio +async def test_add_agent_to_db_drops_a_sentinel_value_instead_of_storing_the_placeholder(): + """A create has nothing stored to restore behind a redaction marker, so a + sensitive key submitted as the literal marker is dropped rather than + persisted as the placeholder string itself.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + created_agent = MagicMock() + created_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + created_agent.object_permission = None + mock_create = AsyncMock(return_value=created_agent) + mock_prisma.db.litellm_agentstable.create = mock_create + + await registry.add_agent_to_db( + agent={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": { + "aws_secret_access_key": REDACTED_BY_LITELM_STRING, + "model": "bedrock/agentcore/my-agent", + }, + }, + prisma_client=mock_prisma, + created_by="test-user", + ) + + stored_params: Final = json.loads(mock_create.call_args.kwargs["data"]["litellm_params"]) + assert "aws_secret_access_key" not in stored_params + assert stored_params["model"] == "bedrock/agentcore/my-agent" + + +@pytest.mark.asyncio +async def test_update_agent_in_db_preserves_secret_when_echoed_back_redacted(): + """PUT round-trips the GET response, which shows the secret redacted. Saving + an unrelated field change must not overwrite the real stored credential + with the redaction marker.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=SimpleNamespace( + litellm_params={ + "aws_access_key_id": SENTINEL_AWS_ACCESS_KEY_ID, + "aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, + "model": "bedrock/agentcore/my-agent", + }, + object_permission_id=None, + ) + ) + updated_agent = MagicMock() + updated_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Renamed Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + updated_agent.object_permission = None + mock_update = AsyncMock(return_value=updated_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.update_agent_in_db( + agent_id="agent-123", + agent={ + "agent_name": "Renamed Agent", + "agent_card_params": _sample_agent_card_params(), + # The UI round-tripped the redacted secret and the untouched + # access key id verbatim; only agent_name actually changed. + "litellm_params": { + "aws_access_key_id": SENTINEL_AWS_ACCESS_KEY_ID, + "aws_secret_access_key": REDACTED_BY_LITELM_STRING, + "model": "bedrock/agentcore/my-agent", + }, + }, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"]) + assert stored_params["aws_secret_access_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY + assert stored_params["aws_access_key_id"] == SENTINEL_AWS_ACCESS_KEY_ID + assert stored_params["model"] == "bedrock/agentcore/my-agent" + + +@pytest.mark.asyncio +async def test_update_agent_in_db_preserves_secret_when_key_omitted_entirely(): + """Omitting the sensitive key altogether must fall back to the stored + value too, not just an explicit redaction-marker round-trip.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=SimpleNamespace( + litellm_params={"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY}, + object_permission_id=None, + ) + ) + updated_agent = MagicMock() + updated_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + updated_agent.object_permission = None + mock_update = AsyncMock(return_value=updated_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.update_agent_in_db( + agent_id="agent-123", + agent={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {"model": "bedrock/agentcore/my-agent"}, + }, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"]) + assert stored_params["aws_secret_access_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY + + +@pytest.mark.asyncio +async def test_update_agent_in_db_preserves_secret_nested_under_a_non_sensitive_key(): + """A secret nested inside a dict held by a non-sensitively-named key + (e.g. a per-provider sub-config) must also survive an echoed-back + redaction marker, not just top-level secret keys.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=SimpleNamespace( + litellm_params={ + "provider_config": { + "aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, + "region": "us-east-1", + } + }, + object_permission_id=None, + ) + ) + updated_agent = MagicMock() + updated_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + updated_agent.object_permission = None + mock_update = AsyncMock(return_value=updated_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.update_agent_in_db( + agent_id="agent-123", + agent={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": { + # The GET response redacted the nested secret; the caller + # round-trips it verbatim while changing nothing. + "provider_config": { + "aws_secret_access_key": REDACTED_BY_LITELM_STRING, + "region": "us-west-2", + } + }, + }, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"]) + assert stored_params["provider_config"]["aws_secret_access_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY + assert stored_params["provider_config"]["region"] == "us-west-2" + + +@pytest.mark.asyncio +async def test_update_agent_in_db_clears_secret_on_explicit_empty_value(): + """An explicit empty string is a deliberate clear, distinct from an omitted + key or the redaction marker, and must actually clear the stored secret.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=SimpleNamespace( + litellm_params={"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY}, + object_permission_id=None, + ) + ) + updated_agent = MagicMock() + updated_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + updated_agent.object_permission = None + mock_update = AsyncMock(return_value=updated_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.update_agent_in_db( + agent_id="agent-123", + agent={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {"aws_secret_access_key": ""}, + }, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"]) + assert stored_params["aws_secret_access_key"] == "" + + +@pytest.mark.asyncio +async def test_patch_agent_in_db_preserves_secret_when_litellm_params_omitted(): + """A PATCH that only renames the agent must not touch (let alone drop) the + stored litellm_params secret.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value={ + "agent_id": "agent-123", + "agent_name": "Old Name", + "litellm_params": {"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY}, + "object_permission_id": None, + } + ) + patched_agent = MagicMock() + patched_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "New Name", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY}, + "object_permission": None, + } + patched_agent.object_permission = None + mock_update = AsyncMock(return_value=patched_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.patch_agent_in_db( + agent_id="agent-123", + agent={"agent_name": "New Name"}, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + update_data: Final = mock_update.call_args.kwargs["data"] + assert "litellm_params" not in update_data + + +@pytest.mark.asyncio +async def test_patch_agent_in_db_preserves_secret_when_echoed_back_redacted(): + """A PATCH that includes litellm_params (e.g. to flip an unrelated flag) + with the secret round-tripped as the redaction marker must not clobber + the stored credential.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value={ + "agent_id": "agent-123", + "agent_name": "Test Agent", + "litellm_params": { + "aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, + "is_public": False, + }, + "object_permission_id": None, + } + ) + patched_agent = MagicMock() + patched_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + patched_agent.object_permission = None + mock_update = AsyncMock(return_value=patched_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.patch_agent_in_db( + agent_id="agent-123", + agent={ + "litellm_params": { + "aws_secret_access_key": REDACTED_BY_LITELM_STRING, + "is_public": True, + } + }, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"]) + assert stored_params["aws_secret_access_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY + assert stored_params["is_public"] is True diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py index 3fb09076e5f..daca244c0a1 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py @@ -16,13 +16,12 @@ from litellm.proxy.agent_endpoints.agent_search import ( AgentSearchHits, AgentSearchIndex, AgentSearchNotConfigured, - Vector, agent_search_text, - cosine_similarity, search_agents, ) from litellm.proxy.agent_endpoints.auth.agent_permission_handler import RestrictedAgentAccess from litellm.proxy.agent_endpoints.endpoints import router, user_api_key_auth +from litellm.proxy.common_utils.semantic_text_index import Vector, cosine_similarity from litellm.types.agents import AgentResponse CALLER: Final = UserAPIKeyAuth(api_key="hashed-caller-key", team_id="team-1", user_id="user-1") diff --git a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py index ea196bda529..a78b3238a9a 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py @@ -4,6 +4,7 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient +from litellm.constants import REDACTED_BY_LITELM_STRING from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.agent_endpoints import endpoints as agent_endpoints from litellm.proxy.agent_endpoints.auth.agent_permission_handler import ( @@ -484,11 +485,15 @@ class TestAgentRBACInternalUserViewOnly: assert resp.status_code == 403 +SENTINEL_AGENT_API_KEY = "sk-test-sentinel-do-not-use" + + class TestAgentRBACProxyAdminViewOnly: """Read-only proxy admins go through the object-permission scoped branch on GET /v1/agents (the admin fast path stays full PROXY_ADMIN only, so viewers - cannot fan out health checks beyond their allowlist), and secret unredaction - also stays gated on full PROXY_ADMIN.""" + cannot fan out health checks beyond their allowlist). litellm_params + secrets are redacted for every caller, admin included (LIT-6736); only the + virtual-key/header visibility stays gated on full PROXY_ADMIN.""" @pytest.fixture(autouse=True) def _setup(self, monkeypatch): @@ -501,7 +506,7 @@ class TestAgentRBACProxyAdminViewOnly: agent_id=f"agent-{index}", agent_name=f"Agent {index}", agent_card_params=_sample_agent_card_params(), - litellm_params={"api_key": "sk-super-secret-agent-key"}, + litellm_params={"api_key": SENTINEL_AGENT_API_KEY}, ) for index in (1, 2) ] @@ -544,7 +549,7 @@ class TestAgentRBACProxyAdminViewOnly: def test_should_still_redact_secrets_for_view_only_admin(self): """An unrestricted viewer sees the same agents as an admin but with keys - stripped and litellm_params masked.""" + stripped; litellm_params secrets never appear in either response.""" self.allowed_agents_spy.return_value = UnrestrictedAgentAccess() viewer_resp = self._list_agents(self.viewer_client) admin_resp = self._list_agents(self.admin_client) @@ -553,14 +558,12 @@ class TestAgentRBACProxyAdminViewOnly: viewer_by_id = {agent["agent_id"]: agent for agent in viewer_resp.json()} assert set(viewer_by_id) == {"agent-1", "agent-2"} assert viewer_by_id["agent-1"]["keys"] is None - assert "sk-super-secret-agent-key" not in viewer_resp.text + assert SENTINEL_AGENT_API_KEY not in viewer_resp.text admin_by_id = {agent["agent_id"]: agent for agent in admin_resp.json()} assert admin_by_id["agent-1"]["keys"][0]["token"] == "hash-aaa" - assert ( - admin_by_id["agent-1"]["litellm_params"]["api_key"] - == "sk-super-secret-agent-key" - ) + assert SENTINEL_AGENT_API_KEY not in admin_resp.text + assert admin_by_id["agent-1"]["litellm_params"]["api_key"] == REDACTED_BY_LITELM_STRING class TestAgentRBACProxyAdmin: @@ -616,6 +619,109 @@ class TestAgentRBACProxyAdmin: # Security scheme is the LiteLLM scheme. assert "LiteLLMKey" in stored_card["securitySchemes"] + def test_create_agent_response_never_echoes_secret(self): + """LIT-6736: POST /v1/agents must not echo the stored secret back, even + though it's the caller's own value and even for a proxy admin.""" + with patch("litellm.proxy.proxy_server.prisma_client"): # test-quality-ok: proxy_server module global is the endpoint's only injection point + self.mock_registry.get_agent_by_name = MagicMock(return_value=None) + self.mock_registry.add_agent_to_db = AsyncMock( + return_value=AgentResponse( + agent_id="agent-123", + agent_name="Test Agent", + agent_card_params=_sample_agent_card_params(), + litellm_params={ + "aws_secret_access_key": SENTINEL_AGENT_API_KEY, + "model": "bedrock/agentcore/my-agent", + }, + ) + ) + self.mock_registry.register_agent = MagicMock() + + resp = self.admin_client.post( + "/v1/agents", + json={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": { + "aws_secret_access_key": SENTINEL_AGENT_API_KEY, + "model": "bedrock/agentcore/my-agent", + }, + }, + headers={"Authorization": "Bearer k"}, + ) + + assert resp.status_code == 200 + assert SENTINEL_AGENT_API_KEY not in resp.text + body = resp.json() + assert body["litellm_params"]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert body["litellm_params"]["model"] == "bedrock/agentcore/my-agent" + + def test_update_agent_response_never_echoes_secret(self): + """LIT-6736: PUT /v1/agents/{id} must not echo the stored secret back.""" + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: # test-quality-ok: proxy_server module global is the endpoint's only injection point + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value={ + "agent_id": "agent-123", + "agent_name": "Existing Agent", + "agent_card_params": _sample_agent_card_params(), + } + ) + self.mock_registry.update_agent_in_db = AsyncMock( + return_value=AgentResponse( + agent_id="agent-123", + agent_name="Test Agent", + agent_card_params=_sample_agent_card_params(), + litellm_params={"aws_secret_access_key": SENTINEL_AGENT_API_KEY}, + ) + ) + self.mock_registry.deregister_agent = MagicMock() + self.mock_registry.register_agent = MagicMock() + + resp = self.admin_client.put( + "/v1/agents/agent-123", + json={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {"aws_secret_access_key": REDACTED_BY_LITELM_STRING}, + }, + headers={"Authorization": "Bearer k"}, + ) + + assert resp.status_code == 200 + assert SENTINEL_AGENT_API_KEY not in resp.text + assert resp.json()["litellm_params"]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + + def test_patch_agent_response_never_echoes_secret(self): + """LIT-6736: PATCH /v1/agents/{id} must not echo the stored secret back.""" + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: # test-quality-ok: proxy_server module global is the endpoint's only injection point + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value={ + "agent_id": "agent-123", + "agent_name": "Existing Agent", + "agent_card_params": _sample_agent_card_params(), + } + ) + self.mock_registry.patch_agent_in_db = AsyncMock( + return_value=AgentResponse( + agent_id="agent-123", + agent_name="Renamed Agent", + agent_card_params=_sample_agent_card_params(), + litellm_params={"aws_secret_access_key": SENTINEL_AGENT_API_KEY}, + ) + ) + self.mock_registry.deregister_agent = MagicMock() + self.mock_registry.register_agent = MagicMock() + + resp = self.admin_client.patch( + "/v1/agents/agent-123", + json={"agent_name": "Renamed Agent"}, + headers={"Authorization": "Bearer k"}, + ) + + assert resp.status_code == 200 + assert SENTINEL_AGENT_API_KEY not in resp.text + assert resp.json()["litellm_params"]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + def test_should_allow_admin_to_delete_agent(self): existing = { "agent_id": "agent-123", diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 3dea89ed67b..ffe3e0fee12 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -7448,3 +7448,64 @@ async def test_delete_cache_key_object_is_best_effort_when_the_cache_backend_fai healthy_cache.delete_cache.assert_called_once_with(key=hashed_token) healthy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache.assert_awaited_once_with(key=hashed_token) assert caplog.records == [], "a healthy eviction must stay silent, and must still reach both caches" + + +# --------------------------------------------------------------------------- +# Budget-exceeded error text must not carry a raw virtual key (LIT-5909) +# --------------------------------------------------------------------------- + + +class _BudgetAlertRecorder: + async def budget_alerts(self, type, user_info): + return None + + +async def _run_key_budget_check(key_name: str) -> str: + """Drive the over-budget key path and return the raised message.""" + valid_token = UserAPIKeyAuth( + token="hashed-token", + key_name=key_name, + key_alias="prod-key", + spend=10.0, + max_budget=1.0, + ) + with pytest.raises(litellm.BudgetExceededError, match="Budget has been exceeded") as exc_info: + await _virtual_key_max_budget_check( + valid_token=valid_token, + proxy_logging_obj=_BudgetAlertRecorder(), + ) + await asyncio.sleep(0) + return exc_info.value.message + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_name", + [ + "sk-mx5ous1o9Iezz5fj3pkLuA", + "my-company-key-2026", + "sk-...5LuA-but-longer", + # /key/generate takes a custom key ending in an escape sequence, and this + # message reaches a terminal and a log viewer + "sk-...\x1b[2J", + "sk-...a\x9bm", + ], +) +async def test_key_budget_error_does_not_carry_a_raw_key_name(key_name): + """key_name is written masked, but the column has no enforced shape (a direct DB + write bypasses abbreviate_api_key) and this message is returned to the caller.""" + message = await _run_key_budget_check(key_name) + assert key_name not in message + assert "Key=prod-key Current cost" in message + + +@pytest.mark.asyncio +@pytest.mark.parametrize("key_name", ["sk-...5LuA", "sk-...", "sk-...ke.!", "sk-...café"]) +async def test_key_budget_error_keeps_the_masked_key_name(key_name): + """The masked form is the whole point of naming the key, so it must survive. + + abbreviate_api_key takes the last four characters of the key verbatim, and a + custom key may end in punctuation or a non-ASCII character, so those masked + names are just as valid as the alphanumeric ones.""" + message = await _run_key_budget_check(key_name) + assert f"Key=prod-key ({key_name}) Current cost" in message diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 90be51cfa5b..21e0b83791f 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -9,6 +9,7 @@ from prisma import errors as prisma_errors from prisma.engine.errors import ( BinaryNotFoundError, EngineConnectionError, + EngineRequestError, MismatchedVersionsError, ) from prisma.errors import ( @@ -32,6 +33,12 @@ from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_exception_handler import UserAPIKeyAuthExceptionHandler +class _EngineHttp500: + """The response half of an EngineRequestError: the query engine answered a request with HTTP 500.""" + + status = 500 + + @pytest.mark.asyncio @pytest.mark.parametrize( "db_error", @@ -113,6 +120,90 @@ async def test_handle_authentication_error_permanent_fault_gets_no_fallback_iden assert exc_info.value.code == str(status.HTTP_503_SERVICE_UNAVAILABLE) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "prisma_error", + [ + pytest.param(BinaryNotFoundError("query engine binary not found"), id="BinaryNotFoundError"), + pytest.param(MismatchedVersionsError(expected="1", got="2"), id="MismatchedVersionsError"), + pytest.param(EngineRequestError(_EngineHttp500(), "query engine crashed"), id="EngineRequestError"), + pytest.param(PrismaError(), id="bare_PrismaError"), + ], +) +async def test_handle_authentication_error_permanent_fault_503_is_not_worded_as_transient(prisma_error): + """The 503 for a fault that never heals must not say the database is + "temporarily unreachable" and ask the caller to retry. The status is right + (the service is at fault) but that wording sends the operator to wait out an + outage that is not one, so the message has to say retrying will not help and + name the engine fault.""" + handler = UserAPIKeyAuthExceptionHandler() + + with patch( # test-quality-ok: the handler reads general_settings off the proxy module, no injection seam + "litellm.proxy.proxy_server.general_settings", {"allow_requests_on_db_unavailable": False} + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error(prisma_error, MagicMock(), {}, "/test", None, "test-key") + + assert exc_info.value.code == str(status.HTTP_503_SERVICE_UNAVAILABLE) + assert exc_info.value.type == ProxyErrorTypes.no_db_connection + assert "temporarily unreachable" not in exc_info.value.message + assert "retry shortly" not in exc_info.value.message.lower() + assert "will not clear by retrying" in exc_info.value.message + assert type(prisma_error).__name__ in exc_info.value.message + + +@pytest.mark.asyncio +async def test_handle_authentication_error_transport_error_raised_over_a_permanent_fault_names_the_fault(): + """A reconnect attempt that fails because the engine binary is missing surfaces as a transport + error with the BinaryNotFoundError as __context__. The response must describe the binary, which is + what keeps the database down, rather than promise the connection will come back.""" + try: + raise BinaryNotFoundError("query engine binary not found") + except BinaryNotFoundError: + try: + raise httpx.ConnectError("All connection attempts failed") + except httpx.ConnectError as surfaced: + transport_over_fault = surfaced + handler = UserAPIKeyAuthExceptionHandler() + + with patch( # test-quality-ok: the handler reads general_settings off the proxy module, no injection seam + "litellm.proxy.proxy_server.general_settings", {"allow_requests_on_db_unavailable": False} + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error(transport_over_fault, MagicMock(), {}, "/test", None, "k") + + assert exc_info.value.code == str(status.HTTP_503_SERVICE_UNAVAILABLE) + assert "temporarily unreachable" not in exc_info.value.message + assert "BinaryNotFoundError" in exc_info.value.message + assert "will not clear by retrying" in exc_info.value.message + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "db_error", + [ + pytest.param(httpx.ConnectError("All connection attempts failed"), id="ConnectError"), + pytest.param(EngineConnectionError(), id="EngineConnectionError"), + pytest.param(PrismaError("can't reach database server"), id="P1001_text"), + ], +) +async def test_handle_authentication_error_transient_outage_503_keeps_retry_wording(db_error): + """A genuine outage is expected to come back, so its 503 keeps telling the + caller the database is temporarily unreachable and to retry.""" + handler = UserAPIKeyAuthExceptionHandler() + + with patch( # test-quality-ok: the handler reads general_settings off the proxy module, no injection seam + "litellm.proxy.proxy_server.general_settings", {"allow_requests_on_db_unavailable": False} + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error(db_error, MagicMock(), {}, "/test", None, "test-key") + + assert exc_info.value.code == str(status.HTTP_503_SERVICE_UNAVAILABLE) + assert exc_info.value.message == ( + "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( "prisma_error", diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 9301176f3ed..f513f397b64 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -3169,7 +3169,7 @@ class TestIsRequestBodySafeChecksBracketNotationMetadata: class TestHasUserSetupSso: - """_has_user_setup_sso must treat SAML IdP metadata as SSO configured. + """has_user_setup_sso must treat SAML IdP metadata as SSO configured. Regression: UI discovery used this helper for sso_configured, but it only checked OAuth client IDs, so SAML-only setups left the login button gray. @@ -3187,29 +3187,167 @@ class TestHasUserSetupSso: monkeypatch.delenv(key, raising=False) def test_false_when_no_sso_env(self): - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso - assert _has_user_setup_sso() is False + assert has_user_setup_sso() is False def test_true_for_oauth_client_ids(self, monkeypatch): - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso monkeypatch.setenv("GOOGLE_CLIENT_ID", "google-client") - assert _has_user_setup_sso() is True + assert has_user_setup_sso() is True def test_true_for_saml_metadata_url(self, monkeypatch): - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso monkeypatch.setenv( "SAML_IDP_METADATA_URL", "https://idp.example.com/metadata.xml" ) - assert _has_user_setup_sso() is True + assert has_user_setup_sso() is True def test_true_for_saml_metadata_xml(self, monkeypatch): - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso monkeypatch.setenv("SAML_IDP_METADATA_XML", "") - assert _has_user_setup_sso() is True + assert has_user_setup_sso() is True + + +class TestIsSsoProviderFullyConfigured: + """A lone client id must not read as ready: `has_user_setup_sso()` only + checks the client id (correct for a UI-discovery "show the login button" + decision), but a gate that BLOCKS the password fallback needs every + companion setting the provider requires, or an incomplete setup locks + every admin out with no working login path at all.""" + + @pytest.fixture(autouse=True) + def _clear_sso_env(self, monkeypatch): + for key in ( + "GOOGLE_CLIENT_ID", + "GOOGLE_CLIENT_SECRET", + "MICROSOFT_CLIENT_ID", + "MICROSOFT_CLIENT_SECRET", + "MICROSOFT_TENANT", + "GENERIC_CLIENT_ID", + "GENERIC_CLIENT_SECRET", + "GENERIC_AUTHORIZATION_ENDPOINT", + "GENERIC_TOKEN_ENDPOINT", + "GENERIC_USERINFO_ENDPOINT", + "SAML_IDP_METADATA_URL", + "SAML_IDP_METADATA_XML", + ): + monkeypatch.delenv(key, raising=False) + + def test_false_when_nothing_configured(self): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + assert is_sso_provider_fully_configured() is False + + def test_google_client_id_alone_is_not_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GOOGLE_CLIENT_ID", "google-client") + assert is_sso_provider_fully_configured() is False + + def test_google_with_secret_is_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GOOGLE_CLIENT_ID", "google-client") + monkeypatch.setenv("GOOGLE_CLIENT_SECRET", "google-secret") + assert is_sso_provider_fully_configured() is True + + def test_microsoft_client_id_alone_is_not_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("MICROSOFT_CLIENT_ID", "ms-client") + assert is_sso_provider_fully_configured() is False + + def test_microsoft_missing_tenant_is_not_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("MICROSOFT_CLIENT_ID", "ms-client") + monkeypatch.setenv("MICROSOFT_CLIENT_SECRET", "ms-secret") + assert is_sso_provider_fully_configured() is False + + def test_microsoft_with_secret_and_tenant_is_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("MICROSOFT_CLIENT_ID", "ms-client") + monkeypatch.setenv("MICROSOFT_CLIENT_SECRET", "ms-secret") + monkeypatch.setenv("MICROSOFT_TENANT", "ms-tenant") + assert is_sso_provider_fully_configured() is True + + def test_generic_client_id_alone_is_not_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GENERIC_CLIENT_ID", "generic-client") + assert is_sso_provider_fully_configured() is False + + def test_generic_missing_one_endpoint_is_not_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GENERIC_CLIENT_ID", "generic-client") + monkeypatch.setenv("GENERIC_CLIENT_SECRET", "generic-secret") + monkeypatch.setenv("GENERIC_AUTHORIZATION_ENDPOINT", "https://idp.example.com/authorize") + monkeypatch.setenv("GENERIC_TOKEN_ENDPOINT", "https://idp.example.com/token") + # GENERIC_USERINFO_ENDPOINT deliberately left unset. + assert is_sso_provider_fully_configured() is False + + def test_generic_with_every_endpoint_is_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GENERIC_CLIENT_ID", "generic-client") + monkeypatch.setenv("GENERIC_CLIENT_SECRET", "generic-secret") + monkeypatch.setenv("GENERIC_AUTHORIZATION_ENDPOINT", "https://idp.example.com/authorize") + monkeypatch.setenv("GENERIC_TOKEN_ENDPOINT", "https://idp.example.com/token") + monkeypatch.setenv("GENERIC_USERINFO_ENDPOINT", "https://idp.example.com/userinfo") + assert is_sso_provider_fully_configured() is True + + def test_saml_metadata_url_is_ready_when_runtime_installed(self, monkeypatch): + from litellm.proxy.auth import auth_utils + + monkeypatch.setenv("SAML_IDP_METADATA_URL", "https://idp.example.com/metadata.xml") + monkeypatch.setattr(auth_utils.importlib.util, "find_spec", lambda name: object()) + assert auth_utils.is_sso_provider_fully_configured() is True + + def test_saml_metadata_url_is_not_ready_without_runtime(self, monkeypatch): + """Regression: python3-saml (``onelogin.saml2``) is an optional + dependency; SAMLAuthHandler fails closed on every request when it is + not installed, so IdP metadata alone must not read as ready.""" + from litellm.proxy.auth import auth_utils + + monkeypatch.setenv("SAML_IDP_METADATA_URL", "https://idp.example.com/metadata.xml") + monkeypatch.setattr(auth_utils.importlib.util, "find_spec", lambda name: None) + assert auth_utils.is_sso_provider_fully_configured() is False + + def test_saml_check_does_not_raise_when_package_entirely_absent(self, monkeypatch): + """Regression: `importlib.util.find_spec("onelogin.saml2.auth")` + raises ModuleNotFoundError (not merely returns None) when the + TOP-LEVEL `onelogin` package is not installed at all, which is + exactly the real-world "optional extra not installed" case. If the + gate does not catch this, every password login 500s instead of + falling back, on a deployment that configured SAML metadata but + skipped the extra.""" + from litellm.proxy.auth import auth_utils + + def _raise(name: str): + raise ModuleNotFoundError("No module named 'onelogin'") + + monkeypatch.setenv("SAML_IDP_METADATA_URL", "https://idp.example.com/metadata.xml") + monkeypatch.setattr(auth_utils.importlib.util, "find_spec", _raise) + assert auth_utils.is_sso_provider_fully_configured() is False + + def test_incomplete_earlier_provider_does_not_mask_a_ready_later_one(self, monkeypatch): + """Regression: a stray GOOGLE_CLIENT_ID with no secret (e.g. a + leftover from a migration) must not stop the check from reaching a + fully configured Microsoft provider set alongside it — every + provider is evaluated independently, not in a first-match order.""" + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GOOGLE_CLIENT_ID", "google-client") + monkeypatch.setenv("MICROSOFT_CLIENT_ID", "ms-client") + monkeypatch.setenv("MICROSOFT_CLIENT_SECRET", "ms-secret") + monkeypatch.setenv("MICROSOFT_TENANT", "ms-tenant") + assert is_sso_provider_fully_configured() is True class TestIsRequestBodySafeBlocksAwsIdentitySelectors: diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index 1c66acf8678..8d93d801bfd 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -6,6 +6,7 @@ to login_utils.py for better reusability. """ import os +from contextlib import ExitStack from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -598,3 +599,203 @@ class TestEncodeUiSessionJwt: request.cookies = {"token": token} with patch("litellm.proxy.proxy_server.master_key", "sk-master-for-tests"): assert _user_id_from_session_cookie(request) == "cornell-user" + + +def _patch_sso_configured(stack: ExitStack, *, configured: bool) -> None: + stack.enter_context( + patch( # test-quality-ok: no HTTP boundary here; same internal the pre-existing tests above already mock + "litellm.proxy.auth.login_utils.is_sso_provider_fully_configured", return_value=configured + ) + ) + + +def _patch_successful_admin_login_deps(stack: ExitStack) -> None: + """The collaborators a real admin login exercises past the SSO gate: + generating the UI session key, syncing the admin role, and reading the + experimental-login flag. Shared so the two "still allowed" tests below + don't each repeat the same three-mock wiring.""" + stack.enter_context( + patch( # test-quality-ok: internal orchestration, no HTTP boundary; matches pre-existing tests + "litellm.proxy.auth.login_utils.generate_key_helper_fn", + new_callable=AsyncMock, + return_value={"token": "test-token", "user_id": LITELLM_PROXY_ADMIN_NAME}, + ) + ) + stack.enter_context( + patch( # test-quality-ok: internal orchestration, no HTTP boundary; matches pre-existing tests + "litellm.proxy.auth.login_utils.user_update", + new_callable=AsyncMock, + return_value=None, + ) + ) + stack.enter_context( + patch( # test-quality-ok: internal orchestration, no HTTP boundary; matches pre-existing tests + "litellm.proxy.auth.login_utils.get_secret_bool", + return_value=False, + ) + ) + + +class TestDisablePasswordLoginWhenSSOEnabled: + """`disable_password_login_when_sso_enabled` must reject every + username/password login attempt (including the UI_USERNAME/UI_PASSWORD + admin fallback) once SSO is configured, so SSO becomes the only way to + reach the Admin UI. It must not affect logins when SSO is unconfigured, + so admins can never lock themselves out with no SSO to fall back to.""" + + @pytest.mark.asyncio + async def test_rejects_correct_admin_credentials_when_sso_configured(self): + master_key = "sk-1234" + ui_username = "admin" + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + with patch.dict(os.environ, {"UI_USERNAME": ui_username, "UI_PASSWORD": master_key}): + with ExitStack() as stack: + _patch_sso_configured(stack, configured=True) + with pytest.raises(ProxyException) as exc_info: + await authenticate_user( + username=ui_username, + password=master_key, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={"disable_password_login_when_sso_enabled": True}, + ) + + assert exc_info.value.type == ProxyErrorTypes.auth_error + assert exc_info.value.code == "403" + # The credential comparison must never even run. + mock_prisma_client.db.litellm_usertable.find_first.assert_not_called() + + @pytest.mark.asyncio + async def test_rejects_correct_db_user_credentials_when_sso_configured(self): + master_key = "sk-1234" + user_email = "test@example.com" + password = "correct-password" + + mock_user = LiteLLM_UserTable( + user_id="test-user-123", + user_email=user_email, + password=hash_token(token=password), + user_role=LitellmUserRoles.INTERNAL_USER, + ) + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=mock_user) + + with patch.dict(os.environ, {"UI_USERNAME": "admin", "UI_PASSWORD": "unrelated"}): + with ExitStack() as stack: + _patch_sso_configured(stack, configured=True) + with pytest.raises(ProxyException) as exc_info: + await authenticate_user( + username=user_email, + password=password, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={"disable_password_login_when_sso_enabled": True}, + ) + + assert exc_info.value.code == "403" + mock_prisma_client.db.litellm_usertable.find_first.assert_not_called() + + @pytest.mark.asyncio + async def test_allows_password_login_when_setting_enabled_but_sso_not_configured(self): + """The setting alone must not lock out an admin who has not actually + configured SSO — there would be no fallback left.""" + master_key = "sk-1234" + ui_username = "admin" + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + with patch.dict( + os.environ, + { + "UI_USERNAME": ui_username, + "UI_PASSWORD": master_key, + "DATABASE_URL": "postgresql://test:test@localhost/test", + }, + clear=True, + ): + with ExitStack() as stack: + _patch_sso_configured(stack, configured=False) + _patch_successful_admin_login_deps(stack) + result = await authenticate_user( + username=ui_username, + password=master_key, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={"disable_password_login_when_sso_enabled": True}, + ) + + assert isinstance(result, LoginResult) + assert result.user_id == LITELLM_PROXY_ADMIN_NAME + + @pytest.mark.asyncio + async def test_allows_password_login_when_sso_env_is_incomplete(self): + """Regression: a lone MICROSOFT_CLIENT_ID with no client secret or + tenant makes has_user_setup_sso() True, but a real SSO sign-in would + fail. The gate must read the real env (no is_sso_provider_fully_configured + mock here) and still let password login through, or an admin who set + one env var by mistake is locked out with no way in.""" + master_key = "sk-1234" + ui_username = "admin" + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + with patch.dict( + os.environ, + { + "UI_USERNAME": ui_username, + "UI_PASSWORD": master_key, + "DATABASE_URL": "postgresql://test:test@localhost/test", + "MICROSOFT_CLIENT_ID": "ms-client-id-only", + }, + clear=True, + ): + with ExitStack() as stack: + _patch_successful_admin_login_deps(stack) + result = await authenticate_user( + username=ui_username, + password=master_key, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={"disable_password_login_when_sso_enabled": True}, + ) + + assert isinstance(result, LoginResult) + assert result.user_id == LITELLM_PROXY_ADMIN_NAME + + @pytest.mark.asyncio + async def test_allows_password_login_when_sso_configured_but_setting_not_enabled(self): + """SSO being configured must not, by itself, disable the password + fallback: the setting is opt-in.""" + master_key = "sk-1234" + ui_username = "admin" + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + with patch.dict( + os.environ, + { + "UI_USERNAME": ui_username, + "UI_PASSWORD": master_key, + "DATABASE_URL": "postgresql://test:test@localhost/test", + }, + clear=True, + ): + with ExitStack() as stack: + _patch_sso_configured(stack, configured=True) + _patch_successful_admin_login_deps(stack) + result = await authenticate_user( + username=ui_username, + password=master_key, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={}, + ) + + assert isinstance(result, LoginResult) + assert result.user_id == LITELLM_PROXY_ADMIN_NAME diff --git a/tests/test_litellm/proxy/auth/test_mcp_ip_filtering.py b/tests/test_litellm/proxy/auth/test_mcp_ip_filtering.py index e0585ab04f1..4c219760762 100644 --- a/tests/test_litellm/proxy/auth/test_mcp_ip_filtering.py +++ b/tests/test_litellm/proxy/auth/test_mcp_ip_filtering.py @@ -591,3 +591,100 @@ class TestFilterServerIdsByIpWithInfo: ) assert allowed == [] assert blocked == 2 + + +def _make_scheme_request( + scheme: str, client_host: str = "203.0.113.5", headers: dict[str, str] | None = None +) -> Request: + request = MagicMock(spec=Request) + request.client = MagicMock() + request.client.host = client_host + request.headers = headers or {} + request.url = MagicMock() + request.url.scheme = scheme + return request + + +class TestIsRequestHttps: + """Regression tests for the cookie Secure trust-boundary resolution. + + litellm only sees a plain-HTTP hop when TLS terminates at a reverse + proxy, so a cookie's Secure attribute must not be derived from the + literal request scheme alone. It must also not blindly trust a + client-spoofable X-Forwarded-Proto header with no trust boundary. + """ + + def test_direct_https_is_secure(self, monkeypatch): + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + request = _make_scheme_request("https") + assert IPAddressUtils.is_request_https(request, general_settings={}) is True + + def test_direct_http_is_not_secure(self, monkeypatch): + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + request = _make_scheme_request("http") + assert IPAddressUtils.is_request_https(request, general_settings={}) is False + + def test_spoofed_forwarded_proto_without_trusted_proxy_config_is_ignored( + self, monkeypatch + ): + # Regression: an internal HTTP hop with an attacker-supplied + # X-Forwarded-Proto: https must NOT flip Secure on, because no + # trust boundary (use_x_forwarded_for + mcp_trusted_proxy_ranges) + # is configured. Blindly trusting this header is itself a + # vulnerability. + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + request = _make_scheme_request( + "http", headers={"X-Forwarded-Proto": "https"} + ) + assert IPAddressUtils.is_request_https(request, general_settings={}) is False + + def test_forwarded_proto_honored_only_from_trusted_proxy(self, monkeypatch): + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + request = _make_scheme_request( + "http", + client_host="10.0.0.5", + headers={"X-Forwarded-Proto": "https"}, + ) + general_settings = { + "use_x_forwarded_for": True, + "mcp_trusted_proxy_ranges": ["10.0.0.0/8"], + } + assert IPAddressUtils.is_request_https(request, general_settings=general_settings) is True + + def test_forwarded_proto_http_from_trusted_proxy_is_not_secure(self, monkeypatch): + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + request = _make_scheme_request( + "https", + client_host="10.0.0.5", + headers={"X-Forwarded-Proto": "http"}, + ) + general_settings = { + "use_x_forwarded_for": True, + "mcp_trusted_proxy_ranges": ["10.0.0.0/8"], + } + assert IPAddressUtils.is_request_https(request, general_settings=general_settings) is False + + def test_untrusted_direct_peer_falls_back_to_literal_scheme(self, monkeypatch): + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + request = _make_scheme_request( + "http", + client_host="203.0.113.5", + headers={"X-Forwarded-Proto": "https"}, + ) + general_settings = { + "use_x_forwarded_for": True, + "mcp_trusted_proxy_ranges": ["10.0.0.0/8"], + } + assert IPAddressUtils.is_request_https(request, general_settings=general_settings) is False + + def test_proxy_base_url_https_overrides_literal_http_scheme(self, monkeypatch): + monkeypatch.setenv("PROXY_BASE_URL", "https://litellm.example.com") + request = _make_scheme_request("http") + assert IPAddressUtils.is_request_https(request, general_settings={}) is True + + def test_proxy_base_url_http_overrides_literal_https_scheme(self, monkeypatch): + # An explicit operator-configured plain-http public origin wins over + # the literal connection scheme, same as the https direction above. + monkeypatch.setenv("PROXY_BASE_URL", "http://litellm.internal") + request = _make_scheme_request("https") + assert IPAddressUtils.is_request_https(request, general_settings={}) is False diff --git a/tests/test_litellm/proxy/auth/test_onboarding.py b/tests/test_litellm/proxy/auth/test_onboarding.py index d55a5472af1..524b655b465 100644 --- a/tests/test_litellm/proxy/auth/test_onboarding.py +++ b/tests/test_litellm/proxy/auth/test_onboarding.py @@ -231,7 +231,7 @@ async def test_claim_token_rejects_already_used_link(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with patch("litellm.proxy.proxy_server.prisma_client", prisma): @@ -254,7 +254,7 @@ async def test_claim_token_rejects_expired_link(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with patch("litellm.proxy.proxy_server.prisma_client", prisma): @@ -275,7 +275,7 @@ async def test_claim_token_rejects_mismatched_user_id(): data = InvitationClaim( invitation_link="invite-abc", user_id="wrong-user", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with patch("litellm.proxy.proxy_server.prisma_client", prisma): @@ -296,7 +296,7 @@ async def test_claim_token_rejects_missing_onboarding_token(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with ( @@ -322,7 +322,7 @@ async def test_claim_token_rejects_wrong_onboarding_session(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) request = _make_claim_request( _make_onboarding_token(invitation_link="other-invite") @@ -351,7 +351,7 @@ async def test_claim_token_rejects_invalid_bearer_token(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) request = _make_claim_request("sk-regular-key") @@ -380,7 +380,7 @@ async def test_claim_token_rejects_concurrent_reuse_before_password_write(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with ( @@ -418,7 +418,7 @@ async def test_claim_token_sets_accepted_at_after_password_written(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) mock_token_response = {"token": "sk-generated-key", "user_id": "user-123"} @@ -477,7 +477,7 @@ async def test_claim_token_rolls_back_invite_when_session_key_mint_fails(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with ( diff --git a/tests/test_litellm/proxy/auth/test_password_policy.py b/tests/test_litellm/proxy/auth/test_password_policy.py new file mode 100644 index 00000000000..f6e7d443907 --- /dev/null +++ b/tests/test_litellm/proxy/auth/test_password_policy.py @@ -0,0 +1,136 @@ +""" +Tests for the configurable password-strength policy in +`litellm.proxy.auth.password_policy`, enforced on every path that persists a +new or changed password for a locally-managed user. +""" + +import pytest + +from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.proxy.auth.password_policy import ( + DEFAULT_MIN_LENGTH, + MIN_ALLOWED_LENGTH, + PasswordPolicy, + get_password_policy, + validate_password_policy, +) + +STRONG_PASSWORD = "Str0ng!Passw0rd" + + +def test_get_password_policy_defaults_to_pif_baseline(): + policy = get_password_policy({}) + assert policy == PasswordPolicy( + min_length=DEFAULT_MIN_LENGTH, + require_uppercase=True, + require_lowercase=True, + require_numbers=True, + require_special_characters=True, + ) + + +def test_get_password_policy_reads_overrides_from_general_settings(): + policy = get_password_policy( + { + "password_policy_min_length": 20, + "password_policy_require_uppercase": False, + "password_policy_require_lowercase": False, + "password_policy_require_numbers": False, + "password_policy_require_special_characters": False, + } + ) + assert policy == PasswordPolicy( + min_length=20, + require_uppercase=False, + require_lowercase=False, + require_numbers=False, + require_special_characters=False, + ) + + +def test_validate_password_policy_accepts_strong_password(): + assert validate_password_policy(STRONG_PASSWORD, {}) is None + + +@pytest.mark.parametrize( + "password,expected_fragment", + [ + ("Sh0rt!Pw", "12 characters"), + ("weakpassword123!", "uppercase"), + ("WEAKPASSWORD123!", "lowercase"), + ("WeakPassword!!!!", "number"), + ("WeakPassword12345", "special character"), + ], +) +def test_validate_password_policy_rejects_each_missing_class(password, expected_fragment): + with pytest.raises(ProxyException) as exc_info: + validate_password_policy(password, {}) + assert exc_info.value.code == "400" + assert exc_info.value.type == ProxyErrorTypes.validation_error + assert exc_info.value.param == "password" + assert expected_fragment in exc_info.value.message + + +def test_validate_password_policy_reports_every_violation_at_once(): + with pytest.raises(ProxyException) as exc_info: + validate_password_policy("weak", {}) + assert "12 characters" in exc_info.value.message + assert "uppercase" in exc_info.value.message + assert "number" in exc_info.value.message + assert "special character" in exc_info.value.message + + +def test_validate_password_policy_honors_relaxed_config(): + general_settings = { + "password_policy_min_length": MIN_ALLOWED_LENGTH, + "password_policy_require_special_characters": False, + } + # 8 chars, has upper/lower/number, no special char: fails default policy, + # passes the relaxed one above. + validate_password_policy("Abcd1234", general_settings) + with pytest.raises(ProxyException): + validate_password_policy("Abcd1234", {}) + + +def test_validate_password_policy_honors_stricter_min_length(): + general_settings = {"password_policy_min_length": 20} + with pytest.raises(ProxyException) as exc_info: + validate_password_policy(STRONG_PASSWORD, general_settings) + assert "20 characters" in exc_info.value.message + + +@pytest.mark.parametrize("configured_min_length", [0, -1, -100, 1, 7]) +def test_get_password_policy_floors_nonpositive_or_too_low_min_length(configured_min_length): + """A misconfigured min_length must never disable the length check + entirely: it floors at MIN_ALLOWED_LENGTH instead of passing through.""" + policy = get_password_policy({"password_policy_min_length": configured_min_length}) + assert policy.min_length == MIN_ALLOWED_LENGTH + + +def test_validate_password_policy_rejects_short_password_even_with_zero_min_length_configured(): + general_settings = {"password_policy_min_length": 0} + with pytest.raises(ProxyException) as exc_info: + validate_password_policy("a", general_settings) + assert f"{MIN_ALLOWED_LENGTH} characters" in exc_info.value.message + + +def test_get_password_policy_ignores_boolean_min_length(): + """`bool` is a subclass of `int` in Python; a stray `true`/`false` value + must not silently coerce into a min_length of 1 or 0.""" + policy = get_password_policy({"password_policy_min_length": False}) + assert policy.min_length == DEFAULT_MIN_LENGTH + + +def test_validate_password_policy_rejects_unicode_letter_as_special_character(): + """Regression: an ASCII-only `[^A-Za-z0-9]` check would miscount an + accented letter as the required special character, so a letters-and- + digits-only password like this one (no real symbol) must still be + rejected.""" + with pytest.raises(ProxyException) as exc_info: + validate_password_policy("Passwörd1234", {}) + assert "special character" in exc_info.value.message + + +def test_validate_password_policy_accepts_real_special_character_with_unicode_letters(): + """Same base password as the rejection test above, plus an actual symbol.""" + assert validate_password_policy("Passwörd1234!", {}) is None diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index 1d0a99b8e0a..821323e722c 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -59,6 +59,7 @@ def _mock_cli_sso_start_response( login_id: str = "cli-session-uuid-456", poll_secret: str = "poll-secret", user_code: str = "ABCD-EFGH", + **extra_fields: object, ) -> Mock: mock_response = Mock() mock_response.status_code = 200 @@ -66,6 +67,7 @@ def _mock_cli_sso_start_response( "login_id": login_id, "poll_secret": poll_secret, "user_code": user_code, + **extra_fields, } mock_response.raise_for_status = Mock() return mock_response @@ -333,7 +335,9 @@ class TestLoginCommand: call_args = mock_browser.call_args[0][0] assert "https://test.example.com/sso/key/generate" in call_args assert "cli-test-uuid-123" in call_args + assert "user_code" not in call_args assert "Verification code: ABCD-EFGH" in result.output + assert "pre-filled in the browser" not in result.output mock_post.assert_called_once() mock_get.assert_called() assert mock_get.call_args.kwargs["headers"] == {"x-litellm-cli-poll-secret": "poll-secret"} @@ -347,6 +351,72 @@ class TestLoginCommand: # Verify commands were shown mock_show_commands.assert_called_once() + def test_login_prefills_the_code_in_the_browser_when_the_proxy_advertises_it( + self, isolated_home, secret_vault_factory + ) -> None: + vault = secret_vault_factory() + poll_response = Mock() + poll_response.status_code = 200 + poll_response.json.return_value = { + "status": "ready", + "key": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.jwt", + "user_id": "test-user-123", + "team_id": "team-1", + "teams": ["team-1"], + } + start_response = _mock_cli_sso_start_response( + login_id="cli-test-uuid-123", + verification_uri_complete=( + "https://internal-hostname.example.com/sso/key/generate" + "?source=litellm-cli&key=cli-test-uuid-123&user_code=ABCD-EFGH" + ), + ) + + with ( + patch("webbrowser.open") as mock_browser, + patch("requests.post", return_value=start_response), + patch("requests.get", return_value=poll_response), + ): + result = self.runner.invoke(login, obj={"base_url": "https://test.example.com", "secret_vault": vault}) + + assert result.exit_code == 0, result.output + assert json.loads(vault.blob)["key"] == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.jwt" + assert json.loads((isolated_home / ".litellm" / "token.json").read_text())["user_id"] == "test-user-123" + opened_url = mock_browser.call_args[0][0] + assert opened_url.startswith("https://test.example.com/sso/key/generate?") + assert "internal-hostname" not in opened_url + assert "key=cli-test-uuid-123" in opened_url + assert "user_code=ABCD-EFGH" in opened_url + assert "Verification code: ABCD-EFGH (pre-filled in the browser, check it matches)" in result.output + + def test_login_keeps_the_code_out_of_the_url_when_the_proxy_sends_a_non_url_verification_uri( + self, secret_vault_factory + ) -> None: + poll_response = Mock() + poll_response.status_code = 200 + poll_response.json.return_value = { + "status": "ready", + "key": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.jwt", + "user_id": "test-user-123", + "team_id": "team-1", + "teams": ["team-1"], + } + for advertised in (None, True): + start_response = _mock_cli_sso_start_response(verification_uri_complete=advertised) + + with ( + patch("webbrowser.open") as mock_browser, + patch("requests.post", return_value=start_response), + patch("requests.get", return_value=poll_response), + ): + result = self.runner.invoke( + login, obj={"base_url": "https://test.example.com", "secret_vault": secret_vault_factory()} + ) + + assert result.exit_code == 0, result.output + assert "user_code" not in mock_browser.call_args[0][0] + assert "pre-filled in the browser" not in result.output + def test_login_timeout(self): """Test login timeout scenario""" mock_context = Mock() diff --git a/tests/test_litellm/proxy/db/test_exception_handler.py b/tests/test_litellm/proxy/db/test_exception_handler.py index d80e3acb4b8..c685d778c0e 100644 --- a/tests/test_litellm/proxy/db/test_exception_handler.py +++ b/tests/test_litellm/proxy/db/test_exception_handler.py @@ -1,12 +1,14 @@ import asyncio import json import sys +from typing import Final from unittest.mock import MagicMock, patch import httpx import pytest from fastapi import HTTPException, Request from prisma import errors as prisma_errors +from prisma.engine.errors import BinaryNotFoundError, EngineConnectionError from prisma.errors import ( ClientNotConnectedError, DataError, @@ -317,6 +319,43 @@ def test_is_database_service_unavailable_error_in_chain_sees_through_wrapping(): assert PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(ValueError("nope")) is False +def test_find_database_service_unavailable_error_in_chain_returns_the_wrapped_outage_itself(): + """Wording a 503 by the kind of outage needs the wrapped database error, not the ValueError + get_user_object wrapped it in, so the finder must hand back the inner exception.""" + outage = _wrapped_like_get_user_object(ConnectionError("can't reach database server")) + found = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(outage) + assert isinstance(found, ConnectionError) + assert found is outage.__context__ + missing_user = _wrapped_like_get_user_object(Exception()) + assert PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(missing_user) is None + + +def _raised_while_handling(inner, outer): + try: + raise inner + except BaseException: + try: + raise outer + except BaseException as surfaced: + return surfaced + + +def test_permanent_fault_outranks_the_transient_error_that_surfaced_it(): + """A reconnect that dies on a missing engine binary raises the transport error last, with the + BinaryNotFoundError left as __context__. The binary is what keeps the database down, so both the + finder and the 503 wording must pick it over the outer transient error, whichever way they nest.""" + permanent = BinaryNotFoundError("query engine binary not found") + transient_over_permanent = _raised_while_handling(permanent, httpx.ConnectError("connection refused")) + permanent_over_transient = _raised_while_handling(httpx.ConnectError("connection refused"), permanent) + + for chain in (transient_over_permanent, permanent_over_transient): + assert PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(chain) is permanent + message = PrismaDBExceptionHandler.database_unavailable_message(chain) + assert "BinaryNotFoundError" in message + assert "will not clear by retrying" in message + assert "temporarily unreachable" not in message + + def test_is_database_service_unavailable_error_in_chain_terminates_on_a_cause_cycle(): """The walk must terminate on a pathological __cause__ cycle rather than hang. Neither link is an outage, so the bounded walk returns False instead of looping forever.""" @@ -508,6 +547,51 @@ def test_permanent_prisma_faults_are_still_reported_as_service_problems(prisma_e assert PrismaDBExceptionHandler.is_database_service_unavailable_error(prisma_error) is True +RECONNECTABLE_CLIENT_STATE_FAULTS = (prisma_errors.ClientNotConnectedError, prisma_errors.HTTPClientClosedError) + + +@pytest.mark.parametrize("prisma_error", PERMANENT_PRISMA_FAULTS) +def test_permanent_prisma_faults_are_worded_as_not_retryable(prisma_error): + """A 503 for a fault that never heals must not tell the operator to wait. + + The status stays 503 (the service is at fault), but the message has to say + the outage is not transient and name the engine fault, or an operator + watching a version-skewed engine keeps retrying a request that can never + succeed. The two client-state faults a reconnect can repair keep the retry + wording.""" + reconnectable = isinstance(prisma_error, RECONNECTABLE_CLIENT_STATE_FAULTS) + message = PrismaDBExceptionHandler.database_unavailable_message(prisma_error) + + assert PrismaDBExceptionHandler.is_permanent_database_fault(prisma_error) is (not reconnectable) + assert message.startswith("Service Unavailable") + assert ("temporarily unreachable" in message) is reconnectable + assert ("Please retry shortly" in message) is reconnectable + assert ("will not clear by retrying" in message) is (not reconnectable) + assert (type(prisma_error).__name__ in message) is (not reconnectable) + + +@pytest.mark.parametrize( + "transient_error", + [ + pytest.param(httpx.ConnectError("All connection attempts failed"), id="ConnectError"), + pytest.param(ConnectionError("connection refused"), id="ConnectionError"), + pytest.param(EngineConnectionError(), id="EngineConnectionError"), + pytest.param(prisma_errors.PrismaError("can't reach database server"), id="P1001_text"), + pytest.param( + ProxyException(message="no db", type=ProxyErrorTypes.no_db_connection, param=None, code=503), + id="ProxyException", + ), + ], +) +def test_transient_outages_keep_the_retry_wording(transient_error): + """A genuine outage is expected to come back, so the retry guidance is the + right message and must not be replaced by the permanent-fault text.""" + assert PrismaDBExceptionHandler.is_permanent_database_fault(transient_error) is False + assert PrismaDBExceptionHandler.database_unavailable_message(transient_error) == ( + "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." + ) + + @pytest.mark.parametrize( "transient_error", [ @@ -579,3 +663,40 @@ def test_is_deadlock_error_matches_postgres_deadlock(error): def test_is_deadlock_error_excludes_non_deadlocks(error): """Non-deadlock prisma errors, connectivity failures, and non-prisma exceptions are not treated as deadlocks.""" assert PrismaDBExceptionHandler.is_deadlock_error(error) is False + + +MOCKED_PRISMA_PREDICATES: Final = ( + PrismaDBExceptionHandler.is_database_infrastructure_error, + PrismaDBExceptionHandler.is_database_transport_error, + PrismaDBExceptionHandler.is_deadlock_error, + PrismaDBExceptionHandler.is_prisma_engine_internal_error, + PrismaDBExceptionHandler.is_database_service_unavailable_error, +) + + +@pytest.mark.parametrize("predicate", MOCKED_PRISMA_PREDICATES, ids=lambda p: p.__name__) +def test_predicates_answer_false_for_a_plain_exception_when_prisma_is_mocked(predicate): + """Suites that swap ``sys.modules["prisma"]`` for a ``MagicMock`` hand the + predicates mocks in place of prisma's error classes. ``isinstance`` against + a mock raises ``TypeError``; the predicate must instead answer for the + non-prisma checks it still has.""" + with patch.dict(sys.modules, {"prisma": MagicMock()}): + assert predicate(Exception("db connection dropped")) is False + + +def test_infrastructure_error_still_recognizes_transport_errors_when_prisma_is_mocked(): + """Skipping the prisma classes must not skip the checks that do not need them.""" + with patch.dict(sys.modules, {"prisma": MagicMock()}): + no_db: Final = ProxyException(message="no db", type=ProxyErrorTypes.no_db_connection, param=None, code=503) + assert PrismaDBExceptionHandler.is_database_infrastructure_error(httpx.ConnectError("refused")) is True + assert PrismaDBExceptionHandler.is_database_infrastructure_error(no_db) is True + + +def test_connection_error_answers_when_prisma_is_mocked_after_import(): + """``prisma.engine`` is already loaded in a real process, so a mock parent + still resolves ``prisma.engine.errors``; its classes are then mocks too.""" + import prisma.engine.errors # noqa: F401 + + with patch.dict(sys.modules, {"prisma": MagicMock()}): + assert PrismaDBExceptionHandler.is_database_connection_error(Exception("x")) is False + assert PrismaDBExceptionHandler.is_database_connection_error(httpx.ConnectError("refused")) is True diff --git a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py index f4da8c941a4..c3f7b0100d8 100644 --- a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py +++ b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py @@ -18,7 +18,7 @@ def test_ui_discovery_endpoints_with_defaults(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): @@ -41,7 +41,7 @@ def test_ui_discovery_endpoints_with_custom_server_root_path(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/litellm"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): @@ -66,7 +66,7 @@ def test_ui_discovery_endpoints_with_proxy_base_url_when_set(): "litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com", ), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): @@ -91,7 +91,7 @@ def test_ui_discovery_endpoints_with_sso_configured_and_auto_redirect_enabled(): "litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com", ), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch.dict( os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true", "DISABLE_ADMIN_UI": "false"}, @@ -121,7 +121,7 @@ def test_ui_discovery_endpoints_with_sso_configured_and_auto_redirect_not_set_de "litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com", ), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): # Ensure AUTO_REDIRECT_UI_LOGIN_TO_SSO is not set (simulate default) @@ -148,7 +148,7 @@ def test_ui_discovery_endpoints_with_sso_configured_but_auto_redirect_disabled() "litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com", ), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch.dict( os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "false", "DISABLE_ADMIN_UI": "false"}, @@ -174,7 +174,7 @@ def test_ui_discovery_endpoints_with_sso_not_configured_but_auto_redirect_enable with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict( os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true", "DISABLE_ADMIN_UI": "false"}, @@ -203,7 +203,7 @@ def test_ui_discovery_endpoints_both_routes_return_same_data(): "litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com", ), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch.dict( os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true", "DISABLE_ADMIN_UI": "false"}, @@ -228,7 +228,7 @@ def test_ui_discovery_endpoints_with_auto_redirect_via_general_settings(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch( "litellm.proxy.proxy_server.general_settings", {"auto_redirect_ui_login_to_sso": True}, @@ -254,7 +254,7 @@ def test_ui_discovery_endpoints_with_auto_redirect_env_var_overrides_general_set with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch( "litellm.proxy.proxy_server.general_settings", {"auto_redirect_ui_login_to_sso": False}, @@ -281,7 +281,7 @@ def test_ui_discovery_endpoints_with_admin_ui_disabled(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "true"}, clear=False), ): @@ -311,7 +311,7 @@ def test_ui_discovery_endpoints_is_control_plane_true_when_workers_configured(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch("litellm.proxy.proxy_server.proxy_config", mock_config), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): @@ -336,7 +336,7 @@ def test_ui_discovery_endpoints_hide_default_credentials_hint_default_false(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): os.environ.pop("LITELLM_HIDE_DEFAULT_CREDENTIALS_HINT", None) @@ -357,7 +357,7 @@ def test_ui_discovery_endpoints_hide_default_credentials_hint_via_env_var(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict( os.environ, { @@ -384,7 +384,7 @@ def test_ui_discovery_endpoints_hide_default_credentials_hint_via_general_settin with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch( "litellm.proxy.proxy_server.general_settings", {"hide_default_credentials_hint": True}, @@ -411,7 +411,7 @@ def test_ui_discovery_endpoints_is_control_plane_false_when_no_workers(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch("litellm.proxy.proxy_server.proxy_config", mock_config), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py index 476d443d8d8..cb6772977ec 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py @@ -310,8 +310,8 @@ def _make_stream_chunk(content: str, finish_reason=None): @pytest.mark.asyncio async def test_openai_moderation_streaming_default_uses_sampled_cadence(): """Default config samples every 5th streamed chunk and runs a final aggregate - pass after the stream ends. 10 chunks → sampled at chunks 5 and 10 → 2 in-stream - calls, plus 1 final = 3 total. + pass after the stream ends. 10 chunks are sampled at 5 and 10; the end-of-stream + round is skipped because chunk 10 already scanned the full text, for 2 total calls """ import litellm @@ -370,8 +370,9 @@ async def test_openai_moderation_streaming_default_uses_sampled_cadence(): ): pass - assert patched_make_request.await_count == 3, ( - f"Expected 3 moderation calls (2 sampled at chunks 5 / 10 + 1 final), " + assert patched_make_request.await_count == 2, ( + f"Expected 2 moderation calls (2 sampled at chunks 5 / 10; " + f"the end-of-stream round is skipped because chunk 10 already scanned the full text), " f"got {patched_make_request.await_count}" ) @@ -448,7 +449,8 @@ async def test_openai_moderation_streaming_end_of_stream_only_opt_in_calls_moder @pytest.mark.asyncio async def test_openai_moderation_streaming_sampled_when_end_of_stream_only_disabled(): """With streaming_end_of_stream_only=False and streaming_sampling_rate=2, - moderation runs every 2nd chunk during the stream, plus once more at end. + moderation runs every 2nd chunk during the stream. The terminal chunk scan covers + the final aggregate, for 3 total calls """ import litellm @@ -509,9 +511,8 @@ async def test_openai_moderation_streaming_sampled_when_end_of_stream_only_disab ): pass - # 6 chunks, sampling_rate=2 → in-stream calls at chunks 2, 4, 6 (3 calls), - # plus the final aggregate pass after the stream ends (1 call) = 4 total. - assert patched_make_request.await_count == 4, ( - f"Expected 4 moderation calls (3 sampled + 1 final aggregate), " + assert patched_make_request.await_count == 3, ( + f"Expected 3 moderation calls (3 sampled; the end-of-stream round is skipped " + f"because chunk 6 already scanned the full text), " f"got {patched_make_request.await_count}" ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py index ec7854b9a35..a07157396df 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_crowdstrike_aidr.py @@ -3,7 +3,9 @@ from unittest.mock import patch import httpx import pytest from fastapi import HTTPException +from pydantic import ValidationError +import litellm from litellm.exceptions import Timeout from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket from litellm.proxy.guardrails.guardrail_hooks.crowdstrike_aidr import initialize_guardrail @@ -12,8 +14,8 @@ from litellm.proxy.guardrails.guardrail_hooks.crowdstrike_aidr.crowdstrike_aidr CrowdStrikeAIDRHandler, ) from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 -from litellm.types.guardrails import Guardrail, LitellmParams -from litellm.types.utils import GenericGuardrailAPIInputs, ModelResponse +from litellm.types.guardrails import Guardrail, GuardrailEventHooks, LitellmParams +from litellm.types.utils import Delta, GenericGuardrailAPIInputs, ModelResponse, ModelResponseStream @pytest.fixture @@ -1578,3 +1580,142 @@ async def test_unparseable_transformed_response_fails_closed_under_fail_open() - assert exc_info.value.status_code == 500 assert "failing closed" in exc_info.value.detail["error"] + + +def _initialize_from_config(**litellm_params_kwargs: object) -> CrowdStrikeAIDRHandler: + litellm_params = LitellmParams( + guardrail="crowdstrike_aidr", + api_key="pts_crowdstrike_tokenid", + api_base="https://api.crowdstrike.com/aidr/aiguard", + default_on=True, + **litellm_params_kwargs, + ) + guardrail = Guardrail(guardrail_name="crowdstrike-aidr-guard", litellm_params=litellm_params) + return initialize_guardrail(litellm_params=litellm_params, guardrail=guardrail) + + +@pytest.mark.parametrize( + ("mode", "runs_pre_call", "runs_post_call"), + [("post_call", False, True), ("pre_call", True, False), (["pre_call", "post_call"], True, True)], +) +def test_initialize_guardrail_honors_configured_mode( + mode: str | list[str], runs_pre_call: bool, runs_post_call: bool +) -> None: + handler = _initialize_from_config(mode=mode) + + assert handler.should_run_guardrail({}, GuardrailEventHooks.pre_call) is runs_pre_call + assert handler.should_run_guardrail({}, GuardrailEventHooks.post_call) is runs_post_call + + +def test_initialize_guardrail_rejects_unsupported_mode_instead_of_running_other_hooks() -> None: + with pytest.raises(ValueError, match="during_call is not in the supported event hooks"): + _initialize_from_config(mode="during_call") + + +def test_initialize_guardrail_defaults_streaming_params() -> None: + handler = _initialize_from_config(mode="post_call") + + assert handler.streaming_end_of_stream_only is False + assert handler.streaming_sampling_rate == 5 + + +@pytest.mark.parametrize( + "configured", + [ + {"streaming_end_of_stream_only": True, "streaming_sampling_rate": 50}, + {"optional_params": {"streaming_end_of_stream_only": True, "streaming_sampling_rate": 50}}, + ], +) +def test_initialize_guardrail_forwards_streaming_params(configured: dict[str, object]) -> None: + handler = _initialize_from_config(mode="post_call", **configured) + + assert handler.streaming_end_of_stream_only is True + assert handler.streaming_sampling_rate == 50 + + +def test_initialize_guardrail_rejects_non_positive_sampling_rate() -> None: + with pytest.raises(ValidationError): + _initialize_from_config(mode="post_call", streaming_sampling_rate=0) + + +def test_update_in_memory_litellm_params_reapplies_streaming_params() -> None: + handler = _initialize_from_config(mode="post_call") + + handler.update_in_memory_litellm_params( + LitellmParams( + guardrail="crowdstrike_aidr", + mode="post_call", + streaming_end_of_stream_only=True, + streaming_sampling_rate=7, + ) + ) + + assert handler.streaming_end_of_stream_only is True + assert handler.streaming_sampling_rate == 7 + + +def _stream_chunk(content: str, finish_reason: str | None) -> ModelResponseStream: + return ModelResponseStream( + model="gpt-4", + choices=[ + litellm.StreamingChoices( + index=0, delta=Delta(role="assistant", content=content), finish_reason=finish_reason + ) + ], + ) + + +async def _guard_calls_for_stream(handler: CrowdStrikeAIDRHandler, chunk_texts: list[str]) -> int: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import UnifiedLLMGuardrails + + async def stream(): + for i, content in enumerate(chunk_texts): + yield _stream_chunk(content, "stop" if i == len(chunk_texts) - 1 else None) + + calls = 0 + + def _allow(request: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + return httpx.Response( + status_code=200, json={"result": {"blocked": False, "transformed": False}}, request=request + ) + + request_data = { + "messages": [{"role": "user", "content": "hi"}], + "guardrail_to_apply": handler, + "metadata": {"guardrails": ["crowdstrike-aidr-guard"]}, + } + async with httpx.AsyncClient(transport=httpx.MockTransport(_allow)) as client: + await handler.async_handler.close() + handler.async_handler.client = client + async for _ in UnifiedLLMGuardrails().async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test", request_route="/chat/completions"), + response=stream(), + request_data=request_data, + ): + pass + return calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("configured", "expected_calls"), + [ + ({}, 2), + ({"streaming_sampling_rate": 2}, 5), + ({"streaming_end_of_stream_only": True}, 1), + ({"streaming_end_of_stream_only": True, "streaming_sampling_rate": 2}, 1), + ], +) +async def test_streaming_params_from_config_control_output_scan_cadence( + configured: dict[str, object], expected_calls: int +) -> None: + """10 chunks: default samples at 5 and 10, rate 2 samples 5 times, end-of-stream scans once. + + The final pass is skipped because chunk 10 already scanned the complete output. + """ + handler = _initialize_from_config(mode="post_call", **configured) + + assert await _guard_calls_for_stream(handler, list("ABCDEFGHIJ")) == expected_calls diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index 523ec1a37b4..83cc9ae8bb9 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -1517,7 +1517,9 @@ class TestGenericGuardrailAPIStreamingViaUnified: @pytest.mark.asyncio async def test_streaming_default_uses_sampled_cadence(self): - """Default samples every 5th chunk + final pass: 10 chunks → calls at 5, 10, and final = 3.""" + """Default samples every 5th chunk. For 10 chunks, sampled scans at 5 and 10 + cover the full text, so the end-of-stream round is skipped and there are 2 calls + """ from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( UnifiedLLMGuardrails, ) @@ -1566,8 +1568,9 @@ class TestGenericGuardrailAPIStreamingViaUnified: ): pass - assert mock_post.await_count == 3, ( - f"Expected 3 guardrail calls (2 sampled at chunks 5 / 10 + 1 final), " + assert mock_post.await_count == 2, ( + f"Expected 2 guardrail calls (2 sampled at chunks 5 / 10; " + f"the end-of-stream round is skipped because chunk 10 already scanned the full text), " f"got {mock_post.await_count}" ) for call in mock_post.await_args_list: @@ -1631,7 +1634,9 @@ class TestGenericGuardrailAPIStreamingViaUnified: @pytest.mark.asyncio async def test_streaming_sampling_rate_override(self): - """sampling_rate=2 on 6 chunks → in-stream at 2,4,6 plus final = 4 calls.""" + """sampling_rate=2 on 6 chunks. Scans at 2, 4, and 6 cover the full text, so + the end-of-stream round is skipped and there are 3 calls + """ from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( UnifiedLLMGuardrails, ) @@ -1680,8 +1685,9 @@ class TestGenericGuardrailAPIStreamingViaUnified: ): pass - assert mock_post.await_count == 4, ( - f"Expected 4 guardrail calls (3 sampled + 1 final aggregate), " + assert mock_post.await_count == 3, ( + f"Expected 3 guardrail calls (3 sampled; the end-of-stream round is skipped " + f"because chunk 6 already scanned the full text), " f"got {mock_post.await_count}" ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index 1fbc975e40a..c04fb7b30ec 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -2199,6 +2199,164 @@ async def test_history_is_still_compressed(guardrail: HeadroomGuardrail): assert has_headroom_retrieve_tool(result.get("tools") or []) +# --------------------------------------------------------------------------- +# #38558: a client that runs its own tool loop (e.g. Claude Code via the MCP +# gateway) executes headroom_retrieve and echoes the recovered original content +# back as a tool result. Compressing that row re-derives the same content hash +# it was just retrieved from -- the marker returns and the agent loops. The +# retrieved row must be held back from the compression service. +# --------------------------------------------------------------------------- + +RETRIEVE_ECHO_MESSAGES = [ + {"role": "system", "content": "You are Claude Code. " + "S" * 5000}, + {"role": "user", "content": "H" * 5000}, + { + "role": "assistant", + "content": "Expanding the marker.", + "tool_calls": [ + { + "id": "hr_1", + "type": "function", + "function": { + "name": "mcp__headroom__headroom_retrieve", + "arguments": '{"hash": "b573993006976af767214fac"}', + }, + } + ], + }, + {"role": "tool", "tool_call_id": "hr_1", "content": "RETRIEVED BODY " + "R" * 5000}, + {"role": "assistant", "content": "Older answer. " + "O" * 5000}, + {"role": "user", "content": "now summarize the description"}, +] + + +@pytest.mark.asyncio +async def test_retrieved_content_is_never_recompressed(guardrail: HeadroomGuardrail): + """The tool result carrying headroom_retrieve output is held back, so it can + never collapse back to the hash it was just retrieved from.""" + wire, result = await _wire_and_result(guardrail, RETRIEVE_ECHO_MESSAGES) + + assert not any(row.get("tool_call_id") == "hr_1" for row in wire) + assert not any("RETRIEVED BODY" in json.dumps(row) for row in wire) + # Reaches the model byte-identical, so no marker stands in for the expansion. + assert result["structured_messages"][3] == RETRIEVE_ECHO_MESSAGES[3] + # Negative control: unrelated history is still compressed, not a no-op. + assert any(row.get("content") == "H" * 5000 for row in wire) + + +@pytest.mark.asyncio +async def test_retrieved_content_guard_matches_direct_tool_name(guardrail: HeadroomGuardrail): + """Server-side the tool is named headroom_retrieve (no MCP prefix); its + result must be protected the same way.""" + messages = [ + {"role": "system", "content": "sys " + "S" * 5000}, + {"role": "user", "content": "H" * 5000}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "hr_direct", + "type": "function", + "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME, "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "hr_direct", "content": "RETRIEVED BODY " + "R" * 5000}, + {"role": "assistant", "content": "Older. " + "O" * 5000}, + {"role": "user", "content": "summarize"}, + ] + wire, result = await _wire_and_result(guardrail, messages) + + assert not any(row.get("tool_call_id") == "hr_direct" for row in wire) + assert result["structured_messages"][3] == messages[3] + + +@pytest.mark.asyncio +async def test_retrieved_content_protected_when_mcp_tool_name_is_truncated(guardrail: HeadroomGuardrail): + """A long mcp____headroom_retrieve name is truncated past 64 chars in + the OpenAI-translated view the guardrail scans, dropping the suffix. The call + id read from the request's own Anthropic tool_use (never truncated) still + pairs the retrieved row so it is held back.""" + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + truncate_tool_name, + ) + + long_name = "mcp__" + "s" * 45 + "__" + HEADROOM_RETRIEVE_TOOL_NAME + assert len(long_name) > 64 + truncated = truncate_tool_name(long_name) + assert not truncated.endswith(HEADROOM_RETRIEVE_TOOL_NAME) + + # What the guardrail scans: OpenAI-translated messages with the truncated name. + structured = [ + {"role": "system", "content": "sys " + "S" * 5000}, + {"role": "user", "content": "H" * 5000}, + { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "hr_long", "type": "function", "function": {"name": truncated, "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "hr_long", "content": "RETRIEVED BODY " + "R" * 5000}, + {"role": "assistant", "content": "Older. " + "O" * 5000}, + {"role": "user", "content": "summarize"}, + ] + # The request's own messages, untranslated: Anthropic tool_use carries the full name. + raw_messages = [ + {"role": "assistant", "content": [{"type": "tool_use", "id": "hr_long", "name": long_name, "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "hr_long", "content": "RETRIEVED BODY"}]}, + ] + + inputs = GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(structured))) + sent: dict = {} + + def _echo(**kwargs): + sent["messages"] = kwargs["json"]["messages"] + return _make_compress_response(json.loads(json.dumps(kwargs["json"]["messages"]))) + + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_echo): + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data={"model": "claude-sonnet-4-5-20250929", "messages": raw_messages}, + input_type="request", + ) + + assert not any(row.get("tool_call_id") == "hr_long" for row in sent["messages"]) + assert result["structured_messages"][3] == structured[3] + assert any(row.get("content") == "H" * 5000 for row in sent["messages"]) + + +def test_raw_retrieve_call_ids_covers_both_shapes_and_ignores_others(): + """Retrieve ids are read from OpenAI tool_calls and Anthropic tool_use blocks; + non-retrieve calls, non-tool_use blocks, string content, and non-list inputs + yield nothing.""" + from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import _raw_retrieve_call_ids + + messages = [ + { + "role": "assistant", + "tool_calls": [ + {"id": "oa1", "function": {"name": HEADROOM_RETRIEVE_TOOL_NAME}}, + {"id": "other", "function": {"name": "get_weather"}}, + {"id": "malformed", "function": {"name": 123}}, + {"id": "nofunc"}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "an1", "name": "mcp__hr__headroom_retrieve", "input": {}}, + {"type": "tool_use", "id": "an2", "name": "jira_get_issue", "input": {}}, + {"type": "text", "text": "noise"}, + ], + }, + {"role": "user", "content": "plain string content, not a list"}, + ] + + assert _raw_retrieve_call_ids(messages) == frozenset({"oa1", "an1"}) + assert _raw_retrieve_call_ids("not a list") == frozenset() + assert _raw_retrieve_call_ids(None) == frozenset() + + @pytest.mark.asyncio async def test_nothing_compressible_returns_inputs_untouched(guardrail: HeadroomGuardrail): """A single-turn request is all protected, so there is nothing to send and diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index da66c36328e..47089b7b1b1 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -15,6 +15,7 @@ import litellm.types.utils from litellm._logging import verbose_proxy_logger from litellm.caching import DualCache from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError +from litellm.proxy.guardrails.anthropic_sse import anthropic_sse_error_frames from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.model_armor import ModelArmorGuardrail from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ( @@ -3778,3 +3779,1153 @@ async def test_moderation_hook_skips_chat_traffic_when_configured_for_during_mcp assert result == data mock_post.assert_not_called() + + +_ANTHROPIC_SSE_CHUNKS = ( + b'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_1","type":"message",' + b'"role":"assistant","model":"claude","content":[],"usage":{"input_tokens":5,"output_tokens":0}}}\n\n', + b'event: content_block_start\ndata: {"type":"content_block_start","index":0,' + b'"content_block":{"type":"text","text":""}}\n\n', + b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,' + b'"delta":{"type":"text_delta","text":"my card is 4111-1111-1111-1111"}}\n\n', + b'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', + b'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},' + b'"usage":{"output_tokens":9}}\n\n', + b'event: message_stop\ndata: {"type":"message_stop"}\n\n', +) + +_MODEL_ARMOR_CLEAN = {"sanitizationResult": {"filterMatchState": "NO_MATCH_FOUND"}} + +_MODEL_ARMOR_BLOCK = { + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "sdp": { + "sdpFilterResult": { + "inspectResult": { + "matchState": "MATCH_FOUND", + "findings": [ + {"infoType": "CREDIT_CARD_NUMBER", "likelihood": "VERY_LIKELY"} + ], + } + } + } + }, + } +} + +# The root-level sanitizedText fallback in _get_sanitized_content, i.e. a rewrite that trips no +# named filter +_MODEL_ARMOR_SANITIZED = { + "sanitizedText": "my card is [REDACTED]", + "sanitizationResult": {"filterMatchState": "NO_MATCH_FOUND"}, +} + +# The shape a real de-identify template returns: the SDP filter both matches and hands back the +# rewritten text, so whether it blocks or masks is decided by allow_sanitization alone +_MODEL_ARMOR_DEIDENTIFIED = { + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "sdp": { + "sdpFilterResult": { + "deidentifyResult": { + "matchState": "MATCH_FOUND", + "data": {"text": "my card is [REDACTED]"}, + } + } + } + }, + } +} + + +def _chat_completion_chunks(): + """The chat-completions surface: typed ModelResponseStream chunks.""" + return ( + litellm.types.utils.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + index=0, + delta=litellm.types.utils.Delta(content="my card is 4111-1111-1111-1111"), + ) + ] + ), + litellm.types.utils.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + index=0, + delta=litellm.types.utils.Delta(content=""), + finish_reason="stop", + ) + ] + ), + ) + + +def _surface_guardrail(**kwargs): + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + **kwargs, + ) + guardrail._ensure_access_token_async = AsyncMock( + return_value=("test-token", "test-project") + ) + return guardrail + + +def _armor_post_mock(payload): + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.json = AsyncMock(return_value=payload) + return AsyncMock(return_value=mock_response) + + +async def _anthropic_sse_stream(): + for chunk in _ANTHROPIC_SSE_CHUNKS: + yield chunk + + +def _responses_api_events(): + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + completed = ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-4o-mini", + object="response", + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "my card is 4111-1111-1111-1111"}], + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + return ( + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta="my card is 4111-1111-1111-1111", + ), + ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=completed, + ), + ) + + +async def _drain_surface_hook(guardrail, chunks, request_data=None): + async def _stream(): + for chunk in chunks: + yield chunk + + return [ + item + async for item in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_stream(), + request_data=request_data + if request_data is not None + else { + "model": "claude-haiku", + "messages": [{"role": "user", "content": "show me a card"}], + "metadata": {"guardrails": ["model-armor-test"]}, + }, + ) + ] + + +@pytest.mark.asyncio +async def test_streaming_hook_scans_raw_anthropic_sse_instead_of_crashing(): + """A /v1/messages stream arrives as raw SSE bytes and must be assembled, then scanned. + + Regression for the 500 `Error building chunks for logging/streaming usage calculation`: + stream_chunk_builder calls .get() on each chunk, which raises on bytes. + """ + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, _ANTHROPIC_SSE_CHUNKS) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert "my card is 4111-1111-1111-1111" in scanned + assert tuple(delivered) == _ANTHROPIC_SSE_CHUNKS + + +@pytest.mark.asyncio +async def test_streaming_hook_scans_responses_api_events_instead_of_crashing(): + """A /v1/responses stream arrives as typed Responses events, which stream_chunk_builder + cannot subscript. The final response.completed event carries the text to scan.""" + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + events = _responses_api_events() + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, events) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert scanned == "my card is 4111-1111-1111-1111" + assert tuple(delivered) == events + + +@pytest.mark.asyncio +async def test_streaming_block_emits_anthropic_error_frame(): + """A block on /v1/messages must terminate the stream in Anthropic's error format. + + The OpenAI-shaped `data: {"error": ...}` frame the chat surface uses is rejected by + Anthropic clients. + """ + guardrail = _surface_guardrail() + + with patch.object( + guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_BLOCK) + ): + delivered = await _drain_surface_hook(guardrail, _ANTHROPIC_SSE_CHUNKS) + + body = b"".join(delivered) + assert b"event: error" in body + assert b'"type": "error"' in body + assert b"guardrail_error" in body + assert b"Streaming response blocked by Model Armor" in body + assert b"4111-1111-1111-1111" not in body + + +@pytest.mark.asyncio +async def test_streaming_block_emits_responses_api_error_event(): + """A block on /v1/responses must terminate the stream with a Responses ErrorEvent.""" + from litellm.types.llms.openai import ErrorEvent + + guardrail = _surface_guardrail() + + with patch.object( + guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_BLOCK) + ): + delivered = await _drain_surface_hook(guardrail, _responses_api_events()) + + assert len(delivered) == 1 + error_event = delivered[0] + assert isinstance(error_event, ErrorEvent) + assert error_event.error.type == "guardrail_error" + assert error_event.error.code == "400" + assert error_event.error.message == "Streaming response blocked by Model Armor" + + +@pytest.mark.asyncio +async def test_streaming_masking_re_emits_anthropic_sse_with_sanitized_text(): + """mask_response_content on /v1/messages must ship the sanitized text, not the original.""" + guardrail = _surface_guardrail(mask_response_content=True) + + with patch.object( + guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_SANITIZED) + ): + delivered = await _drain_surface_hook(guardrail, _ANTHROPIC_SSE_CHUNKS) + + body = b"".join(delivered) + assert b"[REDACTED]" in body + assert b"4111-1111-1111-1111" not in body + + +@pytest.mark.asyncio +async def test_streaming_masking_blocks_responses_api_stream(): + """A Responses event stream cannot be rebuilt from sanitized text, so releasing it would + ship the content the guardrail just rewrote. It is blocked instead.""" + from litellm.types.llms.openai import ErrorEvent + + guardrail = _surface_guardrail(mask_response_content=True) + + with patch.object( + guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_SANITIZED) + ): + delivered = await _drain_surface_hook(guardrail, _responses_api_events()) + + assert len(delivered) == 1 + assert isinstance(delivered[0], ErrorEvent) + assert delivered[0].error.code == "400" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("surface", ["anthropic_sse", "responses"]) +async def test_streaming_api_failure_frames_error_per_surface(surface): + """A Model Armor outage with fail_on_error must terminate the stream in the endpoint's + own error format rather than leaking an OpenAI SSE frame onto it.""" + from litellm.types.llms.openai import ErrorEvent + + guardrail = _surface_guardrail(fail_on_error=True) + chunks = _ANTHROPIC_SSE_CHUNKS if surface == "anthropic_sse" else _responses_api_events() + + mock_response = AsyncMock() + mock_response.status_code = 500 + mock_response.text = "Internal Server Error" + + with patch.object( + guardrail.async_handler, "post", AsyncMock(return_value=mock_response) + ): + delivered = await _drain_surface_hook(guardrail, chunks) + + assert len(delivered) >= 1 + if surface == "anthropic_sse": + assert b"event: error" in b"".join(delivered) + else: + assert isinstance(delivered[0], ErrorEvent) + assert delivered[0].error.code == "500" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "chunks", + [ + pytest.param( + (b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,' + b'"delta":{"type":"text_delta","text":"hi"}}\n\n',), + id="anthropic-sse-without-message-start", + ), + pytest.param(None, id="responses-stream-without-completed-event"), + pytest.param("created", id="responses-stream-cut-off-after-response-created"), + ], +) +async def test_streaming_hook_fails_closed_when_a_surface_stream_cannot_be_assembled(chunks): + """Forwarding an unscannable /v1/messages or /v1/responses stream would silently disable the + guardrail, so the stream is refused in its own wire format instead of released unscanned.""" + from litellm.types.llms.openai import ( + ErrorEvent, + OutputTextDeltaEvent, + ResponsesAPIStreamEvents, + ) + + if chunks is None or chunks == "created": + delta = OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta="my card is 4111-1111-1111-1111", + ) + # response.created carries a ResponsesAPIResponse too, but an empty one: reading the body + # off it would scan "" and release every buffered delta unscanned + chunks = (delta,) if chunks is None else (_responses_created_event(), delta) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + assert tuple(delivered) != tuple(chunks) + if isinstance(chunks[0], bytes): + joined = b"".join(item.encode() if isinstance(item, str) else item for item in delivered).decode() + assert "event: error" in joined + assert "could not be assembled for scanning" in joined + return + assert len(delivered) == 1 + assert isinstance(delivered[0], ErrorEvent) + assert "could not be assembled for scanning" in delivered[0].error.message + + +@pytest.mark.asyncio +async def test_streaming_hook_forwards_a_preceding_guardrails_error_item(): + """A guardrail earlier in the post_call chain replaces the stream with its own terminal + error item. That item is not a chat delta, and feeding it to stream_chunk_builder is what + surfaced the ticket's 500, so it has to be forwarded untouched instead.""" + from litellm.types.llms.openai import ( + ErrorEvent, + ErrorEventError, + ResponsesAPIStreamEvents, + ) + + chunks = ( + ErrorEvent( + type=ResponsesAPIStreamEvents.ERROR, + sequence_number=1, + error=ErrorEventError( + type="guardrail_error", + code="400", + message="Streaming response blocked by Model Armor", + param=None, + ), + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + assert tuple(delivered) == chunks + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "chunks", + [ + pytest.param(None, id="anthropic-error-event"), + pytest.param( + ('data: {"error": {"message": "Streaming response blocked by the first guardrail", "code": "400"}}\n\n',), + id="chat-completions-error-payload", + ), + ], +) +async def test_streaming_hook_forwards_a_preceding_guardrails_error_frame(chunks): + """Chained post_call guardrails hand each other their output. An earlier guardrail's error + frame carries no message to assemble, and replacing it would hide the real refusal.""" + if chunks is None: + chunks = anthropic_sse_error_frames("Streaming response blocked by the first guardrail") + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + assert tuple(delivered) == chunks + + +@pytest.mark.asyncio +async def test_streaming_responses_error_falls_back_to_sse_when_the_handler_declines(): + """build_stream_error_items may return None, which must not swallow the block into a clean + 200: the refusal falls back to the chat-completions SSE form that still carries the status.""" + from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ( + _StreamSurface, + ) + + class _DecliningGuardrail(ModelArmorGuardrail): + @staticmethod + def _build_responses_error_items(exc): + return None + + guardrail = _DecliningGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + exc = HTTPException(status_code=400, detail={"message": "blocked"}) + + items = guardrail._stream_error_items(exc, surface=_StreamSurface.RESPONSES) + + assert len(items) == 1 + assert '"code": "400"' in items[0] + assert "blocked" in items[0] + + +def _responses_created_event(): + from litellm.types.llms.openai import ( + ResponseCreatedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + return ResponseCreatedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_CREATED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-4o-mini", + object="response", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + + +@pytest.mark.asyncio +async def test_streaming_hook_refuses_an_opaque_sse_stream_without_anthropic_framing(): + """/v1/messages is not the only endpoint that streams raw SSE: the Google generateContent + route marks its own stream raw too. Its frames carry no Anthropic event types, so refusing + them in Anthropic's format would hand a Google client a body it cannot parse.""" + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + chunks = (b'data: {"candidates":[{"content":{"parts":[{"text":"my card is 4111"}]}}]}\n\n',) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + assert tuple(delivered) != chunks + body = "".join(item.decode() if isinstance(item, bytes) else item for item in delivered) + assert "could not be assembled for scanning" in body + assert "event: error" not in body + assert '"code": "500"' in body + + +@pytest.mark.asyncio +async def test_streaming_unassemblable_stream_is_forwarded_when_fail_on_error_is_disabled(): + """fail_on_error: false is a deliberate choice to degrade open, and it governs every other + path in this hook. The fail-closed refusal has to honour it too.""" + guardrail = _surface_guardrail(fail_on_error=False) + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + chunks = ( + b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,' + b'"delta":{"type":"text_delta","text":"hi"}}\n\n', + ) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + assert tuple(delivered) == chunks + + +@pytest.mark.asyncio +async def test_streaming_fail_closed_records_the_applied_guardrail(): + """A refusal that no header or log attributes to the guardrail leaves on-call unable to tell + a guardrail block apart from a provider failure.""" + guardrail = _surface_guardrail() + request_data = { + "model": "claude-haiku", + "messages": [{"role": "user", "content": "show me a card"}], + "metadata": {"guardrails": ["model-armor-test"]}, + } + chunks = ( + b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,' + b'"delta":{"type":"text_delta","text":"hi"}}\n\n', + ) + + with patch.object(guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_CLEAN)): + await _drain_surface_hook(guardrail, chunks, request_data=request_data) + + assert request_data["metadata"]["applied_guardrails"] == ["model-armor-test"] + + +@pytest.mark.asyncio +async def test_streaming_responses_tool_call_output_is_scanned(): + """An agentic /v1/responses turn can carry its whole payload in tool-call arguments, which + is what the chat surface already folds into the scanned text.""" + from litellm.types.llms.openai import ( + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-4o-mini", + object="response", + output=[ + { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "send_email", + "arguments": '{"body": "my card is 4111-1111-1111-1111"}', + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, (completed,)) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert "4111-1111-1111-1111" in scanned + assert tuple(delivered) == (completed,) + + +@pytest.mark.asyncio +async def test_streaming_hook_refuses_a_content_stream_that_ends_with_an_error_frame(): + """The chain-aware passthrough must stay narrow. A stream carrying real content plus a + trailing error frame is not a bare refusal to forward: the assembler cannot read it, and + releasing it would ship the buffered content unscanned.""" + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + chunks = (*_ANTHROPIC_SSE_CHUNKS, *anthropic_sse_error_frames("upstream gave up")) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + body = b"".join(delivered) + assert b"4111-1111-1111-1111" not in body + assert b"could not be assembled for scanning" in body + + +@pytest.mark.parametrize( + "chunks, expected, case", + [ + (anthropic_sse_error_frames("blocked upstream"), True, "anthropic-error-frames-only"), + ((f"data: {json.dumps({'error': {'message': 'blocked'}})}\n\n",), True, "chat-error-payload-only"), + ((), False, "empty-stream"), + ( + (b'event: message_delta\ndata: {"type":"message_delta","error":null}\n\n',), + False, + "content-event-carrying-a-null-error-field", + ), + ( + ( + litellm.types.utils.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + index=0, + delta=litellm.types.utils.Delta(content="my card is 4111-1111-1111-1111"), + ) + ] + ), + *anthropic_sse_error_frames("upstream gave up"), + ), + False, + "typed-content-chunks-plus-a-trailing-error-frame", + ), + ], +) +def test_is_sse_error_stream_only_matches_a_stream_that_is_nothing_but_refusals(chunks, expected, case): + """The chain-aware passthrough turns on this predicate, so anything it calls error-only is + forwarded to the client untouched. A stream that still carries content must not qualify: the + frames-only join drops typed chunks, and a content event may carry an empty ``error`` field.""" + from litellm.proxy.guardrails.anthropic_sse import is_sse_error_stream + + assert is_sse_error_stream(chunks) is expected, case + + +@pytest.mark.asyncio +async def test_streaming_hook_does_not_forward_typed_chunks_that_end_with_an_error_frame(): + """A stream mixing buffered content with a trailing refusal is not the bare refusal the chain + passthrough exists for. Forwarding it would release the content no scanner ever saw.""" + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + chunks = ( + litellm.types.utils.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + index=0, + delta=litellm.types.utils.Delta(content="my card is 4111-1111-1111-1111"), + ) + ] + ), + *anthropic_sse_error_frames("upstream gave up"), + ) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_not_called() + body = b"".join(item if isinstance(item, bytes) else str(item).encode() for item in delivered) + assert b"4111-1111-1111-1111" not in body + assert b"could not be assembled for scanning" in body + + +def _delivered_bytes(delivered): + return b"".join( + item + if isinstance(item, bytes) + else item.encode() + if isinstance(item, str) + else str(item.model_dump() if hasattr(item, "model_dump") else item).encode() + for item in delivered + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("chunks, case", [(None, "chat_completions"), (_ANTHROPIC_SSE_CHUNKS, "anthropic_sse")]) +async def test_streaming_deidentify_match_masks_when_masking_is_enabled(chunks, case): + """A de-identify template reports MATCH_FOUND for every redaction it makes, so reading that + match as a refusal makes mask_response_content unusable on a stream: the client gets an error + where its non-streaming sibling gets redacted text. The block check has to allow sanitization + exactly as the non-streaming hook does.""" + guardrail = _surface_guardrail(mask_response_content=True) + post = _armor_post_mock(_MODEL_ARMOR_DEIDENTIFIED) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook( + guardrail, _chat_completion_chunks() if chunks is None else chunks + ) + + body = _delivered_bytes(delivered) + assert b"[REDACTED]" in body, case + assert b"4111-1111-1111-1111" not in body, case + assert b"blocked by Model Armor" not in body, case + + +@pytest.mark.asyncio +@pytest.mark.parametrize("chunks, case", [(None, "chat_completions"), (_ANTHROPIC_SSE_CHUNKS, "anthropic_sse")]) +async def test_streaming_deidentify_match_still_blocks_when_masking_is_disabled(chunks, case): + """Without mask_response_content there is nowhere to put the rewritten text, so the same + de-identify match must still end the stream rather than release the original.""" + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_DEIDENTIFIED) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook( + guardrail, _chat_completion_chunks() if chunks is None else chunks + ) + + body = _delivered_bytes(delivered) + assert b"Streaming response blocked by Model Armor" in body, case + assert b"4111-1111-1111-1111" not in body, case + + +@pytest.mark.asyncio +async def test_streaming_deidentify_match_logs_masked_run_as_success_not_blocked(): + """The status stamped on request metadata feeds the spend log, so it has to agree with what + the client actually received: a masked stream is a success, not a block.""" + guardrail = _surface_guardrail(mask_response_content=True) + request_data = { + "model": "claude-haiku", + "messages": [{"role": "user", "content": "show me a card"}], + "metadata": {"guardrails": ["model-armor-test"]}, + } + + with patch.object(guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_DEIDENTIFIED)): + await _drain_surface_hook(guardrail, _chat_completion_chunks(), request_data=request_data) + + assert request_data["metadata"]["_model_armor_status"] == "success" + + +# A de-identify template that matched but handed back no rewrite, e.g. because the transformation +# itself failed. The match still says the buffered original carries what it matched on +_MODEL_ARMOR_DEIDENTIFIED_NO_TEXT = { + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "sdp": {"sdpFilterResult": {"deidentifyResult": {"matchState": "MATCH_FOUND"}}} + }, + } +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("chunks, case", [(None, "chat_completions"), (_ANTHROPIC_SSE_CHUNKS, "anthropic_sse")]) +async def test_streaming_deidentify_match_without_a_rewrite_fails_closed(chunks, case): + """Allowing sanitization past the block check is a promise to apply the redaction. When Model + Armor matches but returns no sanitized text there is nothing to apply, and yielding the + buffered chunks would hand back exactly what it matched on.""" + guardrail = _surface_guardrail(mask_response_content=True) + post = _armor_post_mock(_MODEL_ARMOR_DEIDENTIFIED_NO_TEXT) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook( + guardrail, _chat_completion_chunks() if chunks is None else chunks + ) + + body = _delivered_bytes(delivered) + assert b"4111-1111-1111-1111" not in body, case + assert b"Streaming response blocked by Model Armor" in body, case + + +@pytest.mark.asyncio +async def test_streaming_status_records_a_surface_that_cannot_carry_the_rewrite_as_blocked(): + """The Responses surface has no assembled body to rewrite, so a de-identify match ends as a + refusal. The status stamped on metadata feeds the spend log and has to say so rather than + reporting the success the block check alone would have implied.""" + guardrail = _surface_guardrail(mask_response_content=True) + request_data = { + "model": "gpt-4o-mini", + "input": "show me a card", + "metadata": {"guardrails": ["model-armor-test"]}, + } + + with patch.object(guardrail.async_handler, "post", _armor_post_mock(_MODEL_ARMOR_DEIDENTIFIED)): + delivered = await _drain_surface_hook( + guardrail, _responses_api_events(), request_data=request_data + ) + + body = _delivered_bytes(delivered) + assert b"4111-1111-1111-1111" not in body + assert b"Streaming response blocked by Model Armor" in body + assert request_data["metadata"]["_model_armor_status"] == "blocked" + + +def _responses_api_events_truncated(terminal: str): + """A /v1/responses stream whose text went out as deltas and whose terminal event reports no body. + + ``response.failed`` and ``response.incomplete`` are terminal like ``response.completed``, but a + turn that broke mid-generation reports an empty ``output`` while the deltas ahead of it already + spelled the answer out to the client. + """ + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + empty_body = ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-4o-mini", + object="response", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + terminal_event = ( + ResponseFailedEvent(type=ResponsesAPIStreamEvents.RESPONSE_FAILED, response=empty_body) + if terminal == "failed" + else ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=empty_body) + ) + return ( + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta="my card is 4111-1111-1111-1111", + ), + terminal_event, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("terminal", ["failed", "incomplete"]) +async def test_streaming_responses_terminal_event_without_a_body_still_scans_the_deltas(terminal): + """A /v1/responses turn that broke mid-generation has still delivered its deltas. + + Reading only the terminal body would find nothing to scan and hand every buffered delta to the + client untouched, so the deltas themselves are what gets scanned. + """ + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_BLOCK) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, _responses_api_events_truncated(terminal)) + + post.assert_called_once() + assert "4111-1111-1111-1111" in post.call_args.kwargs["json"]["modelResponseData"]["text"] + rendered = "".join(str(item) for item in delivered) + assert "4111-1111-1111-1111" not in rendered + assert "Streaming response blocked by Model Armor" in rendered + + +@pytest.mark.asyncio +async def test_streaming_responses_mcp_argument_deltas_are_scanned_when_the_body_is_empty(): + """A turn that only streamed MCP tool arguments still handed the client a payload. + + The delta fallback is read off the event enum rather than listed by hand, so an argument event + that carries no `output_text` cannot fall out of the scan. + """ + from litellm.types.llms.openai import ( + MCPCallArgumentsDeltaEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + chunks = ( + MCPCallArgumentsDeltaEvent( + type=ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA, + output_index=0, + item_id="mcp_1", + delta='{"note": "my card is 4111-1111-1111-1111"}', + sequence_number=0, + ), + ResponseIncompleteEvent( + type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-4o-mini", + object="response", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_BLOCK) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, chunks) + + post.assert_called_once() + assert "4111-1111-1111-1111" in post.call_args.kwargs["json"]["modelResponseData"]["text"] + rendered = "".join(str(item) for item in delivered) + assert "4111-1111-1111-1111" not in rendered + assert "Streaming response blocked by Model Armor" in rendered + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "with_output_text_delta", + [True, False], + ids=["summary-and-text-deltas", "summary-delta-only"], +) +async def test_streaming_responses_reasoning_summary_deltas_are_scanned_alongside_the_body(with_output_text_delta): + """A reasoning turn streams its summary in deltas the terminal body never repeats. + + Reading only the body scans the visible answer and hands the client every summary delta + unscanned, so the body and the deltas are scanned together. + """ + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ReasoningSummaryTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + answer = "the weather is fine" + summary_delta = ReasoningSummaryTextDeltaEvent( + type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, + item_id="rs_1", + output_index=0, + delta="the user said my card is 4111-1111-1111-1111", + ) + text_deltas = ( + ( + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=1, + content_index=0, + delta=answer, + ), + ) + if with_output_text_delta + else () + ) + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-5-mini", + object="response", + output=[ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": answer, "annotations": []}], + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_BLOCK) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, (summary_delta, *text_deltas, completed)) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert "4111-1111-1111-1111" in scanned + assert answer in scanned + assert scanned.count(answer) == 1 + rendered = "".join(str(item) for item in delivered) + assert "4111-1111-1111-1111" not in rendered + assert "Streaming response blocked by Model Armor" in rendered + + +@pytest.mark.asyncio +async def test_streaming_responses_deltas_of_separate_fields_do_not_form_a_finding_across_their_boundary(): + """Two fields of a turn are separate text, so what runs across their boundary is not model output. + + A reasoning summary ending in half a card number and an answer opening with the other half + each carry nothing to find, and joining them without a break would invent one. + """ + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ReasoningSummaryTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + answer = "1111-1111 is not a full card" + summary_delta = ReasoningSummaryTextDeltaEvent( + type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, + item_id="rs_1", + output_index=0, + delta="the prefix they gave me is 4111-1111-", + ) + text_delta = OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=1, + content_index=0, + delta=answer, + ) + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-5-mini", + object="response", + output=[ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": answer, "annotations": []}], + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, (summary_delta, text_delta, completed)) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert "4111-1111-" in scanned + assert answer in scanned + assert "4111-1111-1111-1111" not in scanned + rendered = "".join(str(item) for item in delivered) + assert "Streaming response blocked by Model Armor" not in rendered + + +@pytest.mark.asyncio +async def test_streaming_responses_one_fields_deltas_still_join_into_a_single_finding(): + """A card number split across two deltas of one field is still one card number to scan.""" + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + halves = ("my card is 4111-1111-", "1111-1111") + text_deltas = tuple( + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta=half, + ) + for half in halves + ) + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-5-mini", + object="response", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_BLOCK) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, (*text_deltas, completed)) + + post.assert_called_once() + assert "4111-1111-1111-1111" in post.call_args.kwargs["json"]["modelResponseData"]["text"] + rendered = "".join(str(item) for item in delivered) + assert "4111-1111-1111-1111" not in rendered + assert "Streaming response blocked by Model Armor" in rendered + + +@pytest.mark.asyncio +async def test_streaming_responses_fields_the_body_repeats_are_not_scanned_a_second_time(): + """A turn whose visible fields all reach the terminal body is scanned once, not twice. + + Two output_text fields stream as deltas and come back in the completed body, so scanning the + deltas on top of the body would send Model Armor two copies of everything the client sees. + """ + from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ) + + paragraphs = ("the first thing to know", "a second and separate point") + text_deltas = tuple( + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id=f"msg_{index}", + output_index=index, + content_index=0, + delta=paragraph, + ) + for index, paragraph in enumerate(paragraphs) + ) + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=0, + model="gpt-5-mini", + object="response", + output=[ + { + "type": "message", + "id": f"msg_{index}", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": paragraph, "annotations": []}], + } + for index, paragraph in enumerate(paragraphs) + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ), + ) + guardrail = _surface_guardrail() + post = _armor_post_mock(_MODEL_ARMOR_CLEAN) + + with patch.object(guardrail.async_handler, "post", post): + delivered = await _drain_surface_hook(guardrail, (*text_deltas, completed)) + + post.assert_called_once() + scanned = post.call_args.kwargs["json"]["modelResponseData"]["text"] + assert [scanned.count(paragraph) for paragraph in paragraphs] == [1, 1] + rendered = "".join(str(item) for item in delivered) + assert all(paragraph in rendered for paragraph in paragraphs) + + +def test_every_responses_delta_event_is_in_the_scanned_set(): + """Every ``.delta`` the Responses event enum defines is model output on its way to the client.""" + from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ( + _RESPONSES_DELTA_EVENT_TYPES, + ) + from litellm.types.llms.openai import ResponsesAPIStreamEvents + + missing = { + event.value + for event in ResponsesAPIStreamEvents + if event.value.endswith(".delta") and event.value not in _RESPONSES_DELTA_EVENT_TYPES + } + assert not missing + assert "response.mcp_call_arguments.delta" in _RESPONSES_DELTA_EVENT_TYPES diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 4ee6741ee02..84f7611c0c0 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -842,24 +842,37 @@ async def test_presidio_filter_scope_initializer(monkeypatch): params_input = LitellmParams(guardrail="presidio", mode="pre_call", presidio_filter_scope="input") guardrail_dict = {"guardrail_name": "g1"} - cb = initialize_presidio(params_input, guardrail_dict) - assert cb is created[0] + callbacks = initialize_presidio(params_input, guardrail_dict) + assert callbacks == (created[0],) assert created[0].apply_to_output is False # output-only created.clear() params_output = LitellmParams(guardrail="presidio", mode="pre_call", presidio_filter_scope="output") - cb = initialize_presidio(params_output, guardrail_dict) + callbacks = initialize_presidio(params_output, guardrail_dict) assert len(created) == 1 + assert callbacks == (created[0],) assert created[0].apply_to_output is True - # both -> expect two callbacks (input + output) + # both -> expect two callbacks (input + output), both returned, input first created.clear() params_both = LitellmParams(guardrail="presidio", mode="pre_call", presidio_filter_scope="both") - cb = initialize_presidio(params_both, guardrail_dict) + callbacks = initialize_presidio(params_both, guardrail_dict) assert len(created) == 2 - assert any(not c.apply_to_output for c in created) - assert any(c.apply_to_output for c in created) + assert callbacks == tuple(created) + assert callbacks[0].apply_to_output is False + assert callbacks[1].apply_to_output is True + + # both + output_parse_pii -> three callbacks, all returned, input first + created.clear() + params_all = LitellmParams( + guardrail="presidio", mode="pre_call", presidio_filter_scope="both", output_parse_pii=True + ) + callbacks = initialize_presidio(params_all, guardrail_dict) + assert len(created) == 3 + assert callbacks == tuple(created) + assert callbacks[0].apply_to_output is False + assert mgr.added[-3:] == list(created) @pytest.mark.asyncio @@ -3116,6 +3129,18 @@ def test_update_in_memory_applies_analyze_chunk_size(): assert guardrail.presidio_analyze_chunk_size_bytes == 99_000 +def test_update_in_memory_keeps_output_masker_from_unmasking(): + masker = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True, output_parse_pii=False) + unmasker = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True) + params = LitellmParams(guardrail="presidio", mode="pre_call", output_parse_pii=True) + + masker.update_in_memory_litellm_params(params) + unmasker.update_in_memory_litellm_params(params) + + assert (masker.apply_to_output, masker.output_parse_pii) == (True, False) + assert (unmasker.apply_to_output, unmasker.output_parse_pii) == (False, True) + + def test_merge_drops_truncated_same_type_fragment_from_overlap(): """A boundary entity seen truncated by chunk 1 and whole by chunk 2 must merge to the single full span; keeping both overlapping spans corrupts the diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 8cad1c634a9..a28a2a71613 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -1971,3 +1971,271 @@ class TestStreamingGuardrailInformationBucket: assert recorded[0]["guardrail_name"] == "audit-recorder" assert recorded[0]["guardrail_status"] == "success" assert request_data["metadata"]["user_api_key_user_id"] == "user-1" + + +class _ScanCountingGuardrail(CustomGuardrail): + """Pass-through guardrail that records every response-side scan payload.""" + + def __init__(self, *, sampling_rate=5, end_of_stream_only=False, buffer_until_moderated=False): + super().__init__(guardrail_name="scan-counter") + self.streaming_sampling_rate = sampling_rate + self.streaming_end_of_stream_only = end_of_stream_only + self.streaming_buffer_until_moderated = buffer_until_moderated + self.guardrail_config = {} + self.scans: tuple[dict[str, object], ...] = () + + def should_run_guardrail(self, data, event_type): # type: ignore[override] + return True + + async def apply_guardrail(self, inputs, request_data, input_type, **kwargs): + self.scans = ( + *self.scans, + { + "texts": list(inputs.get("texts") or []), + "tool_calls": list(inputs.get("tool_calls") or []), + "model": inputs.get("model"), + }, + ) + return inputs + + +def _responses_delta(sequence_number, text): + return { + "type": "response.output_text.delta", + "sequence_number": sequence_number, + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": text, + } + + +def _responses_tail(sequence_number, text): + return [ + { + "type": "response.output_text.done", + "sequence_number": sequence_number, + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "text": text, + }, + { + "type": "response.completed", + "sequence_number": sequence_number + 1, + "response": { + "model": "gpt-5.6", + "output": [{"type": "message", "content": [{"type": "output_text", "text": text}]}], + }, + }, + ] + + +class TestStreamingScanDedup: + """A sampled round whose scan payload matches the previous round (or carries + no text yet) is skipped, so a stream is never re-scanned for output the + guardrail already cleared. Regression for LIT-6692.""" + + @pytest.fixture(autouse=True) + def _use_real_mappings(self, monkeypatch): + monkeypatch.setattr( + unified_module, + "endpoint_guardrail_translation_mappings", + load_guardrail_translation_mappings(), + ) + + @pytest.mark.asyncio + async def test_chat_terminal_chunk_on_sampled_index_is_scanned_once(self): + guardrail = _ScanCountingGuardrail(sampling_rate=3) + chunks = [_stream_chunk("a"), _stream_chunk("b"), _stream_chunk("c", finish_reason="stop")] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert len(out) == 3 + assert [scan["texts"] for scan in guardrail.scans] == [["abc"]] + + @pytest.mark.asyncio + async def test_chat_round_with_unchanged_text_is_skipped(self): + guardrail = _ScanCountingGuardrail(sampling_rate=3) + chunks = [ + _stream_chunk("a"), + _stream_chunk("b"), + _stream_chunk("c"), + _stream_chunk(None), + _stream_chunk(None), + _stream_chunk(None), + _stream_chunk("d", finish_reason="stop"), + ] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert len(out) == 7 + assert [scan["texts"] for scan in guardrail.scans] == [["abc"], ["abcd"]] + + @pytest.mark.asyncio + async def test_chat_finish_chunk_right_after_a_sampled_round_is_not_rescanned(self): + guardrail = _ScanCountingGuardrail(sampling_rate=3) + chunks = [_stream_chunk("a"), _stream_chunk("b"), _stream_chunk("c"), _stream_chunk(None, finish_reason="stop")] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert len(out) == 4 + assert [scan["texts"] for scan in guardrail.scans] == [["abc"]] + + @pytest.mark.asyncio + async def test_chat_finish_chunk_carrying_tool_calls_is_still_scanned(self): + from litellm.types.utils import ChatCompletionDeltaToolCall, Function + + guardrail = _ScanCountingGuardrail(sampling_rate=3) + tool_call = ChatCompletionDeltaToolCall( + id="call_1", index=0, type="function", function=Function(name="get_weather", arguments='{"city": "Paris"}') + ) + finish = ModelResponseStream( + choices=[ + StreamingChoices(index=0, delta=Delta(content=None, tool_calls=[tool_call]), finish_reason="tool_calls") + ] + ) + chunks = [_stream_chunk("a"), _stream_chunk("b"), _stream_chunk("c"), finish] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert len(out) == 4 + assert [scan["texts"] for scan in guardrail.scans] == [["abc"], ["abc"]] + assert [call["function"]["name"] for call in guardrail.scans[1]["tool_calls"]] == ["get_weather"] + + @pytest.mark.asyncio + async def test_chat_second_choice_finishing_later_still_gets_the_end_scan(self): + guardrail = _ScanCountingGuardrail(sampling_rate=3) + chunks = [ + _stream_chunk("a", index=0), + _stream_chunk("x", index=1), + _stream_chunk("b", finish_reason="stop", index=0), + _stream_chunk("y", index=1), + _stream_chunk("z", finish_reason="stop", index=1), + ] + + await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert len(guardrail.scans) == 2 + assert any("yz" in text for text in guardrail.scans[-1]["texts"]) + + @pytest.mark.asyncio + async def test_responses_completed_event_on_sampled_index_is_scanned_once(self): + guardrail = _ScanCountingGuardrail(sampling_rate=5) + deltas = [_responses_delta(i, f"t{i}") for i in range(8)] + full_text = "".join(f"t{i}" for i in range(8)) + chunks = deltas + _responses_tail(8, full_text) + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses") + + assert len(out) == 10 + assert [scan["texts"] for scan in guardrail.scans] == [["t0t1t2t3t4"], [full_text]] + assert guardrail.scans[-1]["model"] == "gpt-5.6" + + @pytest.mark.asyncio + async def test_responses_completed_right_after_a_sampled_round_is_not_rescanned(self): + guardrail = _ScanCountingGuardrail(sampling_rate=5) + deltas = [_responses_delta(i, f"t{i}") for i in range(5)] + chunks = deltas + _responses_tail(5, "t0t1t2t3t4") + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses") + + assert len(out) == 7 + assert [scan["texts"] for scan in guardrail.scans] == [["t0t1t2t3t4"]] + + @pytest.mark.asyncio + async def test_responses_completed_carrying_a_function_call_is_still_scanned(self): + guardrail = _ScanCountingGuardrail(sampling_rate=5) + deltas = [_responses_delta(i, f"t{i}") for i in range(5)] + completed = { + "type": "response.completed", + "sequence_number": 5, + "response": { + "model": "gpt-5.6", + "output": [ + {"type": "message", "content": [{"type": "output_text", "text": "t0t1t2t3t4"}]}, + { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "status": "completed", + }, + ], + }, + } + chunks = deltas + [completed] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses") + + assert len(out) == 6 + assert [scan["texts"] for scan in guardrail.scans] == [["t0t1t2t3t4"], ["t0t1t2t3t4"]] + assert [call["function"]["name"] for call in guardrail.scans[1]["tool_calls"]] == ["get_weather"] + + @pytest.mark.asyncio + async def test_responses_round_with_unchanged_text_is_skipped(self): + guardrail = _ScanCountingGuardrail(sampling_rate=5) + deltas = [_responses_delta(i, f"t{i}") for i in range(5)] + quiet = [{"type": "response.in_progress", "sequence_number": i} for i in range(5, 10)] + chunks = deltas + quiet + _responses_tail(10, "t0t1t2t3t4") + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses") + + assert len(out) == 12 + assert guardrail.scans == ({"texts": ["t0t1t2t3t4"], "tool_calls": [], "model": None},) + + @pytest.mark.asyncio + async def test_responses_tool_call_done_event_is_still_scanned(self): + guardrail = _ScanCountingGuardrail(sampling_rate=2) + tool_call_done = { + "type": "response.output_item.done", + "sequence_number": 1, + "output_index": 1, + "item": { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "status": "completed", + }, + } + chunks = [_responses_delta(0, "hi"), tool_call_done] + _responses_tail(2, "hi") + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses") + + assert len(out) == 4 + assert len(guardrail.scans) == 2 + assert [call["function"]["name"] for call in guardrail.scans[0]["tool_calls"]] == ["get_weather"] + assert guardrail.scans[1]["texts"] == ["hi"] + + @pytest.mark.asyncio + async def test_anthropic_skips_empty_round_and_terminal_duplicate(self): + guardrail = _ScanCountingGuardrail(sampling_rate=2) + chunks = _anthropic_message_chunks(["hello ", "world"]) + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/messages") + + assert out == chunks + assert [scan["texts"] for scan in guardrail.scans] == [["hello world"]] + + @pytest.mark.asyncio + async def test_end_of_stream_only_still_scans_exactly_once(self): + guardrail = _ScanCountingGuardrail(sampling_rate=2, end_of_stream_only=True) + chunks = _anthropic_message_chunks(["hello ", "world"]) + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/messages") + + assert out == chunks + assert [scan["texts"] for scan in guardrail.scans] == [["hello world"]] + + @pytest.mark.asyncio + async def test_buffer_until_moderated_still_scans_exactly_once_and_releases_every_chunk(self): + guardrail = _ScanCountingGuardrail(sampling_rate=1, buffer_until_moderated=True) + chunks = [_stream_chunk("a"), _stream_chunk("b"), _stream_chunk("c", finish_reason="stop")] + + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks) + + assert out == chunks + assert [scan["texts"] for scan in guardrail.scans] == [["abc"]] diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index 9511732fd50..d85c6da659b 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -104,7 +104,7 @@ def mock_in_memory_handler(mocker): mock_handler.get_guardrail_by_id.return_value = MOCK_CONFIG_GUARDRAIL mock_handler.get_source.return_value = "config" mock_handler.initialize_guardrail = mocker.Mock() - mock_handler.update_in_memory_guardrail = mocker.Mock() + mock_handler.sync_guardrail_from_db = mocker.Mock() mock_handler.delete_in_memory_guardrail = mocker.Mock() mock_handler.reconcile_db_guardrails = mocker.Mock(return_value=[]) return mock_handler @@ -1047,13 +1047,15 @@ async def test_create_guardrail_endpoint( "scenario,expected_result,expected_exception", [ ("success_with_sync", "test-db-guardrail", None), - ("success_sync_fails", "test-db-guardrail", None), + ("success_sync_fails_unexpected_error", "test-db-guardrail", None), + ("sync_fails_invalid_config", None, HTTPException), ("database_failure", None, HTTPException), ("no_prisma_client", None, HTTPException), ], ids=[ "success_with_immediate_sync", - "success_but_sync_fails", + "success_but_sync_fails_with_unexpected_error", + "sync_rejects_invalid_config", "database_error", "missing_prisma_client", ], @@ -1073,6 +1075,7 @@ async def test_update_guardrail_endpoint( mock_logger = None if scenario == "success_with_sync": mock_prisma_client = mocker.Mock() + mock_in_memory_handler.sync_guardrail_from_db = mocker.Mock() mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch( "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", @@ -1083,10 +1086,13 @@ async def test_update_guardrail_endpoint( mock_in_memory_handler, ) - elif scenario == "success_sync_fails": + elif scenario == "success_sync_fails_unexpected_error": + # A non-ValueError/TypeError failure is not a config-rejection signal, + # so it keeps the pre-existing swallow-and-warn behavior rather than + # rolling back the DB write. mock_prisma_client = mocker.Mock() - mock_in_memory_handler.update_in_memory_guardrail.side_effect = Exception( - "Sync failed" + mock_in_memory_handler.sync_guardrail_from_db = mocker.Mock( + side_effect=Exception("Sync failed") ) mock_logger = mocker.patch( "litellm.proxy.guardrails.guardrail_endpoints.verbose_proxy_logger" @@ -1102,6 +1108,25 @@ async def test_update_guardrail_endpoint( mock_in_memory_handler, ) + elif scenario == "sync_fails_invalid_config": + # Regression for the PUT half of the fix: a TypeError from the sync (the + # in-place update_in_memory_guardrail raised exactly this on every PUT) + # must roll back the DB write and surface a 422, not persist the + # rejected config with a 200. + mock_prisma_client = mocker.Mock() + mock_in_memory_handler.sync_guardrail_from_db = mocker.Mock( + side_effect=TypeError("vars() argument must have __dict__ attribute") + ) + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) # test-quality-ok: reused pattern + mocker.patch( # test-quality-ok: reused pattern + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", + mock_guardrail_registry, + ) + mocker.patch( # test-quality-ok: reused pattern + "litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", + mock_in_memory_handler, + ) + elif scenario == "database_failure": mock_prisma_client = mocker.Mock() mock_guardrail_registry.update_guardrail_in_db.side_effect = Exception( @@ -1130,6 +1155,16 @@ async def test_update_guardrail_endpoint( assert "Database error" in str(exc_info.value.detail) elif scenario == "no_prisma_client": assert "Prisma client not initialized" in str(exc_info.value.detail) + elif scenario == "sync_fails_invalid_config": + assert exc_info.value.status_code == 422 + assert "update rejected" in str(exc_info.value.detail) + # Rolled back: update_guardrail_in_db is called once for the + # rejected write and once more to restore the previous config. + assert mock_guardrail_registry.update_guardrail_in_db.call_count == 2 + assert ( + mock_guardrail_registry.update_guardrail_in_db.call_args.kwargs["guardrail"] + == MOCK_DB_GUARDRAIL + ) else: result = await update_guardrail( @@ -1145,11 +1180,11 @@ async def test_update_guardrail_endpoint( prisma_client=mocker.ANY, ) - mock_in_memory_handler.update_in_memory_guardrail.assert_called_once_with( - guardrail_id="test-guardrail-id", guardrail=mocker.ANY + mock_in_memory_handler.sync_guardrail_from_db.assert_called_once_with( + guardrail=mocker.ANY ) - if scenario == "success_sync_fails": + if scenario == "success_sync_fails_unexpected_error": assert mock_logger is not None mock_logger.warning.assert_called_once() assert "Failed to update" in str(mock_logger.warning.call_args) diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 2c0735970d3..56661b5b843 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -1,3 +1,4 @@ +from collections.abc import Iterable from unittest.mock import AsyncMock, MagicMock import pytest @@ -491,6 +492,144 @@ def test_repeated_db_sync_does_not_accumulate_runner_instances(): cb_list[:] = snapshot +PRESIDIO_SIBLINGS_GID = "55555555-5555-5555-5555-555555555555" +PRESIDIO_SIBLINGS_NAME = "presidio-siblings" + + +def _presidio_db_guardrail(pii_entities_config: dict[str, str]) -> Guardrail: + return Guardrail( + guardrail_id=PRESIDIO_SIBLINGS_GID, + guardrail_name=PRESIDIO_SIBLINGS_NAME, + litellm_params={ + "guardrail": "presidio", + "mode": "pre_call", + "default_on": True, + "output_parse_pii": True, + "presidio_filter_scope": "both", + "presidio_analyzer_api_base": "https://fakelink.com/v1/presidio/analyze", + "presidio_anonymizer_api_base": "https://fakelink.com/v1/presidio/anonymize", + "pii_entities_config": pii_entities_config, + }, + ) + + +def _presidio_callbacks_in(cb_list: Iterable[object]) -> list[CustomGuardrail]: + return [ + callback + for callback in cb_list + if isinstance(callback, CustomGuardrail) and getattr(callback, "guardrail_name", None) == PRESIDIO_SIBLINGS_NAME + ] + + +def test_presidio_siblings_are_tracked_and_deleted_together(): + """ + A presidio guardrail scoped to both stages registers the pre_call primary plus + the post_call unmask and mask-output siblings. Deleting the guardrail must remove + all three from every callback list, not just the primary. + """ + import litellm + + handler = InMemoryGuardrailHandler() + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + handler.initialize_guardrail(_presidio_db_guardrail({"EMAIL_ADDRESS": "MASK"})) + + registered = _presidio_callbacks_in(litellm.callbacks) + assert len(registered) == 3 + primary = handler.guardrail_id_to_custom_guardrail[PRESIDIO_SIBLINGS_GID] + siblings = handler.guardrail_id_to_sibling_callbacks[PRESIDIO_SIBLINGS_GID] + assert primary is registered[0] + assert siblings == tuple(registered[1:]) + assert [sibling.event_hook for sibling in siblings] == [GuardrailEventHooks.post_call] * 2 + + for cb_list in lists[1:]: + cb_list.extend(registered) + + handler.delete_in_memory_guardrail(PRESIDIO_SIBLINGS_GID) + + for cb_list in lists: + assert _presidio_callbacks_in(cb_list) == [] + assert PRESIDIO_SIBLINGS_GID not in handler.guardrail_id_to_custom_guardrail + assert PRESIDIO_SIBLINGS_GID not in handler.guardrail_id_to_sibling_callbacks + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + +def test_update_in_memory_guardrail_reaches_presidio_siblings_and_keeps_their_stage(): + import litellm + + handler = InMemoryGuardrailHandler() + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + handler.initialize_guardrail(_presidio_db_guardrail({"EMAIL_ADDRESS": "MASK", "IP_ADDRESS": "MASK"})) + tracked = _presidio_callbacks_in(litellm.callbacks) + roles_before = [ + (callback.apply_to_output, callback.output_parse_pii, callback.event_hook) for callback in tracked + ] + assert roles_before == [ + (False, True, [GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call]), + (False, True, GuardrailEventHooks.post_call), + (True, False, GuardrailEventHooks.post_call), + ] + + updated = Guardrail( + guardrail_id=PRESIDIO_SIBLINGS_GID, + guardrail_name=PRESIDIO_SIBLINGS_NAME, + litellm_params=LitellmParams( + guardrail="presidio", + mode="pre_call", + default_on=True, + output_parse_pii=True, + presidio_filter_scope="both", + presidio_analyzer_api_base="https://fakelink.com/v1/presidio/analyze", + presidio_anonymizer_api_base="https://fakelink.com/v1/presidio/anonymize", + pii_entities_config={"EMAIL_ADDRESS": "MASK"}, + ), + ) + handler.update_in_memory_guardrail(guardrail_id=PRESIDIO_SIBLINGS_GID, guardrail=updated) + + assert [callback.pii_entities_config for callback in tracked] == [{"EMAIL_ADDRESS": "MASK"}] * 3 + assert [ + (callback.apply_to_output, callback.output_parse_pii, callback.event_hook) for callback in tracked + ] == roles_before + assert _presidio_callbacks_in(litellm.callbacks) == tracked + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + +def test_repeated_db_sync_replaces_presidio_siblings_instead_of_leaking_stale_ones(): + """ + The callback manager dedupes custom loggers by their scalar attributes, so a + leaked post_call sibling blocks the re-initialized sibling from registering and + keeps serving the previous entity config. After every DB re-sync, each callback + list must hold exactly the three current instances, all on the latest config. + """ + import litellm + + handler = InMemoryGuardrailHandler() + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + entity_configs = [{"EMAIL_ADDRESS": "MASK"}, {"EMAIL_ADDRESS": "MASK", "IP_ADDRESS": "MASK"}] + for cycle in range(4): + latest = entity_configs[cycle % 2] + handler.sync_guardrail_from_db(_presidio_db_guardrail(latest)) + for cb_list in lists[1:]: + cb_list.extend(_presidio_callbacks_in(litellm.callbacks)) + + for cb_list in lists: + current = _presidio_callbacks_in(cb_list) + assert len({id(callback) for callback in current}) == 3 + assert all(callback.pii_entities_config == latest for callback in current) + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + def _judge_guardrail(guardrail_id: str) -> Guardrail: return Guardrail( guardrail_id=guardrail_id, @@ -774,3 +913,96 @@ def test_reinitialize_guardrail_restores_previous_on_failure(): assert restored.guardrail_name == "restore-me" finally: registry_module.guardrail_initializer_registry.pop("restore_test", None) + + +def test_reinitialize_guardrail_raises_value_error_for_non_value_error_init_failures(): + """Regression for the LIT-6479 fix's 422 path: a constructor failure that is not + already a ValueError/TypeError (re.error from an invalid regex has neither in its + MRO) must still surface as ValueError, so the PUT/PATCH endpoints' rollback+422 + catch is exhaustive instead of warn-and-200 persisting a broken config.""" + import re + + from litellm.proxy.guardrails import guardrail_registry as registry_module + + def _initializer(litellm_params, guardrail): + if litellm_params.api_key == "bad-regex": + re.compile("([") + return CustomGuardrail( + guardrail_name=guardrail["guardrail_name"], + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + + registry_module.guardrail_initializer_registry["regex_test"] = _initializer + try: + handler = InMemoryGuardrailHandler() + created = handler.initialize_guardrail( + guardrail={ + "guardrail_name": "regex-me", + "litellm_params": {"guardrail": "regex_test", "mode": "pre_call", "api_key": "ok"}, + }, + ) + guardrail_id = created["guardrail_id"] + + with pytest.raises(ValueError, match="Guardrail initialization failed") as excinfo: + handler.reinitialize_guardrail( + guardrail={ + "guardrail_id": guardrail_id, + "guardrail_name": "regex-me", + "litellm_params": {"guardrail": "regex_test", "mode": "pre_call", "api_key": "bad-regex"}, + }, + ) + + assert isinstance(excinfo.value.__cause__, re.error) + assert guardrail_id in handler.IN_MEMORY_GUARDRAILS + restored = handler.guardrail_id_to_custom_guardrail[guardrail_id] + assert restored is not None and restored.guardrail_name == "regex-me" + finally: + registry_module.guardrail_initializer_registry.pop("regex_test", None) + + +def test_sync_guardrail_from_db_applies_db_dict_params_to_live_instance(): + """ + Regression for PUT /guardrails/{id}: the DB row arrives with litellm_params as + a plain jsonb dict, and the in-place update_in_memory_guardrail cast it to + LitellmParams without constructing one, so vars() raised and the running proxy + kept enforcing the stale config forever. The PUT endpoint now routes through + sync_guardrail_from_db, which must rebuild the live instance from the dict: + new blocked words compiled in, old ones gone, and the event hook re-derived + from mode (the base-class setattr path wrote self.mode while dispatch reads + self.event_hook, so only a full re-init applies a mode change). + """ + from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, + ) + + handler = InMemoryGuardrailHandler() + gid = "66666666-6666-6666-6666-666666666666" + + def db_guardrail(word: str, mode: str) -> Guardrail: + return Guardrail( + guardrail_id=gid, + guardrail_name="cf-put-sync", + litellm_params={ + "guardrail": "litellm_content_filter", + "mode": mode, + "default_on": True, + "blocked_words": [{"keyword": word, "action": "BLOCK"}], + }, + ) + + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + handler.sync_guardrail_from_db(db_guardrail("foobarblock", "pre_call")) + handler.sync_guardrail_from_db(db_guardrail("quxnewblock", "during_call")) + + instance = handler.guardrail_id_to_custom_guardrail[gid] + assert isinstance(instance, ContentFilterGuardrail) + assert instance._check_blocked_words("hello QUXNEWBLOCK") is not None + assert instance._check_blocked_words("hello FOOBARBLOCK") is None + assert instance.event_hook == GuardrailEventHooks.during_call + assert instance.should_run_guardrail(data={}, event_type=GuardrailEventHooks.during_call) is True + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 8bce967b316..f231eb66a50 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4220,3 +4220,88 @@ async def test_user_new_persists_model_max_budget( ) assert captured["user_data"].get("model_max_budget") == expected_written + + +@pytest.fixture +def _admin_prisma(mocker): + """A mocked prisma_client wired in as proxy_server's module globals, for + the password-policy tests below (mirrors the pattern every other test in + this file repeats per-test; consolidated here since these three share it + verbatim).""" + mock_prisma_client = mocker.MagicMock() + mocker.patch( # test-quality-ok: same module-global mocking every test in this file already uses + "litellm.proxy.proxy_server.prisma_client", mock_prisma_client + ) + mocker.patch( # test-quality-ok: same module-global mocking every test in this file already uses + "litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin" + ) + return mock_prisma_client + + +@pytest.mark.asyncio +async def test_user_update_rejects_weak_password(_admin_prisma): + """/user/update must reject a password that fails the configured + policy before it ever reaches the DB write.""" + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + user_request = UpdateUserRequest(user_id="target-user", password="short1!") + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + with pytest.raises(ProxyException) as exc_info: + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) + + assert exc_info.value.code == "400" + _admin_prisma.db.litellm_usertable.find_first.assert_not_called() + + +@pytest.mark.asyncio +async def test_user_update_rejects_weak_password_against_configured_policy(_admin_prisma, mocker): + """A password that meets the default policy but not a stricter + admin-configured one must still be rejected.""" + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + mocker.patch( # test-quality-ok: same module-global mocking every test in this file already uses + "litellm.proxy.proxy_server.general_settings", + {"password_policy_min_length": 24}, + ) + + user_request = UpdateUserRequest(user_id="target-user", password="Str0ng!Passw0rd") + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + with pytest.raises(ProxyException) as exc_info: + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) + + assert "24 characters" in exc_info.value.message + + +@pytest.mark.asyncio +async def test_user_update_hashes_and_persists_strong_password(_admin_prisma, mocker): + """A password meeting the policy is hashed (never stored in plaintext) + and reaches the DB write.""" + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + mock_prisma_client = _admin_prisma + existing_user = mocker.MagicMock() + existing_user.model_dump.return_value = {"user_id": "target-user"} + existing_user.user_id = "target-user" + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user"}) + mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) + + strong_password = "Str0ng!Passw0rd" + user_request = UpdateUserRequest(user_id="target-user", password=strong_password) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) + + written_data = mock_prisma_client.update_data.call_args.kwargs["data"] + assert written_data.get("password") is not None + assert written_data["password"] != strong_password diff --git a/tests/test_litellm/proxy/management_endpoints/test_saml_sso.py b/tests/test_litellm/proxy/management_endpoints/test_saml_sso.py index 57decc7d458..36d6414ba9f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_saml_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_saml_sso.py @@ -642,3 +642,77 @@ async def test_read_acs_post_data_rejects_oversized_stream_without_content_lengt with pytest.raises(HTTPException) as exc: await SAMLAuthHandler.read_acs_post_data(cast(Request, request)) assert exc.value.status_code == 413 + + +def _fake_request_with_scheme(scheme, headers=None, client_host="203.0.113.5"): + """A fuller fake Request than ``_fake_request``: adds ``url``, ``headers`` and + ``client``, which ``IPAddressUtils.is_request_https`` reads directly instead of + going through ``PROXY_BASE_URL``.""" + return type( + "Req", + (), + { + "base_url": URL(f"{scheme}://proxy.example.com/"), + "url": URL(f"{scheme}://proxy.example.com/sso/saml/login"), + "query_params": {}, + "cookies": {}, + "headers": headers or {}, + "client": type("Client", (), {"host": client_host})(), + }, + )() + + +class TestSAMLAuthnCookieSecureFlag: + """Regression tests for the litellm_saml_authn cookie's Secure attribute. + litellm only sees a plain-HTTP hop whenever TLS terminates at a reverse + proxy, so Secure must not be derived from the literal request scheme alone.""" + + @pytest.mark.asyncio + async def test_secure_over_direct_https(self, saml_env, monkeypatch): + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + cache = DualCache() + request = _fake_request_with_scheme("https") + redirect = await SAMLAuthHandler.build_login_redirect(request, cache) + cookie = redirect.headers["set-cookie"] + assert "Secure" in cookie + assert "SameSite=none" in cookie + + @pytest.mark.asyncio + async def test_not_secure_over_direct_http(self, saml_env, monkeypatch): + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + cache = DualCache() + request = _fake_request_with_scheme("http") + redirect = await SAMLAuthHandler.build_login_redirect(request, cache) + cookie = redirect.headers["set-cookie"] + assert "Secure" not in cookie + assert "SameSite=lax" in cookie + + @pytest.mark.asyncio + async def test_secure_behind_trusted_tls_terminating_proxy(self, saml_env, monkeypatch): + """THE regression: TLS terminates at a reverse proxy, litellm only sees a + plain-HTTP hop, but the cookie must still be marked Secure when the operator + has configured a trusted proxy reporting X-Forwarded-Proto: https.""" + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"use_x_forwarded_for": True, "mcp_trusted_proxy_ranges": ["10.0.0.0/8"]}, + ) + cache = DualCache() + request = _fake_request_with_scheme( + "http", headers={"X-Forwarded-Proto": "https"}, client_host="10.0.0.5" + ) + redirect = await SAMLAuthHandler.build_login_redirect(request, cache) + cookie = redirect.headers["set-cookie"] + assert "Secure" in cookie + + @pytest.mark.asyncio + async def test_untrusted_spoofed_forwarded_proto_is_ignored(self, saml_env, monkeypatch): + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + cache = DualCache() + request = _fake_request_with_scheme( + "http", headers={"X-Forwarded-Proto": "https"}, client_host="203.0.113.5" + ) + redirect = await SAMLAuthHandler.build_login_redirect(request, cache) + cookie = redirect.headers["set-cookie"] + assert "Secure" not in cookie diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index e648bd09734..dd8c752a868 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -7604,6 +7604,112 @@ class TestPKCEStateCookieBinding: assert cookie_str is not None assert "Secure" not in cookie_str + @pytest.mark.asyncio + async def test_redirect_response_sets_secure_flag_behind_trusted_tls_terminating_proxy( + self, monkeypatch + ): + """Regression: litellm sees a plain-HTTP hop when TLS terminates at a reverse + proxy. The Secure flag must still be set when the direct peer is a configured + trusted proxy and it reports X-Forwarded-Proto: https -- but NOT from an + unconfigured/untrusted caller spoofing the same header (see the sibling test + below).""" + from fastapi.responses import RedirectResponse + + from litellm.proxy.management_endpoints.ui_sso import ( + SSOAuthenticationHandler, + ) + + mock_redirect = RedirectResponse( + url="http://idp.internal/authorize?state=behind-proxy-state" + ) + mock_generic_sso = MagicMock() + mock_generic_sso.__enter__ = MagicMock(return_value=mock_generic_sso) + mock_generic_sso.__exit__ = MagicMock(return_value=None) + mock_generic_sso.get_login_redirect = AsyncMock(return_value=mock_redirect) + + proxied_request = MagicMock(spec=Request) + proxied_request.url.scheme = "http" + proxied_request.headers = {"X-Forwarded-Proto": "https"} + proxied_request.client = MagicMock() + proxied_request.client.host = "10.0.0.5" + + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"use_x_forwarded_for": True, "mcp_trusted_proxy_ranges": ["10.0.0.0/8"]}, + ) + + with patch.dict( + os.environ, + { + "GENERIC_CLIENT_STATE": "behind-proxy-state", + "GENERIC_CLIENT_USE_PKCE": "true", + }, + ): + response = await SSOAuthenticationHandler.get_generic_sso_redirect_response( + generic_sso=mock_generic_sso, + state=None, + generic_authorization_endpoint="http://idp.internal/authorize", + request=proxied_request, + ) + + cookie_headers = response.headers.getlist("set-cookie") + cookie_str = next( + (c for c in cookie_headers if "litellm_oauth_state=" in c), None + ) + assert cookie_str is not None + assert "Secure" in cookie_str + + @pytest.mark.asyncio + async def test_redirect_response_ignores_spoofed_forwarded_proto_without_trust_config( + self, monkeypatch + ): + """The same X-Forwarded-Proto: https header must NOT flip Secure on when no + trusted-proxy config is present -- honoring it unconditionally would let any + client spoof the header and would not itself be the vulnerability the ticket + warns against.""" + from fastapi.responses import RedirectResponse + + from litellm.proxy.management_endpoints.ui_sso import ( + SSOAuthenticationHandler, + ) + + mock_redirect = RedirectResponse( + url="http://idp.internal/authorize?state=spoofed-state" + ) + mock_generic_sso = MagicMock() + mock_generic_sso.__enter__ = MagicMock(return_value=mock_generic_sso) + mock_generic_sso.__exit__ = MagicMock(return_value=None) + mock_generic_sso.get_login_redirect = AsyncMock(return_value=mock_redirect) + + spoofed_request = MagicMock(spec=Request) + spoofed_request.url.scheme = "http" + spoofed_request.headers = {"X-Forwarded-Proto": "https"} + spoofed_request.client = MagicMock() + spoofed_request.client.host = "203.0.113.5" + + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + + with patch.dict( + os.environ, + { + "GENERIC_CLIENT_STATE": "spoofed-state", + "GENERIC_CLIENT_USE_PKCE": "true", + }, + ): + response = await SSOAuthenticationHandler.get_generic_sso_redirect_response( + generic_sso=mock_generic_sso, + state=None, + generic_authorization_endpoint="http://idp.internal/authorize", + request=spoofed_request, + ) + + cookie_headers = response.headers.getlist("set-cookie") + cookie_str = next( + (c for c in cookie_headers if "litellm_oauth_state=" in c), None + ) + assert cookie_str is not None + assert "Secure" not in cookie_str + @pytest.mark.asyncio async def test_pkce_callback_rejects_missing_cookie(self): """When PKCE is enabled and a code_verifier is in the cache, the @@ -8586,6 +8692,24 @@ class TestSameOriginReturnPath: assert _is_same_origin_return_path("") is False +def _make_https_request() -> Request: + request = MagicMock(spec=Request) + request.url.scheme = "https" + request.headers = {} + request.client = MagicMock() + request.client.host = "203.0.113.5" + return request + + +def _make_http_request() -> Request: + request = MagicMock(spec=Request) + request.url.scheme = "http" + request.headers = {} + request.client = MagicMock() + request.client.host = "203.0.113.5" + return request + + class TestPersistReturnToCookieSharedHelper: """The single shared return_to helper used by EVERY sign-in branch (SSO / Okta / generic AND the username/password form). It must be best-effort and NEVER raise — a bad return_to can never block @@ -8603,7 +8727,7 @@ class TestPersistReturnToCookieSharedHelper: monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) resp = Response() - _persist_return_to_cookie(resp, "/mcp/authorize?client_id=llm_dcrc_abc") + _persist_return_to_cookie(resp, "/mcp/authorize?client_id=llm_dcrc_abc", _make_https_request()) assert "litellm_cp_return_to=" in self._cookie(resp) def test_bad_absolute_with_control_plane_configured_does_not_raise_and_is_not_stored(self, monkeypatch): @@ -8617,7 +8741,7 @@ class TestPersistReturnToCookieSharedHelper: "litellm.proxy.proxy_server.general_settings", {"control_plane_url": "https://cp.example.com"} ) resp = Response() - _persist_return_to_cookie(resp, "https://evil.example.com/steal") # must not raise + _persist_return_to_cookie(resp, "https://evil.example.com/steal", _make_https_request()) # must not raise assert "litellm_cp_return_to=" not in self._cookie(resp) def test_none_return_to_is_a_noop(self): @@ -8626,7 +8750,7 @@ class TestPersistReturnToCookieSharedHelper: from litellm.proxy.management_endpoints.ui_sso import _persist_return_to_cookie resp = Response() - _persist_return_to_cookie(resp, None) + _persist_return_to_cookie(resp, None, _make_https_request()) assert "litellm_cp_return_to=" not in self._cookie(resp) def test_control_plane_matching_absolute_is_stored(self, monkeypatch): @@ -8638,5 +8762,126 @@ class TestPersistReturnToCookieSharedHelper: "litellm.proxy.proxy_server.general_settings", {"control_plane_url": "https://cp.example.com"} ) resp = Response() - _persist_return_to_cookie(resp, "https://cp.example.com/ui?page=models") + _persist_return_to_cookie(resp, "https://cp.example.com/ui?page=models", _make_https_request()) assert "litellm_cp_return_to=" in self._cookie(resp) + + def test_cookie_is_secure_and_httponly_over_https(self, monkeypatch): + from fastapi import Response + + from litellm.proxy.management_endpoints.ui_sso import _persist_return_to_cookie + + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + resp = Response() + _persist_return_to_cookie(resp, "/mcp/authorize", _make_https_request()) + cookie = self._cookie(resp) + assert "Secure" in cookie + assert "HttpOnly" in cookie + assert "SameSite=lax" in cookie + + def test_cookie_is_not_secure_over_plain_http_direct(self, monkeypatch): + from fastapi import Response + + from litellm.proxy.management_endpoints.ui_sso import _persist_return_to_cookie + + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + resp = Response() + _persist_return_to_cookie(resp, "/mcp/authorize", _make_http_request()) + assert "Secure" not in self._cookie(resp) + + def test_cookie_is_secure_behind_trusted_tls_terminating_proxy(self, monkeypatch): + """Regression for the reported bug: TLS terminates at a reverse proxy, litellm only + sees a plain-HTTP hop, but a trusted X-Forwarded-Proto: https must still mark the + cookie Secure.""" + from fastapi import Response + + from litellm.proxy.management_endpoints.ui_sso import _persist_return_to_cookie + + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"use_x_forwarded_for": True, "mcp_trusted_proxy_ranges": ["10.0.0.0/8"]}, + ) + resp = Response() + request = _make_http_request() + request.client.host = "10.0.0.5" + request.headers = {"X-Forwarded-Proto": "https"} + _persist_return_to_cookie(resp, "/mcp/authorize", request) + assert "Secure" in self._cookie(resp) + + +class TestSessionTokenCookie: + """Regression tests for the ``token`` session cookie set by every sign-in path + (username/password login, SSO callback, the CLI /v2, /v3 login exchange helpers). + It was previously set with no Secure/HttpOnly/SameSite attributes at all -- always + sent over plain HTTP and readable by any script on the page. HttpOnly must stay off + deliberately: the dashboard reads this cookie via document.cookie.""" + + @staticmethod + def _cookie(resp) -> str: + return resp.headers.get("set-cookie", "") + + def test_secure_over_direct_https(self, monkeypatch): + from fastapi import Response + + from litellm.proxy.management_endpoints.ui_sso import set_session_token_cookie + + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + resp = Response() + set_session_token_cookie(resp, _make_https_request(), "jwt-token-value") + cookie = self._cookie(resp) + assert "token=jwt-token-value" in cookie + assert "Secure" in cookie + assert "SameSite=lax" in cookie + assert "HttpOnly" not in cookie + + def test_not_secure_over_direct_http(self, monkeypatch): + from fastapi import Response + + from litellm.proxy.management_endpoints.ui_sso import set_session_token_cookie + + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + resp = Response() + set_session_token_cookie(resp, _make_http_request(), "jwt-token-value") + assert "Secure" not in self._cookie(resp) + + def test_secure_behind_trusted_tls_terminating_proxy(self, monkeypatch): + """THE regression: TLS terminates at a reverse proxy, litellm only sees a + plain-HTTP hop, but the session cookie must still be marked Secure when the + operator has configured a trusted proxy that reports X-Forwarded-Proto: https.""" + from fastapi import Response + + from litellm.proxy.management_endpoints.ui_sso import set_session_token_cookie + + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"use_x_forwarded_for": True, "mcp_trusted_proxy_ranges": ["10.0.0.0/8"]}, + ) + request = _make_http_request() + request.client.host = "10.0.0.5" + request.headers = {"X-Forwarded-Proto": "https"} + resp = Response() + set_session_token_cookie(resp, request, "jwt-token-value") + assert "Secure" in self._cookie(resp) + + def test_untrusted_spoofed_forwarded_proto_is_ignored(self, monkeypatch): + from fastapi import Response + + from litellm.proxy.management_endpoints.ui_sso import set_session_token_cookie + + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + request = _make_http_request() + request.headers = {"X-Forwarded-Proto": "https"} + resp = Response() + set_session_token_cookie(resp, request, "jwt-token-value") + assert "Secure" not in self._cookie(resp) + + def test_proxy_base_url_https_overrides_literal_http_scheme(self, monkeypatch): + from fastapi import Response + + from litellm.proxy.management_endpoints.ui_sso import set_session_token_cookie + + monkeypatch.setenv("PROXY_BASE_URL", "https://litellm.example.com") + resp = Response() + set_session_token_cookie(resp, _make_http_request(), "jwt-token-value") + assert "Secure" in self._cookie(resp) diff --git a/tests/test_litellm/proxy/management_helpers/test_access_group_key_sync.py b/tests/test_litellm/proxy/management_helpers/test_access_group_key_sync.py new file mode 100644 index 00000000000..60c36e33e09 --- /dev/null +++ b/tests/test_litellm/proxy/management_helpers/test_access_group_key_sync.py @@ -0,0 +1,57 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.proxy.db.prisma_client import PrismaWrapper +from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper +from litellm.proxy.management_helpers.access_group_key_sync import ( + sync_key_access_group_membership, + sync_key_regeneration_access_group_membership, +) + + +def _routed_prisma_client(): + writer_inner = MagicMock(name="writer_prisma") + reader_inner = MagicMock(name="reader_prisma") + writer_inner.query_raw = AsyncMock(return_value=[]) + reader_inner.query_raw = AsyncMock(return_value=[]) + writer = PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False) + reader = PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False) + routing = RoutingPrismaWrapper(writer=writer, reader=reader) + return SimpleNamespace(db=routing), writer_inner, reader_inner + + +@pytest.mark.asyncio +async def test_regeneration_repoint_update_runs_on_the_writer(): + prisma_client, writer_inner, reader_inner = _routed_prisma_client() + + await sync_key_regeneration_access_group_membership( + prisma_client=prisma_client, + previous_key_token="old-token", + new_key_token="new-token", + data=None, + existing_key_row=MagicMock(), + ) + + writer_inner.query_raw.assert_awaited_once() + assert writer_inner.query_raw.await_args.args[0].startswith('UPDATE "LiteLLM_AccessGroupTable"') + reader_inner.query_raw.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_membership_attach_and_detach_updates_run_on_the_writer(): + prisma_client, writer_inner, reader_inner = _routed_prisma_client() + + await sync_key_access_group_membership( + prisma_client=prisma_client, + key_token="token", + previous_access_group_ids=["ag-old"], + updated_access_group_ids=["ag-new"], + ) + + assert writer_inner.query_raw.await_count == 2 + assert all( + call.args[0].startswith('UPDATE "LiteLLM_AccessGroupTable"') for call in writer_inner.query_raw.await_args_list + ) + reader_inner.query_raw.assert_not_awaited() diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index 87e0319f6a1..bca97915347 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -4555,3 +4555,114 @@ def test_create_file_async_pre_call_hook_rejection_blocks_upload(monkeypatch, ll assert response.status_code == 400, response.text assert "file upload not allowed" in response.text assert provider_route.call_count == 0 + + +def test_create_file_non_batch_over_max_file_size_mb_rejected_before_forwarding(monkeypatch, llm_router: Router): + """max_file_size_mb applies to every purpose, unlike the batch-only max_batch_file_size_mb.""" + import litellm.proxy.proxy_server as ps + + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + monkeypatch.setitem(ps.general_settings, "max_file_size_mb", 1) + + oversized = b"x" * (2 * 1024 * 1024) + try: + response = client.post( + "/v1/files", + files={"file": ("labels.jsonl", oversized, "application/octet-stream")}, + data={"purpose": "user_data"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 413, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "file" + assert "max_file_size_mb" in error["message"] + assert "1 MB" in error["message"] + assert forwarded_calls == [] + + +def test_create_file_non_batch_under_max_file_size_mb_forwards(monkeypatch, llm_router: Router): + import litellm.proxy.proxy_server as ps + + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + monkeypatch.setitem(ps.general_settings, "max_file_size_mb", 1) + + try: + response = client.post( + "/v1/files", + files={"file": ("labels.jsonl", b"small content", "application/octet-stream")}, + data={"purpose": "user_data"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 200, response.text + assert len(forwarded_calls) == 1 + + +def test_create_file_blocked_extension_rejected_before_forwarding(monkeypatch, llm_router: Router): + import litellm.proxy.proxy_server as ps + + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + monkeypatch.setitem(ps.general_settings, "blocked_file_extensions", [".exe", ".sh"]) + + try: + response = client.post( + "/v1/files", + files={"file": ("payload.exe", b"MZ\x90\x00", "application/octet-stream")}, + data={"purpose": "user_data"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "file" + assert ".exe" in error["message"] + assert "blocked_file_extensions" in error["message"] + assert forwarded_calls == [] + + +def test_create_file_blocked_extension_unset_allows_everything(monkeypatch, llm_router: Router): + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + + try: + response = client.post( + "/v1/files", + files={"file": ("payload.exe", b"MZ\x90\x00", "application/octet-stream")}, + data={"purpose": "user_data"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 200, response.text + assert len(forwarded_calls) == 1 + + +def test_create_file_path_traversal_filename_rejected_before_forwarding(monkeypatch, llm_router: Router): + """A filename carrying a directory-traversal component must never reach storage or the provider.""" + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + + try: + response = client.post( + "/v1/files", + files={"file": ("../../etc/passwd", b"malicious content", "text/plain")}, + data={"purpose": "user_data"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "file" + assert "traversal" in error["message"].lower() + assert forwarded_calls == [] diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_general_upload_validation.py b/tests/test_litellm/proxy/openai_files_endpoint/test_general_upload_validation.py new file mode 100644 index 00000000000..9f7dd914e4a --- /dev/null +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_general_upload_validation.py @@ -0,0 +1,134 @@ +import io + +import pytest + +from litellm.proxy._types import ProxyException +from litellm.proxy.openai_files_endpoints.general_upload_validation import ( + MB, + UploadedFileBlockedExtension, + UploadedFileTooLarge, + UploadedFileUnsafeFilename, + check_blocked_extension, + check_unsafe_filename, + check_upload_file_size, + raise_upload_validation_failure, +) + + +def test_size_under_cap_allowed(): + assert check_upload_file_size(b"x" * 100, 1) is None + + +def test_size_over_cap_rejected_for_bytes(): + content = b"x" * (2 * MB) + assert check_upload_file_size(content, 1) == UploadedFileTooLarge(size_bytes=len(content), limit_mb=1) + + +def test_size_over_cap_rejected_for_binaryio_and_restores_caller_position(): + """The handle is caller-owned; inspecting its size must not discard where the caller had it.""" + content = b"x" * (2 * MB) + handle = io.BytesIO(content) + handle.seek(17) + assert check_upload_file_size(handle, 1) == UploadedFileTooLarge(size_bytes=len(content), limit_mb=1) + assert handle.tell() == 17 + + +def test_size_under_cap_allowed_for_binaryio_restores_caller_position(): + handle = io.BytesIO(b"x" * 100) + handle.seek(42) + assert check_upload_file_size(handle, 1) is None + assert handle.tell() == 42 + + +def test_size_exactly_at_cap_allowed(): + content = b"x" * MB + assert check_upload_file_size(content, 1) is None + + +def test_no_cap_skips_size_check(): + assert check_upload_file_size(b"x" * (10 * MB), None) is None + + +@pytest.mark.parametrize("cap", [0, -3]) +def test_nonpositive_cap_disables_size_check(cap): + assert check_upload_file_size(b"x" * (10 * MB), cap) is None + + +def test_blocked_extension_rejected(): + assert check_blocked_extension("payload.exe", (".exe", ".sh")) == UploadedFileBlockedExtension(extension=".exe") + + +def test_blocked_extension_match_is_case_insensitive(): + assert check_blocked_extension("payload.EXE", (".exe",)) == UploadedFileBlockedExtension(extension=".exe") + + +def test_blocked_extension_match_is_case_insensitive_for_configured_value(): + """A config entry like blocked_file_extensions: ['.EXE'] must still catch a lowercase upload.""" + assert check_blocked_extension("payload.exe", (".EXE",)) == UploadedFileBlockedExtension(extension=".exe") + + +def test_extension_not_in_blocklist_allowed(): + assert check_blocked_extension("report.pdf", (".exe", ".sh")) is None + + +def test_empty_blocklist_allows_everything(): + assert check_blocked_extension("payload.exe", ()) is None + + +def test_no_filename_skips_extension_check(): + assert check_blocked_extension(None, (".exe",)) is None + + +def test_path_traversal_filename_rejected(): + assert check_unsafe_filename("../../etc/passwd") == UploadedFileUnsafeFilename(filename="../../etc/passwd") + + +def test_windows_style_path_traversal_filename_rejected(): + assert check_unsafe_filename("..\\..\\windows\\system32\\config") == UploadedFileUnsafeFilename( + filename="..\\..\\windows\\system32\\config" + ) + + +def test_traversal_embedded_after_extension_rejected(): + assert check_unsafe_filename("report.jsonl/../../etc/cron.d/evil") == UploadedFileUnsafeFilename( + filename="report.jsonl/../../etc/cron.d/evil" + ) + + +def test_null_byte_filename_rejected(): + assert check_unsafe_filename("report.pdf\x00.exe") == UploadedFileUnsafeFilename(filename="report.pdf\x00.exe") + + +@pytest.mark.parametrize("filename", ["report.pdf", ".env", "a.b.c.jsonl", "my file (1).csv", None]) +def test_ordinary_filenames_allowed(filename): + assert check_unsafe_filename(filename) is None + + +@pytest.mark.parametrize( + "failure, expected_code, expected_fragments", + [ + ( + UploadedFileTooLarge(size_bytes=15728640, limit_mb=10), + "413", + ("15.0 MB", "max_file_size_mb", "10 MB", "not forwarded"), + ), + ( + UploadedFileBlockedExtension(extension=".exe"), + "400", + (".exe", "blocked_file_extensions", "not forwarded"), + ), + ( + UploadedFileUnsafeFilename(filename="../../etc/passwd"), + "400", + ("../../etc/passwd", "traversal", "not forwarded"), + ), + ], +) +def test_failures_map_to_openai_shaped_proxy_exceptions(failure, expected_code, expected_fragments): + with pytest.raises(ProxyException) as exc_info: + raise_upload_validation_failure(failure) + assert exc_info.value.code == expected_code + assert exc_info.value.type == "invalid_request_error" + assert exc_info.value.param == "file" + for fragment in expected_fragments: + assert fragment in exc_info.value.message diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index ad3c470acf3..dcb63b8ca82 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -60,6 +60,141 @@ def test_config_update_happy_admin(client, auth_as, mock_prisma, monkeypatch): assert normalize(response.json()) == {"message": "Config updated successfully"} +def test_config_update_persists_optional_pre_call_checks(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + fake_proxy_config = MagicMock() + fake_proxy_config.add_deployment = AsyncMock() + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"optional_pre_call_checks": ["prompt_caching"]}}, + ) + + assert response.status_code == 200 + persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"]) + assert persisted["optional_pre_call_checks"] == ["prompt_caching"] + + +def test_config_update_persists_model_group_affinity_config(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + fake_proxy_config = MagicMock() + fake_proxy_config.add_deployment = AsyncMock() + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + model_group_affinity_config = {"gpt-4": ["session_affinity"]} + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"model_group_affinity_config": model_group_affinity_config}}, + ) + + assert response.status_code == 200 + persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"]) + assert persisted["model_group_affinity_config"] == model_group_affinity_config + + +def test_config_update_persists_disable_cooldowns(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + fake_proxy_config = MagicMock() + fake_proxy_config.add_deployment = AsyncMock() + monkeypatch.setattr(ps, "proxy_config", fake_proxy_config) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"disable_cooldowns": True}}, + ) + + assert response.status_code == 200 + persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"]) + assert persisted["disable_cooldowns"] is True + + +def test_config_update_rejects_assistants_config(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"assistants_config": {"enabled": True}}}, + ) + + assert response.status_code == 400 + assert "assistants_config" in response.json()["error"]["message"] + table.upsert.assert_not_called() + + +def test_config_update_rejects_router_general_settings(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"router_general_settings": {"async_only_mode": True}}}, + ) + + assert response.status_code == 400 + assert "router_general_settings" in response.json()["error"]["message"] + table.upsert.assert_not_called() + + +def test_config_update_rejects_unknown_router_setting(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + table = _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.post( + "/config/update", + json={"router_settings": {"optional_precall_checks": ["prompt_caching"]}}, + ) + + assert response.status_code == 400 + assert "optional_precall_checks" in response.json()["error"]["message"] + table.upsert.assert_not_called() + + +def test_config_update_unknown_router_setting_non_admin_forbidden(client, auth_as, mock_prisma, monkeypatch): + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _install_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + with auth_as(LitellmUserRoles.INTERNAL_USER): + response = client.post( + "/config/update", + json={"router_settings": {"optional_precall_checks": ["prompt_caching"]}}, + ) + + assert response.status_code == 403 + assert "admin" in response.json()["error"]["message"].lower() + + def test_config_update_non_admin_forbidden(client, auth_as, mock_prisma, monkeypatch): """POST /config/update by a non-admin caller is rejected; the error surfaces as a ProxyException with the admin-only message.""" diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py index af37dbe85fe..45460dcecf1 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py @@ -29,7 +29,7 @@ def _install_login_mocks(monkeypatch, raise_on_auth: bool = False) -> None: """ from litellm.proxy import proxy_server as ps - async def _fake_auth(username, password, master_key, prisma_client): + async def _fake_auth(username, password, master_key, prisma_client, general_settings=None): if raise_on_auth: raise Exception("boom-auth-failure") fake = MagicMock() diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_models.py b/tests/test_litellm/proxy/proxy_server/test_routes_models.py index 2b126b1ea95..bc6106a06f8 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_models.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_models.py @@ -45,6 +45,7 @@ def patched_models(monkeypatch): deployment = MagicMock() deployment.litellm_params.model = "gpt-4" router.get_deployment_by_model_group_name = MagicMock(return_value=deployment) + router.get_configured_display_name = MagicMock(return_value=None) monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) @@ -187,6 +188,83 @@ def test_anthropic_format_carries_router_configured_token_limits(client, auth_as assert (claude["max_input_tokens"], claude["max_tokens"]) == (500000, 4096) +@pytest.mark.parametrize("path", ["/v1/models", "/models"]) +def test_anthropic_format_uses_configured_display_name(client, auth_as, patched_models, path): + """A deployment's ``model_info.display_name`` becomes the Anthropic-native + ``display_name`` so Claude Code's picker shows a clean name while the id keeps + routing; models without one keep the id fallback, and the OpenAI-shaped + listing carries no display_name either way.""" + + def _configured(model_name): + return "Kimi K3" if model_name == "gpt-4" else None + + patched_models.get_configured_display_name = MagicMock(side_effect=_configured) + + with auth_as(): + anthropic_response = client.get(path, headers={"anthropic-version": "2023-06-01"}) + openai_response = client.get(path) + + assert anthropic_response.status_code == 200 + gpt_4, claude = anthropic_response.json()["data"] + assert (gpt_4["id"], gpt_4["display_name"]) == ("gpt-4", "Kimi K3") + assert (claude["id"], claude["display_name"]) == ("claude-sonnet", "claude-sonnet") + + assert openai_response.status_code == 200 + openai_models = openai_response.json()["data"] + assert [m["id"] for m in openai_models] == ["gpt-4", "claude-sonnet"] + assert all("display_name" not in m for m in openai_models) + + +@pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) +def test_anthropic_display_name_resolved_via_internal_team_key( + client, auth_as, patched_models, monkeypatch, params +): + """For a team-scoped row the configured display name must be looked up by the + internal routing key while the entry itself is keyed by the public name, so + the clean name lands on the id the client actually sees.""" + from litellm.proxy import utils as proxy_utils + from litellm.proxy.auth import model_checks + + internal_name = "model_name_team-1_c0ffee" + + patched_models.get_model_list = MagicMock( + return_value=[ + { + "model_name": internal_name, + "model_info": { + "team_id": "team-1", + "team_public_model_name": "gpt-4-team", + }, + } + ] + ) + patched_models.get_model_names = MagicMock(return_value=[internal_name]) + patched_models.get_configured_display_name = MagicMock( + side_effect=lambda model_name: "Team GPT" if model_name == internal_name else None + ) + + async def _fake_get_available_models_for_user(**kwargs): + return [internal_name] + + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + _fake_get_available_models_for_user, + ) + monkeypatch.setattr( + model_checks, "get_complete_model_list", lambda **kwargs: [internal_name] + ) + + with auth_as(): + response = client.get( + "/v1/models", params=params, headers={"anthropic-version": "2023-06-01"} + ) + + assert response.status_code == 200 + (entry,) = response.json()["data"] + assert (entry["id"], entry["display_name"]) == ("gpt-4-team", "Team GPT") + + @pytest.mark.parametrize("path", ["/v1/models", "/models"]) def test_get_models_invalid_scope_returns_400(client, auth_as, patched_models, path): """Pins: ``GET /v1/models``, ``GET /models`` (error path: invalid scope).""" diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py b/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py index 5cc22cca7a0..778acc1baab 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py @@ -234,7 +234,7 @@ def test_claim_onboarding_link_happy(client, monkeypatch, mock_prisma): json={ "invitation_link": "inv-123", "user_id": "user-abc", - "password": "hunter2", + "password": "Hunter2Strong!", }, headers={"Authorization": f"Bearer {onboarding_jwt}"}, ) @@ -260,7 +260,7 @@ def test_claim_onboarding_link_invalid_invite_401(client, monkeypatch, mock_pris json={ "invitation_link": "missing", "user_id": "user-abc", - "password": "hunter2", + "password": "Hunter2Strong!", }, headers={"Authorization": "Bearer irrelevant"}, ) @@ -287,7 +287,7 @@ def test_claim_onboarding_link_user_id_mismatch_401( json={ "invitation_link": "inv-123", "user_id": "user-attacker", - "password": "hunter2", + "password": "Hunter2Strong!", }, headers={"Authorization": "Bearer irrelevant"}, ) @@ -339,7 +339,7 @@ def test_claim_onboarding_link_bad_onboarding_jwt_401( json={ "invitation_link": "inv-123", "user_id": "user-abc", - "password": "hunter2", + "password": "Hunter2Strong!", }, headers={"Authorization": f"Bearer {bogus_jwt}"}, ) diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index aa35fd64f18..0fb9b1a6d88 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -19,7 +19,10 @@ from litellm.proxy._types import ( LitellmUserRoles, UserAPIKeyAuth, ) -from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator +from litellm.proxy.common_utils.model_listing_utils import ( + TeamModelNameTranslator, + configured_display_names, +) from litellm.proxy.proxy_server import ( _get_proxy_model_info, _translate_model_name_for_response, @@ -1391,6 +1394,27 @@ def test_resolve_public_name_respects_legacy_flag(): ) +def test_configured_display_names_keyed_by_response_id(): + """The map is keyed by the public response id while the router lookup uses + the internal routing key, and entries without a configured name are omitted.""" + router = MagicMock() + router.get_configured_display_name = MagicMock( + side_effect=lambda model_name: "Team Sonnet" if model_name == "model_name_team-abc-123_4a6b8" else None + ) + + assert configured_display_names( + entries=[ + ("team-claude-sonnet", "model_name_team-abc-123_4a6b8"), + ("gpt-4o", "gpt-4o"), + ], + llm_router=router, + ) == {"team-claude-sonnet": "Team Sonnet"} + + +def test_configured_display_names_empty_without_router(): + assert configured_display_names(entries=[("gpt-4o", "gpt-4o")], llm_router=None) == {} + + @pytest.mark.asyncio async def test_retrieve_model_by_public_name_returns_200(monkeypatch): """Regression: `GET /v1/models/{public_name}` must NOT 404. The listing diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 31430da71e8..551e27a18f5 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -1,3 +1,4 @@ +import re from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch @@ -756,6 +757,45 @@ def test_public_agent_hub_returns_empty_when_no_public_groups(): assert response.json() == [] +# --------------------------------------------------------------------------- +# /public/agents/fields +# --------------------------------------------------------------------------- + + +def test_bedrock_agentcore_runtime_arn_validation_pattern_accepts_full_resource_path(): + """Regression for LIT-6737: the AgentCore agent_runtime_arn field's + validation_pattern must accept a complete runtime ARN whose resource part + is itself multi-segment (``runtime/``), and reject the exact + truncated shape a naive split("/")-by-position parse used to produce (the + ARN cut off right after the ``runtime`` resource type). + """ + app_instance = FastAPI() + app_instance.include_router(router) + test_client = TestClient(app_instance) + + response = test_client.get("/public/agents/fields") + assert response.status_code == 200 + agents = response.json() + + bedrock_agentcore = next((a for a in agents if a["agent_type"] == "bedrock_agentcore"), None) + assert bedrock_agentcore is not None, "bedrock_agentcore agent type not found" + assert bedrock_agentcore["model_template"] == "bedrock/agentcore/{agent_runtime_arn}" + + fields_by_key = {f["key"]: f for f in bedrock_agentcore["credential_fields"]} + arn_field = fields_by_key["agent_runtime_arn"] + assert arn_field["required"] is True + assert arn_field["include_in_litellm_params"] is False + + pattern = arn_field.get("validation_pattern") + assert pattern, "agent_runtime_arn must ship a validation_pattern so the UI can reject a truncated ARN" + + full_arn = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime/hosted_agent_4vm3i-BaTdfOELAs" + truncated_arn = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime" + + assert re.match(pattern, full_arn), "the validator must accept a complete runtime ARN" + assert not re.match(pattern, truncated_arn), "the validator must reject the truncated ARN" + + # --------------------------------------------------------------------------- # /public/endpoints # --------------------------------------------------------------------------- @@ -1037,3 +1077,243 @@ def test_public_mcp_hub_does_not_expose_upstream_url(): assert all("url" not in item for item in data) assert secret_url not in response.text app.dependency_overrides.clear() + + + +@pytest.fixture +def reset_autorouter_presets_cache(): + from litellm.proxy.public_endpoints.public_endpoints import _AutoRouterPresetsCache + + _AutoRouterPresetsCache.presets = None + _AutoRouterPresetsCache.lock = None + yield + _AutoRouterPresetsCache.presets = None + _AutoRouterPresetsCache.lock = None + + +def test_get_autorouter_presets_local_mode_serves_bundled_catalog( + monkeypatch, reset_autorouter_presets_cache +): + monkeypatch.setenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "True") + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + response = client.get("/public/autorouter_presets") + + assert response.status_code == 200 + payload = response.json() + assert "anthropic_family" in payload + for preset in payload.values(): + assert isinstance(preset["label"], str) + assert isinstance(preset["description"], str) + assert "tiers" in preset["complexity_router_config"] + + +@pytest.mark.asyncio +async def test_get_autorouter_presets_fetches_once_per_process( + monkeypatch, reset_autorouter_presets_cache +): + from litellm.proxy.public_endpoints.public_endpoints import ( + _AUTOROUTER_PRESETS_ADAPTER, + get_autorouter_presets, + ) + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + remote = _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "remote_only": { + "label": "Remote Only", + "description": "from the remote catalog", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + } + } + ) + calls = [] + + async def fake_fetch(url): + calls.append(url) + return remote + + first = await get_autorouter_presets(url="https://example.test/presets.json", fetch=fake_fetch) + second = await get_autorouter_presets(url="https://example.test/presets.json", fetch=fake_fetch) + + assert first == remote + assert second == remote + assert calls == ["https://example.test/presets.json"] + + +@pytest.mark.asyncio +async def test_get_autorouter_presets_single_flight_on_concurrent_cold_start( + monkeypatch, reset_autorouter_presets_cache +): + import asyncio + + from litellm.proxy.public_endpoints.public_endpoints import ( + _AUTOROUTER_PRESETS_ADAPTER, + get_autorouter_presets, + ) + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + remote = _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "remote_only": { + "label": "Remote Only", + "description": "from the remote catalog", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + } + } + ) + calls = [] + + async def slow_fetch(url): + calls.append(url) + await asyncio.sleep(0.05) + return remote + + results = await asyncio.gather( + get_autorouter_presets(url="https://example.test/presets.json", fetch=slow_fetch), + get_autorouter_presets(url="https://example.test/presets.json", fetch=slow_fetch), + get_autorouter_presets(url="https://example.test/presets.json", fetch=slow_fetch), + ) + + assert all(result == remote for result in results) + assert len(calls) == 1 + + +@pytest.mark.asyncio +async def test_get_autorouter_presets_caches_bundled_fallback_on_remote_failure( + monkeypatch, reset_autorouter_presets_cache +): + from litellm.proxy.public_endpoints.public_endpoints import get_autorouter_presets + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + calls = [] + + async def broken_fetch(url): + calls.append(url) + raise ValueError("remote catalog unavailable") + + first = await get_autorouter_presets(url="https://example.test/presets.json", fetch=broken_fetch) + second = await get_autorouter_presets(url="https://example.test/presets.json", fetch=broken_fetch) + + assert "anthropic_family" in first + assert second == first + assert len(calls) == 1 + + +@pytest.mark.asyncio +async def test_autorouter_presets_adapter_rejects_wrong_shapes(): + from pydantic import ValidationError + + from litellm.proxy.public_endpoints.public_endpoints import _AUTOROUTER_PRESETS_ADAPTER + + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python({"bad": {"label": "no description or config"}}) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python(["not", "a", "mapping"]) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + {"no_tiers": {"label": "L", "description": "D", "complexity_router_config": {}}} + ) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "missing_builtin_tier": { + "label": "L", + "description": "D", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"]}}, + } + } + ) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "unknown_tier_name": { + "label": "L", + "description": "D", + "complexity_router_config": { + "tiers": { + "SIMPLE": ["m1"], + "MEDIUM": ["m2"], + "COMPLEX": ["m3"], + "REASONING": ["m4"], + "ULTRA": ["m5"], + } + }, + } + } + ) + with pytest.raises(ValidationError): + _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "bad_tiers": { + "label": "L", + "description": "D", + "complexity_router_config": {"tiers": "not-a-mapping"}, + } + } + ) + + +def test_get_autorouter_presets_passes_unknown_catalog_fields_through( + monkeypatch, reset_autorouter_presets_cache +): + from litellm.proxy.public_endpoints.public_endpoints import ( + _AUTOROUTER_PRESETS_ADAPTER, + _AutoRouterPresetsCache, + ) + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + _AutoRouterPresetsCache.presets = _AUTOROUTER_PRESETS_ADAPTER.validate_python( + { + "future_preset": { + "label": "Future", + "description": "carries fields this proxy version does not know", + "complexity_router_config": { + "tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}, + "future_config_knob": 3, + }, + "icon": "sparkles", + } + } + ) + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + response = client.get("/public/autorouter_presets") + + assert response.status_code == 200 + served = response.json()["future_preset"] + assert served["icon"] == "sparkles" + assert served["complexity_router_config"]["future_config_knob"] == 3 + assert served["complexity_router_config"]["tiers"]["SIMPLE"] == ["m1"] + + +@pytest.mark.asyncio +async def test_fetch_remote_autorouter_presets_parses_and_rejects_empty(monkeypatch): + import litellm.llms.custom_httpx.http_handler as http_handler_module + from litellm.proxy.public_endpoints.public_endpoints import _fetch_remote_autorouter_presets + + catalog = { + "remote_only": { + "label": "Remote Only", + "description": "from the remote catalog", + "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + } + } + response = MagicMock() + response.raise_for_status = MagicMock() + response.json = MagicMock(return_value=catalog) + client = MagicMock() + client.get = AsyncMock(return_value=response) + monkeypatch.setattr(http_handler_module, "get_async_httpx_client", lambda llm_provider: client) + + presets = await _fetch_remote_autorouter_presets("https://example.test/presets.json") + assert presets["remote_only"].label == "Remote Only" + response.raise_for_status.assert_called_once() + + response.json = MagicMock(return_value={}) + with pytest.raises(ValueError, match="empty"): + await _fetch_remote_autorouter_presets("https://example.test/presets.json") diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index abbf6892a98..0085b6ebd36 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -324,6 +324,127 @@ def test_rag_query_stream_returns_event_stream(client_internal_user): assert "data: [DONE]" in response.text +def test_rag_query_merges_managed_store_params(client_internal_user): + """ + Regression: /v1/rag/query must consult the managed vector store registry + (like the direct /v1/vector_stores/{id}/search endpoint does) so that + provider, region, embedding model, etc. don't have to be repeated in + retrieval_config. Pre-fix the registry was never read, so managed S3 + Vectors stores failed with "aws_region_name is required". + """ + import litellm + from litellm.types.utils import ModelResponse + + mock_vector_store = { + "vector_store_id": "s3-store", + "custom_llm_provider": "s3_vectors", + "litellm_params": { + "aws_region_name": "eu-west-1", + "embedding_model": "my-embed", + "vector_bucket_name": "bkt", + }, + } + mock_registry = MagicMock() + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = mock_vector_store + + mock_response = ModelResponse( + id="chatcmpl-test", + choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + model="gpt-4o-mini", + ) + + with patch( # test-quality-ok: aquery is the endpoint's downstream boundary; the forwarded config is what the test asserts + "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and grants access so real store resolution runs + "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", + new=AsyncMock(return_value=True), + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store"}, + }, + ) + + assert response.status_code == 200, response.json() + mock_aquery.assert_awaited_once() + forwarded_config = mock_aquery.await_args.kwargs["retrieval_config"] + assert forwarded_config["vector_store_id"] == "s3-store" + assert forwarded_config["custom_llm_provider"] == "s3_vectors" + assert forwarded_config["aws_region_name"] == "eu-west-1" + assert forwarded_config["embedding_model"] == "my-embed" + assert forwarded_config["vector_bucket_name"] == "bkt" + + +def test_rag_query_store_params_win_over_user_retrieval_config(client_internal_user): + """Registry values must win over user-supplied retrieval_config keys so callers cannot override store credentials.""" + import litellm + from litellm.types.utils import ModelResponse + + mock_vector_store = { + "vector_store_id": "s3-store", + "custom_llm_provider": "s3_vectors", + "litellm_params": {"aws_region_name": "eu-west-1"}, + } + mock_registry = MagicMock() + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = mock_vector_store + + mock_response = ModelResponse( + id="chatcmpl-test", + choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + model="gpt-4o-mini", + ) + + with patch( # test-quality-ok: aquery is the endpoint's downstream boundary; the forwarded config is what the test asserts + "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and grants access so real store resolution runs + "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", + new=AsyncMock(return_value=True), + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store", "aws_region_name": "us-east-1"}, + }, + ) + + assert response.status_code == 200, response.json() + forwarded_config = mock_aquery.await_args.kwargs["retrieval_config"] + assert forwarded_config["aws_region_name"] == "eu-west-1" + + +@pytest.mark.parametrize( + "blocked_key", + ["embedding_model", "litellm_embedding_model", "litellm_embedding_config", "litellm_credential_name"], +) +def test_rag_query_rejects_caller_embedding_selection_params(client_internal_user, blocked_key): + """ + Regression: a caller must not pick the embedding model or credential used at + search time. Those resolve through the Router with the proxy's credentials, + bypassing the key's model permissions, so they may only come from the + managed store's server-side registration. + """ + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store", blocked_key: "attacker-choice"}, + }, + ) + + assert response.status_code == 400, response.json() + assert blocked_key in str(response.json()) + + EICAR = r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" INGEST_REQUEST = '{"ingest_options":{"vector_store":{"custom_llm_provider":"openai"}}}' diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index a0dcbf802ef..30b086bab61 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3959,18 +3959,18 @@ async def test_build_ui_spend_logs_response_dict_rows_session_counts(): ] mock_prisma = MagicMock() - mock_prisma.db.litellm_spendlogs.group_by = AsyncMock( - return_value=[ - {"session_id": session_id, "_count": {"session_id": 2}}, - ] - ) + mock_prisma.db.litellm_spendlogs.group_by = AsyncMock() mock_prisma.db.query_raw = AsyncMock( return_value=[ { "session_id": session_id, + "api_key": api_key, + "session_total_count": 2, "session_total_spend": 15.0, "mcp_tool_call_count": 1, "mcp_tool_call_spend": 10.0, + "session_llm_count": 1, + "session_agent_count": 0, } ] ) @@ -3995,6 +3995,8 @@ async def test_build_ui_spend_logs_response_dict_rows_session_counts(): assert rows[0]["mcp_tool_call_spend"] == 10.0 assert rows[1]["mcp_tool_call_count"] == 1 assert rows[1]["mcp_tool_call_spend"] == 10.0 + assert rows[0]["session_llm_count"] == 1 + assert rows[0]["session_agent_count"] == 0 # Every row in the session carries the full session spend, not just its own assert rows[0]["session_total_spend"] == 15.0 @@ -4003,13 +4005,126 @@ async def test_build_ui_spend_logs_response_dict_rows_session_counts(): # Row without a session_id defaults to 1 assert rows[2]["session_total_count"] == 1 - # group_by should have been called with the session_id - mock_prisma.db.litellm_spendlogs.group_by.assert_called_once_with( - by=["session_id"], - where={"session_id": {"in": [session_id]}}, - count={"session_id": True}, + # The count is folded into the single aggregate query; no separate group_by call. + mock_prisma.db.litellm_spendlogs.group_by.assert_not_called() + + +@pytest.mark.asyncio +async def test_build_ui_spend_logs_response_key_split_session_gets_per_key_aggregates(): + """ + Two keys reusing one session id are separate rows under grouped pagination, + and each row must carry ITS key's totals, never the combined session's: + the aggregate query and its lookup are keyed by (session_id, api_key). + """ + from litellm.proxy.spend_tracking.spend_management_endpoints import ( + _build_ui_spend_logs_response, ) + session_id = "sess-shared" + dict_rows = [ + {"request_id": "req-a", "session_id": session_id, "call_type": "completion", "api_key": "key-a"}, + {"request_id": "req-b", "session_id": session_id, "call_type": "completion", "api_key": "key-b"}, + ] + + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + { + "session_id": session_id, + "api_key": "key-a", + "session_total_count": 2, + "session_total_spend": 0.2, + "mcp_tool_call_count": 0, + "mcp_tool_call_spend": 0.0, + "session_cache_hit_count": 1, + "session_llm_count": 2, + "session_agent_count": 0, + }, + { + "session_id": session_id, + "api_key": "key-b", + "session_total_count": 1, + "session_total_spend": 0.7, + "mcp_tool_call_count": 0, + "mcp_tool_call_spend": 0.0, + "session_cache_hit_count": 0, + "session_llm_count": 1, + "session_agent_count": 0, + }, + ] + ) + + result = await _build_ui_spend_logs_response( + prisma_client=mock_prisma, + data=dict_rows, + total_records=2, + page=1, + page_size=50, + total_pages=1, + enrich_session_counts=True, + ) + + rows = result["data"] + assert [(r["session_total_count"], r["session_total_spend"]) for r in rows] == [(2, 0.2), (1, 0.7)] + assert [r["session_cache_hit_count"] for r in rows] == [1, 0] + assert [r["session_llm_count"] for r in rows] == [2, 1] + + aggregate_sql = mock_prisma.db.query_raw.mock_calls[0][1][0] + assert "GROUP BY session_id, api_key" in aggregate_sql + + +@pytest.mark.asyncio +async def test_build_ui_spend_logs_response_empty_api_key_keeps_session_aggregates(): + """ + The spend-log schema defaults api_key to an empty string, which is a real + group value and not a missing one: a multi-call session logged under an + empty key must keep its count and spend instead of degrading to a plain + single-call row. + """ + from litellm.proxy.spend_tracking.spend_management_endpoints import ( + _build_ui_spend_logs_response, + ) + + session_id = "sess-keyless" + dict_rows = [ + {"request_id": "req-1", "session_id": session_id, "call_type": "completion", "api_key": ""}, + ] + + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock( + return_value=[ + { + "session_id": session_id, + "api_key": "", + "session_total_count": 3, + "session_total_spend": 0.09, + "mcp_tool_call_count": 0, + "mcp_tool_call_spend": 0.0, + "session_cache_hit_count": 0, + "session_llm_count": 3, + "session_agent_count": 0, + } + ] + ) + + result = await _build_ui_spend_logs_response( + prisma_client=mock_prisma, + data=dict_rows, + total_records=1, + page=1, + page_size=50, + total_pages=1, + enrich_session_counts=True, + ) + + row = result["data"][0] + assert row["session_total_count"] == 3 + assert row["session_total_spend"] == 0.09 + + # The empty key must reach the aggregate's authorized-keys filter too. + _, call_args, _ = mock_prisma.db.query_raw.mock_calls[0] + assert call_args[2] == [""] + @pytest.mark.asyncio async def test_build_ui_spend_logs_response_sums_multi_round_session_spend(): @@ -4033,14 +4148,13 @@ async def test_build_ui_spend_logs_response_sums_multi_round_session_spend(): ] mock_prisma = MagicMock() - mock_prisma.db.litellm_spendlogs.group_by = AsyncMock( - return_value=[{"session_id": session_id, "_count": {"session_id": 3}}] - ) # The raw aggregate query returns the full session spend (0.01 + 0.02 + 0.03). mock_prisma.db.query_raw = AsyncMock( return_value=[ { "session_id": session_id, + "api_key": api_key, + "session_total_count": 3, "session_total_spend": 0.06, "mcp_tool_call_count": 0, "mcp_tool_call_spend": 0.0, @@ -4089,13 +4203,12 @@ async def test_build_ui_spend_logs_response_session_cache_hit_count(): ] mock_prisma = MagicMock() - mock_prisma.db.litellm_spendlogs.group_by = AsyncMock( - return_value=[{"session_id": session_id, "_count": {"session_id": 2}}] - ) mock_prisma.db.query_raw = AsyncMock( return_value=[ { "session_id": session_id, + "api_key": api_key, + "session_total_count": 2, "session_total_spend": 0.05, "mcp_tool_call_count": 0, "mcp_tool_call_spend": 0.0, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py index ef68d9ce178..27e633099f0 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py @@ -274,6 +274,9 @@ async def test_spend_logs_ui_uses_bounded_count_not_full_scan(monkeypatch): "the page query must not carry a window count that forces a full-window " f"scan. SQL was:\n{page_sql}" ) + assert "GROUP BY" not in count_sql and "DISTINCT ON" not in page_sql, ( + "without group_by_session the endpoint must keep raw per-call pagination" + ) assert response["total"] == 137 assert response["total_is_capped"] is False @@ -499,3 +502,106 @@ async def test_global_spend_report_team_group_forwards_team_id(monkeypatch): params = mock_prisma.db.query_raw.call_args[0][1:] assert "team_x" in params, "team_id must be forwarded into the DB query params" assert "sl.team_id = $3" in sql, f"team query must filter on team_id. SQL was:\n{sql}" + + +@pytest.mark.asyncio +async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): + """ + With group_by_session=true, /spend/logs/ui must page and count SESSIONS, + not raw calls: the page query returns one representative row per session + (DISTINCT ON the session group key, preferring non-MCP calls, newest + first) and the bounded count counts groups. Otherwise the UI collapses a + server page of N calls into fewer visible rows while the footer still + claims N (issue #38060). + """ + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.spend_tracking.spend_management_endpoints import ( + SPEND_LOGS_PAGINATION_COUNT_CAP, + ui_view_spend_logs, + ) + + page_rows = [ + {"request_id": "req-1", "metadata": "{}", "session_id": None}, + {"request_id": "req-2", "metadata": "{}", "session_id": None}, + ] + mock_prisma = _make_ui_spend_logs_mock(count_total=12, page_rows=page_rows) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin") + mock_request = MagicMock() + mock_request.url.path = "/spend/logs/ui" + + response = await ui_view_spend_logs( + request=mock_request, + api_key=None, + user_id=None, + request_id=None, + start_date="2026-02-16 00:00:00", + end_date="2026-02-16 23:59:59", + page=1, + page_size=50, + sort_by="startTime", + sort_order="desc", + user_api_key_dict=auth, + group_by_session=True, + ) + + group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key" + + count_call = mock_prisma.db.query_raw.call_args_list[0] + count_sql = count_call[0][0] + assert f"GROUP BY {group_key}" in count_sql, f"grouped total must count sessions. SQL was:\n{count_sql}" + assert "COUNT(*) OVER ()" not in count_sql + assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" + assert count_call[0][-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1 + + page_sql = mock_prisma.db.query_raw.call_args_list[1][0][0] + assert f"DISTINCT ON ({group_key})" in page_sql, f"page must return one row per session. SQL was:\n{page_sql}" + assert f"ORDER BY {group_key}, call_type IN ('call_mcp_tool', 'list_mcp_tools'), \"startTime\" DESC" in page_sql, ( + "the session representative must prefer the newest non-MCP call" + ) + assert "COUNT(*) OVER ()" not in page_sql + + assert response["total"] == 12 + assert response["total_is_capped"] is False + assert response["total_pages"] == 1 + + +@pytest.mark.asyncio +async def test_spend_logs_ui_request_id_lookup_with_grouping_returns_exact_row(monkeypatch): + """ + A request_id lookup with group_by_session=true must still resolve the + exact requested row: the filter runs before grouping, so the row is its + own group's representative and deep links keep working. + """ + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.spend_tracking.spend_management_endpoints import ui_view_spend_logs + + target_row = {"request_id": "req-deep-link", "metadata": "{}", "session_id": None} + mock_prisma = _make_ui_spend_logs_mock(count_total=1, page_rows=[target_row]) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin") + mock_request = MagicMock() + mock_request.url.path = "/spend/logs/ui" + + response = await ui_view_spend_logs( + request=mock_request, + api_key=None, + user_id=None, + request_id="req-deep-link", + start_date=None, + end_date=None, + page=1, + page_size=1, + sort_by="startTime", + sort_order="desc", + user_api_key_dict=auth, + group_by_session=True, + ) + + page_call = mock_prisma.db.query_raw.call_args_list[1] + assert "request_id = $" in page_call[0][0], "the request_id equality filter must survive grouping" + assert "req-deep-link" in page_call[0] + assert [row["request_id"] for row in response["data"]] == ["req-deep-link"] + assert response["total"] == 1 diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index df14224af5c..6d6aad22ca3 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -3013,7 +3013,7 @@ class TestHandleLLMApiExceptionDictDetail: assert "NotFoundError" in proxy_exc.message async def test_exception_with_status_code_propagates(self): - """Exception with a statically-set status_code should propagate it.""" + """Exception with a statically-set status_code should propagate it and its message.""" from litellm.llms.vertex_ai.common_utils import VertexAIError exc = VertexAIError( @@ -3022,12 +3022,30 @@ class TestHandleLLMApiExceptionDictDetail: ) proxy_exc = await self._invoke(exc) assert proxy_exc.code == "429" + assert proxy_exc.message == "Rate limit exceeded" async def test_exception_without_status_code_defaults_to_500(self): - """Exception with no status_code attribute defaults to 500.""" + """Exception with no status_code attribute defaults to 500; a message with nothing + to redact still reaches the client, since routes raise plain exceptions as validation text.""" exc = ValueError("Something broke") proxy_exc = await self._invoke(exc) assert proxy_exc.code == "500" + assert proxy_exc.message == "Something broke" + + async def test_unclassified_exception_redacts_internal_details_from_client_message(self): + """Regression for LIT-6747: an unclassified exception's credential, path, and host + must not reach the client.""" + exc = RuntimeError( + "Failed to connect to postgresql://litellm_internal:S3cr3tPGPass@10.20.30.40:5432/litellm_prod " + "(config file /etc/litellm/secrets/db.yaml)" + ) + proxy_exc = await self._invoke(exc) + assert proxy_exc.code == "500" + assert "S3cr3tPGPass" not in proxy_exc.message + assert "litellm_internal" not in proxy_exc.message + assert "10.20.30.40" not in proxy_exc.message + assert "/etc/litellm/secrets/db.yaml" not in proxy_exc.message + assert "REDACTED" in proxy_exc.message async def test_already_normalized_proxy_exception_is_honored(self): """A ProxyException raised mid-request (e.g. a guardrail block) is already @@ -3244,6 +3262,42 @@ class TestStreamCloseOnDisconnect: assert upstream.aclosed + async def test_async_streaming_data_generator_redacts_internal_details_on_error( + self, + ): + """Regression for LIT-6747: a mid-stream exception must not hand its raw text or a + traceback to serialize_error.""" + + class FailingUpstream: + def __aiter__(self): + return self + + async def __anext__(self): + raise RuntimeError( + "Failed to connect to postgresql://litellm_internal:S3cr3tPGPass@10.20.30.40:5432/litellm_prod " + "(config file /etc/litellm/secrets/db.yaml)" + ) + + ProxyLogging._callback_capabilities_cache.clear() + captured: list = [] + gen = ProxyBaseLLMRequestProcessing.async_streaming_data_generator( + response=FailingUpstream(), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + request_data={"model": "mock-model"}, + proxy_logging_obj=ProxyLogging(user_api_key_cache=MagicMock()), + serialize_chunk=lambda c: "data: x\n\n", + serialize_error=lambda e: captured.append(e) or "data: error\n\n", + ) + + await gen.__anext__() + + assert len(captured) == 1 + message = captured[0].message + assert "S3cr3tPGPass" not in message + assert "10.20.30.40" not in message + assert "/etc/litellm/secrets/db.yaml" not in message + assert "Traceback (most recent call last)" not in message + @staticmethod def _request_that_disconnects() -> Request: async def receive(): diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index ee0e2014951..8366e5546a9 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -41,7 +41,9 @@ from litellm.litellm_core_utils.get_provider_specific_headers import ( from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( TRUSTED_CALLBACK_VARS_FIELD, ) +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.types.utils import CredentialItem @@ -7719,3 +7721,177 @@ def test_stamped_model_access_groups_survive_the_litellm_metadata_merge(): } assert get_litellm_metadata_from_kwargs(kwargs)[MODEL_ACCESS_GROUP_METADATA_KEY] == ["tier-a"] + + +def _request_for(path: str) -> MagicMock: + request = MagicMock(spec=Request) + request.scope = {"path": path} + request.url = MagicMock() + request.url.path = path + request.url.__str__.return_value = f"http://localhost{path}" + request.method = "POST" + request.query_params = {} + request.headers = {"Content-Type": "application/json"} + request.client = MagicMock() + request.client.host = "127.0.0.1" + return request + + +def _spend_log_session_id(data: dict[str, object]) -> str: + """Resolve session_id the way LiteLLM_SpendLogs does: standard_logging_payload.trace_id.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.proxy.spend_tracking.spend_tracking_utils import _get_session_id_for_spend_log + + metadata = data["metadata"] + assert isinstance(metadata, dict) + litellm_params = get_litellm_params( + litellm_session_id=str(data["litellm_session_id"]) if "litellm_session_id" in data else None, + litellm_trace_id=str(data["litellm_trace_id"]) if "litellm_trace_id" in data else None, + metadata=metadata, + ) + trace_id = StandardLoggingPayloadSetup.get_standard_logging_payload_trace_id( + logging_obj=SimpleNamespace(litellm_trace_id="per-call-random-trace-id"), + litellm_params=litellm_params, + ) + return _get_session_id_for_spend_log(kwargs={}, standard_logging_payload={"trace_id": trace_id}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_correlation_in_logs", [False, True]) +async def test_missing_session_id_generate_makes_spend_log_and_callback_session_ids_agree( + monkeypatch: pytest.MonkeyPatch, request_correlation_in_logs: bool +): + """Without a session header, SpendLogs.session_id and the metadata.session_id that Langfuse logs + must be the same generated id, so cross-referencing the two by session_id works. The id is marked + as generated so affinity consumers (Fireworks x-session-affinity, router session pins) skip it.""" + monkeypatch.setattr(litellm, "request_correlation_in_logs", request_correlation_in_logs) + data = {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]} + + updated = await add_litellm_data_to_request( + data=data, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "generate"}, + ) + + callback_session_id = updated["metadata"]["session_id"] + assert isinstance(callback_session_id, str) and len(callback_session_id) == 36 + assert _spend_log_session_id(updated) == callback_session_id + assert updated["metadata"][SESSION_ID_GENERATED_METADATA_KEY] is True + assert get_fireworks_session_id( + {"litellm_session_id": updated["litellm_session_id"], "metadata": updated["metadata"]} + ) is None + + +@pytest.mark.asyncio +async def test_missing_session_id_unset_keeps_legacy_divergence(): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + ) + + assert "session_id" not in updated["metadata"] + assert "litellm_session_id" not in updated + assert _spend_log_session_id(updated) == "per-call-random-trace-id" + + +@pytest.mark.asyncio +async def test_missing_session_id_generate_reuses_traceparent_trace_id(): + """A W3C traceparent already decides SpendLogs.session_id, so the callback session id must reuse it.""" + request = _request_for("/v1/chat/completions") + request.headers = {"traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"} + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "generate"}, + ) + + assert updated["metadata"]["session_id"] == "4bf92f3577b34da6a3ce929d0e0e4736" + assert _spend_log_session_id(updated) == "4bf92f3577b34da6a3ce929d0e0e4736" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["generate", "reject"]) +async def test_missing_session_id_policy_keeps_client_supplied_session_id(policy: str): + request = _request_for("/v1/chat/completions") + request.headers = {"x-litellm-session-id": "client-session-1"} + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": policy}, + ) + + assert updated["litellm_session_id"] == "client-session-1" + assert updated["metadata"]["session_id"] == "client-session-1" + assert _spend_log_session_id(updated) == "client-session-1" + assert SESSION_ID_GENERATED_METADATA_KEY not in updated["metadata"] + assert ( + get_fireworks_session_id({"litellm_session_id": "client-session-1", "metadata": updated["metadata"]}) + == "client-session-1" + ) + + +@pytest.mark.asyncio +async def test_missing_session_id_reject_accepts_body_metadata_session_id(): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": [], "metadata": {"session_id": "body-session-1"}}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "reject"}, + ) + + assert updated["metadata"]["session_id"] == "body-session-1" + + +@pytest.mark.asyncio +async def test_missing_session_id_reject_returns_400_without_session_id(): + with pytest.raises(ProxyException) as exc_info: + await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "reject"}, + ) + + assert exc_info.value.code == "400" + assert exc_info.value.param == "session_id" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/mcp/", "/mcp/tools", "/key/health"]) +async def test_missing_session_id_policy_skips_non_inference_routes(path: str): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o"}, + request=_request_for(path), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "reject"}, + ) + + assert "session_id" not in updated["metadata"] + + +@pytest.mark.asyncio +async def test_missing_session_id_unknown_value_is_ignored(): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "typo"}, + ) + + assert "session_id" not in updated["metadata"] diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 3e70dee23b7..7b3528f3a68 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -95,6 +95,7 @@ class TestProxyInitializationHelpers: assert args["app"] == "litellm.proxy.proxy_server:app" assert args["host"] == "localhost" assert args["port"] == 8000 + assert args["server_header"] is False # Test with log_config args = ProxyInitializationHelpers._get_default_unvicorn_init_args( diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 91fca8f1e27..4ed6a468371 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -130,6 +130,7 @@ def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch): password="secret", master_key="test-master-key", prisma_client=mock_prisma_client, + general_settings={}, ) mock_create_ui_token_object.assert_called_once_with( login_result=mock_login_result, @@ -147,6 +148,72 @@ def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch): assert mock_jwt_encode.call_args.kwargs == {"algorithm": "HS256"} +def _mock_login_v2_deps(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.authenticate_user", + AsyncMock(return_value={"user_id": "test-user"}), + ) + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.create_ui_token_object", + MagicMock(return_value={"user_id": "test-user"}), + ) + monkeypatch.setattr("jwt.encode", MagicMock(return_value="signed-token")) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "test-master-key") + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock()) + monkeypatch.setattr("litellm.proxy.utils.get_server_root_path", lambda: "") + monkeypatch.setattr("litellm.proxy.utils.get_proxy_base_url", lambda: None) + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + + +def test_login_v2_sets_secure_cookie_over_direct_https(monkeypatch): + """Regression: the token cookie previously carried no Secure/HttpOnly/SameSite + attributes at all, so it was always sent over plain HTTP.""" + _mock_login_v2_deps(monkeypatch) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + + client = TestClient(app, base_url="https://testserver") + response = client.post("/v2/login", json={"username": "alice", "password": "secret"}) + + assert response.status_code == 200 + cookie = response.headers.get("set-cookie") + assert "Secure" in cookie + assert "HttpOnly" not in cookie # deliberate: the dashboard reads this cookie via JS + assert "samesite=lax" in cookie.lower() + + +def test_login_v2_does_not_set_secure_cookie_over_direct_http(monkeypatch): + _mock_login_v2_deps(monkeypatch) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + + client = TestClient(app, base_url="http://testserver") + response = client.post("/v2/login", json={"username": "alice", "password": "secret"}) + + assert response.status_code == 200 + assert "Secure" not in response.headers.get("set-cookie") + + +def test_login_v2_sets_secure_cookie_behind_trusted_tls_terminating_proxy(monkeypatch): + """THE regression: litellm only sees a plain-HTTP hop when TLS terminates at a + reverse proxy, but the token cookie must still be Secure when the direct peer is + a configured trusted proxy reporting X-Forwarded-Proto: https.""" + _mock_login_v2_deps(monkeypatch) + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"use_x_forwarded_for": True, "mcp_trusted_proxy_ranges": ["10.0.0.0/8"]}, + ) + + client = TestClient(app, base_url="http://testserver", client=("10.0.0.5", 50000)) + response = client.post( + "/v2/login", + json={"username": "alice", "password": "secret"}, + headers={"X-Forwarded-Proto": "https"}, + ) + + assert response.status_code == 200 + assert "Secure" in response.headers.get("set-cookie") + + def test_login_v2_returns_json_on_proxy_exception(monkeypatch): """Test that /v2/login returns JSON error when ProxyException is raised""" from litellm.proxy._types import ProxyErrorTypes, ProxyException @@ -355,6 +422,51 @@ def test_login_v3_exchange_happy_path(monkeypatch): assert exchange_response.cookies.get("token") == "signed-token" +def test_login_v3_exchange_sets_secure_cookie_behind_trusted_tls_terminating_proxy(monkeypatch): + """Regression: /v3/login/exchange's token cookie must be Secure behind a trusted + TLS-terminating reverse proxy even though litellm only sees a plain-HTTP hop.""" + mock_prisma_client = MagicMock() + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.authenticate_user", + AsyncMock(return_value={"user_id": "test-user"}), + ) + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.create_ui_token_object", + MagicMock(return_value={"user_id": "test-user"}), + ) + monkeypatch.setattr("jwt.encode", MagicMock(return_value="signed-token")) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "test-master-key") + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + { + "control_plane_url": "https://cp.example.com", + "use_x_forwarded_for": True, + "mcp_trusted_proxy_ranges": ["10.0.0.0/8"], + }, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mock_config = MagicMock() + mock_config.worker_registry = [] + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_config", mock_config) + monkeypatch.setattr("litellm.proxy.utils.get_server_root_path", lambda: "") + monkeypatch.setattr("litellm.proxy.utils.get_proxy_base_url", lambda: None) + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + + client = TestClient(app, base_url="http://testserver", client=("10.0.0.5", 50000)) + + login_response = client.post("/v3/login", json={"username": "alice", "password": "secret"}) + code = login_response.json()["code"] + + exchange_response = client.post( + "/v3/login/exchange", + json={"code": code}, + headers={"X-Forwarded-Proto": "https"}, + ) + assert exchange_response.status_code == 200 + assert "Secure" in exchange_response.headers.get("set-cookie") + + def test_login_v3_exchange_single_use(monkeypatch): """Code can only be redeemed once.""" mock_prisma_client = MagicMock() @@ -10782,11 +10894,54 @@ def test_update_config_redacts_all_environment_variable_values(_update_config_se class _EnvBuiltRedisCache(RedisCache): """RedisCache stand-in that records its constructor kwargs and never opens a network connection, so tests can assert which connection params - the proxy used to build its coordination Redis.""" + the proxy used to build its coordination Redis. `ping()` reports reachable + by default, matching a real Redis the env fallback should adopt.""" def __init__(self, **kwargs): self.init_kwargs = kwargs + async def ping(self) -> bool: + return True + + +class _UnreachableRedisCache(_EnvBuiltRedisCache): + """Same stand-in, but `ping()` fails like a REDIS_* env var naming a Redis + that is not actually reachable (wrong host, no service running, ...).""" + + async def ping(self) -> bool: + raise ConnectionError("connection refused") + + +@contextlib.contextmanager +def _patched_coordination_redis_module_state( + *, + spend_cache: DualCache, + config_cache: types.SimpleNamespace, + redis_cache_class: type = _EnvBuiltRedisCache, +): + """Stub every `litellm.proxy.proxy_server` global that + `_attach_redis_usage_cache` (and its callers) can write to, shared by the + whole coordination-Redis test family below. + + Centralizing this is not just DRY: `_attach_redis_usage_cache` always sets + `cli_sso_session_cache.redis_cache` unconditionally, and a call site that + forgets to patch that one real (persistent) global leaks a throwaway + Redis stand-in into it for the rest of the pytest session, breaking + unrelated tests that run later. One patched-state helper means a new call + site cannot forget a global this family already knows to isolate. + """ + with ( + patch.object(proxy_server_module, "redis_usage_cache", None), + patch.object(proxy_server_module, "spend_counter_cache", spend_cache), + patch.object(proxy_server_module, "user_api_key_cache", DualCache()), + patch.object(proxy_server_module, "cli_sso_session_cache", DualCache()), + patch.object(proxy_server_module, "llm_router", None), + patch.object(proxy_server_module, "litellm_config_cache", config_cache), + patch.object(proxy_server_module, "RedisCache", redis_cache_class), + patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + ): + yield + def _run_init_cache_with_backend(cache_backend, redis_env_kwargs): """Run ProxyConfig._init_cache with a stubbed response-cache backend and a @@ -10798,12 +10953,7 @@ def _run_init_cache_with_backend(cache_backend, redis_env_kwargs): fresh_config_cache = types.SimpleNamespace(redis_cache=None) with ( - patch.object(proxy_server_module, "redis_usage_cache", None), - patch.object(proxy_server_module, "spend_counter_cache", fresh_spend_cache), - patch.object(proxy_server_module, "user_api_key_cache", DualCache()), - patch.object(proxy_server_module, "llm_router", None), - patch.object(proxy_server_module, "litellm_config_cache", fresh_config_cache), - patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + _patched_coordination_redis_module_state(spend_cache=fresh_spend_cache, config_cache=fresh_config_cache), patch( "litellm._redis._redis_kwargs_from_environment", return_value=redis_env_kwargs, @@ -10878,12 +11028,7 @@ def _run_init_coordination_redis(config, env=None): fresh_config_cache = types.SimpleNamespace(redis_cache=None) with ( - patch.object(proxy_server_module, "redis_usage_cache", None), - patch.object(proxy_server_module, "spend_counter_cache", fresh_spend_cache), - patch.object(proxy_server_module, "user_api_key_cache", DualCache()), - patch.object(proxy_server_module, "litellm_config_cache", fresh_config_cache), - patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), - patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + _patched_coordination_redis_module_state(spend_cache=fresh_spend_cache, config_cache=fresh_config_cache), mock.patch.dict(os.environ, env or {}, clear=False), ): built = proxy_server_module.ProxyConfig()._init_coordination_redis(config=config) @@ -10971,13 +11116,7 @@ def test_explicit_coordination_redis_takes_precedence_over_cache_backend(): mock_litellm_cache.cache = cache_backend with ( - patch.object(proxy_server_module, "redis_usage_cache", None), - patch.object(proxy_server_module, "spend_counter_cache", fresh_spend_cache), - patch.object(proxy_server_module, "user_api_key_cache", DualCache()), - patch.object(proxy_server_module, "llm_router", None), - patch.object(proxy_server_module, "litellm_config_cache", fresh_config_cache), - patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), - patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + _patched_coordination_redis_module_state(spend_cache=fresh_spend_cache, config_cache=fresh_config_cache), patch("litellm.Cache", return_value=mock_litellm_cache), ): litellm.cache = None @@ -10995,6 +11134,95 @@ def test_explicit_coordination_redis_takes_precedence_over_cache_backend(): assert fresh_spend_cache.redis_cache is usage_cache +async def _run_init_coordination_redis_env_fallback( + litellm_settings, redis_env_kwargs, redis_cache_class=_EnvBuiltRedisCache +): + """Run ProxyConfig._init_coordination_redis_env_fallback against a + stubbed module state and a controlled REDIS_* environment, returning + (built, spend_counter redis).""" + fresh_spend_cache = DualCache() + fresh_config_cache = types.SimpleNamespace(redis_cache=None) + + with ( + _patched_coordination_redis_module_state( + spend_cache=fresh_spend_cache, config_cache=fresh_config_cache, redis_cache_class=redis_cache_class + ), + patch( + "litellm._redis._redis_kwargs_from_environment", + return_value=redis_env_kwargs, + ), + ): + built = await proxy_server_module.ProxyConfig._init_coordination_redis_env_fallback( + litellm_settings=litellm_settings + ) + return built, fresh_spend_cache.redis_cache + + +@pytest.mark.asyncio +async def test_init_coordination_redis_env_fallback_builds_from_environment(): + """A deployment with no coordination_redis block and no litellm_settings.cache + but with bare REDIS_HOST/REDIS_PORT env vars must still get a coordination + Redis: otherwise spend counters, budget-window enforcement, and the + reset_spend cache-eviction broadcast stay per-pod local and a reset issued + on one pod never clears another pod's stale enforcement.""" + built, spend_redis = await _run_init_coordination_redis_env_fallback( + litellm_settings={}, + redis_env_kwargs={"host": "env-fallback-host", "port": "6390"}, + ) + + assert isinstance(built, _EnvBuiltRedisCache) + assert built.init_kwargs["host"] == "env-fallback-host" + assert spend_redis is built + + +@pytest.mark.asyncio +async def test_init_coordination_redis_env_fallback_without_redis_env_returns_none(): + """With no REDIS_* connection info at all, the fallback must leave the + coordination Redis unset rather than building a broken client.""" + built, spend_redis = await _run_init_coordination_redis_env_fallback( + litellm_settings={}, + redis_env_kwargs={}, + ) + + assert built is None + assert spend_redis is None + + +@pytest.mark.asyncio +async def test_init_coordination_redis_env_fallback_unreachable_stays_in_memory(): + """REDIS_* env vars can name a Redis that is not actually reachable (wrong + host, leftover from an unrelated job/service). Guessing "coordination + available" from bare env vars must not turn a previously harmless + in-memory-only proxy into one that raises on its next cache write, so an + unreachable ping must leave everything exactly as if no REDIS_* vars were + set at all.""" + built, spend_redis = await _run_init_coordination_redis_env_fallback( + litellm_settings={}, + redis_env_kwargs={"host": "unreachable-host", "port": "6390"}, + redis_cache_class=_UnreachableRedisCache, + ) + + assert built is None + assert spend_redis is None + + +@pytest.mark.asyncio +async def test_init_coordination_redis_env_fallback_malformed_cluster_nodes_stays_in_memory(): + """REDIS_CLUSTER_NODES can be set to a malformed value nothing here ever + asked to be parsed. Unlike the explicit coordination_redis block (a + deliberate opt-in, so a bad value there should fail loudly), this + inferred fallback must not abort proxy startup over it -- it has to + decline the same way it does for an absent or unreachable Redis.""" + with mock.patch.dict(os.environ, {"REDIS_CLUSTER_NODES": "not-valid-json"}, clear=False): + built, spend_redis = await _run_init_coordination_redis_env_fallback( + litellm_settings={}, + redis_env_kwargs={}, + ) + + assert built is None + assert spend_redis is None + + def test_env_fallback_builds_cluster_client_from_cluster_nodes_env(): """A deployment whose only Redis env is REDIS_CLUSTER_NODES must still get a coordination Redis from the env fallback, and it must be a cluster @@ -11036,11 +11264,7 @@ async def test_startup_applies_coordination_redis_saved_in_database(): fresh_config_cache = types.SimpleNamespace(redis_cache=None) with ( - patch.object(proxy_server_module, "spend_counter_cache", fresh_spend_cache), - patch.object(proxy_server_module, "user_api_key_cache", DualCache()), - patch.object(proxy_server_module, "litellm_config_cache", fresh_config_cache), - patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), - patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + _patched_coordination_redis_module_state(spend_cache=fresh_spend_cache, config_cache=fresh_config_cache), patch.object( proxy_server_module, "get_persisted_coordination_redis_settings", @@ -12189,3 +12413,97 @@ async def test_load_config_router_authorizes_fallback_targets_against_the_callin router, _, _ = await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) assert router.fallback_access_check is router_fallback_access_check + + +def test_docs_redoc_openapi_are_reachable_by_default(): + """ + LIT-6745: the interactive/machine-readable docs surfaces are on by + default (the customer-facing production toggle is opt-in, not opt-out). + """ + client = TestClient(app) + + assert client.get("/redoc").status_code == 200 + openapi_response = client.get("/openapi.json") + assert openapi_response.status_code == 200 + assert "paths" in openapi_response.json() + + +def test_production_app_docs_urls_are_wired_to_the_real_env_helpers(): + """ + LIT-6745: pins the actual `FastAPI(docs_url=..., redoc_url=..., openapi_url=...)` + construction in proxy_server.py to _get_docs_url/_get_redoc_url/_get_openapi_url, + so a hardcoded or drifted value at that call site fails this test even though + the helpers themselves are covered separately. + """ + from litellm.proxy import utils as proxy_utils + + assert app.docs_url == proxy_utils._get_docs_url() + assert app.redoc_url == proxy_utils._get_redoc_url() + assert app.openapi_url == proxy_utils._get_openapi_url() + + +def _build_app_with_docs_env(monkeypatch, *, disabled: bool) -> FastAPI: + from litellm.proxy import utils as proxy_utils + from litellm.proxy.health_endpoints._health_endpoints import router as health_router + + for flag in ("DOCS_URL", "REDOC_URL", "OPENAPI_URL"): + monkeypatch.delenv(flag, raising=False) + for flag in ("NO_DOCS", "NO_REDOC", "NO_OPENAPI"): + if disabled: + monkeypatch.setenv(flag, "True") + else: + monkeypatch.delenv(flag, raising=False) + + # Mirrors the exact FastAPI() construction in proxy_server.py, so this + # exercises the real gating mechanism rather than a reimplementation of it. + app_under_test = FastAPI( + docs_url=proxy_utils._get_docs_url(), + redoc_url=proxy_utils._get_redoc_url(), + openapi_url=proxy_utils._get_openapi_url(), + ) + app_under_test.include_router(health_router) + return app_under_test + + +def test_docs_endpoints_enabled_when_env_unset(monkeypatch): + app_under_test = _build_app_with_docs_env(monkeypatch, disabled=False) + assert app_under_test.docs_url == "/" + assert app_under_test.redoc_url == "/redoc" + assert app_under_test.openapi_url == "/openapi.json" + + client = TestClient(app_under_test) + assert client.get(app_under_test.docs_url).status_code == 200 + assert client.get(app_under_test.redoc_url).status_code == 200 + assert client.get(app_under_test.openapi_url).status_code == 200 + + +def test_no_docs_no_redoc_no_openapi_disable_every_documentation_surface(monkeypatch): + """ + LIT-6745: NO_DOCS, NO_REDOC and NO_OPENAPI must each 404 their surface + with no schema in the body, so a production/air-gapped deployment can + restrict every doc route consistently. + """ + app_under_test = _build_app_with_docs_env(monkeypatch, disabled=True) + assert app_under_test.docs_url is None + assert app_under_test.redoc_url is None + assert app_under_test.openapi_url is None + + client = TestClient(app_under_test) + for route in ("/", "/redoc", "/openapi.json"): + response = client.get(route) + assert response.status_code == 404 + assert "openapi" not in response.text.lower() + assert "paths" not in response.text.lower() + + +def test_disabling_docs_does_not_disable_other_routes(monkeypatch): + """ + LIT-6745: disabling the doc surfaces must not affect inference/management + routes, since NO_DOCS/NO_REDOC/NO_OPENAPI only remove the routes FastAPI + itself auto-registers for docs_url/redoc_url/openapi_url. + """ + app_under_test = _build_app_with_docs_env(monkeypatch, disabled=True) + client = TestClient(app_under_test) + + assert client.get("/redoc").status_code == 404 + assert client.get("/health/liveliness").status_code == 200 diff --git a/tests/test_litellm/proxy/test_redis_auth_cache_flag.py b/tests/test_litellm/proxy/test_redis_auth_cache_flag.py index 772b08bc9d0..573bfc40c96 100644 --- a/tests/test_litellm/proxy/test_redis_auth_cache_flag.py +++ b/tests/test_litellm/proxy/test_redis_auth_cache_flag.py @@ -5,7 +5,7 @@ Verifies that _init_cache attaches Redis to user_api_key_cache only when the flag is explicitly set to True, and leaves it in-memory-only otherwise. """ -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager import json from unittest.mock import MagicMock, patch @@ -167,3 +167,25 @@ class TestRedisAuthCacheFlag: f"cli_sso_session_cache must always get Redis " f"(enable_redis_auth_cache={flag_value!r})" ) + + def test_flag_absent_still_shares_the_model_budget_counters_over_redis(self): + """ + Per-model budget counters are spend counters: the limiter must be able to + push and read them through Redis without the auth-cache opt-in, or every + worker enforces and reports its own share of a key's spend + """ + fake_redis = _FakeRedisCache() + limiter_cache = ps.model_max_budget_limiter.dual_cache + touched_caches = ( + limiter_cache, + ps.spend_counter_cache, + ps.cli_sso_session_cache, + ps.user_api_key_cache, + ps.litellm_config_cache, + ) + with ExitStack() as detached: + for cache in touched_caches: + detached.enter_context(patch.object(cache, "redis_cache", None)) + ps._attach_redis_usage_cache(fake_redis, enable_redis_auth_cache=False) + assert limiter_cache.redis_cache is fake_redis + assert ps.user_api_key_cache.redis_cache is None diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index dc256ccf718..709447d23c0 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -3006,6 +3006,128 @@ def test_update_mcp_semantic_filter_settings_requires_proxy_admin(monkeypatch): app.dependency_overrides.pop(user_api_key_auth, None) +class TestMcpToolSearchSettingsEndpoints: + """`litellm_settings.mcp_tool_search` drives the native `mcp_tool_search` virtual tool, so the UI must round-trip it.""" + + @staticmethod + def _override_auth(role: LitellmUserRoles): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="u", api_key="hashed", user_role=role + ) + + def test_get_returns_stored_values_and_field_schema(self, mock_proxy_config, mock_auth, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", object()) + mock_proxy_config["config"]["litellm_settings"]["mcp_tool_search"] = { + "embedding_model": "text-embedding-3-small", + "core_tools": ["treasury-get_rates"], + } + + resp = client.get("/get/mcp_tool_search_settings") + + assert resp.status_code == 200, resp.text + assert resp.json()["values"] == { + "embedding_model": "text-embedding-3-small", + "top_k": 5, + "similarity_threshold": 0.0, + "core_tools": ["treasury-get_rates"], + } + assert resp.json()["field_schema"]["properties"]["core_tools"]["type"] == "array" + + def test_update_requires_proxy_admin(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + self._override_auth(LitellmUserRoles.INTERNAL_USER) + try: + resp = client.patch("/update/mcp_tool_search_settings", json={"top_k": 3}) + finally: + app.dependency_overrides.clear() + assert resp.status_code == 403 + + def test_update_persists_and_applies_in_memory(self, mock_proxy_config, monkeypatch): + import litellm + + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "mcp_tool_search", None) + self._override_auth(LitellmUserRoles.PROXY_ADMIN) + payload = { + "embedding_model": "text-embedding-3-small", + "top_k": 3, + "similarity_threshold": 0.25, + "core_tools": ["treasury-get_rates"], + } + try: + resp = client.patch("/update/mcp_tool_search_settings", json=payload) + finally: + app.dependency_overrides.clear() + + assert resp.status_code == 200, resp.text + assert mock_proxy_config["save_call_count"]() == 1 + assert litellm.mcp_tool_search == payload + assert mock_proxy_config["config"]["litellm_settings"]["mcp_tool_search"] == payload + + def test_update_rejects_out_of_range_top_k(self, mock_proxy_config, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + self._override_auth(LitellmUserRoles.PROXY_ADMIN) + try: + resp = client.patch("/update/mcp_tool_search_settings", json={"top_k": 0}) + finally: + app.dependency_overrides.clear() + assert resp.status_code == 422 + assert mock_proxy_config["save_call_count"]() == 0 + + +def test_upload_logo_requires_proxy_admin(monkeypatch): + """Any authenticated key could previously write a file to the server's disk here.""" + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + async def _internal_user_auth(): + return UserAPIKeyAuth( + user_id="internal-user-1", + api_key="hashed-internal-key", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + app.dependency_overrides[user_api_key_auth] = _internal_user_auth + try: + resp = client.post( + "/upload/logo", + files={"file": ("logo.png", b"\x89PNG\r\n\x1a\n" + b"x" * 32, "image/png")}, + ) + assert resp.status_code == 403 + assert "proxy admin" in resp.json()["detail"].lower() + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_upload_logo_allows_proxy_admin(monkeypatch, tmp_path): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="admin-1", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.post( + "/upload/logo", + files={"file": ("logo.png", b"\x89PNG\r\n\x1a\n" + b"x" * 32, "image/png")}, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["status"] == "success" + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + uploaded_path = resp.json().get("file_path") + if uploaded_path and os.path.exists(uploaded_path): + os.remove(uploaded_path) + + class TestPtuCostAttributionUISetting: """``enable_ptu_cost_attribution`` is derived from the environment on every GET. diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index eae6f90863a..1abbbe91e97 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -2,29 +2,24 @@ from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest -from fastapi import Request - - -from fastapi import HTTPException +from fastapi import HTTPException, Request import litellm from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( LiteLLM_ManagedVectorStore, ) +from litellm.llms.base_llm.vector_store.transformation import ( + LiteLLMVectorStoreEmbeddingExecutor, + RouterVectorStoreEmbeddingExecutor, +) from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.vector_store_endpoints.endpoints import ( _update_request_data_with_litellm_managed_vector_store_registry, index_create, index_list, ) -from litellm.proxy.vector_store_files_endpoints.endpoints import ( - _update_request_data_with_model_routing_hint, -) from litellm.proxy.vector_store_endpoints.management_endpoints import ( _check_vector_store_access, - _resolve_embedding_config, - _resolve_embedding_config_from_db, - _resolve_embedding_config_from_router, create_vector_store_in_db, new_vector_store, ) @@ -33,8 +28,12 @@ from litellm.proxy.vector_store_endpoints.utils import ( is_allowed_to_call_vector_store_endpoint, is_allowed_to_call_vector_store_files_endpoint, ) +from litellm.proxy.vector_store_files_endpoints.endpoints import ( + _update_request_data_with_model_routing_hint, +) +from litellm.types.utils import EmbeddingResponse, LlmProviders from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse -from litellm.types.utils import LlmProviders +from litellm.vector_stores.main import _direct_vector_store_embedding_executor def _serialize_litellm_params(litellm_params): @@ -51,17 +50,113 @@ def _serialize_litellm_params(litellm_params): return json.dumps(litellm_params or {}) -@pytest.fixture(autouse=True) -def _reset_embedding_config_cache(): - """The use-time embedding-config resolver caches results in process - memory across calls. Reset it before every test so the resolver - actually exercises the router/DB path under test instead of returning - a value cached by an earlier test.""" - from litellm.proxy.vector_store_endpoints import management_endpoints +def test_direct_vector_store_embedding_executor_rejects_invalid_value(): + with pytest.raises(TypeError, match="Invalid direct vector store embedding executor"): + _direct_vector_store_embedding_executor(object(), None, {}) - management_endpoints._embedding_config_cache = None - yield - management_endpoints._embedding_config_cache = None + +def test_router_vector_store_search_injects_executor_and_request_metadata(): + router = litellm.Router(model_list=[]) + original = MagicMock(return_value="searched") + wrapped = router.factory_function(original, call_type="vector_store_search") + + assert ( + wrapped( + vector_store_id="store", + query="query", + custom_llm_provider="valkey", + litellm_metadata={"user_api_key_team_id": "team-a"}, + ) + == "searched" + ) + + call_kwargs = original.call_args.kwargs + assert call_kwargs["custom_llm_provider"] == "valkey" + executor = call_kwargs["_direct_vector_store_embedding_executor"] + assert isinstance(executor, RouterVectorStoreEmbeddingExecutor) + assert executor.metadata == {"user_api_key_team_id": "team-a"} + assert litellm.Router._vector_store_request_metadata({"metadata": {"user_api_key_team_id": "team-b"}}) == { + "user_api_key_team_id": "team-b" + } + assert litellm.Router._vector_store_request_metadata({}) == {} + + with patch.object( # test-quality-ok: fallback dispatch is the boundary this wrapper delegates to + router, "_generic_api_call_with_fallbacks", return_value="routed" + ) as fallback: + assert wrapped(model="vector-alias", vector_store_id="store", query="query") == "routed" + assert fallback.call_args.kwargs["model"] == "vector-alias" + assert fallback.call_args.kwargs["original_function"] is original + + create_original = MagicMock(return_value="created") + wrapped_create = router.factory_function(create_original, call_type="vector_store_create") + assert wrapped_create(name="store") == "created" + create_original.assert_called_once_with(name="store") + with patch.object( # test-quality-ok: fallback dispatch is the boundary this wrapper delegates to + router, "_generic_api_call_with_fallbacks", return_value="created-through-router" + ) as fallback: + assert wrapped_create(model="vector-alias", name="store") == "created-through-router" + fallback.assert_called_once_with(original_function=create_original, model="vector-alias", name="store") + + +@pytest.mark.asyncio +async def test_vector_store_embedding_executors_preserve_explicit_configuration(): + response = EmbeddingResponse(data=[{"embedding": [0.1], "index": 0, "object": "embedding"}]) + sdk_executor = LiteLLMVectorStoreEmbeddingExecutor() + + with ( + patch( # test-quality-ok: isolates SDK dispatch from external embedding providers + "litellm.embedding", return_value=response + ) as embedding, + patch( # test-quality-ok: isolates async SDK dispatch from external embedding providers + "litellm.aembedding", new=AsyncMock(return_value=response) + ) as aembedding, + ): + assert sdk_executor.embed("openai/model", "sync", {"api_key": "explicit"}) is response + assert await sdk_executor.aembed("openai/model", "async", {"api_key": "explicit"}) is response + + embedding.assert_called_once_with(model="openai/model", input=["sync"], api_key="explicit") + aembedding.assert_awaited_once_with(model="openai/model", input=["async"], api_key="explicit") + + mock_router = MagicMock() + mock_router.embedding.return_value = response + mock_router.aembedding = AsyncMock(return_value=response) + router_executor = RouterVectorStoreEmbeddingExecutor( + router=mock_router, + metadata={"user_api_key_team_id": "team-a"}, + ) + + assert router_executor.embed("team-alias", "query", {}) is response + mock_router.embedding.assert_called_once_with( + model="team-alias", + input=["query"], + metadata={"user_api_key_team_id": "team-a"}, + ) + + with ( + patch( # test-quality-ok: verifies explicit store configuration at the SDK boundary + "litellm.embedding", return_value=response + ) as explicit_embedding, + patch( # test-quality-ok: verifies async explicit store configuration at the SDK boundary + "litellm.aembedding", new=AsyncMock(return_value=response) + ) as explicit_aembedding, + ): + assert router_executor.embed("openai/model", "query", {"api_key": "store-key"}) is response + assert await router_executor.aembed("openai/model", "query", {"api_key": "store-key"}) is response + + explicit_embedding.assert_not_called() + explicit_aembedding.assert_not_awaited() + assert mock_router.embedding.call_args.kwargs == { + "model": "openai/model", + "input": ["query"], + "api_key": "store-key", + "metadata": {"user_api_key_team_id": "team-a"}, + } + mock_router.aembedding.assert_awaited_once_with( + model="openai/model", + input=["query"], + api_key="store-key", + metadata={"user_api_key_team_id": "team-a"}, + ) @pytest.mark.asyncio @@ -82,10 +177,11 @@ async def test_router_avector_store_search_passes_correct_args(): } # Call router's avector_store_search - result = await router.avector_store_search( + await router.avector_store_search( vector_store_id="test_store_id", query="test query", custom_llm_provider="bedrock", + metadata={"user_api_key_team_id": "team-a"}, ) # Verify the internal method was called with correct args @@ -96,6 +192,38 @@ async def test_router_avector_store_search_passes_correct_args(): assert call_args[1]["vector_store_id"] == "test_store_id" assert call_args[1]["query"] == "test query" assert call_args[1]["custom_llm_provider"] == "bedrock" + executor = call_args[1]["_direct_vector_store_embedding_executor"] + assert isinstance(executor, RouterVectorStoreEmbeddingExecutor) + assert executor.metadata["user_api_key_team_id"] == "team-a" + + +@pytest.mark.asyncio +async def test_vector_store_embedding_executor_uses_team_scoped_router_deployment(): + router = litellm.Router( + model_list=[ + { + "model_name": "shared-embedding", + "litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "team-a-key"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "shared-embedding"}, + }, + { + "model_name": "shared-embedding", + "litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "team-b-key"}, + "model_info": {"team_id": "team-b", "team_public_model_name": "shared-embedding"}, + }, + ] + ) + executor = RouterVectorStoreEmbeddingExecutor( + router=router, + metadata={"user_api_key_team_id": "team-b"}, + ) + response = EmbeddingResponse(data=[{"embedding": [0.1], "index": 0, "object": "embedding"}]) + + with patch("litellm.aembedding", new=AsyncMock(return_value=response)) as mock_aembedding: + result = await executor.aembed("shared-embedding", "query", {}) + + assert result is response + assert mock_aembedding.await_args.kwargs["api_key"] == "team-b-key" @pytest.mark.asyncio @@ -502,91 +630,30 @@ async def test_update_request_data_with_litellm_managed_vector_store_registry(): @pytest.mark.asyncio -async def test_update_request_data_resolves_embedding_config_at_use_time(): - """When the persisted vector store row carries only a - ``litellm_embedding_model`` reference (the new behaviour after - moving the auto-resolve out of write time), the request-handling - layer must resolve the embedding config so the downstream embed - call still has ``api_key`` / ``api_base`` / ``api_version``. The - resolved config lives in this per-request data dict only — never - persisted.""" - mock_vector_store: LiteLLM_ManagedVectorStore = { +async def test_managed_vector_store_keeps_embedding_reference_and_explicit_config(): + explicit_config = {"api_key": "store-specific-key", "api_base": "https://embedding.example"} + managed_vector_store: LiteLLM_ManagedVectorStore = { "vector_store_id": "test_store", - "custom_llm_provider": "azure_ai", + "custom_llm_provider": "valkey", "litellm_params": { - "litellm_embedding_model": "azure/text-embedding-3-large", - # Note: no litellm_embedding_config persisted + "litellm_embedding_model": "team-embedding-alias", + "litellm_embedding_config": explicit_config, }, } - mock_registry = MagicMock() - mock_registry.get_litellm_managed_vector_store_from_registry.return_value = ( - mock_vector_store - ) + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = managed_vector_store - resolved = { - "api_key": "use-time-resolved-key", - "api_base": "https://my-azure.example", - "api_version": "2024-09-01", - } - - with ( - patch.object(litellm, "vector_store_registry", mock_registry), - patch( - "litellm.proxy.vector_store_endpoints.endpoints._resolve_embedding_config", - new=AsyncMock(return_value=resolved), - ), - ): + with patch.object(litellm, "vector_store_registry", mock_registry): result = await _update_request_data_with_litellm_managed_vector_store_registry( - data={}, vector_store_id="test_store" + data={}, + vector_store_id="test_store", ) - assert result["litellm_embedding_model"] == "azure/text-embedding-3-large" - assert result["litellm_embedding_config"] == resolved + assert result["litellm_embedding_model"] == "team-embedding-alias" + assert result["litellm_embedding_config"] == explicit_config + assert managed_vector_store["litellm_params"]["litellm_embedding_config"] == explicit_config -@pytest.mark.asyncio -async def test_update_request_data_passes_through_legacy_embedding_config(): - """A vector store row created by an older proxy version may already - carry a fully-resolved ``litellm_embedding_config`` in its persisted - ``litellm_params`` (the very leak this PR closes). Those legacy rows - must still work — the use-time resolver skips re-resolution when - the config is already present so the embed call keeps succeeding.""" - legacy_config = { - "api_key": "legacy-cleartext-key", - "api_base": "https://legacy-azure.example", - "api_version": "2024-01-01", - } - mock_vector_store: LiteLLM_ManagedVectorStore = { - "vector_store_id": "legacy_store", - "custom_llm_provider": "azure_ai", - "litellm_params": { - "litellm_embedding_model": "azure/text-embedding-3-large", - "litellm_embedding_config": legacy_config, - }, - } - - mock_registry = MagicMock() - mock_registry.get_litellm_managed_vector_store_from_registry.return_value = ( - mock_vector_store - ) - - resolve_mock = AsyncMock() - - with ( - patch.object(litellm, "vector_store_registry", mock_registry), - patch( - "litellm.proxy.vector_store_endpoints.endpoints._resolve_embedding_config", - new=resolve_mock, - ), - ): - result = await _update_request_data_with_litellm_managed_vector_store_registry( - data={}, vector_store_id="legacy_store" - ) - - assert result["litellm_embedding_config"] == legacy_config - resolve_mock.assert_not_awaited() - class TestCheckVectorStorePermission: """Test suite for check_vector_store_permission function.""" @@ -2003,57 +2070,7 @@ async def test_vector_store_update_and_list_synchronization(): @pytest.mark.asyncio -async def test_resolve_embedding_config_from_db(): - """Test that _resolve_embedding_config_from_db correctly resolves embedding config from database.""" - mock_prisma_client = MagicMock() - - # Mock database model with litellm_params - mock_db_model = MagicMock() - mock_db_model.litellm_params = { - "api_key": "test-api-key", - "api_base": "https://api.openai.com", - "api_version": "2024-01-01", - } - - mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock( - return_value=mock_db_model - ) - - with patch( - "litellm.proxy.vector_store_endpoints.management_endpoints.decrypt_value_helper", - side_effect=lambda value, key, return_original_value: value, - ): - result = await _resolve_embedding_config_from_db( - embedding_model="text-embedding-ada-002", prisma_client=mock_prisma_client - ) - - assert result is not None - assert result["api_key"] == "test-api-key" - assert result["api_base"] == "https://api.openai.com" - assert result["api_version"] == "2024-01-01" - mock_prisma_client.db.litellm_proxymodeltable.find_first.assert_called_once_with( - where={"model_name": "text-embedding-ada-002"} - ) - - # Test with empty embedding_model - result_empty = await _resolve_embedding_config_from_db( - embedding_model="", prisma_client=mock_prisma_client - ) - assert result_empty is None - - # Test with model not found - mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock( - return_value=None - ) - result_not_found = await _resolve_embedding_config_from_db( - embedding_model="non-existent-model", prisma_client=mock_prisma_client - ) - assert result_not_found is None - - -@pytest.mark.asyncio -async def test_new_vector_store_auto_resolves_embedding_config(): - """Test that new_vector_store auto-resolves embedding config when embedding_model is provided but config is not.""" +async def test_new_vector_store_persists_embedding_reference_without_credentials(): import json from litellm.types.vector_stores import LiteLLM_ManagedVectorStore @@ -2070,14 +2087,6 @@ async def test_new_vector_store_auto_resolves_embedding_config(): }, } - # Mock database model lookup for embedding config resolution - mock_db_model = MagicMock() - mock_db_model.litellm_params = { - "api_key": "resolved-api-key", - "api_base": "https://api.openai.com", - "api_version": "2024-01-01", - } - # Mock user API key mock_user_api_key = MagicMock(spec=UserAPIKeyAuth) mock_user_api_key.user_role = None @@ -2088,10 +2097,6 @@ async def test_new_vector_store_auto_resolves_embedding_config(): mock_prisma_client.db.litellm_managedvectorstorestable.find_unique = AsyncMock( return_value=None # Vector store doesn't exist yet ) - mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock( - return_value=mock_db_model - ) - # Track what was passed to create captured_create_data = {} @@ -2112,261 +2117,21 @@ async def test_new_vector_store_auto_resolves_embedding_config(): mock_registry = MagicMock() mock_registry.add_vector_store_to_registry = MagicMock() - # Mock router to return None (so it falls back to DB resolution) - mock_router = MagicMock() - mock_router.get_deployment_by_model_group_name.return_value = None - with ( patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), - patch("litellm.proxy.proxy_server.llm_router", mock_router), - patch( - "litellm.proxy.vector_store_endpoints.management_endpoints.decrypt_value_helper", - side_effect=lambda value, key, return_original_value: value, - ), patch.object(litellm, "vector_store_registry", mock_registry), ): - result = await new_vector_store( - vector_store=vector_store_data, user_api_key_dict=mock_user_api_key - ) + result = await new_vector_store(vector_store=vector_store_data, user_api_key_dict=mock_user_api_key) assert result["status"] == "success" - # Auto-resolve no longer happens at create time — the persisted row - # carries only the model reference, never the resolved cleartext - # credential. Resolution now happens at request-handling time inside - # ``_update_request_data_with_litellm_managed_vector_store_registry``, - # where the resolved config lives in per-request memory and is never - # written to the database. litellm_params_json = captured_create_data.get("litellm_params") assert litellm_params_json is not None litellm_params_dict = json.loads(litellm_params_json) assert "litellm_embedding_config" not in litellm_params_dict assert litellm_params_dict["litellm_embedding_model"] == "text-embedding-ada-002" - # The response must also not echo a cleartext credential — even on - # the create response, where redaction guards against caller-supplied - # cleartext or pre-existing rows that were created by an earlier - # proxy version. response_vs = result["vector_store"] - assert "resolved-api-key" not in _serialize_litellm_params( - response_vs.get("litellm_params") - ) - - -def test_resolve_embedding_config_from_router(): - """Test that _resolve_embedding_config_from_router correctly extracts credentials from config-defined models.""" - from litellm.types.router import Deployment, LiteLLM_Params - - # Create a mock router with a model - mock_router = MagicMock() - - # Create a mock deployment with litellm_params - mock_litellm_params = MagicMock(spec=LiteLLM_Params) - mock_litellm_params.api_key = "config-api-key" - mock_litellm_params.api_base = "https://config-api-base.com" - mock_litellm_params.api_version = "2024-02-01" - - mock_deployment = MagicMock(spec=Deployment) - mock_deployment.litellm_params = mock_litellm_params - - mock_router.get_deployment_by_model_group_name.return_value = mock_deployment - - # Test resolution - result = _resolve_embedding_config_from_router( - embedding_model="text-embedding-ada-002", llm_router=mock_router - ) - - assert result is not None - assert result["api_key"] == "config-api-key" - assert result["api_base"] == "https://config-api-base.com" - assert result["api_version"] == "2024-02-01" - - mock_router.get_deployment_by_model_group_name.assert_called_once_with( - model_group_name="text-embedding-ada-002" - ) - - -def test_resolve_embedding_config_from_router_with_provider_prefix(): - """Test that _resolve_embedding_config_from_router handles provider prefixes like 'azure/model-name'.""" - from litellm.types.router import Deployment, LiteLLM_Params - - # Create a mock router - mock_router = MagicMock() - - # Create a mock deployment - mock_litellm_params = MagicMock(spec=LiteLLM_Params) - mock_litellm_params.api_key = "azure-api-key" - mock_litellm_params.api_base = "https://azure-endpoint.openai.azure.com" - mock_litellm_params.api_version = "2024-02-15" - - mock_deployment = MagicMock(spec=Deployment) - mock_deployment.litellm_params = mock_litellm_params - - # First call with full name returns None, second call with stripped name returns deployment - mock_router.get_deployment_by_model_group_name.side_effect = [None, mock_deployment] - - result = _resolve_embedding_config_from_router( - embedding_model="azure/text-embedding-3-large", llm_router=mock_router - ) - - assert result is not None - assert result["api_key"] == "azure-api-key" - assert result["api_base"] == "https://azure-endpoint.openai.azure.com" - assert result["api_version"] == "2024-02-15" - - # Should have tried both the full name and stripped name - assert mock_router.get_deployment_by_model_group_name.call_count == 2 - - -def test_resolve_embedding_config_from_router_returns_none_when_not_found(): - """Test that _resolve_embedding_config_from_router returns None when model is not in router.""" - mock_router = MagicMock() - mock_router.get_deployment_by_model_group_name.return_value = None - - result = _resolve_embedding_config_from_router( - embedding_model="nonexistent-model", llm_router=mock_router - ) - - assert result is None - - -def test_resolve_embedding_config_from_router_handles_os_environ(): - """Test that _resolve_embedding_config_from_router handles os.environ/ prefixed values.""" - from litellm.types.router import Deployment, LiteLLM_Params - - mock_router = MagicMock() - - mock_litellm_params = MagicMock(spec=LiteLLM_Params) - mock_litellm_params.api_key = "os.environ/OPENAI_API_KEY" - mock_litellm_params.api_base = "https://direct-url.com" - mock_litellm_params.api_version = None - - mock_deployment = MagicMock(spec=Deployment) - mock_deployment.litellm_params = mock_litellm_params - - mock_router.get_deployment_by_model_group_name.return_value = mock_deployment - - with patch( - "litellm.proxy.vector_store_endpoints.management_endpoints.get_secret", - return_value="resolved-from-env", - ) as mock_get_secret: - result = _resolve_embedding_config_from_router( - embedding_model="text-embedding-ada-002", llm_router=mock_router - ) - - assert result is not None - assert result["api_key"] == "resolved-from-env" - assert result["api_base"] == "https://direct-url.com" - assert "api_version" not in result - - mock_get_secret.assert_called_once_with("os.environ/OPENAI_API_KEY") - - -@pytest.mark.asyncio -async def test_resolve_embedding_config_tries_router_then_db(): - """Test that _resolve_embedding_config tries router first, then falls back to DB.""" - from litellm.types.router import Deployment, LiteLLM_Params - - mock_prisma_client = MagicMock() - mock_router = MagicMock() - - # Router has the model - mock_litellm_params = MagicMock(spec=LiteLLM_Params) - mock_litellm_params.api_key = "router-api-key" - mock_litellm_params.api_base = "https://router-api-base.com" - mock_litellm_params.api_version = None - - mock_deployment = MagicMock(spec=Deployment) - mock_deployment.litellm_params = mock_litellm_params - - mock_router.get_deployment_by_model_group_name.return_value = mock_deployment - - # DB should NOT be called since router has the model - mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock() - - result = await _resolve_embedding_config( - embedding_model="text-embedding-ada-002", - prisma_client=mock_prisma_client, - llm_router=mock_router, - ) - - assert result is not None - assert result["api_key"] == "router-api-key" - - # DB should NOT have been called since router found the model - mock_prisma_client.db.litellm_proxymodeltable.find_first.assert_not_called() - - -@pytest.mark.asyncio -async def test_resolve_embedding_config_caches_result(): - """The first lookup should hit the router/DB; subsequent lookups for - the same model name should return the cached value without touching - the router or the database.""" - from litellm.types.router import Deployment, LiteLLM_Params - - mock_prisma_client = MagicMock() - mock_router = MagicMock() - - mock_litellm_params = MagicMock(spec=LiteLLM_Params) - mock_litellm_params.api_key = "router-api-key" - mock_litellm_params.api_base = "https://router-api-base.com" - mock_litellm_params.api_version = None - - mock_deployment = MagicMock(spec=Deployment) - mock_deployment.litellm_params = mock_litellm_params - mock_router.get_deployment_by_model_group_name.return_value = mock_deployment - - first = await _resolve_embedding_config( - embedding_model="cached-model", - prisma_client=mock_prisma_client, - llm_router=mock_router, - ) - assert first is not None - assert mock_router.get_deployment_by_model_group_name.call_count == 1 - - second = await _resolve_embedding_config( - embedding_model="cached-model", - prisma_client=mock_prisma_client, - llm_router=mock_router, - ) - assert second == first - # Router (and by extension the DB) was not consulted again. - assert mock_router.get_deployment_by_model_group_name.call_count == 1 - - -@pytest.mark.asyncio -async def test_resolve_embedding_config_falls_back_to_db(): - """Test that _resolve_embedding_config falls back to DB when router doesn't have the model.""" - mock_prisma_client = MagicMock() - mock_router = MagicMock() - - # Router doesn't have the model - mock_router.get_deployment_by_model_group_name.return_value = None - - # DB has the model - mock_db_model = MagicMock() - mock_db_model.litellm_params = { - "api_key": "db-api-key", - "api_base": "https://db-api-base.com", - } - mock_prisma_client.db.litellm_proxymodeltable.find_first = AsyncMock( - return_value=mock_db_model - ) - - with patch( - "litellm.proxy.vector_store_endpoints.management_endpoints.decrypt_value_helper", - side_effect=lambda value, key, return_original_value: value, - ): - result = await _resolve_embedding_config( - embedding_model="text-embedding-ada-002", - prisma_client=mock_prisma_client, - llm_router=mock_router, - ) - - assert result is not None - assert result["api_key"] == "db-api-key" - - # DB should have been called since router didn't find the model - mock_prisma_client.db.litellm_proxymodeltable.find_first.assert_called() + assert "api_key" not in _serialize_litellm_params(response_vs.get("litellm_params")) @pytest.mark.asyncio @@ -2425,9 +2190,7 @@ async def test_new_vector_store_auto_resolves_from_router(): } return mock_created_vector_store - mock_prisma_client.db.litellm_managedvectorstorestable.create = AsyncMock( - side_effect=mock_create - ) + mock_prisma_client.db.litellm_managedvectorstorestable.create = AsyncMock(side_effect=mock_create) mock_registry = MagicMock() mock_registry.add_vector_store_to_registry = MagicMock() @@ -3158,3 +2921,35 @@ class TestAzureAIAnalyzeNamedIndexClassification: user_api_key_dict=self._team_member("analyze", ["read"]), ) assert result is True + + +@pytest.mark.parametrize( + "blocked_key", + ["embedding_model", "litellm_embedding_model", "litellm_embedding_config", "litellm_credential_name"], +) +def test_vector_store_search_rejects_caller_embedding_selection_params(blocked_key): + """ + Regression: the search request body must not pick the embedding model or + credential used to embed the query. Those resolve through the Router with + the proxy's credentials, bypassing the key's model permissions, so they may + only come from the managed store's server-side registration. + """ + from fastapi.testclient import TestClient + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + + mock_auth = UserAPIKeyAuth(user_id="test_internal_user", user_role=LitellmUserRoles.INTERNAL_USER.value) + original_overrides = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: mock_auth + try: + client = TestClient(app) + response = client.post( + "/v1/vector_stores/s3-store/search", + json={"query": "hello", blocked_key: "attacker-choice"}, + ) + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 400, response.json() + assert blocked_key in str(response.json()) diff --git a/tests/test_litellm/rag/test_main.py b/tests/test_litellm/rag/test_main.py index 2d1b460513f..51d03544910 100644 --- a/tests/test_litellm/rag/test_main.py +++ b/tests/test_litellm/rag/test_main.py @@ -259,6 +259,135 @@ async def test_aquery_streaming_bills_sub_call_costs_into_final_event(): assert standard_logging_object["response_cost"] >= 0.003 +@pytest.mark.asyncio +async def test_aquery_forwards_provider_retrieval_config_and_router_to_search(): + """ + Regression: provider-specific retrieval_config keys (aws_region_name, + embedding_model, vector_bucket_name, ...) and the router must be forwarded + to the vector store search call. Pre-fix they were silently dropped, so + /v1/rag/query failed with provider config errors (e.g. S3 Vectors + "aws_region_name is required") even when the caller supplied them. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}, + } + ] + ) + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): # test-quality-ok: asearch is the boundary the forwarding contract under test targets + response = await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={ + "vector_store_id": "bkt:idx", + "custom_llm_provider": "s3_vectors", + "top_k": 5, + "aws_region_name": "eu-west-1", + "embedding_model": "my-embed", + "vector_bucket_name": "bkt", + }, + router=router, + mock_response="hi", + ) + + assert isinstance(response, ModelResponse) + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["vector_store_id"] == "bkt:idx" + assert search_kwargs["custom_llm_provider"] == "s3_vectors" + assert search_kwargs["max_num_results"] == 5 + assert search_kwargs["router"] is router + # provider-specific extras forwarded + assert search_kwargs["aws_region_name"] == "eu-west-1" + assert search_kwargs["embedding_model"] == "my-embed" + assert search_kwargs["vector_bucket_name"] == "bkt" + # consumed keys are not duplicated into the spread + assert "top_k" not in search_kwargs + + +@pytest.mark.asyncio +async def test_aquery_minimal_retrieval_config_forwards_no_extras(): + """ + A minimal retrieval_config must not leak consumed keys (or invent extras) + into the vector store search call. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): # test-quality-ok: asearch is the boundary the forwarding contract under test targets + await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={"vector_store_id": "vs_test_123", "custom_llm_provider": "openai"}, + mock_response="hi", + ) + + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["vector_store_id"] == "vs_test_123" + assert search_kwargs["custom_llm_provider"] == "openai" + assert search_kwargs["router"] is None + leaked = {"top_k", "filters", "retrieval_filter", "aws_region_name", "embedding_model", "vector_bucket_name"} + assert not (leaked & set(search_kwargs.keys())) + + +@pytest.mark.asyncio +async def test_aquery_does_not_forward_connection_override_keys_to_search(): + """ + Only allowlisted retrieval_config keys may reach the vector store search + call. Caller-controlled connection overrides (api_base, api_key, arbitrary + extras) must be dropped, otherwise a caller could redirect store + credentials to an attacker-chosen host. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): # test-quality-ok: asearch is the boundary the forwarding contract under test targets + await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={ + "vector_store_id": "bkt:idx", + "custom_llm_provider": "s3_vectors", + "aws_region_name": "eu-west-1", + "api_base": "https://attacker.example.com", + "api_key": "attacker-key", + "arbitrary_extra": "nope", + }, + mock_response="hi", + ) + + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["aws_region_name"] == "eu-west-1" + blocked = {"api_base", "api_key", "arbitrary_extra"} + assert not (blocked & set(search_kwargs.keys())) + + def test_rag_call_types_are_registered(): """ query/aquery/ingest/aingest are @client-decorated entry points, so their diff --git a/tests/test_litellm/rerank_api/test_main.py b/tests/test_litellm/rerank_api/test_main.py index 587be59c550..2b6cfeda2c2 100644 --- a/tests/test_litellm/rerank_api/test_main.py +++ b/tests/test_litellm/rerank_api/test_main.py @@ -111,6 +111,99 @@ def test_together_rerank_honors_api_base(respx_mock: respx.MockRouter): assert mock_route.calls[0].request.headers["authorization"] == "Bearer fake-together-key" +DASHSCOPE_404_BODY = { + "error": { + "message": "The model `does-not-exist` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": None, + "code": "model_not_found", + }, + "request_id": "mock-request-id", +} + + +def test_rerank_error_names_provider_and_keeps_body(respx_mock: respx.MockRouter, monkeypatch): + """Regression for the rerank error path mapping with the unresolved provider param: + a provider 404 surfaced as 'None - ' instead of naming the provider and its error body.""" + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False) + + mock_route = respx_mock.post("https://dashscope.example/v1/reranks") + mock_route.return_value = httpx.Response(404, json=DASHSCOPE_404_BODY) + + with pytest.raises(litellm.NotFoundError) as exc_info: + litellm.rerank( + model="dashscope/does-not-exist", + query=MARKER_QUERY, + documents=[MARKER_DOC], + api_key="fake-dashscope-key", + api_base="https://dashscope.example/v1", + ) + + assert mock_route.called + assert "DashscopeException" in str(exc_info.value) + assert "does not exist or you do not have access to it" in str(exc_info.value) + assert "None - " not in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_arerank_error_is_mapped_to_litellm_exception(respx_mock: respx.MockRouter, monkeypatch): + """Regression for arerank's bare re-raise: provider errors escaped as raw + provider exception classes instead of the mapped litellm exception contract.""" + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + + mock_route = respx_mock.post("https://dashscope.example/v1/reranks") + mock_route.return_value = httpx.Response(404, json=DASHSCOPE_404_BODY) + + with pytest.raises(litellm.NotFoundError) as exc_info: + await litellm.arerank( + model="dashscope/does-not-exist", + query=MARKER_QUERY, + documents=[MARKER_DOC], + api_key="fake-dashscope-key", + api_base="https://dashscope.example/v1", + ) + + assert mock_route.called + assert "DashscopeException" in str(exc_info.value) + assert "does not exist or you do not have access to it" in str(exc_info.value) + assert "None - " not in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_arerank_declared_authenticating_provider_skips_resolution(monkeypatch): + """Regression for the event-loop hazard in arerank's provider pre-resolution: + get_llm_provider runs the blocking OAuth device flow for github_copilot/chatgpt, + so arerank must adopt the declared provider instead of resolving it, while the + except path still maps with that declared provider.""" + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + resolution_calls = [] + + def record_resolution(*args, **kwargs): + resolution_calls.append((args, kwargs)) + return "gpt-4o", "github_copilot", None, None + + def rerank_raises_provider_error(*args, **kwargs): + raise BaseLLMException(status_code=401, message='{"error":"bad key"}') + + monkeypatch.setattr(litellm, "get_llm_provider", record_resolution) + monkeypatch.setattr("litellm.rerank_api.main.rerank", rerank_raises_provider_error) + + with pytest.raises(litellm.AuthenticationError) as exc_info: + await litellm.arerank( + model="github_copilot/gpt-4o", + query=MARKER_QUERY, + documents=[MARKER_DOC], + ) + + assert resolution_calls == [] + assert "Github_copilotException" in str(exc_info.value) + assert "None - " not in str(exc_info.value) + + @pytest.mark.asyncio async def test_together_rerank_async_honors_env_api_base(respx_mock: respx.MockRouter, monkeypatch): """Regression: TOGETHER_AI_API_BASE was honored by chat but ignored by rerank.""" diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index b2b8eb5da80..2068f10ea2d 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1928,6 +1928,19 @@ class TestToolTransformation: assert result_tool["function"]["parameters"]["type"] == "object" assert "properties" in result_tool["function"]["parameters"] + def test_transform_function_tools_parameters_keep_client_key_order(self): + tools = [ + {"type": "function", "name": "a", "parameters": {"properties": {"arg": {"type": "string"}}, "required": ["arg"]}}, + {"type": "function", "name": "b", "parameters": {"type": "object", "properties": {}}}, + ] + + result_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) + + assert list(result_tools[0]["function"]["parameters"]) == ["properties", "required", "type"] + assert list(result_tools[1]["function"]["parameters"]) == ["type", "properties"] + def test_transform_function_tools_empty_parameters(self): """Test that empty parameters get 'type': 'object' added""" function_tool = { diff --git a/tests/test_litellm/responses/test_rust_bridge_websocket.py b/tests/test_litellm/responses/test_rust_bridge_websocket.py index c9a5b988be6..1233ddf1785 100644 --- a/tests/test_litellm/responses/test_rust_bridge_websocket.py +++ b/tests/test_litellm/responses/test_rust_bridge_websocket.py @@ -3,7 +3,7 @@ from __future__ import annotations import pytest from litellm.llms.custom_httpx.llm_http_handler import _rust_responses_websocket_enabled -from litellm.rust_bridge import responses_websocket +from litellm.rust_bridge import configuration, responses_websocket from litellm.types.router import GenericLiteLLMParams @@ -39,12 +39,33 @@ class _FakeNativeBridge: return _FakeNativeConnection() +@pytest.fixture(autouse=True) +def reset_responses_websocket(): + responses_websocket.set_rust_responses_websocket(connection=None) + configuration.reset_rust_configuration() + yield + responses_websocket.set_rust_responses_websocket(connection=None) + configuration.reset_rust_configuration() + + def test_rust_websocket_bridge_is_disabled_without_flag() -> None: assert not _rust_responses_websocket_enabled("openai", GenericLiteLLMParams()) assert not _rust_responses_websocket_enabled("anthropic", GenericLiteLLMParams(rust=True)) assert _rust_responses_websocket_enabled("openai", GenericLiteLLMParams(rust=True)) +def test_explicit_false_overrides_process_enable() -> None: + configuration.use_litellm_rust(True) + + assert not _rust_responses_websocket_enabled("openai", GenericLiteLLMParams(rust=False)) + + +def test_process_enable_applies_without_request_override() -> None: + configuration.use_litellm_rust(True) + + assert _rust_responses_websocket_enabled("openai", GenericLiteLLMParams()) + + @pytest.mark.asyncio async def test_adapter_raises_clean_close_when_rust_connection_ends() -> None: adapter = responses_websocket._ConnectionAdapter(_ClosedNativeConnection()) diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index 677faf7f655..c226c0b4d09 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -6,7 +6,7 @@ completion_start_time = end_time.""" import json from datetime import datetime from typing import Optional -from unittest.mock import Mock +from unittest.mock import Mock, patch import httpx import pytest @@ -326,3 +326,214 @@ def test_run_post_success_hooks_does_not_report_generation_time_as_overhead(): assert iterator.completed_response._hidden_params["_response_ms"] == 10000.0 assert "litellm_overhead_time_ms" not in iterator.completed_response._hidden_params + + +def _responses_api_response_with_usage() -> ResponsesAPIResponse: + from litellm.types.llms.openai import ResponseAPIUsage + + return ResponsesAPIResponse( + id="resp_lit6427", + created_at=int(datetime(2025, 1, 1).timestamp()), + status="completed", + model="mantle-claude", + object="response", + output=[], + usage=ResponseAPIUsage(input_tokens=20, output_tokens=60, total_tokens=80), + ) + + +def test_stamp_responses_usage_cost_stamps_computed_cost(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + + response = _responses_api_response_with_usage() + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj._response_cost_calculator.return_value = 0.000704 + + _stamp_responses_usage_cost(response, logging_obj) + + assert getattr(response.usage, "cost", None) == pytest.approx(0.000704) + logging_obj._response_cost_calculator.assert_called_once_with(result=response) + + +def test_stamp_responses_usage_cost_keeps_provider_reported_cost(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + + response = _responses_api_response_with_usage() + setattr(response.usage, "cost", 0.5) + logging_obj = Mock(spec=LiteLLMLoggingObj) + + _stamp_responses_usage_cost(response, logging_obj) + + assert getattr(response.usage, "cost", None) == pytest.approx(0.5) + logging_obj._response_cost_calculator.assert_not_called() + + +def test_stamp_responses_usage_cost_survives_calculator_failure(): + from litellm.responses.streaming_iterator import _stamp_responses_usage_cost + + response = _responses_api_response_with_usage() + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj._response_cost_calculator.side_effect = RuntimeError("cost map unavailable") + + _stamp_responses_usage_cost(response, logging_obj) + + assert getattr(response.usage, "cost", None) is None + + +def _capture_dispatch(logged: list): + """Record the object handed to the success handlers. + + ``Mock(spec=LiteLLMLoggingObj).dispatch_success_handlers`` is an AsyncMock whose side effect + only runs when the coroutine is awaited, so capture with a plain function instead. + """ + + async def _noop() -> None: + return None + + def _dispatch(result, **kwargs): + logged.append(result) + return _noop() + + return _dispatch + + +def _headers_config(*, transform_hidden_params: Optional[dict] = None) -> Mock: + """Config whose completed event carries a real ResponsesAPIResponse, so the logging copy + performs a genuine model_dump/model_validate round trip.""" + mock_config = Mock(spec=BaseResponsesAPIConfig) + + def _transform(model, parsed_chunk, logging_obj): + evt_type = parsed_chunk.get("type") + if evt_type != "response.completed": + stub = Mock() + stub.type = evt_type + return stub + response = ResponsesAPIResponse( + id="resp_headers", + created_at=1, + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + if transform_hidden_params is not None: + response._hidden_params.update(transform_hidden_params) + return ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=response, + ) + + mock_config.transform_streaming_response.side_effect = _transform + return mock_config + + +def _make_header_iterator( + *, + headers: dict, + config: Mock, + logging_obj: LiteLLMLoggingObj, +) -> ResponsesAPIStreamingIterator: + async def aiter_bytes(): + yield _sse_event({"type": "response.completed"}) + + mock_response = Mock() + mock_response.headers = headers + mock_response.aiter_bytes = aiter_bytes + + return ResponsesAPIStreamingIterator( + response=mock_response, + model="gpt-4o-mini", + responses_api_provider_config=config, + logging_obj=logging_obj, + litellm_metadata={}, + custom_llm_provider="azure", + ) + + +@pytest.mark.asyncio +async def test_streaming_logging_response_carries_provider_response_headers(): + """LIT-6055: the provider headers the iterator captured must reach the logged response, so + custom loggers can read Azure's apim-request-id from the callback payload.""" + logging_obj = _logging_obj_stub() + logged: list[object] = [] + logging_obj.dispatch_success_handlers = _capture_dispatch(logged) + + logging_obj._on_deferred_stream_complete = None + + iterator = _make_header_iterator( + headers={"apim-request-id": "azure-correlation-1", "x-ms-region": "East US 2"}, + config=_headers_config(), + logging_obj=logging_obj, + ) + async for _ in iterator: + pass + + assert len(logged) == 1 + hidden_params = logged[0].response._hidden_params + assert hidden_params["additional_headers"]["llm_provider-apim-request-id"] == "azure-correlation-1" + assert hidden_params["additional_headers"]["llm_provider-x-ms-region"] == "East US 2" + assert hidden_params["headers"]["apim-request-id"] == "azure-correlation-1" + # the proxy builds the client's response headers from the iterator's own dict, so the logged + # response must hold copies rather than alias it + assert hidden_params["additional_headers"] is not iterator._hidden_params["additional_headers"] + assert hidden_params["headers"] is not iterator._raw_response_headers + + +@pytest.mark.asyncio +async def test_streaming_logging_copy_preserves_transform_hidden_params(): + """LIT-6055: model_validate(model_dump()) drops pydantic private attributes, so headers a + provider transform already set on the response (fake_stream) must be re-applied.""" + logging_obj = _logging_obj_stub() + logged: list[object] = [] + logging_obj.dispatch_success_handlers = _capture_dispatch(logged) + + logging_obj._on_deferred_stream_complete = None + + iterator = _make_header_iterator( + headers={}, + config=_headers_config( + transform_hidden_params={ + "additional_headers": {"llm_provider-apim-request-id": "from-transform"}, + "headers": {"apim-request-id": "from-transform"}, + "response_cost": 0.5, + } + ), + logging_obj=logging_obj, + ) + async for _ in iterator: + pass + + assert len(logged) == 1 + hidden_params = logged[0].response._hidden_params + assert hidden_params["additional_headers"]["llm_provider-apim-request-id"] == "from-transform" + assert hidden_params["headers"]["apim-request-id"] == "from-transform" + assert iterator.completed_response is not logged[0] + # only the header keys travel: response_cost would short-circuit the cost calculator + assert "response_cost" not in hidden_params + + +@pytest.mark.asyncio +async def test_streaming_logging_copy_fallback_leaves_caller_event_untouched(): + """LIT-6055: when the logging copy falls back to the original event, the header restore must + not stamp logging-only state onto the object the caller is iterating.""" + logging_obj = _logging_obj_stub() + logged: list[object] = [] + logging_obj.dispatch_success_handlers = _capture_dispatch(logged) + logging_obj._on_deferred_stream_complete = None + + iterator = _make_header_iterator( + headers={"apim-request-id": "azure-correlation-1"}, + config=_headers_config(), + logging_obj=logging_obj, + ) + async for _ in iterator: + pass + + assert len(logged) == 1 + iterator._completed_response_logged = False + logged.clear() + with patch.object(type(iterator.completed_response), "model_dump", side_effect=ValueError("cannot serialize")): + iterator._log_completed_response(is_async=True) + + assert logged == [iterator.completed_response] + assert iterator.completed_response.response._hidden_params == {} diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 1ec8be88c9b..3ecccb673f3 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -12,12 +12,11 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from pydantic import ValidationError - import litellm from litellm import Router from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache -from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY +from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.router_strategy.complexity_router.complexity_router import ( _CLASSIFICATION_CURRENT_MESSAGE_ONLY, _CLASSIFICATION_WITH_CONVERSATION, @@ -34,10 +33,14 @@ from litellm.router_strategy.complexity_router.config import ( DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, DEFAULT_COMPLEXITY_CONFIG, DEFAULT_TECHNICAL_KEYWORDS, + ClassificationRubric, ClassifierLLMConfig, ComplexityRouterConfig, ComplexityTier, - ClassificationRubric, +) +from litellm.router_strategy.complexity_router.tier_predictor import ( + TierGlobalStatistic, + TrainedTierArtifact, ) from litellm.types.router import ( Deployment, @@ -46,6 +49,16 @@ from litellm.types.router import ( ) +def _heuristic_v2_artifact() -> TrainedTierArtifact: + return TrainedTierArtifact( + global_statistics=tuple( + TierGlobalStatistic(tier=tier, successes=successes, observations=100) + for tier, successes in enumerate((10, 20, 90, 99), start=1) + ), + routing_threshold=0.8, + ) + + @pytest.fixture def mock_router_instance(): """Create a mock LiteLLM Router instance.""" @@ -1696,6 +1709,59 @@ class TestLLMClassifier: assert outcome.cause == "heuristic_scorer" assert outcome.score is not None + @pytest.mark.asyncio + async def test_heuristic_v2_routes_directly_to_predicted_builtin_tier(self, mock_router_instance): + router = ComplexityRouter( + model_name="tier-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "classifier_type": "heuristic_v2", + "heuristic_v2_artifact": _heuristic_v2_artifact(), + "tiers": { + "SIMPLE": "simple-model", + "MEDIUM": "medium-model", + "COMPLEX": "complex-model", + "REASONING": "reasoning-model", + }, + }, + ) + + response = await router.async_pre_routing_hook( + model="tier-router", + request_kwargs={}, + messages=[{"role": "user", "content": "Handle this new request"}], + ) + + assert response is not None + assert response.model == "complex-model" + assert response.routing_decision["tier"] == "COMPLEX" + assert response.routing_decision["cause"] == "heuristic_v2" + assert response.routing_decision["signals"] == [ + "request-type:general", + "tier-probability:simple=0.107843", + "tier-probability:medium=0.205882", + "tier-probability:complex=0.892157", + "tier-probability:reasoning=0.980392", + ] + + def test_heuristic_v2_needs_no_classifier_model(self): + config = ComplexityRouterConfig(classifier_type="heuristic_v2") + + assert config.classifier_llm_config is None + assert config.heuristic_v2_artifact == "ultrafeedback" + + def test_heuristic_v2_rejects_custom_tier_definitions(self): + with pytest.raises(ValidationError, match="as does heuristic_v2"): + ComplexityRouterConfig( + classifier_type="heuristic_v2", + tier_definitions=( + {"name": "low", "description": "easy work"}, + {"name": "high", "description": "hard work"}, + ), + tiers={"low": "cheap", "high": "expensive"}, + fallback_tier="high", + ) + @pytest.mark.asyncio async def test_aclassify_llm_success_routes_by_llm_verdict(self, llm_complexity_router, mock_router_instance): """A well-formed structured LLM response should decide the tier directly. @@ -4208,6 +4274,26 @@ class TestSessionAffinity: assert first.model == "o1-preview" assert second.model == "gpt-4o-mini" + @pytest.mark.asyncio + async def test_proxy_generated_session_id_never_pins(self, mock_router_instance, session_affinity_config): + """A session id the proxy generated for a request that had none is per request, so + it must not create a pin even with session_affinity enabled.""" + mock_router_instance.cache = DualCache() + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=session_affinity_config, + ) + request_kwargs = {"metadata": {"session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True}} + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs=request_kwargs, messages=self.REASONING_MESSAGE + ) + second = await router.async_pre_routing_hook( + model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE_MESSAGE + ) + assert first.model == "o1-preview" + assert second.model == "gpt-4o-mini" + @pytest.mark.asyncio async def test_can_be_enabled_to_pin_every_later_turn(self, mock_router_instance, session_affinity_config): """Regression: session_affinity=True is the opt-in, so a shared session_id reuses the @@ -10023,6 +10109,207 @@ class TestHeuristicFirst: assert outcome.cause == "default_model_fallback" +# Scores 0.175 with one signal, so it sits 0.025 from simple_medium: the pair of tiers either side of +# that boundary are different model pools, and a hair's difference in score picks the other one. +NEAR_BOUNDARY_PROMPT = ( + "design a distributed cache with consistent hashing, then explain the failure modes step by step" +) + +# Scores 0.075 with signals, the far side of any margin under 0.075: the scorer is decided here. +CLEAR_OF_BOUNDARY_PROMPT = "explain step by step how consistent hashing rebalances keys" + + +def _hybrid_router(mock_router_instance, **config_overrides): + config = { + "tiers": dict(HEURISTIC_FIRST_TIERS), + "tier_boundaries": dict(HEURISTIC_FIRST_BOUNDARIES), + "classifier_type": "hybrid", + "hybrid_boundary_margin": 0.03, + "classifier_llm_config": {"model": "haiku-classifier", "timeout_ms": 400}, + **config_overrides, + } + return ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + + +class TestHybridConfig: + """Config validation for classifier_type='hybrid'.""" + + @pytest.mark.parametrize( + "overrides, expected", + [ + ({"classifier_llm_config": None}, "classifier_llm_config is required"), + ({"hybrid_boundary_margin": None}, "hybrid_boundary_margin is required"), + ({"hybrid_boundary_margin": -0.01}, "greater than or equal to 0"), + ({"hybrid_boundary_margin": 1.01}, "less than or equal to 1"), + ], + ) + def test_rejects_incoherent_config(self, overrides, expected): + config = { + "tiers": dict(HEURISTIC_FIRST_TIERS), + "classifier_type": "hybrid", + "hybrid_boundary_margin": 0.03, + "classifier_llm_config": {"model": "haiku-classifier"}, + **overrides, + } + with pytest.raises(ValidationError, match=expected): + ComplexityRouterConfig(**config) + + @pytest.mark.parametrize("classifier_type", ["heuristic", "llm", "custom", "heuristic_first"]) + def test_margin_rejected_on_every_other_classifier_type(self, classifier_type): + """A margin on a router that never compares a score to a boundary is a silent no-op, so it is + refused rather than accepted and ignored. heuristic_first is in this list on purpose: its + ceiling is a different question from proximity, and accepting both on one router would make + two modes out of one classifier_type.""" + config: dict[str, object] = { + "tiers": dict(HEURISTIC_FIRST_TIERS), + "classifier_type": classifier_type, + "hybrid_boundary_margin": 0.03, + } + if classifier_type in ("llm", "heuristic_first"): + config["classifier_llm_config"] = {"model": "haiku-classifier"} + if classifier_type == "heuristic_first": + config["heuristic_first_max_tier"] = "SIMPLE" + if classifier_type == "custom": + config["classifier_plugin"] = _FixedTierClassifier("SIMPLE") + with pytest.raises(ValidationError, match="hybrid_boundary_margin is set but classifier_type"): + ComplexityRouterConfig(**config) + + def test_the_cheap_tier_ceiling_is_rejected_here(self): + """The two modes are told apart by which knob they take, so the ceiling is refused on hybrid + exactly as the margin is refused on heuristic_first.""" + with pytest.raises(ValidationError, match="heuristic_first_max_tier is set but classifier_type"): + ComplexityRouterConfig( + tiers=dict(HEURISTIC_FIRST_TIERS), + classifier_type="hybrid", + hybrid_boundary_margin=0.03, + heuristic_first_max_tier="SIMPLE", + classifier_llm_config={"model": "haiku-classifier"}, + ) + + def test_custom_tier_set_is_rejected(self): + """The scorer only emits the four built-in tiers, so it cannot judge proximity on a replaced set.""" + with pytest.raises(ValidationError, match="tier_definitions requires classifier_type"): + ComplexityRouterConfig( + classifier_type="hybrid", + hybrid_boundary_margin=0.03, + classifier_llm_config={"model": "haiku-classifier"}, + tier_definitions=[{"name": "lo", "description": "x"}, {"name": "hi", "description": "y"}], + tiers={"lo": "gpt-4o-mini", "hi": "gpt-4o"}, + ) + + def test_classifier_model_is_a_dependency(self): + config = ComplexityRouterConfig( + tiers=dict(HEURISTIC_FIRST_TIERS), + classifier_type="hybrid", + hybrid_boundary_margin=0.03, + classifier_llm_config={"model": "haiku-classifier"}, + ) + assert config.uses_llm_classifier is True + + +class TestHybrid: + """Behavior of the hybrid chain: the scorer keeps its tier unless the score is near a boundary.""" + + @pytest.mark.asyncio + async def test_near_boundary_prompt_escalates(self, mock_router_instance): + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}')) + router = _hybrid_router(mock_router_instance) + + _tier, score, signals, _cause = router._score_and_classify(NEAR_BOUNDARY_PROMPT) + assert signals and abs(score - HEURISTIC_FIRST_BOUNDARIES["simple_medium"]) < 0.03 + + outcome = await router.aclassify(NEAR_BOUNDARY_PROMPT) + mock_router_instance.acompletion.assert_awaited_once() + assert outcome.tier == ComplexityTier.COMPLEX + assert outcome.cause == "llm_classifier" + + @pytest.mark.asyncio + async def test_score_clear_of_every_boundary_keeps_the_heuristic_tier(self, mock_router_instance): + mock_router_instance.acompletion = AsyncMock() + router = _hybrid_router(mock_router_instance) + outcome = await router.aclassify(CLEAR_OF_BOUNDARY_PROMPT) + mock_router_instance.acompletion.assert_not_called() + assert outcome.tier == ComplexityTier.SIMPLE + assert outcome.cause == "hybrid_short_circuit" + + @pytest.mark.asyncio + async def test_an_expensive_tier_short_circuits_too(self, mock_router_instance): + """This is the whole difference from heuristic_first, which would have escalated this by tier + alone. Hybrid asks whether the score is DECIDED, not whether the tier is cheap.""" + mock_router_instance.acompletion = AsyncMock() + router = _hybrid_router( + mock_router_instance, + tier_boundaries={"simple_medium": -0.9, "medium_complex": -0.8, "complex_reasoning": -0.7}, + ) + + tier, _score, signals, _cause = router._score_and_classify(CLEAR_OF_BOUNDARY_PROMPT) + assert (tier, bool(signals)) == (ComplexityTier.REASONING, True) + + outcome = await router.aclassify(CLEAR_OF_BOUNDARY_PROMPT) + mock_router_instance.acompletion.assert_not_called() + assert outcome.tier == ComplexityTier.REASONING + assert outcome.cause == "hybrid_short_circuit" + + @pytest.mark.asyncio + async def test_widening_the_margin_escalates_what_a_narrow_one_kept(self, mock_router_instance): + """The margin is the knob: the same prompt short-circuits at 0.03 and escalates at 0.08.""" + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "MEDIUM"}')) + router = _hybrid_router(mock_router_instance, hybrid_boundary_margin=0.08) + outcome = await router.aclassify(CLEAR_OF_BOUNDARY_PROMPT) + mock_router_instance.acompletion.assert_awaited_once() + assert outcome.cause == "llm_classifier" + + @pytest.mark.asyncio + async def test_a_zero_margin_escalates_only_an_exact_boundary_score(self, mock_router_instance): + """0 is a real margin, not an off switch: a score sitting exactly on the line still escalates. + + The boundary is spelled as the scorer's own accumulated float rather than the 0.075 it prints + as, because the comparison is on raw floats: a boundary written 0.075 sits 1.4e-17 away from + this score and a zero margin correctly declines to call that exact.""" + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "MEDIUM"}')) + on_the_line = 0.07499999999999998 + router = _hybrid_router( + mock_router_instance, + tier_boundaries={"simple_medium": on_the_line, "medium_complex": 0.35, "complex_reasoning": 0.60}, + hybrid_boundary_margin=0, + ) + + _tier, score, _signals, _cause = router._score_and_classify(CLEAR_OF_BOUNDARY_PROMPT) + assert score == on_the_line + + outcome = await router.aclassify(CLEAR_OF_BOUNDARY_PROMPT) + mock_router_instance.acompletion.assert_awaited_once() + assert outcome.cause == "llm_classifier" + + @pytest.mark.asyncio + async def test_no_signal_prompt_escalates_however_far_from_a_boundary(self, mock_router_instance): + """The scorer with no opinion has no tier to be confident about, so proximity cannot save it.""" + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}')) + router = _hybrid_router(mock_router_instance) + + tier, score, signals, _cause = router._score_and_classify(NO_SIGNAL_PROMPT) + assert (tier, score, signals) == (ComplexityTier.SIMPLE, 0.0, ()) + + outcome = await router.aclassify(NO_SIGNAL_PROMPT) + mock_router_instance.acompletion.assert_awaited_once() + assert outcome.cause == "llm_classifier" + + @pytest.mark.asyncio + async def test_classifier_failure_falls_back_to_the_scorer(self, mock_router_instance): + mock_router_instance.acompletion = AsyncMock(side_effect=RuntimeError("classifier exploded")) + router = _hybrid_router(mock_router_instance) + expected_tier, expected_score, expected_signals, _cause = router._score_and_classify(NEAR_BOUNDARY_PROMPT) + + outcome = await router.aclassify(NEAR_BOUNDARY_PROMPT) + + assert (outcome.tier, outcome.score, outcome.signals) == (expected_tier, expected_score, expected_signals) + assert outcome.cause == "heuristic_scorer" + + def _windowed_router(*deployments: tuple) -> Router: """Real Router; each deployment is (group, provider_model, declared window or None). None means no declared override on a model the cost map does not know: unresolvable.""" @@ -10268,7 +10555,8 @@ class TestContextWindowEscalation: litellm_router_instance=_windowed_router(_SMALL, _BIG), complexity_router_config=_tier_config(session_affinity=True), ) - session_kwargs = lambda: {"metadata": {"session_id": "s-1", "user_api_key_hash": "k-1"}} # noqa: E731 + def session_kwargs() -> dict[str, object]: + return {"metadata": {"session_id": "s-1", "user_api_key_hash": "k-1"}} first = await router.async_pre_routing_hook( model="test-router", request_kwargs=session_kwargs(), messages=_OVERSIZED_TURNS @@ -10291,7 +10579,8 @@ class TestContextWindowEscalation: litellm_router_instance=_windowed_router(_SMALL, _BIG), complexity_router_config=_tier_config(session_affinity=True), ) - session_kwargs = lambda: {"metadata": {"session_id": "s-2", "user_api_key_hash": "k-2"}} # noqa: E731 + def session_kwargs() -> dict[str, object]: + return {"metadata": {"session_id": "s-2", "user_api_key_hash": "k-2"}} pinned = await router.async_pre_routing_hook( model="test-router", request_kwargs=session_kwargs(), messages=[{"role": "user", "content": "ok continue"}] @@ -10508,6 +10797,9 @@ class TestModalityRouting: ("custom_tiers_walk", "premium-model", "modality_escalation"), ("pin_kept_bypasses", "text-cheap", "session_affinity_pin"), ("pin_replacement_gated", "vision-big", "modality_escalation"), + ("pin_override_escalates", "vision-mid", "modality_pin_override"), + ("pin_override_same_tier", "vision-cheap", "modality_pin_override"), + ("pin_override_inert_without_modality_routing", "text-cheap", "session_affinity_pin"), ("adaptive_pick_rewritten", "vision-mid", "modality_escalation"), ], ) @@ -10558,7 +10850,7 @@ class TestModalityRouting: messages = [ {"role": "user", "content": [{"type": "text", "text": "quick lookup: what is this?"}, IMG_PART]} ] - elif path in ("pin_kept_bypasses", "pin_replacement_gated"): + elif path.startswith(("pin_kept", "pin_replacement", "pin_override")): cache = AsyncMock() cache.async_get_cache = AsyncMock(return_value={"model": "text-cheap", "tier": "SIMPLE"}) mock_router_instance.cache = cache @@ -10570,6 +10862,13 @@ class TestModalityRouting: messages = [ {"role": "user", "content": [{"type": "text", "text": "LITELLM ESCALATE describe this"}, IMG_PART]} ] + elif path == "pin_override_same_tier": + config["modality_pin_override"] = True + config["tiers"]["SIMPLE"] = ["text-cheap", "vision-cheap"] + vision["vision-cheap"] = True + elif path == "pin_override_inert_without_modality_routing": + config["modality_routing"] = False + config["modality_pin_override"] = path.startswith("pin_override") elif path == "adaptive_pick_rewritten": config["adaptive"] = True mock_router_instance.model_list = [] @@ -10736,4 +11035,60 @@ class TestModalityRouting: from litellm.router_strategy.complexity_router.complexity_router import _decision_is_pinnable assert _decision_is_pinnable({"cause": "modality_escalation"}) is False + assert _decision_is_pinnable({"cause": "modality_pin_override"}) is False assert _decision_is_pinnable({"cause": "heuristic_scorer"}) is True + + @pytest.mark.asyncio + async def test_pin_override_serves_the_image_turn_without_repinning(self, mock_router_instance): + """The override is for one request: the session keeps the model it was pinned to.""" + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value={"model": "text-cheap", "tier": "SIMPLE"}) + mock_router_instance.cache = cache + router = self._router( + mock_router_instance, + { + "tiers": dict(self.BASE_TIERS), + "modality_routing": True, + "modality_pin_override": True, + "session_affinity": True, + }, + dict(self.BASE_VISION), + ) + request_kwargs = {"metadata": {"session_id": "s1"}} + + image_turn = await router.async_pre_routing_hook( + model="m", request_kwargs=request_kwargs, messages=self.IMAGE_MESSAGE + ) + assert image_turn.model == "vision-mid" + assert image_turn.routing_decision["cause"] == "modality_pin_override" + assert "modality_escalated_from:SIMPLE" in image_turn.routing_decision["signals"] + + assert cache.async_set_cache.await_args.kwargs["value"] == {"model": "text-cheap", "tier": "SIMPLE"} + + text_turn = await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "s1"}}, messages=[{"role": "user", "content": "hi"}] + ) + assert text_turn.model == "text-cheap" + assert text_turn.routing_decision["cause"] == "session_affinity_pin" + + @pytest.mark.asyncio + async def test_pin_override_with_no_capable_model_rejects_and_keeps_the_pin(self, mock_router_instance): + """The clear 400 replaces the provider's, and a rejected turn must not cost the session its pin.""" + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value={"model": "text-cheap", "tier": "SIMPLE"}) + mock_router_instance.cache = cache + router = self._router( + mock_router_instance, + { + "tiers": {"SIMPLE": "text-cheap", "COMPLEX": "text-big"}, + "modality_routing": True, + "modality_pin_override": True, + "session_affinity": True, + }, + {"text-cheap": False, "text-big": False}, + ) + with pytest.raises(litellm.BadRequestError, match="no model"): + await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "s1"}}, messages=self.IMAGE_MESSAGE + ) + assert cache.async_set_cache.await_args.kwargs["value"] == {"model": "text-cheap", "tier": "SIMPLE"} diff --git a/tests/test_litellm/router_strategy/test_complexity_tier_predictor.py b/tests/test_litellm/router_strategy/test_complexity_tier_predictor.py new file mode 100644 index 00000000000..5bbe0fb5669 --- /dev/null +++ b/tests/test_litellm/router_strategy/test_complexity_tier_predictor.py @@ -0,0 +1,91 @@ +from typing import Final + +import pytest + +from litellm.router_strategy.complexity_router.tier_predictor import ( + TierCohortStatistic, + TierDomainStatistic, + TierGlobalStatistic, + TierSuccessPredictor, + TrainedTierArtifact, + resolve_tier_artifact, + similarity_cohort, +) +from litellm.types.router import RequestType + + +def _artifact( + global_successes: tuple[float, float, float, float] = (4.0, 5.0, 6.0, 7.0), + threshold: float = 0.75, + domain_statistics: tuple[TierDomainStatistic, ...] = (), + cohort_statistics: tuple[TierCohortStatistic, ...] = (), +) -> TrainedTierArtifact: + return TrainedTierArtifact( + global_statistics=tuple( + TierGlobalStatistic(tier=tier, successes=successes, observations=10.0) + for tier, successes in enumerate(global_successes, start=1) + ), + domain_statistics=domain_statistics, + cohort_statistics=cohort_statistics, + domain_prior_mass=10.0, + cohort_prior_mass=10.0, + routing_threshold=threshold, + ) + + +def test_predictions_are_monotonic_across_tiers() -> None: + predictor: Final = TierSuccessPredictor(_artifact(global_successes=(9.0, 2.0, 7.0, 6.0))) + + prediction: Final = predictor.predict("hello", RequestType.GENERAL) + + probabilities: Final = tuple(prediction.probabilities.values()) + assert probabilities == tuple(sorted(probabilities)) + + +def test_domain_and_cohort_statistics_back_off_hierarchically() -> None: + matching_cohort: Final = similarity_cohort("hello", RequestType.GENERAL) + artifact: Final = _artifact( + global_successes=(1.0, 5.0, 6.0, 7.0), + domain_statistics=( + TierDomainStatistic( + tier=1, + request_type=RequestType.GENERAL, + successes=10.0, + observations=10.0, + ), + ), + cohort_statistics=( + TierCohortStatistic( + tier=1, + cohort=matching_cohort, + successes=0.0, + observations=10.0, + ), + ), + ) + predictor: Final = TierSuccessPredictor(artifact) + + cohort_probability: Final = predictor.predict("hello", RequestType.GENERAL).probabilities[1] + domain_probability: Final = predictor.predict("hello " * 100, RequestType.GENERAL).probabilities[1] + global_probability: Final = predictor.predict("hello", RequestType.WRITING).probabilities[1] + + assert cohort_probability == pytest.approx(7.0 / 24.0) + assert domain_probability == pytest.approx(7.0 / 12.0) + assert global_probability == pytest.approx(1.0 / 6.0) + + +def test_selects_first_tier_above_probability_threshold() -> None: + predictor: Final = TierSuccessPredictor(_artifact(global_successes=(4.0, 6.0, 8.0, 9.0), threshold=0.7)) + + prediction: Final = predictor.predict("hello", RequestType.GENERAL) + + assert prediction.required_tier == 3 + + +def test_builtin_ultrafeedback_artifact_is_loadable() -> None: + artifact: Final = resolve_tier_artifact("ultrafeedback") + + assert artifact.routing_threshold == 0.75 + assert artifact.domain_prior_mass == 200.0 + assert artifact.cohort_prior_mass == 20.0 + assert artifact.datasets[0].license == "MIT" diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py b/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py index a3772a276fa..cf48888600e 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py @@ -7,7 +7,7 @@ import json import litellm from litellm.caching.dual_cache import DualCache -from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY +from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, ) @@ -180,6 +180,47 @@ async def test_async_session_id_affinity_priority_over_user_key(): assert filtered[0]["model_info"]["id"] == "deployment-2" +@pytest.mark.asyncio +async def test_proxy_generated_session_id_does_not_pin_a_deployment(): + """A session id the proxy generated for a request that had none is per request, so a + pin stored under it must be ignored and none must be written.""" + cache = DualCache() + callback = DeploymentAffinityCheck( + cache=cache, + ttl_seconds=123, + enable_user_key_affinity=False, + enable_responses_api_affinity=False, + enable_session_id_affinity=True, + ) + healthy_deployments = [ + {"model_name": "model_group", "litellm_params": {"model": "model_1"}, "model_info": {"id": "deployment-1"}}, + {"model_name": "model_group", "litellm_params": {"model": "model_2"}, "model_info": {"id": "deployment-2"}}, + ] + await cache.async_set_cache( + DeploymentAffinityCheck.get_session_affinity_cache_key("model_group", "generated-1", user_key="user1"), + {"model_id": "deployment-2"}, + ) + request_kwargs = { + "metadata": {"user_api_key_hash": "user1", "session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True} + } + + filtered = await callback.async_filter_deployments( + model="model_group", healthy_deployments=healthy_deployments, messages=[], request_kwargs=request_kwargs + ) + await callback.async_pre_call_deployment_hook( + kwargs={ + "metadata": {**request_kwargs["metadata"], "deployment_model_name": "model_group"}, + "model_info": {"id": "deployment-1"}, + }, + call_type=None, + ) + + assert len(filtered) == 2 + assert await cache.async_get_cache( + DeploymentAffinityCheck.get_session_affinity_cache_key("model_group", "generated-1", user_key="user1") + ) == {"model_id": "deployment-2"} + + MOCK_RESPONSES_API_RESPONSE = { "id": "resp_mock-resp-456", "object": "response", diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 894b2d9e74f..a4965c49f07 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -614,6 +614,27 @@ async def test_run_async_fallback_forwards_attempted_model_groups_to_nested_call ) +@pytest.mark.asyncio +async def test_run_async_fallback_can_target_the_requested_group_when_a_pre_router_replaced_it(): + """The requested group was never called when a pre-router selected a tier, so a + tier fallback may legitimately target that originally requested group.""" + router = RecordingRouter() + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["requested-model"], + original_model_group="requested-model", + original_exception=RuntimeError("selected tier failed"), + max_fallbacks=3, + fallback_depth=0, + model="requested-model", + metadata={"pre_routing_selected_model": "selected-tier"}, + ) + + assert router.received_kwargs["model"] == "requested-model" + assert router.received_kwargs["attempted_targets"].keys == frozenset({"selected-tier", "requested-model"}) + + @pytest.mark.asyncio @pytest.mark.parametrize( "entry", @@ -1199,6 +1220,28 @@ class TestOrderedFallbackLookupGroups: assert fallback_lookup_groups({}, "smart-router") == ("smart-router",) assert fallback_lookup_groups({}, None) == () + def test_session_remap_keeps_the_bound_router_between_tier_and_requested_group(self): + from litellm.router_utils.fallback_event_handlers import ( + PRE_ROUTING_SELECTED_MODEL_KEY, + fallback_lookup_groups, + ) + + kwargs = { + "litellm_metadata": { + PRE_ROUTING_SELECTED_MODEL_KEY: "tier1", + "model_group": "smart-router", + } + } + + assert fallback_lookup_groups(kwargs, "requested-model") == ( + "tier1", + "smart-router", + "requested-model", + ) + assert fallback_lookup_groups({"metadata": {"model_group": []}}, "requested-model") == ( + "requested-model", + ) + def test_first_resolving_group_wins_and_generic_idx_survives_a_miss(self): from litellm.router_utils.fallback_event_handlers import ( get_fallback_model_group_for_lookup_groups, diff --git a/tests/test_litellm/rust_bridge/native_route_wheel_test.py b/tests/test_litellm/rust_bridge/native_route_wheel_test.py new file mode 100644 index 00000000000..0ce49d51aed --- /dev/null +++ b/tests/test_litellm/rust_bridge/native_route_wheel_test.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +import asyncio +import importlib.util +import json +import os +import signal +import subprocess +import sys +import tempfile +import threading +import zipfile +from http.client import HTTPMessage +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from socket import socket as Socket +from typing import Final + +REQUEST_STARTED: Final = threading.Event() +REQUEST_CANCELLED: Final = threading.Event() + +ANTHROPIC_RESPONSE: Final = ( + b'{"id":"msg_native","type":"message","role":"assistant",' + b'"model":"claude-sonnet-4-5","content":[{"type":"text","text":"native-message"}],' + b'"stop_reason":"end_turn","stop_sequence":null,' + b'"usage":{"input_tokens":2,"output_tokens":3}}' +) + + +class NativeRouteHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + content_length: Final = int(self.headers.get("content-length", "0")) + body: Final = json.loads(self.rfile.read(content_length)) + route: Final = self.headers.get("x-test-route") + outcome: Final = self.headers.get("x-test-outcome") + assert_native_request(route, outcome, self.path, self.headers, body) + if outcome == "hang": + REQUEST_STARTED.set() + self.connection.settimeout(5) + if connection_was_cancelled(self.connection): + REQUEST_CANCELLED.set() + return + + status: Final = 429 if outcome == "429" else 200 + response_body: Final = native_response(status, route) + + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(response_body))) + self.send_header("connection", "close") + self.end_headers() + self.wfile.write(response_body) + + def log_message(self, _message_format: str, *_args: object) -> None: + pass + + +def connection_was_cancelled(connection: Socket) -> bool: + try: + return connection.recv(1) == b"" + except TimeoutError: + return False + except OSError: + return True + + +def assert_native_request( + route: str | None, + outcome: str | None, + path: str, + headers: HTTPMessage, + body: object, +) -> None: + if route not in {"ocr", "transcription", "messages", "chat_completions"}: + raise AssertionError(f"unexpected route marker: {route!r}") + if outcome not in {"success", "429", "hang"}: + raise AssertionError(f"unexpected outcome marker: {outcome!r}") + if not isinstance(body, dict): + raise TypeError(f"{route} sent {type(body).__name__}, expected a JSON object") + if route == "ocr": + assert path == "/v1/ocr" + assert headers.get("authorization") == "Bearer sk-native" + assert body["model"] == "mistral-ocr-latest" + assert body["document"]["document_url"] == "https://example.com/document.pdf" + assert body["include_image_base64"] is True + return + if route == "transcription": + assert path == "/model/mistral.voxtral-mini-3b-2507/converse" + assert headers.get("authorization", "").startswith("AWS4-HMAC-SHA256 ") + assert headers.get("x-amz-date") + assert body["messages"][0]["content"][0]["audio"]["source"]["bytes"] == "AQI=" + assert "The audio language is en" in body["messages"][0]["content"][1]["text"] + return + assert path == "/v1/messages" + assert headers.get("x-api-key") == "sk-native" + assert body["model"] == "claude-sonnet-4-5" + if route == "messages": + assert body["max_tokens"] == 16 + assert body["messages"][0]["content"] == "hello-from-messages" + return + assert body["max_tokens"] == 17 + assert body["messages"][0]["content"] == [{"type": "text", "text": "hello-from-chat"}] + + +def native_response(status: int, route: str | None) -> bytes: + if status == 429: + return b'{"error":"native-rate-limit"}' + if route == "ocr": + return b'{"pages":[{"index":0,"markdown":"native-ocr"}]}' + if route == "transcription": + return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}' + return ANTHROPIC_RESPONSE + + +def load_native(native_path: Path) -> object: + module_spec: Final = importlib.util.spec_from_file_location("litellm.rust_bridge._native", native_path) + if module_spec is None or module_spec.loader is None: + raise RuntimeError("cannot create native extension import specification") + native_module: Final = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(native_module) + return native_module + + +def route_kwargs(route: str, api_base: str, outcome: str) -> dict[str, object]: + common: Final = { + "api_base": api_base, + "extra_headers": {"x-test-outcome": outcome, "x-test-route": route}, + "timeout_seconds": 3.0, + } + if route == "ocr": + return common | { + "model": "mistral-ocr-latest", + "document": {"type": "document_url", "document_url": "https://example.com/document.pdf"}, + "api_key": "sk-native", + "custom_llm_provider": "mistral", + "optional_params": {"include_image_base64": True}, + } + if route == "transcription": + return common | { + "model": "mistral.voxtral-mini-3b-2507", + "audio": {"data": "AQI=", "format": "wav", "filename": "audio.wav"}, + "custom_llm_provider": "bedrock", + "optional_params": { + "aws_access_key_id": "native-access-key", + "aws_secret_access_key": "native-secret-key", + "aws_region_name": "us-east-1", + "language": "en", + }, + } + if route == "messages": + return common | { + "model": "claude-sonnet-4-5", + "body": { + "model": "claude-sonnet-4-5", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hello-from-messages"}], + }, + "api_key": "sk-native", + "custom_llm_provider": "anthropic", + } + if route == "chat_completions": + return common | { + "model": "anthropic/claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hello-from-chat"}], + "optional_params": {"max_tokens": 17}, + "api_key": "sk-native", + } + raise AssertionError(f"unknown route: {route}") + + +def assert_success(route: str, response: object) -> None: + if not isinstance(response, dict): + raise TypeError(f"{route} returned {type(response).__name__}, expected dict") + actual: Final = success_value(route, response) + expected: Final = ( + "native-ocr" if route == "ocr" else "native-transcription" if route == "transcription" else "native-message" + ) + if actual != expected: + raise AssertionError(f"{route} returned {actual!r}, expected {expected!r}") + + +def assert_traced_success(route: str, response: object) -> None: + if not isinstance(response, dict): + raise TypeError(f"{route} returned {type(response).__name__}, expected a traced dict") + assert_success(route, response["response"]) + expected_function: Final = "audio_transcription" if route == "transcription" else route + assert response["trace"][0] == {"function": expected_function, "depth": 0} + + +def success_value(route: str, response: dict[object, object]) -> object: + if route == "ocr": + return response["pages"][0]["markdown"] + if route == "transcription": + return response["text"] + if route == "messages": + return response["content"][0]["text"] + return response["choices"][0]["message"]["content"] + + +def assert_rate_limit(native: object, route: str, error: BaseException) -> None: + if route == "chat_completions": + upstream_error: Final = native.RustUpstreamError + if not isinstance(error, upstream_error) or error.args[0] != 429: + raise AssertionError(f"{route} returned the wrong 429 error: {error!r}") + return + if not isinstance(error, RuntimeError) or "429" not in str(error): + raise AssertionError(f"{route} returned the wrong 429 error: {error!r}") + + +def exercise_sync(native: object, api_base: str) -> None: + for route in ("ocr", "transcription", "messages", "chat_completions"): + function: Final = getattr(native, route) + assert_success(route, function(**route_kwargs(route, api_base, "success"))) + assert_traced_success(route, function(**route_kwargs(route, api_base, "success"), trace=True)) + try: + function(**route_kwargs(route, api_base, "429")) + except (RuntimeError, native.RustUpstreamError) as error: + assert_rate_limit(native, route, error) + else: + raise AssertionError(f"{route} accepted a 429 response") + + +async def exercise_async(native: object, api_base: str) -> None: + for route in ("ocr", "transcription", "messages", "chat_completions"): + function: Final = getattr(native, f"a{route}") + assert_success(route, await function(**route_kwargs(route, api_base, "success"))) + assert_traced_success(route, await function(**route_kwargs(route, api_base, "success"), trace=True)) + try: + await function(**route_kwargs(route, api_base, "429")) + except (RuntimeError, native.RustUpstreamError) as error: + assert_rate_limit(native, route, error) + else: + raise AssertionError(f"a{route} accepted a 429 response") + + +async def exercise_async_concurrency(native: object, api_base: str) -> None: + responses: Final = await asyncio.wait_for( + asyncio.gather( + *( + native.amessages(**route_kwargs("messages", api_base, "success")) + for _ in range(32) + ) + ), + timeout=15, + ) + for response in responses: + assert_success("messages", response) + + +def exercise_routes(native_path: Path, api_base: str) -> object: + native: Final = load_native(native_path) + exercise_sync(native, api_base) + asyncio.run(exercise_async(native, api_base)) + asyncio.run(exercise_async_concurrency(native, api_base)) + return native + + +def exercise_signal(native: object, api_base: str) -> int: + try: + native.messages( + **route_kwargs("messages", api_base, "hang"), + ) + except KeyboardInterrupt: + sys.stdout.write("KeyboardInterrupt\n") + sys.stdout.flush() + sys.stdin.read(1) + return 0 + raise AssertionError("sync native route ignored SIGINT") + + +def verify_sigint(native_path: Path, api_base: str) -> None: + REQUEST_STARTED.clear() + REQUEST_CANCELLED.clear() + process: Final = subprocess.Popen( + (sys.executable, __file__, "child", str(native_path), api_base), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + if not REQUEST_STARTED.wait(30): + process.kill() + stdout, stderr = process.communicate(timeout=5) + raise AssertionError( + f"native route matrix did not reach the hanging upstream\nstdout:\n{stdout}\nstderr:\n{stderr}" + ) + os.kill(process.pid, signal.SIGINT) + if not REQUEST_CANCELLED.wait(5): + raise AssertionError("interrupted native route did not cancel its upstream future") + if process.poll() is not None: + raise AssertionError("signal child exited before cancellation was observed") + stdout, stderr = process.communicate(input="\n", timeout=5) + if process.returncode != 0 or stdout != "KeyboardInterrupt\n": + raise AssertionError( + f"signal child exited with status {process.returncode}\nstdout:\n{stdout}\nstderr:\n{stderr}" + ) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + + +def verify_wheel(wheel: Path) -> int: + with tempfile.TemporaryDirectory() as temporary_directory, zipfile.ZipFile(wheel) as archive: + wheel_root: Final = Path(temporary_directory) + for member in archive.infolist(): + target: Final = wheel_root / member.filename + if member.is_dir(): + target.mkdir(parents=True, exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(archive.read(member)) + native_members: Final = tuple( + member + for member in archive.infolist() + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + raise AssertionError(f"expected one native extension, found {len(native_members)}") + native_path: Final = wheel_root / native_members[0].filename + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), NativeRouteHandler) + server_thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + api_base: Final = f"http://127.0.0.1:{server.server_address[1]}" + try: + verify_sigint(native_path, api_base) + finally: + server.shutdown() + server.server_close() + server_thread.join(timeout=5) + return 0 + + +def main() -> int: + if len(sys.argv) == 2: + return verify_wheel(Path(sys.argv[1])) + if len(sys.argv) == 4 and sys.argv[1] == "child": + native: Final = exercise_routes(Path(sys.argv[2]), sys.argv[3]) + return exercise_signal(native, sys.argv[3]) + sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_litellm/rust_bridge/test_bindings.py b/tests/test_litellm/rust_bridge/test_bindings.py new file mode 100644 index 00000000000..88036a5a556 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_bindings.py @@ -0,0 +1,35 @@ +from types import SimpleNamespace +from typing import Final + +import pytest + +from litellm.rust_bridge import bindings + + +def test_binding_distinguishes_disable_from_reset(monkeypatch) -> None: + native = SimpleNamespace(route=lambda: "native") + monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) + binding: bindings.NativeBinding[object] = bindings.NativeBinding("route", validate=lambda value: value) + + assert binding.load() is native.route + + binding.override(None) + assert binding.load() is None + + replacement = object() + binding.override(replacement) + assert binding.load() is replacement + + binding.reset() + assert binding.load() is native.route + + +@pytest.mark.parametrize(("value", "expected"), ((3, 3), ("invalid", None), (None, None))) +def test_binding_validates_native_attribute( + monkeypatch: pytest.MonkeyPatch, value: object, expected: int | None +) -> None: + native: Final = SimpleNamespace(route=value) + monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) + binding: Final = bindings.NativeBinding("route", validate=lambda item: item if isinstance(item, int) else None) + + assert binding.load() == expected diff --git a/tests/test_litellm/rust_bridge/test_chat_completions.py b/tests/test_litellm/rust_bridge/test_chat_completions.py index 47cb66932b7..03921133c77 100644 --- a/tests/test_litellm/rust_bridge/test_chat_completions.py +++ b/tests/test_litellm/rust_bridge/test_chat_completions.py @@ -11,6 +11,7 @@ import pytest import litellm from litellm.rust_bridge import chat_completions as bridge +from litellm.rust_bridge import configuration from litellm.types.utils import ModelResponse RUST_RESPONSE = { @@ -68,13 +69,11 @@ def _hide_native_bridge(monkeypatch): @pytest.fixture(autouse=True) def reset_bridge(): """Every test starts with no injected callables, and leaves none behind.""" - bridge.set_rust_chat_completions( - chat_completions=None, achat_completions=None, decline=None - ) + bridge.set_rust_chat_completions(chat_completions=None, achat_completions=None, decline=None) + configuration.reset_rust_configuration() yield - bridge.set_rust_chat_completions( - chat_completions=None, achat_completions=None, decline=None - ) + bridge.set_rust_chat_completions(chat_completions=None, achat_completions=None, decline=None) + configuration.reset_rust_configuration() class _RecordingDecline: @@ -138,6 +137,18 @@ class TestGate: assert gate.calls[0]["model"] == "claude-sonnet-4-5" assert gate.calls[0]["custom_llm_provider"] == "anthropic" + def test_explicit_false_overrides_process_enable(self): + bridge.set_rust_chat_completions(decline=_RecordingDecline()) + configuration.use_litellm_rust(True) + + assert _accepts(litellm_params={"rust": False}) is False + + def test_process_enable_applies_without_request_override(self): + bridge.set_rust_chat_completions(decline=_RecordingDecline()) + configuration.use_litellm_rust(True) + + assert _accepts(litellm_params={}) is True + def test_the_env_var_opts_in_without_a_per_model_flag(self, monkeypatch): monkeypatch.setenv("LITELLM_RUST", "true") bridge.set_rust_chat_completions(decline=_RecordingDecline()) @@ -253,9 +264,7 @@ class TestSyncCall: assert result.usage.completion_tokens == 4 assert result.usage.total_tokens == 15 assert result._hidden_params["additional_headers"] == {"x-litellm-rust": "true"} - assert result.id == original_id, ( - "the rust path must keep the chatcmpl id litellm already minted" - ) + assert result.id == original_id, "the rust path must keep the chatcmpl id litellm already minted" def test_passes_the_timeout_through_as_seconds(self): native = _RecordingCall() @@ -269,9 +278,7 @@ class TestSyncCall: def test_falls_back_when_the_core_declines_before_calling_the_provider(self, monkeypatch): _fake_native_bridge(monkeypatch) - bridge.set_rust_chat_completions( - chat_completions=_RecordingCall(error=_FakeDeclined("streaming")) - ) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeDeclined("streaming"))) assert bridge.chat_completions(**_call_kwargs(ModelResponse())) is None @@ -290,13 +297,9 @@ class TestAsyncCall: assert await bridge.achat_completions(**_call_kwargs(ModelResponse())) is None @pytest.mark.asyncio - async def test_falls_back_when_the_core_declines_before_calling_the_provider( - self, monkeypatch - ): + async def test_falls_back_when_the_core_declines_before_calling_the_provider(self, monkeypatch): _fake_native_bridge(monkeypatch) - bridge.set_rust_chat_completions( - achat_completions=_RecordingAsyncCall(error=_FakeDeclined("streaming")) - ) + bridge.set_rust_chat_completions(achat_completions=_RecordingAsyncCall(error=_FakeDeclined("streaming"))) assert await bridge.achat_completions(**_call_kwargs(ModelResponse())) is None @@ -310,25 +313,19 @@ class TestAsyncFallbackWrapper: ran.append(True) return "python" - result = await bridge.achat_completions_or_fallback( - **_call_kwargs(ModelResponse()), python_fallback=fallback - ) + result = await bridge.achat_completions_or_fallback(**_call_kwargs(ModelResponse()), python_fallback=fallback) assert result.choices[0].message.content == "hello from rust" assert ran == [] @pytest.mark.asyncio async def test_runs_the_fallback_when_the_core_declines(self, monkeypatch): _fake_native_bridge(monkeypatch) - bridge.set_rust_chat_completions( - achat_completions=_RecordingAsyncCall(error=_FakeDeclined("streaming")) - ) + bridge.set_rust_chat_completions(achat_completions=_RecordingAsyncCall(error=_FakeDeclined("streaming"))) async def fallback(): return "python" - result = await bridge.achat_completions_or_fallback( - **_call_kwargs(ModelResponse()), python_fallback=fallback - ) + result = await bridge.achat_completions_or_fallback(**_call_kwargs(ModelResponse()), python_fallback=fallback) assert result == "python" @pytest.mark.asyncio @@ -338,9 +335,7 @@ class TestAsyncFallbackWrapper: async def fallback(): return "python" - result = await bridge.achat_completions_or_fallback( - **_call_kwargs(ModelResponse()), python_fallback=fallback - ) + result = await bridge.achat_completions_or_fallback(**_call_kwargs(ModelResponse()), python_fallback=fallback) assert result == "python" @@ -353,17 +348,13 @@ class TestFailureClassification: _fake_native_bridge(monkeypatch) def test_a_decline_falls_back_because_nothing_was_sent(self): - bridge.set_rust_chat_completions( - chat_completions=_RecordingCall(error=_FakeDeclined("streaming")) - ) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeDeclined("streaming"))) assert bridge.chat_completions(**_call_kwargs(ModelResponse())) is None def test_an_upstream_failure_is_surfaced_with_its_status(self): from litellm.exceptions import APIError - bridge.set_rust_chat_completions( - chat_completions=_RecordingCall(error=_FakeUpstream(429, "429: rate limited")) - ) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeUpstream(429, "429: rate limited"))) with pytest.raises(APIError) as raised: bridge.chat_completions(**_call_kwargs(ModelResponse())) assert raised.value.status_code == 429 @@ -372,17 +363,13 @@ class TestFailureClassification: def test_a_transport_failure_with_no_response_surfaces_as_a_500(self): from litellm.exceptions import APIError - bridge.set_rust_chat_completions( - chat_completions=_RecordingCall(error=_FakeUpstream(0, "connection reset")) - ) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=_FakeUpstream(0, "connection reset"))) with pytest.raises(APIError) as raised: bridge.chat_completions(**_call_kwargs(ModelResponse())) assert raised.value.status_code == 500 def test_an_unrecognized_error_is_not_swallowed(self): - bridge.set_rust_chat_completions( - chat_completions=_RecordingCall(error=RuntimeError("something else")) - ) + bridge.set_rust_chat_completions(chat_completions=_RecordingCall(error=RuntimeError("something else"))) with pytest.raises(RuntimeError): bridge.chat_completions(**_call_kwargs(ModelResponse())) @@ -390,9 +377,7 @@ class TestFailureClassification: async def test_the_async_wrapper_does_not_fall_back_on_an_upstream_failure(self): from litellm.exceptions import APIError - bridge.set_rust_chat_completions( - achat_completions=_RecordingAsyncCall(error=_FakeUpstream(500, "500: boom")) - ) + bridge.set_rust_chat_completions(achat_completions=_RecordingAsyncCall(error=_FakeUpstream(500, "500: boom"))) ran = [] async def fallback(): @@ -400,9 +385,7 @@ class TestFailureClassification: return "python" with pytest.raises(APIError): - await bridge.achat_completions_or_fallback( - **_call_kwargs(ModelResponse()), python_fallback=fallback - ) + await bridge.achat_completions_or_fallback(**_call_kwargs(ModelResponse()), python_fallback=fallback) assert ran == [], "a request the provider already served must not be re-issued" @pytest.mark.asyncio @@ -414,7 +397,5 @@ class TestFailureClassification: async def fallback(): return "python" - result = await bridge.achat_completions_or_fallback( - **_call_kwargs(ModelResponse()), python_fallback=fallback - ) + result = await bridge.achat_completions_or_fallback(**_call_kwargs(ModelResponse()), python_fallback=fallback) assert result == "python" diff --git a/tests/test_litellm/rust_bridge/test_configuration.py b/tests/test_litellm/rust_bridge/test_configuration.py new file mode 100644 index 00000000000..1c81c1fb624 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_configuration.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from collections.abc import Generator +from concurrent.futures import ThreadPoolExecutor +from typing import Final + +import pytest + +from litellm.rust_bridge import configuration +from litellm.rust_bridge import ocr as rust_ocr + + +class _OcrBridge: + def __call__( + self, + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str | None, + extra_headers: dict[str, object] | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> dict[str, object]: + return {} + + +@pytest.fixture(autouse=True) +def _isolated_configuration( # pyright: ignore[reportUnusedFunction] # pytest discovers fixtures dynamically + monkeypatch: pytest.MonkeyPatch, +) -> Generator[None]: + configuration.reset_rust_configuration() + monkeypatch.delenv("LITELLM_RUST", raising=False) + monkeypatch.delenv("LITELLM_USE_RUST_OCR", raising=False) + rust_ocr.set_rust_ocr(ocr=None, aocr=None) + yield + configuration.reset_rust_configuration() + rust_ocr.set_rust_ocr(ocr=None, aocr=None) + + +@pytest.mark.parametrize( + ("request_override", "process", "environment", "legacy_ocr", "release_default", "expected"), + ( + (False, True, True, True, True, False), + (True, False, False, False, False, True), + (None, False, True, True, True, False), + (None, True, False, False, False, True), + (None, None, False, True, True, False), + (None, None, True, False, False, True), + (None, None, None, False, True, False), + (None, None, None, True, False, True), + (None, None, None, None, False, False), + (None, None, None, None, True, True), + ), +) +def test_resolution_precedence( + request_override: bool | None, + process: bool | None, + environment: bool | None, + legacy_ocr: bool | None, + release_default: bool, + expected: bool, +) -> None: + assert ( + configuration.resolve_rust_enabled( + request_override=request_override, + process_override=process, + environment_override=environment, + legacy_ocr_override=legacy_ocr, + release_default=release_default, + ) + is expected + ) + + +def test_release_default_remains_disabled() -> None: + assert configuration.DEFAULT_RUST_ENABLED is False + assert configuration.rust_enabled() is False + + +def test_process_override_wins_over_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "0") + configuration.use_litellm_rust(True) + + assert configuration.rust_enabled() is True + assert configuration.rust_enabled(request_override=False) is False + + +def test_global_environment_accepts_explicit_false(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "off") + + assert configuration.rust_enabled() is False + + +@pytest.mark.parametrize("value", ("", " ", "sometimes", "2")) +def test_invalid_environment_value_disables_rust(monkeypatch: pytest.MonkeyPatch, value: str) -> None: + monkeypatch.setenv("LITELLM_RUST", value) + monkeypatch.setenv("LITELLM_USE_RUST_OCR", "1") + + assert configuration.rust_enabled() is False + assert configuration.rust_ocr_enabled() is False + + +@pytest.mark.parametrize("value", ("", " ", "sometimes", "2")) +def test_invalid_legacy_environment_value_disables_ocr(monkeypatch: pytest.MonkeyPatch, value: str) -> None: + monkeypatch.setenv("LITELLM_USE_RUST_OCR", value) + + with pytest.warns(DeprecationWarning, match="LITELLM_USE_RUST_OCR is deprecated"): + assert configuration.rust_ocr_enabled() is False + + +def test_process_override_and_reset_apply_to_existing_threads(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "1") + + with ThreadPoolExecutor(max_workers=1) as executor: + assert executor.submit(configuration.rust_enabled).result() is True + configuration.use_litellm_rust(False) + assert executor.submit(configuration.rust_enabled).result() is False + assert executor.submit(configuration.rust_ocr_enabled).result() is False + configuration.reset_rust_configuration() + assert executor.submit(configuration.rust_enabled).result() is True + assert executor.submit(configuration.rust_ocr_enabled).result() is True + + +def test_explicit_override_precedes_invalid_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "sometimes") + + assert configuration.rust_enabled(request_override=False) is False + configuration.use_litellm_rust(True) + assert configuration.rust_enabled() is True + + +def test_legacy_ocr_environment_is_deprecated_and_ocr_only(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_USE_RUST_OCR", "1") + + with pytest.warns(DeprecationWarning, match="LITELLM_USE_RUST_OCR is deprecated"): + assert configuration.rust_ocr_enabled() is True + assert configuration.rust_enabled() is False + + +def test_global_environment_precedes_legacy_ocr_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_RUST", "0") + monkeypatch.setenv("LITELLM_USE_RUST_OCR", "1") + + assert configuration.rust_ocr_enabled() is False + + +def test_deprecated_public_injection_delegates_to_internal_binding() -> None: + bridge: Final = _OcrBridge() + + with pytest.warns(DeprecationWarning, match="Injecting Rust bridge implementations"): + configuration.use_litellm_rust(True, ocr=bridge) + + assert rust_ocr.load_rust_ocr() is bridge + + +@pytest.mark.parametrize(("value", "expected"), (("1", "True"), ("0", "False"))) +def test_environment_controls_startup(value: str, expected: str) -> None: + environment: Final = {**os.environ, "LITELLM_RUST": value} + result: Final = subprocess.run( + ( + sys.executable, + "-c", + "from litellm.rust_bridge.configuration import rust_enabled; print(rust_enabled())", + ), + check=True, + capture_output=True, + text=True, + env=environment, + ) + + assert result.stdout.strip() == expected diff --git a/tests/test_litellm/rust_bridge/test_runtime.py b/tests/test_litellm/rust_bridge/test_runtime.py new file mode 100644 index 00000000000..b0fa510069b --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_runtime.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from litellm.exceptions import APIError +from litellm.rust_bridge import bindings, runtime + + +class RustBridgeDeclined(Exception): + pass + + +class RustUpstreamError(Exception): + pass + + +@pytest.fixture(autouse=True) +def native_exceptions(monkeypatch: pytest.MonkeyPatch) -> None: + native = SimpleNamespace( + RustBridgeDeclined=RustBridgeDeclined, + RustUpstreamError=RustUpstreamError, + ) + monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) + + +def context() -> runtime.BridgeErrorContext: + return runtime.BridgeErrorContext(route="messages", provider="anthropic", model="model") + + +def test_invoke_tags_native_decline_before_running_fallback() -> None: + calls: list[str] = [] + + def decline() -> object: + calls.append("rust") + raise RustBridgeDeclined("unsupported") + + value = runtime.invoke( + native_call=decline, + fallback=lambda: calls.append("python") or "fallback", + adapt=str, + mode=runtime.FallbackMode.PYTHON, + context=context(), + ) + + assert value == "fallback" + assert calls == ["rust", "python"] + + +def test_invoke_translates_upstream_without_fallback() -> None: + def fail() -> object: + raise RustUpstreamError(429, "rate limited") + + with pytest.raises(APIError, match="rate limited") as caught: + runtime.invoke( + native_call=fail, + fallback=lambda: pytest.fail("fallback must not run"), + adapt=str, + mode=runtime.FallbackMode.PYTHON, + context=context(), + ) + + assert caught.value.status_code == 429 + + +@pytest.mark.asyncio +async def test_ainvoke_handles_native_success() -> None: + async def native() -> int: + return 3 + + async def fallback() -> str: + pytest.fail("fallback must not run") + + assert ( + await runtime.ainvoke( + native_call=native, + fallback=fallback, + adapt=str, + mode=runtime.FallbackMode.PYTHON, + context=context(), + ) + == "3" + ) + + +def test_required_mode_rejects_unavailable_bridge() -> None: + with pytest.raises(RuntimeError, match="is unavailable"): + runtime.invoke( + native_call=None, + fallback=lambda: pytest.fail("fallback must not run"), + adapt=str, + mode=runtime.FallbackMode.RUST_REQUIRED, + context=context(), + ) diff --git a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py new file mode 100644 index 00000000000..e449d4392d8 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import importlib.util +import subprocess +import sys +import zipfile +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from types import MappingProxyType, ModuleType +from typing import Final, Protocol, cast + +import pytest + + +class _CommandRunner(Protocol): + def __call__( + self, + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +class _VerifierModule(Protocol): + main: Callable[ + [ + Sequence[str] | None, + Mapping[str, str] | None, + Callable[[Path], ModuleType | None], + _CommandRunner, + ], + int, + ] + + +_REPO_ROOT: Final = Path(__file__).resolve().parents[3] +_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "verify_linux_native_wheel.py" +_SPEC: Final = importlib.util.spec_from_file_location("verify_linux_native_wheel", _MODULE_PATH) +assert _SPEC is not None and _SPEC.loader is not None +_LOADED_VERIFIER: Final = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _LOADED_VERIFIER +_SPEC.loader.exec_module(_LOADED_VERIFIER) +verifier: Final = cast(_VerifierModule, _LOADED_VERIFIER) + +_EXPECTED_TAG: Final = "cp310-abi3-linux_x86_64" +_NATIVE_MEMBER: Final = "litellm/rust_bridge/_native.abi3.so" +_DIST_INFO: Final = "litellm-1.100.0.dist-info" + + +def _write_wheel( + tmp_path: Path, + *, + filename_tag: str, + metadata_tags: tuple[str, ...] | None = (_EXPECTED_TAG,), + dist_info: str = _DIST_INFO, + duplicate_wheel: bool = False, +) -> Path: + wheel: Final = tmp_path / f"litellm-1.100.0-{filename_tag}.whl" + with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(_NATIVE_MEMBER, b"synthetic native extension") + archive.writestr( + f"{dist_info}/METADATA", + "Metadata-Version: 2.1\nName: litellm\nVersion: 1.100.0\n", + ) + archive.writestr( + f"{dist_info}/RECORD", + f"{_NATIVE_MEMBER},,\n{dist_info}/WHEEL,,\n", + ) + if metadata_tags is not None: + wheel_metadata: Final = ( + "Wheel-Version: 1.0\nGenerator: regression-test\nRoot-Is-Purelib: false\n" + + "".join(f"Tag: {tag}\n" for tag in metadata_tags) + ) + archive.writestr(f"{dist_info}/WHEEL", wheel_metadata) + if duplicate_wheel: + archive.writestr(f"{dist_info}/WHEEL", wheel_metadata) + return wheel + + +def _fake_subprocess_run( + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + assert check and capture_output and text + if command == ("rustc", "--version"): + return subprocess.CompletedProcess(command, 0, stdout="rustc 1.98.0 (regression-test)\n", stderr="") + if "--sections" in command: + return subprocess.CompletedProcess(command, 0, stdout="[ 1] .text PROGBITS\n", stderr="") + if "--dyn-syms" in command: + return subprocess.CompletedProcess(command, 0, stdout="PyInit__native\n", stderr="") + raise AssertionError(f"unexpected subprocess command: {command}") + + +class _NativeModuleWithPanicHook(ModuleType): + def _panic_for_test(self) -> None: + return None + + +def _run_verifier( + wheel: Path, + *, + exposes_panic: bool = False, +) -> int: + native_module: Final = ( + _NativeModuleWithPanicHook("litellm.rust_bridge._native") + if exposes_panic + else ModuleType("litellm.rust_bridge._native") + ) + + def _fake_load_native_module(_: Path) -> ModuleType: + return native_module + + environment: Final = MappingProxyType({"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")}) + return verifier.main( + (str(_MODULE_PATH), str(wheel)), + environment, + _fake_load_native_module, + _fake_subprocess_run, + ) + + +def test_accepts_expected_release_wheel_tags(tmp_path: Path) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + + assert _run_verifier(wheel) == 0 + + +def test_rejects_cp312_version_specific_wheel(tmp_path: Path) -> None: + tag: Final = "cp312-cp312-linux_x86_64" + wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) + + assert _run_verifier(wheel) == 1 + + +def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None: + tag: Final = "cp310-abi3-win_amd64" + wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) + + assert _run_verifier(wheel) == 1 + + +@pytest.mark.parametrize( + "metadata_tags", + (None, ("cp312-cp312-linux_x86_64",)), + ids=("missing", "mismatched"), +) +def test_rejects_missing_or_mismatched_wheel_metadata_tag( + tmp_path: Path, + metadata_tags: tuple[str, ...] | None, +) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=metadata_tags) + + assert _run_verifier(wheel) == 1 + + +def test_rejects_wheel_metadata_from_wrong_dist_info_directory( + tmp_path: Path, +) -> None: + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + dist_info="decoy-1.0.0.dist-info", + ) + + assert _run_verifier(wheel) == 1 + + +def test_rejects_duplicate_wheel_metadata_tags(tmp_path: Path) -> None: + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + metadata_tags=(_EXPECTED_TAG, _EXPECTED_TAG), + ) + + assert _run_verifier(wheel) == 1 + + +def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path) -> None: + with pytest.warns(UserWarning, match="Duplicate name"): + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + duplicate_wheel=True, + ) + + assert _run_verifier(wheel) == 1 + + +def test_rejects_production_module_exposing_panic_hook(tmp_path: Path) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + + assert _run_verifier(wheel, exposes_panic=True) == 1 diff --git a/tests/test_litellm/test_bedrock_usgov_pricing.py b/tests/test_litellm/test_bedrock_usgov_pricing.py index 6b3312b5cc4..f7d95ecda01 100644 --- a/tests/test_litellm/test_bedrock_usgov_pricing.py +++ b/tests/test_litellm/test_bedrock_usgov_pricing.py @@ -26,9 +26,7 @@ import pytest @pytest.fixture(scope="module") def model_data(): - json_path = os.path.join( - os.path.dirname(__file__), "../../model_prices_and_context_window.json" - ) + json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json") with open(json_path) as f: return json.load(f) @@ -51,21 +49,14 @@ def test_usgov_sonnet_4_5_pricing(model_data, model_key): info = model_data[model_key] assert info["input_cost_per_token"] == 3.6e-06, ( - f"{model_key}: input_cost_per_token should be $3.60/MTok " - f"(got {info['input_cost_per_token']})" + f"{model_key}: input_cost_per_token should be $3.60/MTok (got {info['input_cost_per_token']})" ) - assert ( - info["output_cost_per_token"] == 1.8e-05 - ), f"{model_key}: output_cost_per_token should be $18.00/MTok" - assert ( - info["cache_creation_input_token_cost"] == 4.5e-06 - ), f"{model_key}: 5m cache write should be $4.50/MTok" - assert ( - info["cache_creation_input_token_cost_above_1hr"] == 7.2e-06 - ), f"{model_key}: 1h cache write should be $7.20/MTok" - assert ( - info["cache_read_input_token_cost"] == 3.6e-07 - ), f"{model_key}: cache read should be $0.36/MTok" + assert info["output_cost_per_token"] == 1.8e-05, f"{model_key}: output_cost_per_token should be $18.00/MTok" + assert info["cache_creation_input_token_cost"] == 4.5e-06, f"{model_key}: 5m cache write should be $4.50/MTok" + assert info["cache_creation_input_token_cost_above_1hr"] == 7.2e-06, ( + f"{model_key}: 1h cache write should be $7.20/MTok" + ) + assert info["cache_read_input_token_cost"] == 3.6e-07, f"{model_key}: cache read should be $0.36/MTok" def test_usgov_carries_20_percent_premium_over_global(model_data): @@ -84,9 +75,7 @@ def test_usgov_carries_20_percent_premium_over_global(model_data): "cache_read_input_token_cost", ): ratio = usgov_info[field] / global_info[field] - assert ( - abs(ratio - 1.2) < 1e-9 - ), f"{field}: us-gov / global ratio is {ratio}, expected 1.2" + assert abs(ratio - 1.2) < 1e-9, f"{field}: us-gov / global ratio is {ratio}, expected 1.2" # The us-gov.anthropic.* cross-region inference profile is the only us-gov @@ -112,9 +101,7 @@ def test_usgov_cross_region_above_200k_carries_gov_premium(model_data, field, ex """ info = model_data[USGOV_CROSS_REGION_KEY] assert field in info, f"{USGOV_CROSS_REGION_KEY}: missing field {field}" - assert ( - info[field] == expected - ), f"{USGOV_CROSS_REGION_KEY}: {field} should be {expected} (got {info[field]})" + assert info[field] == expected, f"{USGOV_CROSS_REGION_KEY}: {field} should be {expected} (got {info[field]})" def test_usgov_cross_region_above_200k_ratio_to_global(model_data): @@ -127,6 +114,176 @@ def test_usgov_cross_region_above_200k_ratio_to_global(model_data): usgov_info = model_data[USGOV_CROSS_REGION_KEY] for field in EXPECTED_USGOV_ABOVE_200K: ratio = usgov_info[field] / global_info[field] - assert ( - abs(ratio - 1.2) < 1e-9 - ), f"{field}: us-gov / global ratio is {ratio}, expected 1.2" + assert abs(ratio - 1.2) < 1e-9, f"{field}: us-gov / global ratio is {ratio}, expected 1.2" + + +CLAUDE_GOV_EXPECTED = { + "anthropic.claude-sonnet-5": { + "input_cost_per_token": 2.4e-06, + "output_cost_per_token": 1.2e-05, + "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_above_1hr": 4.8e-06, + "cache_read_input_token_cost": 2.4e-07, + }, + "anthropic.claude-opus-4-8": { + "input_cost_per_token": 6e-06, + "output_cost_per_token": 3e-05, + "cache_creation_input_token_cost": 7.5e-06, + "cache_creation_input_token_cost_above_1hr": 1.2e-05, + "cache_read_input_token_cost": 6e-07, + }, +} + + +USGOV_CLAUDE_KEY_TEMPLATES = { + "bedrock/us-gov-east-1/{base_key}": "bedrock", + "bedrock/us-gov-west-1/{base_key}": "bedrock", + "us-gov.{base_key}": "bedrock_converse", +} + + +@pytest.mark.parametrize("base_key", CLAUDE_GOV_EXPECTED) +@pytest.mark.parametrize("key_template,expected_provider", USGOV_CLAUDE_KEY_TEMPLATES.items()) +def test_usgov_claude_sonnet5_opus48_pricing(model_data, key_template, expected_provider, base_key): + """Sonnet 5 and Opus 4.8 gov entries, both in-region keys and the us-gov. + geo inference profile the model cards list for GovCloud, must match the + rates AWS publishes on the Bedrock pricing page (1.2x global). + """ + gov_key = key_template.format(base_key=base_key) + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + assert info["litellm_provider"] == expected_provider + for field, expected in CLAUDE_GOV_EXPECTED[base_key].items(): + assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})" + ratio = info[field] / model_data[base_key][field] + assert abs(ratio - 1.2) < 1e-9, f"{gov_key}: {field} gov/global ratio is {ratio}, expected 1.2" + + +CONVERSE_GOV_EXPECTED = { + "nvidia.nemotron-nano-3-30b": (7.2e-08, 2.88e-07), + "nvidia.nemotron-nano-12b-v2": (2.4e-07, 7.2e-07), + "nvidia.nemotron-super-3-120b": (1.8e-07, 7.8e-07), + "openai.gpt-oss-20b-1:0": (8.4e-08, 3.6e-07), + "openai.gpt-oss-120b-1:0": (1.8e-07, 7.2e-07), +} + + +@pytest.mark.parametrize("base_key", CONVERSE_GOV_EXPECTED) +@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"]) +def test_usgov_converse_model_pricing(model_data, region, base_key): + """Nemotron and gpt-oss gov entries must match the AWS Bedrock offer file, + which prices both GovCloud regions identically at 1.2x commercial. + """ + gov_key = f"bedrock/{region}/{base_key}" + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + expected_input, expected_output = CONVERSE_GOV_EXPECTED[base_key] + assert info["input_cost_per_token"] == expected_input + assert info["output_cost_per_token"] == expected_output + assert info["litellm_provider"] == "bedrock" + base = model_data[base_key] + assert abs(info["input_cost_per_token"] / base["input_cost_per_token"] - 1.2) < 1e-9 + assert abs(info["output_cost_per_token"] / base["output_cost_per_token"] - 1.2) < 1e-9 + + +def test_usgov_west_llama3_8b_output_price_fixed(model_data): + """The us-gov-west-1 llama3-8b entry carried the 70B output rate ($2.65/MTok); + the AWS Bedrock offer file prices output at $0.60/MTok. AWS lists the model + in us-gov-west-1 only, so there is no east entry to check. + """ + info = model_data["bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0"] + assert info["input_cost_per_token"] == 3e-07 + assert info["output_cost_per_token"] == 6e-07 + + +MANTLE_GOV_TIERED_EXPECTED = { + "openai.gpt-5.6-luna": { + "input_cost_per_token": 2.64e-07, + "input_cost_per_token_above_272k_tokens": 5.28e-07, + "cache_creation_input_token_cost": 3.3e-07, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-07, + "cache_read_input_token_cost": 2.64e-08, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-08, + "output_cost_per_token": 1.584e-06, + "output_cost_per_token_above_272k_tokens": 2.376e-06, + }, + "openai.gpt-5.6-terra": { + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_above_272k_tokens": 5.28e-06, + "cache_creation_input_token_cost": 3.3e-06, + "cache_creation_input_token_cost_above_272k_tokens": 6.6e-06, + "cache_read_input_token_cost": 2.64e-07, + "cache_read_input_token_cost_above_272k_tokens": 5.28e-07, + "output_cost_per_token": 1.584e-05, + "output_cost_per_token_above_272k_tokens": 2.376e-05, + }, +} + + +@pytest.mark.parametrize("model", MANTLE_GOV_TIERED_EXPECTED) +def test_usgov_west_mantle_terra_luna_pricing(model_data, model): + """Terra and Luna carry 1.2x commercial across every tier in the + us-gov-west-1 offer file; the us-gov-east-1 offer file has no SKUs for them. + """ + gov_key = f"bedrock_mantle/us-gov-west-1/{model}" + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + for field, expected in MANTLE_GOV_TIERED_EXPECTED[model].items(): + assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})" + assert info["litellm_provider"] == "bedrock_mantle" + assert f"bedrock_mantle/us-gov-east-1/{model}" not in model_data + + +@pytest.mark.parametrize("region", ["us-gov-east-1", "us-gov-west-1"]) +def test_usgov_mantle_gpt_5_4_pricing_has_no_long_context_tier(model_data, region): + """gpt-5.4 gov rates come from the offer file, which publishes only the + standard tier in GovCloud: no long-context SKUs exist there, unlike commercial. + """ + gov_key = f"bedrock_mantle/{region}/openai.gpt-5.4" + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + assert info["input_cost_per_token"] == 3.3e-06 + assert info["cache_read_input_token_cost"] == 3.3e-07 + assert info["output_cost_per_token"] == 1.98e-05 + assert not any(field.endswith("_above_272k_tokens") for field in info) + + +def test_usgov_mantle_grok_4_3_west_only(model_data): + """grok-4.3 is priced in the us-gov-west-1 offer file only; the east offer + file carries grok-4.6 instead. + """ + info = model_data["bedrock_mantle/us-gov-west-1/xai.grok-4.3"] + assert info["input_cost_per_token"] == 1.5e-06 + assert info["output_cost_per_token"] == 3e-06 + assert info["cache_read_input_token_cost"] == 2.4e-07 + assert "bedrock_mantle/us-gov-east-1/xai.grok-4.3" not in model_data + + +AZURE_GOV_EXPECTED = { + "azure/us-gov/gpt-5.1": { + "input_cost_per_token": 1.71875e-06, + "cache_read_input_token_cost": 1.71875e-07, + "output_cost_per_token": 1.375e-05, + }, + "azure/us-gov/o3-mini": { + "input_cost_per_token": 1.513e-06, + "cache_read_input_token_cost": 7.57e-07, + "output_cost_per_token": 6.05e-06, + }, + "azure/us-gov/text-embedding-3-large": {"input_cost_per_token": 1.63e-07}, + "azure/us-gov/text-embedding-3-small": {"input_cost_per_token": 2.5e-08}, +} + + +@pytest.mark.parametrize("gov_key", AZURE_GOV_EXPECTED) +def test_azure_usgov_pricing(model_data, gov_key): + """Azure Government meters from the Azure retail prices API + (usgovvirginia/usgovarizona, serviceName 'Foundry Models'). No Government + retirement schedule is published, so these entries carry no deprecation_date. + """ + assert gov_key in model_data, f"Missing model entry: {gov_key}" + info = model_data[gov_key] + for field, expected in AZURE_GOV_EXPECTED[gov_key].items(): + assert info[field] == expected, f"{gov_key}: {field} should be {expected} (got {info[field]})" + assert info["litellm_provider"] == "azure" + assert "deprecation_date" not in info diff --git a/tests/test_litellm/test_check_py310_typing_imports.py b/tests/test_litellm/test_check_py310_typing_imports.py new file mode 100644 index 00000000000..de370326091 --- /dev/null +++ b/tests/test_litellm/test_check_py310_typing_imports.py @@ -0,0 +1,86 @@ +import sys +from pathlib import Path +from typing import Final + +_CODE_COVERAGE_DIR: Final[Path] = Path(__file__).resolve().parents[1] / "code_coverage_tests" +sys.path.insert(0, str(_CODE_COVERAGE_DIR)) # test-quality-ok: required to import checker from its source directory +import check_py310_typing_imports as checker # noqa: E402 # load checker from its source directory + + +def _scan(tmp_path: Path, source: str) -> tuple[object, ...]: + file_path = tmp_path / "fixture.py" + file_path.write_text(source, encoding="utf-8") + return checker.scan_file(file_path) + + +def test_typing_import_flags_python_311_name(tmp_path: Path) -> None: + violations = _scan(tmp_path, "from typing import NotRequired, TypedDict\n") + assert tuple(violation.name for violation in violations) == ("NotRequired",) + + +def test_typing_extensions_import_passes(tmp_path: Path) -> None: + assert _scan(tmp_path, "from typing_extensions import NotRequired\n") == () + + +def test_typing_attribute_flags_python_311_name(tmp_path: Path) -> None: + violations = _scan(tmp_path, "import typing\nx: typing.Self\n") + assert tuple(violation.name for violation in violations) == ("Self",) + + +def test_version_guarded_typing_import_passes(tmp_path: Path) -> None: + source = ( + "import sys\n" + "if sys.version_info >= (3, 11):\n" + " from typing import NotRequired\n" + "else:\n" + " from typing_extensions import NotRequired\n" + ) + assert _scan(tmp_path, source) == () + + +def test_python_310_branch_flags_typing_import(tmp_path: Path) -> None: + source = ( + "import sys\n" + "if sys.version_info >= (3, 11):\n" + " from typing_extensions import NotRequired\n" + "else:\n" + " from typing import NotRequired\n" + ) + violations = _scan(tmp_path, source) + assert tuple(violation.name for violation in violations) == ("NotRequired",) + + +def test_python_310_branch_is_exempt_for_less_than_guard(tmp_path: Path) -> None: + source = ( + "import sys\n" + "if sys.version_info < (3, 11):\n" + " from typing_extensions import NotRequired\n" + "else:\n" + " from typing import NotRequired\n" + ) + assert _scan(tmp_path, source) == () + + +def test_nearest_if_controls_version_guard(tmp_path: Path) -> None: + source = ( + "if sys.version_info >= (3, 11):\n" + " from typing import Self\n" + " x = 1\n" + "if True:\n" + " from typing import Self\n" + ) + violations = _scan(tmp_path, source) + assert tuple((violation.name, violation.line) for violation in violations) == (("Self", 5),) + + +def test_scan_directory_includes_proxy_extras(tmp_path: Path) -> None: + file_path = tmp_path / "litellm-proxy-extras" / "litellm_proxy_extras" / "m.py" + file_path.parent.mkdir(parents=True) + file_path.write_text("from typing import NotRequired\n", encoding="utf-8") + + violations = checker.scan_directory(tmp_path) + assert tuple((violation.name, violation.file) for violation in violations) == (("NotRequired", str(file_path)),) + + +def test_python_310_typing_name_passes(tmp_path: Path) -> None: + assert _scan(tmp_path, "from typing import Optional\n") == () diff --git a/tests/test_litellm/test_circleci_rust_toolchain.py b/tests/test_litellm/test_circleci_rust_toolchain.py index c35ced51e16..800ca21b95d 100644 --- a/tests/test_litellm/test_circleci_rust_toolchain.py +++ b/tests/test_litellm/test_circleci_rust_toolchain.py @@ -17,28 +17,27 @@ Two invariants are pinned here: Windows job, so the check accepts either. A new job that syncs without one falls back to the unpinned path, which is exactly the regression a static check catches at PR time and a green CI run does not. - 2. `install_rust` itself pins what it downloads: an explicit rustup version in - the URL, a verified SHA-256, and an exact toolchain version rather than a - channel name. - -The Windows job predates `install_rust` and provisions its toolchain inline, so -invariant 2 is scoped to `install_rust`; invariant 1 covers both. + 2. Both installers pin what they download: an explicit rustup version, a + verified SHA-256, and the exact toolchain in `rust-toolchain.toml`. """ from __future__ import annotations import re from pathlib import Path +from typing import Final import pytest import yaml REPO_ROOT = Path(__file__).resolve().parents[2] CONFIG = REPO_ROOT / ".circleci" / "config.yml" +TOOLCHAIN: Final = REPO_ROOT / "rust-toolchain.toml" BUILDS_WORKSPACE = re.compile(r"\buv\s+(?:sync|build)\b") RUSTUP_ARCHIVE_URL = re.compile(r"https://static\.rust-lang\.org/rustup/archive/\d+\.\d+\.\d+/") -EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?\d+\.\d+\.\d+\"?") +EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?(\d+\.\d+\.\d+)\"?") +TOOLCHAIN_CHANNEL: Final = re.compile(r'^channel = "(\d+\.\d+\.\d+)"$', re.MULTILINE) def _config() -> dict[str, object]: @@ -57,6 +56,12 @@ def _step_text(step: object) -> str: return "" +def _pinned_toolchain() -> str: + match: Final = TOOLCHAIN_CHANNEL.search(TOOLCHAIN.read_text()) + assert match is not None, "rust-toolchain.toml must pin an exact channel" + return match.group(1) + + def _without_comments(text: str) -> str: return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) @@ -142,7 +147,17 @@ def test_install_rust_verifies_the_installer_checksum(install_rust_command: str) def test_install_rust_pins_an_exact_toolchain_version(install_rust_command: str) -> None: - assert EXACT_TOOLCHAIN.search(install_rust_command), ( - "install_rust must pin an exact toolchain version (e.g. 1.97.1); a channel name like " + match: Final = EXACT_TOOLCHAIN.search(install_rust_command) + assert match is not None, ( + "install_rust must pin an exact toolchain version (e.g. 1.98.0); a channel name like " "stable/beta/nightly makes the compiler drift with whatever upstream published that day" ) + assert match.group(1) == _pinned_toolchain() + + +def test_windows_installer_matches_the_repo_toolchain() -> None: + windows_steps: Final = _step_lists()["job using_litellm_on_windows"] + windows_command: Final = "\n".join(_step_text(step) for step in windows_steps) + match: Final = EXACT_TOOLCHAIN.search(windows_command) + assert match is not None + assert match.group(1) == _pinned_toolchain() diff --git a/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py b/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py index 9ca4515239a..e33bcfb8378 100644 --- a/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py +++ b/tests/test_litellm/test_cloudflare_workers_ai_model_metadata.py @@ -75,6 +75,22 @@ def test_additional_current_models_are_present(): assert entry["output_cost_per_token"] > 0 +@pytest.mark.parametrize( + "key, published_price_per_audio_minute", + [ + ("cloudflare/@cf/openai/whisper", 0.00045), + ("cloudflare/@cf/openai/whisper-large-v3-turbo", 0.00051), + ], +) +def test_whisper_transcription_pricing_is_stored_per_second(key, published_price_per_audio_minute): + entry = litellm.model_cost[key] + assert entry["litellm_provider"] == "cloudflare" + assert entry["mode"] == "audio_transcription" + assert entry["supported_endpoints"] == ["/v1/audio/transcriptions"] + assert entry["output_cost_per_second"] == 0.0 + assert entry["input_cost_per_second"] == pytest.approx(published_price_per_audio_minute / 60) + + def test_root_and_backup_have_identical_cloudflare_keys(): if not os.path.exists(ROOT_MAP): pytest.skip("root cost map only ships in source checkouts") diff --git a/tests/test_litellm/test_logging.py b/tests/test_litellm/test_logging.py index 087a1c8b3ad..bf3757e6886 100644 --- a/tests/test_litellm/test_logging.py +++ b/tests/test_litellm/test_logging.py @@ -1,26 +1,30 @@ import ast import asyncio import json +import logging import re import sys +import time +from io import StringIO from pathlib import Path from typing import List import pytest -import logging - import litellm from litellm._logging import ( _COLOR_LOG_FORMAT, + _MAX_SCRUBBED_ACCESS_ARG, _PLAIN_LOG_FORMAT, ALL_LOGGERS, + AccessLogRedactionFilter, CorrelationContextFilter, CorrelationPlainFormatter, JsonFormatter, LevelRoutingStreamHandler, SecretRedactionFilter, StdoutLogTruncationFilter, + _get_uvicorn_json_log_config, _initialize_loggers_with_handler, _parse_json_logs_env, _plain_log_format, @@ -968,3 +972,209 @@ def test_plain_log_format_survives_none_streams(): """sys.stdout/sys.stderr can be None in embedded interpreters; import must not crash.""" assert _plain_log_format(None, None) == _PLAIN_LOG_FORMAT assert _plain_log_format(_FakeStream(True), None) == _PLAIN_LOG_FORMAT + + +# --------------------------------------------------------------------------- +# Access-log redaction (LIT-5909) +# --------------------------------------------------------------------------- + +_LEAKED_KEY = "sk-mx5ous1o9Iezz5fj3pkLuA" + + +def _access_record(full_path: str) -> logging.LogRecord: + """A record shaped exactly like the one uvicorn.access emits per request.""" + return logging.LogRecord( + name="uvicorn.access", + level=logging.INFO, + pathname="", + lineno=0, + msg='%s - "%s %s HTTP/%s" %d', + args=("127.0.0.1:1", "GET", full_path, "1.1", 200), + exc_info=None, + ) + + +@pytest.mark.parametrize( + "full_path", + [ + f"/key/info?key={_LEAKED_KEY}", + f"/global/spend/report?api_key={_LEAKED_KEY}&start_date=2026-08-01", + f"/key/spend/report?api_key={_LEAKED_KEY}", + f"/spend/logs?api_key={_LEAKED_KEY}", + f"/user/daily/activity?api_key={_LEAKED_KEY}", + f"/gemini/v1beta/models/gemini-2.0-flash:generateContent?key={_LEAKED_KEY}", + ], +) +def test_access_log_filter_redacts_a_credential_query_parameter(full_path): + record = _access_record(full_path) + assert AccessLogRedactionFilter().filter(record) is True + assert _LEAKED_KEY not in record.getMessage() + assert "REDACTED" in record.getMessage() + + +def test_access_log_filter_keeps_the_record_formattable_by_uvicorn(): + """uvicorn's AccessFormatter unpacks record.args, so the filter must scrub the + args in place rather than collapse them the way SecretRedactionFilter does.""" + from uvicorn.logging import AccessFormatter + + record = _access_record(f"/key/info?key={_LEAKED_KEY}") + AccessLogRedactionFilter().filter(record) + assert isinstance(record.args, tuple) + assert len(record.args) == 5 + + formatted = AccessFormatter('%(client_addr)s - "%(request_line)s" %(status_code)s', use_colors=False).format(record) + assert _LEAKED_KEY not in formatted + assert "GET" in formatted + assert "200 OK" in formatted + + +@pytest.mark.parametrize( + "full_path, want", + [ + # The delimiter must survive so the logged request line stays well formed. + (f"/key/info?key={_LEAKED_KEY}&page=2", "/key/info?REDACTED&page=2"), + ("/download?sig=AbCd1234%2Fxy&page=2", "/download?REDACTED&page=2"), + ( + f"/global/spend/report?api_key={_LEAKED_KEY}&start_date=2026-01-01", + "/global/spend/report?REDACTED&start_date=2026-01-01", + ), + ("/sso/callback?client_secret=abcdefgh12345&state=xyz", "/sso/callback?REDACTED&state=xyz"), + (f"/v1/models?token={_LEAKED_KEY}&page=2", "/v1/models?REDACTED&page=2"), + ], +) +def test_access_log_filter_keeps_the_query_delimiter(full_path, want): + record = _access_record(full_path) + AccessLogRedactionFilter().filter(record) + assert record.args[2] == want + + +@pytest.mark.parametrize( + "full_path, want", + [ + # Both the param name and the value are encoded, so neither is literal text + # the patterns can see, yet the request parser decodes it into a working key. + (f"/key/info?k%65y=sk%2D{_LEAKED_KEY[3:]}", "/key/info?REDACTED"), + (f"/key/info?k%65y=sk%2D{_LEAKED_KEY[3:]}&page=2", "/key/info?REDACTED"), + (f"/v1/models/sk%2D{_LEAKED_KEY[3:]}", "REDACTED"), + # A decoded credential must never be echoed back: it can carry a newline and + # forge a following log line. + (f"/v1/models?k%65y=sk%2D{_LEAKED_KEY[3:]}%0AINFO:%20forged", "/v1/models?REDACTED"), + ], +) +def test_access_log_filter_redacts_a_percent_encoded_credential(full_path, want): + record = _access_record(full_path) + AccessLogRedactionFilter().filter(record) + assert record.args[2] == want + + +@pytest.mark.parametrize( + "full_path", + [ + "/v1/models?filter=gpt%2D4o&page=2", + "/gemini/v1beta/models/gemini-2.0-flash%3AgenerateContent", + ], +) +def test_access_log_filter_leaves_harmless_percent_encoding_alone(full_path): + """Decoding is a detector, not a rewrite, so a request line with no credential + in it survives encoded exactly as the client sent it.""" + record = _access_record(full_path) + AccessLogRedactionFilter().filter(record) + assert record.args[2] == full_path + + +def test_access_log_filter_caps_how_much_of_a_request_target_it_scans(): + """The request target is the only input to the secret regex an unauthenticated + caller controls end to end, so it is bounded before it is scanned, and the + dropped tail must not reach the log either.""" + record = _access_record("/v1/models?u=" + "a://" * 8192 + f"&key={_LEAKED_KEY}") + + started = time.perf_counter() + AccessLogRedactionFilter().filter(record) + elapsed = time.perf_counter() - started + + scrubbed = record.args[2] + assert _LEAKED_KEY not in scrubbed + assert len(scrubbed) < 1024 + assert elapsed < 1.0, f"scrubbing one access line took {elapsed:.2f}s" + + +@pytest.mark.parametrize("chars_before_the_cut", range(1, 12)) +def test_access_log_filter_never_logs_a_half_scanned_credential(chars_before_the_cut): + """Cutting mid-value would leave a prefix too short for the key= pattern to match, + and that prefix would then be logged raw, so the cut lands on a param boundary.""" + prefix = "/v1/models?u=" + padding = _MAX_SCRUBBED_ACCESS_ARG - len(prefix) - len("&key=") - chars_before_the_cut + record = _access_record(f"{prefix}{'a' * padding}&key={_LEAKED_KEY}") + + AccessLogRedactionFilter().filter(record) + + assert f"key={_LEAKED_KEY[:chars_before_the_cut]}" not in record.args[2] + + +def test_access_log_filter_leaves_a_credential_free_request_line_intact(): + record = _access_record("/v1/chat/completions") + AccessLogRedactionFilter().filter(record) + assert record.getMessage() == '127.0.0.1:1 - "GET /v1/chat/completions HTTP/1.1" 200' + + +def test_access_log_filter_redacts_a_record_that_carries_no_positional_args(): + record = logging.LogRecord( + name="uvicorn.access", + level=logging.INFO, + pathname="", + lineno=0, + msg=f'127.0.0.1:1 - "GET /key/info?key={_LEAKED_KEY} HTTP/1.1" 200', + args=None, + exc_info=None, + ) + assert AccessLogRedactionFilter().filter(record) is True + assert _LEAKED_KEY not in record.getMessage() + + +def _emit_access_line(full_path: str) -> str: + """Hand one real record to uvicorn.access and return what a handler wrote out.""" + from uvicorn.logging import AccessFormatter + + logger = logging.getLogger("uvicorn.access") + stream = StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(AccessFormatter('%(client_addr)s - "%(request_line)s" %(status_code)s', use_colors=False)) + saved_level, saved_propagate = logger.level, logger.propagate + logger.addHandler(handler) + logger.setLevel(logging.INFO) + logger.propagate = False + try: + logger.handle(_access_record(full_path)) + finally: + logger.removeHandler(handler) + logger.setLevel(saved_level) + logger.propagate = saved_propagate + return stream.getvalue() + + +def test_uvicorn_access_logger_redacts_a_credential_it_is_handed(): + """Registration happens at litellm import; without it the filter never runs.""" + emitted = _emit_access_line(f"/key/info?key={_LEAKED_KEY}") + + assert _LEAKED_KEY not in emitted + assert "REDACTED" in emitted + + +def test_access_redaction_survives_the_uvicorn_json_log_config(): + """litellm hands uvicorn a dictConfig when json_logs is on. dictConfig clears a + logger's handlers but not its filters, so redaction has to still be attached.""" + import logging.config + + names = ("uvicorn", "uvicorn.error", "uvicorn.access") + saved = tuple((logging.getLogger(n), logging.getLogger(n).handlers[:], logging.getLogger(n).level) for n in names) + try: + logging.config.dictConfig(_get_uvicorn_json_log_config()) + emitted = _emit_access_line(f"/key/info?key={_LEAKED_KEY}") + + assert _LEAKED_KEY not in emitted + assert "REDACTED" in emitted + finally: + for lg, handlers, level in saved: + lg.handlers[:] = handlers + lg.setLevel(level) + lg.propagate = True diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 8cf878d05d9..7c2b9d0be05 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -3150,8 +3150,8 @@ def _stream_builder_logging_obj() -> LiteLLMLogging: return logging_obj -def test_stream_chunk_builder_reports_streaming_usage_cost_when_enabled(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) +def test_stream_chunk_builder_stamps_streaming_usage_cost_by_default(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) chunks: Final = [ _stream_builder_text_chunk("gpt-4o", "Hello "), _stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"), @@ -3168,11 +3168,45 @@ def test_stream_chunk_builder_reports_streaming_usage_cost_when_enabled(monkeypa assert response._hidden_params["response_cost"] == pytest.approx(usage_cost) -def test_stream_chunk_builder_defers_cost_to_logging_obj_when_usage_cost_absent(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) +def test_stream_chunk_builder_skips_stamp_when_cost_is_unpriceable(): + import time as time_module + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + logging_obj: Final = LiteLLMLogging( + model="us.anthropic.claude-opus-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="completion", + start_time=time_module.time(), + litellm_call_id="stream-builder-alias-unpriceable", + function_id="1", + ) + logging_obj.model_call_details["custom_llm_provider"] = "bedrock" + logging_obj.optional_params = {} + usage_chunk: Final = _stream_builder_text_chunk("bedrock-claude-opus-5", "") + usage_chunk.usage = Usage(prompt_tokens=40, completion_tokens=5, total_tokens=45) + chunks: Final = [ + _stream_builder_text_chunk("bedrock-claude-opus-5", "Hello ", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=logging_obj + ) + + assert response is not None + assert getattr(response.usage, "cost", None) is None + assert response._hidden_params.get("response_cost") is None + + +def test_stream_chunk_builder_keeps_provider_reported_usage_cost(): + usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "") + usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5) chunks: Final = [ _stream_builder_text_chunk("gpt-4o", "Hello "), _stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"), + usage_chunk, ] response: Final = litellm.stream_chunk_builder( @@ -3180,4 +3214,26 @@ def test_stream_chunk_builder_defers_cost_to_logging_obj_when_usage_cost_absent( ) assert response is not None - assert response._hidden_params.get("response_cost") is None + assert getattr(response.usage, "cost", None) == pytest.approx(0.5) + assert response._hidden_params["response_cost"] == pytest.approx(0.5) + + +def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk(): + from openai.types.completion_usage import CompletionUsage + + usage_chunk: Final = _stream_builder_text_chunk("mantle-claude", "") + usage_chunk.usage = CompletionUsage(prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704) + assert type(usage_chunk.usage) is CompletionUsage + chunks: Final = [ + _stream_builder_text_chunk("mantle-claude", "Hello "), + _stream_builder_text_chunk("mantle-claude", "world.", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder(chunks=chunks, messages=[{"role": "user", "content": "hi"}]) + + assert response is not None + assert response.usage.prompt_tokens == 20 + assert response.usage.completion_tokens == 60 + assert getattr(response.usage, "cost", None) == pytest.approx(0.000704) + assert response._hidden_params["response_cost"] == pytest.approx(0.000704) diff --git a/tests/test_litellm/test_muse_spark_1_3_model_metadata.py b/tests/test_litellm/test_muse_spark_1_3_model_metadata.py new file mode 100644 index 00000000000..1ecd9490f78 --- /dev/null +++ b/tests/test_litellm/test_muse_spark_1_3_model_metadata.py @@ -0,0 +1,107 @@ +import json +from pathlib import Path + +import pytest + +import litellm +from litellm.cost_calculator import cost_per_token +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import StandardBuiltInToolCostTracking + +MUSE_SPARK_STANDARD = "meta/muse-spark-1.3" +MUSE_SPARK_CONTRIBUTOR = "meta/muse-spark-1.3-contributor" +WEB_SEARCH_COST_PER_QUERY = 0.0025 + +PRICING = ( + (MUSE_SPARK_STANDARD, 1.25e-06, 1.5e-07, 4.25e-06), + (MUSE_SPARK_CONTRIBUTOR, 1e-07, 2e-09, 2e-07), +) + + +def _load_cost_map(filename: str = "model_prices_and_context_window.json") -> dict: + with open(Path(__file__).parents[2] / filename) as f: + return json.load(f) + + + +@pytest.mark.parametrize("model, input_cost, cached_cost, output_cost", PRICING) +def test_muse_spark_1_3_model_info(model: str, input_cost: float, cached_cost: float, output_cost: float): + info = _load_cost_map().get(model) + assert info is not None, f"{model} not found in model_prices_and_context_window.json" + + assert info["litellm_provider"] == "meta" + assert info["mode"] == "chat" + + assert info["input_cost_per_token"] == input_cost + assert info["output_cost_per_token"] == output_cost + assert info["cache_read_input_token_cost"] == cached_cost + + assert info["max_input_tokens"] == 1048576 + assert info["max_output_tokens"] == 131072 + assert info["max_tokens"] == 131072 + + assert info["supports_function_calling"] is True + assert info["supports_parallel_function_calling"] is True + assert info["supports_prompt_caching"] is True + assert info["supports_reasoning"] is True + assert info["supports_response_schema"] is True + assert info["supports_tool_choice"] is True + assert info["supports_vision"] is True + assert info["supports_pdf_input"] is True + assert info["supports_web_search"] is True + assert info["supports_minimal_reasoning_effort"] is True + assert info["supports_xhigh_reasoning_effort"] is True + + assert info["supported_endpoints"] == ["/v1/chat/completions", "/v1/responses", "/v1/messages"] + assert info["supported_modalities"] == ["text", "image", "video"] + assert info["supported_output_modalities"] == ["text"] + + assert info["search_context_cost_per_query"] == { + "search_context_size_high": WEB_SEARCH_COST_PER_QUERY, + "search_context_size_low": WEB_SEARCH_COST_PER_QUERY, + "search_context_size_medium": WEB_SEARCH_COST_PER_QUERY, + } + + +@pytest.mark.parametrize("model, input_cost, cached_cost, output_cost", PRICING) +def test_muse_spark_1_3_cost_per_token( + local_model_cost_map, model: str, input_cost: float, cached_cost: float, output_cost: float +): + prompt_cost, completion_cost = cost_per_token(model=model, prompt_tokens=1000, completion_tokens=500) + + assert prompt_cost == pytest.approx(1000 * input_cost) + assert completion_cost == pytest.approx(500 * output_cost) + + +@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR)) +def test_muse_spark_1_3_routes_to_meta_model_api(model: str): + routed_model, provider, _, api_base = get_llm_provider(model=model, api_key="sk-test") + + assert routed_model == model.split("/", 1)[1] + assert provider == "meta" + assert api_base == "https://api.meta.ai/v1" + + +@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR)) +def test_muse_spark_1_3_web_search_cost_per_query(local_model_cost_map, model: str): + info = litellm.get_model_info(model=model) + + assert StandardBuiltInToolCostTracking.get_cost_for_web_search(model_info=info) == WEB_SEARCH_COST_PER_QUERY + + +@pytest.mark.parametrize("model", (MUSE_SPARK_STANDARD, MUSE_SPARK_CONTRIBUTOR)) +def test_muse_spark_1_3_backup_matches_main(model: str): + """Ensure the bundled model cost map stays in sync with the canonical file.""" + main_cost = _load_cost_map() + backup_cost = _load_cost_map("litellm/model_prices_and_context_window_backup.json") + + assert backup_cost.get(model) == main_cost.get(model), f"{model} differs between main and backup model cost maps" + + +def test_muse_spark_contributor_tier_is_cheaper_than_standard(): + cost_map = _load_cost_map() + standard = cost_map[MUSE_SPARK_STANDARD] + contributor = cost_map[MUSE_SPARK_CONTRIBUTOR] + + for field in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost"): + assert contributor[field] < standard[field], f"contributor {field} should undercut the standard tier" diff --git a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py new file mode 100644 index 00000000000..c0860a5b55f --- /dev/null +++ b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py @@ -0,0 +1,156 @@ +import json +from functools import lru_cache +from pathlib import Path + +import pytest + +import litellm + +REPO_ROOT = Path(__file__).parents[2] +MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" +BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" + +FLEX_LONG_CONTEXT = { + "gpt-5.4": { + "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07, + }, + "gpt-5.4-pro": { + "input_cost_per_token_above_272k_tokens_flex": 3e-05, + "output_cost_per_token_above_272k_tokens_flex": 0.000135, + }, + "gpt-5.5": { + "input_cost_per_token_above_272k_tokens_flex": 5e-06, + "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07, + }, +} + +PRIORITY_LONG_CONTEXT = { + "gpt-5.6": { + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, + }, + "gpt-5.6-sol": { + "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, + "output_cost_per_token_above_272k_tokens_priority": 6e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, + }, + "gpt-5.6-terra": { + "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + }, + "gpt-5.6-luna": { + "input_cost_per_token_above_272k_tokens_priority": 8e-07, + "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, + }, +} + +EXPECTED = {**FLEX_LONG_CONTEXT, **PRIORITY_LONG_CONTEXT} + +NO_PUBLISHED_PRIORITY_LONG_CONTEXT = ("gpt-5.4", "gpt-5.5") + + +@pytest.fixture(autouse=True) +def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + +@lru_cache(maxsize=2) +def _load(path: Path) -> dict[str, dict[str, object]]: + with open(path) as f: + return json.load(f) + + +@pytest.mark.parametrize("path", [MAIN_PATH, BACKUP_PATH], ids=["main", "backup"]) +@pytest.mark.parametrize("model", sorted(EXPECTED)) +def test_service_tier_long_context_rates_are_published(model: str, path: Path) -> None: + """Each tier must carry its own above-272K rates, in both price files.""" + info = _load(path).get(model) + assert info is not None, f"{model} not found in {path.name}" + for key, expected in EXPECTED[model].items(): + assert info.get(key) == pytest.approx(expected), f"{model}.{key} is {info.get(key)!r}, expected {expected!r}" + + +@pytest.mark.parametrize("model", sorted(EXPECTED)) +def test_tier_long_context_rate_is_half_or_double_the_standard(model: str) -> None: + """Flex is half the standard long-context rate; priority is double it.""" + info = _load(MAIN_PATH)[model] + tier = "flex" if model in FLEX_LONG_CONTEXT else "priority" + ratio = 0.5 if tier == "flex" else 2.0 + for base in ("input_cost_per_token", "output_cost_per_token"): + standard = info[f"{base}_above_272k_tokens"] + tiered = info[f"{base}_above_272k_tokens_{tier}"] + assert tiered == pytest.approx(standard * ratio), ( + f"{model}.{base}_above_272k_tokens_{tier} is {tiered!r}, " + f"expected {ratio}x the standard long-context rate {standard!r}" + ) + + +@pytest.mark.parametrize("model", NO_PUBLISHED_PRIORITY_LONG_CONTEXT) +def test_no_priority_long_context_rates_where_openai_publishes_none(model: str) -> None: + """Guard against back-filling a rate OpenAI does not publish.""" + info = _load(MAIN_PATH)[model] + assert "input_cost_per_token_above_272k_tokens_priority" not in info + + +LONG_CONTEXT_PROMPT_TOKENS = 300_000 +COMPLETION_TOKENS = 1_000 + +TIERED_COST_CASES = [ + ("gpt-5.4", "flex", 2.5e-06, 1.125e-05), + ("gpt-5.4-pro", "flex", 3e-05, 0.000135), + ("gpt-5.5", "flex", 5e-06, 2.25e-05), + ("gpt-5.6", "priority", 1.6e-05, 6e-05), + ("gpt-5.6-sol", "priority", 1.6e-05, 6e-05), + ("gpt-5.6-terra", "priority", 8e-06, 3.6e-05), + ("gpt-5.6-luna", "priority", 8e-07, 3.6e-06), +] + + +@pytest.mark.parametrize("model,tier,input_rate,output_rate", TIERED_COST_CASES) +def test_cost_per_token_bills_long_context_at_the_tier_rate( + model: str, tier: str, input_rate: float, output_rate: float +) -> None: + """A prompt over 272K on flex or priority must bill at that tier's long-context rate.""" + input_cost, output_cost = litellm.cost_per_token( + model=model, + prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + service_tier=tier, + ) + assert input_cost == pytest.approx(LONG_CONTEXT_PROMPT_TOKENS * input_rate) + assert output_cost == pytest.approx(COMPLETION_TOKENS * output_rate) + + +@pytest.mark.parametrize("model,tier,input_rate,output_rate", TIERED_COST_CASES) +def test_cost_per_token_tier_differs_from_the_standard_long_context_cost( + model: str, tier: str, input_rate: float, output_rate: float +) -> None: + """Flex halves the standard long-context bill and priority doubles it.""" + ratio = 0.5 if tier == "flex" else 2.0 + standard = sum( + litellm.cost_per_token( + model=model, + prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + ) + ) + tiered = sum( + litellm.cost_per_token( + model=model, + prompt_tokens=LONG_CONTEXT_PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + service_tier=tier, + ) + ) + assert tiered == pytest.approx(standard * ratio) diff --git a/tests/test_litellm/test_redact_string_in_error_paths.py b/tests/test_litellm/test_redact_string_in_error_paths.py index 6404db91acf..07d1ec5f523 100644 --- a/tests/test_litellm/test_redact_string_in_error_paths.py +++ b/tests/test_litellm/test_redact_string_in_error_paths.py @@ -172,8 +172,6 @@ class TestLLMHTTPHandlerRealtimeRedaction: class TestProxyStreamingDataGeneratorRedaction: - """Test _redact_string on traceback.format_exc() — the pattern at common_request_processing.py:1733.""" - def test_redact_traceback_format_exc(self): try: raise RuntimeError( diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 84f6344be35..c843a66a1c1 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -6,6 +6,7 @@ import logging import os import threading from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -1537,6 +1538,91 @@ async def test_ageneric_api_call_deployment_model_overrides_alias(): ), f"Expected deployment model 'vertex_ai/gemini-2.5-flash', got '{captured['model']}'" +@pytest.mark.asyncio +async def test_ageneric_api_call_resolves_realtime_session_model(): + """ + Regression for #36742: realtime client secret requests carry the model inside `session` too, and the proxy + fills it with the pre-routing model group name. The underlying litellm function reads session.model first, + so it must see the resolved deployment, while a caller's nested transcription model stays untouched. + """ + routed: Final = AsyncMock(return_value={"result": "ok"}) + + router = litellm.Router( + model_list=[ + { + "model_name": "my-realtime-group", + "litellm_params": { + "model": "openai/gpt-realtime-2.1-mini", + "api_key": "fake-key", + }, + "model_info": {"mode": "realtime"}, + } + ] + ) + + await router._ageneric_api_call_with_fallbacks( + model="my-realtime-group", + original_function=routed, + session={ + "type": "realtime", + "model": "my-realtime-group", + "audio": {"input": {"transcription": {"model": "gpt-4o-transcribe"}}}, + }, + ) + + sent: Final = routed.call_args.kwargs + assert sent["model"] == "openai/gpt-realtime-2.1-mini" + assert sent["session"]["model"] == "openai/gpt-realtime-2.1-mini" + assert sent["session"]["audio"]["input"]["transcription"]["model"] == "gpt-4o-transcribe" + + +@pytest.mark.asyncio +async def test_ageneric_api_call_does_not_add_session_model(): + """ + A session that never carried a model must not gain one from routing: the underlying function then falls back + to the resolved `model` kwarg itself, and the outgoing session body keeps the caller's shape. + """ + routed: Final = AsyncMock(return_value={"result": "ok"}) + + router = litellm.Router( + model_list=[ + { + "model_name": "my-realtime-group", + "litellm_params": { + "model": "openai/gpt-realtime-2.1-mini", + "api_key": "fake-key", + }, + "model_info": {"mode": "realtime"}, + } + ] + ) + + await router._ageneric_api_call_with_fallbacks( + model="my-realtime-group", + original_function=routed, + session={"type": "realtime"}, + ) + + sent: Final = routed.call_args.kwargs + assert sent["model"] == "openai/gpt-realtime-2.1-mini" + assert sent["session"] == {"type": "realtime"} + + +@pytest.mark.parametrize( + "session, expected", + [ + ({"type": "realtime", "model": "my-realtime-group"}, {"session": {"type": "realtime", "model": "resolved"}}), + ({"type": "realtime"}, {}), + (None, {}), + ("not-a-session", {}), + ], +) +def test_with_router_resolved_session_model(session, expected): + from litellm.router import _with_router_resolved_session_model + + assert dict(_with_router_resolved_session_model(session, "resolved")) == expected + + def test_router_get_model_access_groups_team_only_models(): """ Test that Router.get_model_access_groups returns the correct response for team-only models @@ -7271,6 +7357,71 @@ def test_get_configured_token_limits_coerces_numeric_strings(): assert router.get_configured_token_limits("quoted-limits-model") == (32000, 8000) +def test_get_configured_display_name_reads_deployment_model_info(): + router = litellm.Router( + model_list=[ + { + "model_name": "Kimi K3-claude-compatible", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"display_name": "Kimi K3"}, + } + ] + ) + + assert router.get_configured_display_name("Kimi K3-claude-compatible") == "Kimi K3" + + +def test_get_configured_display_name_returns_none_for_unset_or_unknown(): + router = litellm.Router( + model_list=[ + { + "model_name": "no-display-model", + "litellm_params": {"model": "openai/some-unmapped-model"}, + } + ] + ) + + assert router.get_configured_display_name("no-display-model") is None + assert router.get_configured_display_name("not-a-real-model") is None + + +def test_get_configured_display_name_skips_wildcard_pattern_matching(): + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock/*", + "litellm_params": {"model": "bedrock/*"}, + "model_info": {"display_name": "Bedrock"}, + } + ] + ) + + with patch.object( + router.pattern_router, "route", side_effect=AssertionError("pattern route called") + ): + assert ( + router.get_configured_display_name("bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") + is None + ) + + +def test_get_configured_display_name_treats_malformed_values_as_absent(): + malformed = ["", " ", 12345, ["Kimi K3"], {"name": "Kimi K3"}, True] + router = litellm.Router( + model_list=[ + { + "model_name": f"bad-display-{i}", + "litellm_params": {"model": "openai/some-unmapped-model"}, + "model_info": {"display_name": bad}, + } + for i, bad in enumerate(malformed) + ] + ) + + for i in range(len(malformed)): + assert router.get_configured_display_name(f"bad-display-{i}") is None + + @pytest.mark.asyncio async def test_acreate_batch_disable_fallbacks_surfaces_owning_provider_error(): router = litellm.Router( @@ -7510,6 +7661,118 @@ async def test_acreate_batch_request_bedrock_tags_override_deployment_tags(): assert mock_sign.call_args.kwargs["data"]["tags"] == request_tags +@pytest.mark.asyncio +async def test_avector_store_search_injects_router(): + """ + Regression: router.avector_store_search must pass the router down to the + SDK search call so provider transforms can resolve router-managed + embedding models (e.g. S3 Vectors query embeddings). + """ + from litellm.types.vector_stores import VectorStoreSearchResponse + + expected_response = VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + mock_asearch = AsyncMock(return_value=expected_response) + # Router.__init__ binds asearch via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.asearch", new=mock_asearch): # test-quality-ok: the SDK call is the only place the injected router kwarg is observable + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + search_response = await router.avector_store_search( + vector_store_id="v", query="q", custom_llm_provider="s3_vectors" + ) + + assert search_response is expected_response + mock_asearch.assert_awaited_once() + assert mock_asearch.await_args.kwargs["router"] is router + + +@pytest.mark.asyncio +async def test_avector_store_create_does_not_inject_router(): + """The router injection is gated on the search call type: the create path + must keep calling the SDK without a router kwarg.""" + expected_response = {"id": "vs_1", "object": "vector_store"} + mock_acreate = AsyncMock(return_value=expected_response) + # avector_store_create(model=None) resolves acreate via a local import at + # call time, so patching after Router construction works here. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + with patch("litellm.vector_stores.main.acreate", new=mock_acreate): # test-quality-ok: the SDK call is the only place a leaked router kwarg would surface + create_response = await router.avector_store_create(model=None, custom_llm_provider="openai") + + assert create_response is expected_response + mock_acreate.assert_awaited_once() + assert "router" not in mock_acreate.await_args.kwargs + + +def test_vector_store_search_injects_router(): + """ + Sync parity for the router injection: router.vector_store_search must pass + the router down to the SDK search call so provider transforms can resolve + router-managed embedding models, same as avector_store_search. + """ + from litellm.types.vector_stores import VectorStoreSearchResponse + + expected_response = VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + mock_search = MagicMock(return_value=expected_response) + # Router.__init__ binds search via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.search", new=mock_search): # test-quality-ok: the SDK call is the only place the injected router kwarg is observable + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + search_response = router.vector_store_search( + vector_store_id="v", query="q", custom_llm_provider="s3_vectors" + ) + + assert search_response is expected_response + mock_search.assert_called_once() + assert mock_search.call_args.kwargs["router"] is router + assert mock_search.call_args.kwargs["custom_llm_provider"] == "s3_vectors" + + +def test_vector_store_create_does_not_inject_router(): + """The sync create path must keep calling the SDK without a router kwarg.""" + expected_response = {"id": "vs_1", "object": "vector_store"} + mock_create = MagicMock(return_value=expected_response) + # Router.__init__ binds create via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.create", new=mock_create): # test-quality-ok: the SDK call is the only place a leaked router kwarg would surface + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + create_response = router.vector_store_create(custom_llm_provider="openai") + + assert create_response is expected_response + mock_create.assert_called_once() + assert "router" not in mock_create.call_args.kwargs + + class TestPreRoutingStrategyRegistryLifecycle: """ Regression tests: a deployment leaving the model_list must release the @@ -8316,6 +8579,398 @@ class TestConsumedRequestTagsStamp: assert CONSUMED_REQUEST_TAGS_METADATA_KEY not in request_kwargs["metadata"] +class TestClaudeCodeSubagentSessionRouterBinding: + class _RewriteStrategy: + def __init__(self, routed_model: str = "cheap-model") -> None: + self.routed_model = routed_model + + async def async_pre_routing_hook( + self, model, request_kwargs, messages=None, input=None, specific_deployment=False + ): + from litellm.types.router import PreRoutingHookResponse + + return PreRoutingHookResponse( + model=self.routed_model, + messages=messages, + routing_decision={ + "router_model_name": "smart-router", + "router_type": "complexity", + "routed_model": self.routed_model, + "cause": "heuristic_scorer", + }, + ) + + @classmethod + def _router( + cls, + cheap_response: str = "cheap response", + fallbacks: list[dict[str, list[str]]] | None = None, + ) -> "litellm.Router": + from litellm.types.router import TaggedPreRoutingStrategy + + router = litellm.Router( + model_list=[ + { + "model_name": "cheap-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "mock_response": cheap_response}, + }, + { + "model_name": "expensive-model", + "litellm_params": {"model": "openai/gpt-4o", "mock_response": "expensive response"}, + }, + ], + fallbacks=fallbacks, + num_retries=0, + ) + router.complexity_routers = { + "smart-router": (TaggedPreRoutingStrategy(tags=(), strategy=cls._RewriteStrategy()),), + "premium-router": (TaggedPreRoutingStrategy(tags=(), strategy=cls._RewriteStrategy("expensive-model")),), + } + return router + + @staticmethod + def _request_kwargs( + *, + key_hash: str = "key-hash-a", + app: str = "cli", + agent_id: str | None = None, + fallback_depth: int | None = None, + ) -> dict: + headers = { + "X-Claude-Code-Session-Id": "session-1234", + "x-app": app, + **({"x-claude-code-agent-id": agent_id} if agent_id is not None else {}), + } + return { + "metadata": {"user_api_key_hash": key_hash}, + "proxy_server_request": {"headers": headers}, + **({"fallback_depth": fallback_depth} if fallback_depth is not None else {}), + } + + @pytest.mark.asyncio + async def test_subagent_concrete_model_uses_the_main_sessions_router(self): + router = self._router() + + await router.acompletion( + model="smart-router", + messages=[{"role": "user", "content": "main turn"}], + **self._request_kwargs(), + ) + subagent_kwargs = self._request_kwargs(agent_id="agent-1234") + + response = await router.acompletion( + model="expensive-model", + messages=[{"role": "user", "content": "subagent turn"}], + **subagent_kwargs, + ) + + assert response.choices[0].message.content == "cheap response" + assert subagent_kwargs["metadata"]["model_group"] == "smart-router" + assert subagent_kwargs["metadata"]["routing_decision"]["router_model_name"] == "smart-router" + + @pytest.mark.asyncio + async def test_main_thread_side_calls_to_a_plain_model_keep_the_session_router(self): + router = self._router() + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=self._request_kwargs()) + await router.async_pre_routing_hook(model="expensive-model", request_kwargs=self._request_kwargs()) + + response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(agent_id="agent-1234"), + ) + + assert response is not None + assert response.model == "cheap-model" + + @pytest.mark.asyncio + async def test_redis_cleanup_failure_does_not_reject_a_subagent_request(self): + from litellm.caching.caching import RedisCache + + router = self._router() + del router.complexity_routers["smart-router"] + redis_cache = MagicMock(spec=RedisCache) + redis_cache.async_get_cache = AsyncMock(return_value="smart-router") + redis_cache.async_delete_cache = AsyncMock(side_effect=ConnectionError("redis unavailable")) + router._update_redis_cache(cache=redis_cache) + + response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(agent_id="agent-1234"), + ) + + assert response is None + redis_cache.async_delete_cache.assert_awaited_once() + + @pytest.mark.asyncio + async def test_redis_read_failure_does_not_reject_a_subagent_request(self): + from litellm.caching.caching import RedisCache + + router = self._router() + request_kwargs = self._request_kwargs(agent_id="agent-1234") + cache_key = router._claude_code_session_router_cache_key(request_kwargs) + assert cache_key is not None + await router._claude_code_session_router_cache.in_memory_cache.async_set_cache( + cache_key, + "smart-router", + ) + redis_cache = MagicMock(spec=RedisCache) + redis_cache.async_get_cache = AsyncMock(side_effect=Exception("Redis circuit breaker is open")) + router._update_redis_cache(cache=redis_cache) + + response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=request_kwargs, + ) + + assert response is None + assert "model_group" not in request_kwargs["metadata"] + redis_cache.async_get_cache.assert_awaited_once() + + @pytest.mark.asyncio + async def test_redis_write_failures_do_not_reject_main_or_subagent_requests(self): + from litellm.caching.caching import RedisCache + + router = self._router() + redis_cache = MagicMock(spec=RedisCache) + redis_cache.async_get_cache = AsyncMock(return_value="smart-router") + redis_cache.async_set_cache = AsyncMock(side_effect=Exception("redis unavailable")) + router._update_redis_cache(cache=redis_cache) + + main_response = await router.async_pre_routing_hook( + model="smart-router", + request_kwargs=self._request_kwargs(), + ) + subagent_response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(agent_id="agent-1234"), + ) + + assert main_response is not None + assert main_response.model == "cheap-model" + assert subagent_response is not None + assert subagent_response.model == "cheap-model" + assert redis_cache.async_set_cache.await_count == 2 + + @pytest.mark.asyncio + async def test_subagents_follow_the_main_threads_latest_router_across_workers(self): + from types import SimpleNamespace + + from litellm.caching.caching import RedisCache + + shared_binding = SimpleNamespace(value=None) + shared_redis = MagicMock(spec=RedisCache) + shared_redis.async_get_cache = AsyncMock(side_effect=lambda key, **_: shared_binding.value) + shared_redis.async_set_cache = AsyncMock( + side_effect=lambda key, value, **_: setattr(shared_binding, "value", value) + ) + main_worker, subagent_worker = self._router(), self._router() + main_worker._update_redis_cache(cache=shared_redis) + subagent_worker._update_redis_cache(cache=shared_redis) + + await main_worker.async_pre_routing_hook(model="smart-router", request_kwargs=self._request_kwargs()) + first = await subagent_worker.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(agent_id="agent-1234"), + ) + await main_worker.async_pre_routing_hook(model="premium-router", request_kwargs=self._request_kwargs()) + second = await subagent_worker.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(agent_id="agent-1234"), + ) + + assert first is not None + assert first.model == "cheap-model" + assert second is not None + assert second.model == "expensive-model" + assert shared_binding.value == "premium-router" + + @pytest.mark.asyncio + async def test_no_pre_routing_strategies_means_no_session_cache_traffic(self): + from litellm.caching.caching import RedisCache + + router = self._router() + router.complexity_routers.clear() + redis_cache = MagicMock(spec=RedisCache) + redis_cache.async_get_cache = AsyncMock(return_value=None) + redis_cache.async_set_cache = AsyncMock() + redis_cache.async_delete_cache = AsyncMock() + router._update_redis_cache(cache=redis_cache) + + for request_kwargs in (self._request_kwargs(), self._request_kwargs(agent_id="agent-1234")): + response = await router.async_pre_routing_hook(model="expensive-model", request_kwargs=request_kwargs) + assert response is None + + redis_cache.async_get_cache.assert_not_awaited() + redis_cache.async_set_cache.assert_not_awaited() + redis_cache.async_delete_cache.assert_not_awaited() + + @pytest.mark.asyncio + async def test_session_bindings_do_not_evict_router_rate_limit_state(self): + router = self._router() + assert router._update_usage(deployment_id="deployment-id", parent_otel_span=None) == 1 + + for session_index in range(201): + request_kwargs = self._request_kwargs() + request_kwargs["proxy_server_request"]["headers"]["X-Claude-Code-Session-Id"] = ( + f"session-{session_index:04d}" + ) + await router.async_pre_routing_hook(model="smart-router", request_kwargs=request_kwargs) + + assert router._update_usage(deployment_id="deployment-id", parent_otel_span=None) == 2 + + @pytest.mark.asyncio + async def test_background_and_fallback_requests_do_not_clear_the_session_router(self): + router = self._router() + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=self._request_kwargs()) + await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(app="cli-bg"), + ) + await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(fallback_depth=1), + ) + + response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(agent_id="agent-1234"), + ) + + assert response is not None + assert response.model == "cheap-model" + + @pytest.mark.asyncio + async def test_subagent_fallback_does_not_reapply_the_session_router(self): + router = self._router() + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=self._request_kwargs()) + + response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(agent_id="agent-1234", fallback_depth=1), + ) + + assert response is None + + @pytest.mark.asyncio + async def test_subagent_can_fallback_to_its_original_requested_model(self): + router = self._router( + cheap_response="litellm.RateLimitError", + fallbacks=[{"cheap-model": ["expensive-model"]}], + ) + + await router.acompletion( + model="smart-router", + messages=[{"role": "user", "content": "main turn"}], + **self._request_kwargs(), + ) + subagent_kwargs = self._request_kwargs(agent_id="agent-1234") + + response = await router.acompletion( + model="expensive-model", + messages=[{"role": "user", "content": "subagent turn"}], + **subagent_kwargs, + ) + + assert response.choices[0].message.content == "expensive response" + assert subagent_kwargs["metadata"]["routing_decision"]["routed_model"] == "cheap-model" + + @pytest.mark.asyncio + async def test_subagent_can_use_the_bound_router_name_fallback(self): + router = self._router( + cheap_response="litellm.RateLimitError", + fallbacks=[{"smart-router": ["expensive-model"]}], + ) + + await router.acompletion( + model="smart-router", + messages=[{"role": "user", "content": "main turn"}], + **self._request_kwargs(), + ) + + response = await router.acompletion( + model="expensive-model", + messages=[{"role": "user", "content": "subagent turn"}], + **self._request_kwargs(agent_id="agent-1234"), + ) + + assert response.choices[0].message.content == "expensive response" + + @pytest.mark.asyncio + async def test_anthropic_subagent_four_fallback_hops_use_each_current_model_chain(self): + from litellm.types.router import TaggedPreRoutingStrategy + + failing_groups = ("cheap-model", "fallback-1", "fallback-2", "fallback-3") + router = litellm.Router( + model_list=[ + *( + { + "model_name": group, + "litellm_params": { + "model": "anthropic/claude-3-haiku-20240307", + "mock_response": "litellm.RateLimitError", + }, + } + for group in failing_groups + ), + { + "model_name": "requested-model", + "litellm_params": { + "model": "anthropic/claude-3-haiku-20240307", + "mock_response": "requested response", + }, + }, + { + "model_name": "fallback-4", + "litellm_params": { + "model": "anthropic/claude-3-haiku-20240307", + "mock_response": "fourth fallback response", + }, + }, + ], + fallbacks=[ + {"smart-router": ["fallback-1"]}, + {"fallback-1": ["fallback-2"]}, + {"fallback-2": ["fallback-3"]}, + {"fallback-3": ["fallback-4"]}, + ], + num_retries=0, + max_fallbacks=4, + ) + router.complexity_routers = { + "smart-router": (TaggedPreRoutingStrategy(tags=(), strategy=self._RewriteStrategy()),) + } + main_kwargs = self._request_kwargs() + main_kwargs["litellm_metadata"] = main_kwargs.pop("metadata") + await router.async_pre_routing_hook(model="smart-router", request_kwargs=main_kwargs) + subagent_kwargs = self._request_kwargs(agent_id="agent-1234") + subagent_kwargs["litellm_metadata"] = subagent_kwargs.pop("metadata") + + response = await router.aanthropic_messages( + model="requested-model", + messages=[{"role": "user", "content": "subagent turn"}], + max_tokens=64, + **subagent_kwargs, + ) + + assert response["content"][0]["text"] == "fourth fallback response" + + @pytest.mark.asyncio + async def test_session_router_binding_is_scoped_to_the_authenticated_key(self): + router = self._router() + + await router.async_pre_routing_hook(model="smart-router", request_kwargs=self._request_kwargs()) + + response = await router.async_pre_routing_hook( + model="expensive-model", + request_kwargs=self._request_kwargs(key_hash="key-hash-b", agent_id="agent-1234"), + ) + + assert response is None + + class TestAutoRouterMaxInputCharsWiring: """`auto_router_max_input_chars` on the deployment has to reach the AutoRouter that embeds prompts. @@ -11730,3 +12385,55 @@ class TestPreRoutingTierDrivesFallbacks: response = await router.acompletion(model="smart-router", messages=[{"role": "user", "content": "hi"}]) assert response.choices[0].message.content == "from backup-b" + + +@pytest.mark.asyncio +async def test_prompt_management_factory_marks_injection_for_every_deployment(monkeypatch): + """The factory stamps a provisional deployment's model_info into kwargs before the + prompt pass runs, then routes on the returned model, so any deployment can end up + billed. An injection recorded there must carry the every-deployment sentinel, never + the provisional deployment's id, or a differently-billed deployment loses the credit.""" + import time + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + router = litellm.Router( + model_list=[ + { + "model_name": "cached-claude", + "litellm_params": { + "model": "anthropic_cache_control_hook/claude-sonnet-5", + "prompt_id": "cache-points", + }, + "model_info": {"id": "provisional-dep"}, + } + ] + ) + captured: dict = {} + + async def _capture_acompletion(**kwargs): + captured.update(kwargs) + return litellm.ModelResponse() + + monkeypatch.setattr(litellm, "acompletion", _capture_acompletion) + logging_obj = LiteLLMLogging( + model="cached-claude", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + start_time=time.time(), + litellm_call_id="lit-6445", + function_id="f", + ) + await router.acompletion( + model="cached-claude", + messages=[ + {"role": "system", "content": "a static system prompt"}, + {"role": "user", "content": "hi"}, + ], + cache_control_injection_points=[{"location": "message", "role": "system"}], + litellm_logging_obj=logging_obj, + ) + bucket = captured.get("litellm_metadata") or captured["metadata"] + assert captured["model_info"]["id"] == "provisional-dep" + assert bucket["litellm_gateway_injected_cache"] == "" diff --git a/tests/test_litellm/test_router_retry_policy_update.py b/tests/test_litellm/test_router_retry_policy_update.py index 1b98b8c1ae8..be568134763 100644 --- a/tests/test_litellm/test_router_retry_policy_update.py +++ b/tests/test_litellm/test_router_retry_policy_update.py @@ -21,6 +21,7 @@ This file pins both halves of the fix. import json from dataclasses import dataclass +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -28,8 +29,19 @@ from pydantic import ValidationError import litellm +from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck +from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck from litellm.types.router import RetryPolicy, UpdateRouterConfig + +@pytest.fixture(autouse=True) +def isolate_litellm_callbacks(): + callbacks_before: Final = litellm.callbacks.copy() + yield + litellm.callbacks = callbacks_before # test-quality-ok: required callback-state restoration fixture + + # --------------------------------------------------------------------------- # UpdateRouterConfig schema membership (LIT-3152 part 1) # --------------------------------------------------------------------------- @@ -100,6 +112,114 @@ def _build_router() -> litellm.Router: ) +def test_update_settings_adds_optional_pre_call_check_once(): + router = _build_router() + + router.update_settings(num_retries=7, optional_pre_call_checks=["prompt_caching"]) + router.update_settings(optional_pre_call_checks=["prompt_caching"]) + + prompt_caching_callbacks = [ + callback for callback in router.optional_callbacks if isinstance(callback, PromptCachingDeploymentCheck) + ] + assert len(prompt_caching_callbacks) == 1 + assert router.num_retries == 7 + + +def test_update_settings_clears_omitted_toggleable_pre_call_checks(): + router = _build_router() + + router.update_settings(optional_pre_call_checks=["prompt_caching"]) + router.update_settings(optional_pre_call_checks=[]) + + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + + +def test_set_optional_pre_call_checks_reconciles_callback_types(): + router = _build_router() + + router.set_optional_pre_call_checks(["prompt_caching"]) + router.set_optional_pre_call_checks([]) + + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + + +def test_remove_optional_pre_call_check_removes_local_and_global_callbacks(): + router = _build_router() + + router.set_optional_pre_call_checks(["prompt_caching"]) + router._remove_optional_callbacks_of_type(PromptCachingDeploymentCheck) + + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router.optional_callbacks or [])) + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + +def test_remove_optional_pre_call_check_keeps_global_callback_for_another_router(): + router_a = _build_router() + router_b = _build_router() + + router_a.update_settings(optional_pre_call_checks=["prompt_caching"]) + router_b.update_settings(optional_pre_call_checks=["prompt_caching"]) + + router_a.update_settings(optional_pre_call_checks=[]) + + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router_a.optional_callbacks or [])) + assert any(type(callback) is PromptCachingDeploymentCheck for callback in (router_b.optional_callbacks or [])) + assert any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + router_b.update_settings(optional_pre_call_checks=[]) + + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router_b.optional_callbacks or [])) + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + +def test_remove_optional_pre_call_check_keeps_global_callback_when_second_router_clears_first(): + router_a = _build_router() + router_b = _build_router() + + router_a.update_settings(optional_pre_call_checks=["prompt_caching"]) + router_b.update_settings(optional_pre_call_checks=["prompt_caching"]) + + router_b.update_settings(optional_pre_call_checks=[]) + + assert any(type(callback) is PromptCachingDeploymentCheck for callback in (router_a.optional_callbacks or [])) + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in (router_b.optional_callbacks or [])) + assert any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + router_a.update_settings(optional_pre_call_checks=[]) + + assert not any(type(callback) is PromptCachingDeploymentCheck for callback in litellm.callbacks) + + +def test_update_settings_replaces_toggleable_pre_call_checks(): + router = _build_router() + + router.update_settings(optional_pre_call_checks=["prompt_caching"]) + router.update_settings(optional_pre_call_checks=["enforce_model_rate_limits"]) + + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in (router.optional_callbacks or [])) + assert not any(isinstance(callback, PromptCachingDeploymentCheck) for callback in litellm.callbacks) + assert any(isinstance(callback, ModelRateLimitingCheck) for callback in (router.optional_callbacks or [])) + + +@pytest.mark.asyncio +async def test_update_settings_preserves_router_budget_limiting_when_omitted(monkeypatch): + async def _disable_periodic_sync(*args, **kwargs): + return None + + monkeypatch.setattr( + "litellm.router_strategy.budget_limiter.RouterBudgetLimiting.periodic_sync_in_memory_spend_with_redis", + _disable_periodic_sync, + ) + router = _build_router() + + router.add_optional_pre_call_checks(["router_budget_limiting"]) + router.update_settings(optional_pre_call_checks=[]) + + assert any(isinstance(callback, RouterBudgetLimiting) for callback in (router.optional_callbacks or [])) + + def test_update_settings_persists_retry_policy_dict(): """When the proxy's ``_add_router_settings_from_db_config`` calls ``llm_router.update_settings(retry_policy={...})`` after reading the @@ -255,8 +375,12 @@ async def test_config_update_persists_and_reads_back_retry_policy(monkeypatch): RateLimitErrorRetries=7, ) ) + request = MagicMock() + request.json = AsyncMock(return_value={"router_settings": {"retry_policy": posted.model_dump()}}) + await proxy_server.update_config( config_info=ConfigYAML(router_settings=posted), + request=request, user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234"), ) diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index 7d50694c805..9fa748edec1 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -1,8 +1,11 @@ import logging import logging.config import sys +import time +import traceback from collections.abc import Callable from io import StringIO +from typing import Final from unittest.mock import patch import pytest @@ -11,11 +14,12 @@ from litellm._logging import ( JsonFormatter, _redact_string, _secret_filter, + redact_internal_details_from_client_message, verbose_logger, verbose_proxy_logger, verbose_router_logger, ) -from litellm.litellm_core_utils.secret_redaction import redact_string +from litellm.litellm_core_utils.secret_redaction import redact_internal_details, redact_string SECRET = "sk-proj-abc123def456ghi789jklmnopqrst" @@ -67,6 +71,50 @@ def test_redact_string_catches_secret_patterns(): assert redact_string(normal) == normal +@pytest.mark.parametrize( + "connection_string", + [ + "postgres://admin:pass3cret@db.example.com:5432/mydb", + "redis://:pass3cret@cache.example.com:6379", + "postgres://admin:pass/s3cret@db.example.com:5432/mydb", + "amqp://admin:pass:s3cret@rabbit:5672", + "https://ad@min:pass3cret@host", + # An unencoded "@" inside the password, with a ":" after it + "postgresql://admin:p@ss3cret:2026@db.example.com:5432/mydb", + "amqp://guest:gu@st3cret:1@rabbit:5672/", + # An AWS RDS IAM auth token is a presigned query string used as the + # password, so the userinfo runs to several hundred characters. + "postgresql://litellm:host%3A5432%2F%3FAction%3Dconnect%26X-Amz-Signature%3D" + + "f" * 540 + + "s3cret@db.host:5432/litellm", + ], +) +def test_redact_string_still_catches_connection_string_credentials(connection_string): + """The bounded userinfo pattern must keep matching real connection strings.""" + assert "s3cret" not in redact_string(connection_string) + + +def _redaction_cost(url_bytes: int) -> float: + url: Final = "/x?u=" + "a://" * (url_bytes // 4) + + def once() -> float: + started = time.perf_counter() + redact_string(url) + return time.perf_counter() - started + + return min(once() for _ in range(3)) + + +def test_redact_string_stays_sub_quadratic_on_a_long_adversarial_url(): + """Access-log redaction runs on attacker-controlled request lines, so quadrupling + a URL of scheme separators must not multiply the cost by sixteen. Comparing two + sizes rather than asserting a wall-clock ceiling keeps this honest on a slow box: + the unbounded pattern this replaced cost 5s at 4 KB and 314s at 16 KB.""" + growth: Final = _redaction_cost(16 * 1024) / _redaction_cost(4 * 1024) + + assert growth < 11.0, f"cost grew {growth:.1f}x for 4x the URL length" + + def test_redact_string_catches_minimum_length_virtual_key(): """Regression test for LIT-4355: keys at the enforced 16-char minimum (MINIMUM_CUSTOM_KEY_LENGTH) must be treated as key-shaped by the scrubber.""" @@ -611,3 +659,65 @@ def test_json_formatter_redacts_non_string_extra_values(extra): assert output.strip(), "no record captured" assert SECRET not in output, f"non-string extra leaked a secret: {output}" assert "REDACTED" in output + + +@pytest.mark.parametrize( + "text,leaked", + ( + ("config file /etc/litellm/secrets/db.yaml", "/etc/litellm/secrets/db.yaml"), + ("home dir /Users/admin/.litellm/master_key.txt", "/Users/admin/.litellm/master_key.txt"), + ("cache at /var/cache/litellm/tokens.db", "/var/cache/litellm/tokens.db"), + ("path C:\\Users\\admin\\secrets.env", "C:\\Users\\admin\\secrets.env"), + ("connecting to host 10.20.30.40", "10.20.30.40"), + ("connecting to host 192.168.1.5", "192.168.1.5"), + ("connecting to host 172.16.0.9", "172.16.0.9"), + ("connecting to host 127.0.0.1", "127.0.0.1"), + ("connecting to db-primary.internal", "db-primary.internal"), + ("connecting to redis.corp", "redis.corp"), + ), +) +def test_redact_internal_details_catches_paths_and_hostnames(text, leaked): + result = redact_internal_details(text) + assert leaked not in result, f"{leaked!r} was not redacted" + assert "REDACTED" in result + + +def test_redact_internal_details_leaves_public_hostnames_and_routes_alone(): + """litellm's own error messages rely on routes like /v1/models staying legible.""" + safe_strings = ( + "call https://api.openai.com/v1/chat/completions", + "/chat/completions: Invalid model name passed in model=gpt-9", + "Call `/v1/models` to view available models for your key", + "reducto:// file IDs are not accepted through the proxy OCR API", + ) + for text in safe_strings: + assert redact_internal_details(text) == text + + +def test_redact_internal_details_layers_on_top_of_credential_redaction(): + text = "postgresql://litellm_internal:S3cr3tPGPass@10.20.30.40:5432/litellm_prod" + result = redact_internal_details(text) + assert "S3cr3tPGPass" not in result + assert "10.20.30.40" not in result + + +def test_redact_internal_details_drops_embedded_traceback(): + """Regression for LIT-6747: the traceback exception_type() embeds for SDK callers + must never reach an HTTP client.""" + try: + raise RuntimeError("socket hung up") + except RuntimeError: + raw_tb = traceback.format_exc() + message = f"litellm.APIConnectionError: MinimaxException - socket hung up\n{raw_tb}" + + result = redact_internal_details(message) + + assert result == "litellm.APIConnectionError: MinimaxException - socket hung up" + assert "Traceback (most recent call last)" not in result + assert __file__.split("/")[-1] not in result + + +def test_redact_internal_details_from_client_message_respects_disable_flag(): + with patch("litellm._logging._ENABLE_SECRET_REDACTION", False): # test-quality-ok: the opt-out flag is the SUT + text = "config file /etc/litellm/secrets/db.yaml" + assert redact_internal_details_from_client_message(text) == text diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 521e91daded..200cfd02197 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -41,7 +41,6 @@ from litellm.utils import ( _snapshot_exception_for_hook, async_post_call_failure_deployment_hook, client, - get_api_key, get_llm_provider, get_non_default_completion_params, get_optional_params_image_gen, @@ -4655,6 +4654,7 @@ GEMINI_4096_CACHE_MIN_MODELS: Final = tuple( "gemini-3.5-flash", "gemini-3.6-flash", "gemini-3.7-flash", + "gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3.1-pro-preview-customtools", ) @@ -4916,17 +4916,6 @@ def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkey _invalidate_model_cost_lowercase_map() -def test_ai21_api_key_is_resolved_from_the_documented_env_var(monkeypatch: pytest.MonkeyPatch) -> None: - """The ai21 branch resolved a misspelled env var, so the name every other ai21 code path - reads, and the only name documented, was ignored.""" - monkeypatch.setattr(litellm, "api_key", None) - monkeypatch.setattr(litellm, "ai21_key", None) - monkeypatch.delenv("AI211_API_KEY", raising=False) - monkeypatch.setenv("AI21_API_KEY", "sk-ai21-resolved-from-env") - - assert get_api_key(llm_provider="ai21", dynamic_api_key=None) == "sk-ai21-resolved-from-env" - - class _JsonCapture(logging.Handler): def __init__(self): super().__init__() diff --git a/tests/test_litellm/vector_stores/__init__.py b/tests/test_litellm/vector_stores/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/vector_stores/test_main.py b/tests/test_litellm/vector_stores/test_main.py new file mode 100644 index 00000000000..d01e696906a --- /dev/null +++ b/tests/test_litellm/vector_stores/test_main.py @@ -0,0 +1,78 @@ +""" +Tests for litellm/vector_stores/main.py. + +Pins the router threading contract for vector store search: the router is an +explicit named parameter that reaches the HTTP handler, and it must never leak +into litellm_params/kwargs where logging would model_dump() it (the #19550 +serialization trap). +""" + +from unittest.mock import MagicMock, patch + +import litellm.vector_stores.main as vector_stores_main +from litellm.vector_stores.main import search + +MOCK_SEARCH_RESPONSE = { + "object": "vector_store.search_results.page", + "search_query": "q", + "data": [], +} + + +def test_search_threads_router_to_handler(): + """search() must pass its router param through to the HTTP handler""" + mock_router = MagicMock() + logger = MagicMock() + + with ( + patch( # test-quality-ok: stubs provider config resolution; the seam under test is the router kwarg threading + "litellm.vector_stores.main.ProviderConfigManager.get_provider_vector_stores_config", + return_value=MagicMock(), + ), + patch.object( # test-quality-ok: the handler call is the observable boundary for the router kwarg contract + vector_stores_main.base_llm_http_handler, + "vector_store_search_handler", + return_value=MOCK_SEARCH_RESPONSE, + ) as mock_handler, + ): + response = search( + vector_store_id="bkt:idx", + query="q", + custom_llm_provider="s3_vectors", + router=mock_router, + litellm_logging_obj=logger, + ) + + assert response == MOCK_SEARCH_RESPONSE + mock_handler.assert_called_once() + assert mock_handler.call_args.kwargs["router"] is mock_router + + +def test_search_router_not_in_litellm_params(): + """Regression (#19550 class): the router must stay out of GenericLiteLLMParams, + otherwise pre-call logging model_dump()s it and breaks serialization.""" + mock_router = MagicMock() + logger = MagicMock() + + with ( + patch( # test-quality-ok: stubs provider config resolution; the seam under test is litellm_params contents + "litellm.vector_stores.main.ProviderConfigManager.get_provider_vector_stores_config", + return_value=MagicMock(), + ), + patch.object( # test-quality-ok: the handler call is where a leaked router in litellm_params would surface + vector_stores_main.base_llm_http_handler, + "vector_store_search_handler", + return_value=MOCK_SEARCH_RESPONSE, + ) as mock_handler, + ): + search( + vector_store_id="bkt:idx", + query="q", + custom_llm_provider="s3_vectors", + router=mock_router, + litellm_logging_obj=logger, + ) + + litellm_params = mock_handler.call_args.kwargs["litellm_params"] + assert "router" not in litellm_params.model_dump(exclude_none=True) + assert getattr(litellm_params, "router", None) is None diff --git a/tests/test_rust_python_harness.py b/tests/test_rust_python_harness.py new file mode 100644 index 00000000000..6a8fa8d35cc --- /dev/null +++ b/tests/test_rust_python_harness.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import importlib +import json +from pathlib import Path + +import pytest + +catalog = importlib.import_module("tests.rust-python-harness.catalog") +cli = importlib.import_module("tests.rust-python-harness.cli") +ledger_module = importlib.import_module("tests.rust-python-harness.shared.parity.ledger") +mapping_validator = importlib.import_module( + "tests.rust-python-harness.strategies.unit_tests.mapping_validator" +) +models = importlib.import_module("tests.rust-python-harness.models") +runner = importlib.import_module("tests.rust-python-harness.runner") +ui = importlib.import_module("tests.rust-python-harness.ui") + +load_catalog = catalog.load_catalog +load_ledger = ledger_module.load_ledger +ledger_path_for = mapping_validator.ledger_path_for +REPO_ROOT = mapping_validator.REPO_ROOT +audit_ledger = mapping_validator.audit_ledger +build_function_report = mapping_validator.build_function_report +_pick_values = cli._pick_values +_coverage_pytest_args = cli._coverage_pytest_args +_select = cli._select +_validate_ledger = cli._validate_ledger +CaseResult = models.CaseResult +Coverage = models.Coverage +HarnessCase = models.HarnessCase +HarnessRun = models.HarnessRun +RunStatus = models.RunStatus +SDK_FUNCTIONS = models.SDK_FUNCTIONS +section_confidence = models.section_confidence +run_pytest = runner.run_pytest +runnable_selectors = runner.runnable_selectors +selector_matches_node = runner.selector_matches_node +_format_duration = ui._format_duration +_rerun_command = ui._rerun_command +_summary = ui._summary + + +def _case( + *, selectors: tuple[str, ...] = (), coverage: Coverage = Coverage.COMPLETE +) -> HarnessCase: + return HarnessCase( + strategy_id="example", + strategy_label="Example", + sdk_function="messages", + coverage=coverage, + selectors=selectors, + ) + + +def _manifest() -> dict[str, object]: + return { + "order": 1, + "id": "example", + "label": "Example strategy", + "description": "Example description", + "functions": { + function: {"coverage": "planned", "selectors": []} + for function in SDK_FUNCTIONS + }, + } + + +def test_should_load_the_three_harness_strategies_in_order() -> None: + strategies = load_catalog() + + assert [strategy.id for strategy in strategies] == [ + "e2e_fuzz_tests", + "unit_tests_rust", + "validate_sub_methods", + ] + assert all( + tuple(case.sdk_function for case in strategy.cases) == SDK_FUNCTIONS + for strategy in strategies + ) + + +def test_should_reject_a_manifest_missing_an_sdk_function(tmp_path: Path) -> None: + strategy_directory = tmp_path / "example" + strategy_directory.mkdir() + manifest = _manifest() + del manifest["functions"]["count_tokens"] # type: ignore[index] + (strategy_directory / "strategy.json").write_text( + json.dumps(manifest), encoding="utf-8" + ) + + with pytest.raises(ValueError, match="functions must exactly match"): + load_catalog(tmp_path) + + +@pytest.mark.parametrize( + ("selector", "nodeid", "matches"), + [ + ("tests/test_parity.py", "tests/test_parity.py::test_one", True), + ("tests/test_parity.py::test_one", "tests/test_parity.py::test_one", True), + ( + "tests/test_parity.py::test_one", + "tests/test_parity.py::test_one[value]", + True, + ), + ("tests/test_parity.py::test_one", "tests/test_parity.py::test_two", False), + ], +) +def test_should_match_pytest_file_and_node_selectors( + selector: str, nodeid: str, matches: bool +) -> None: + assert selector_matches_node(selector, nodeid) is matches + + +def test_should_only_return_selectors_whose_files_exist(tmp_path: Path) -> None: + existing = tmp_path / "tests" / "test_parity.py" + existing.parent.mkdir() + existing.write_text("", encoding="utf-8") + case = _case( + selectors=("tests/test_parity.py", "tests/test_missing.py::test_missing") + ) + + assert runnable_selectors((case,), tmp_path) == ("tests/test_parity.py",) + + +def test_should_mark_planned_and_not_applicable_cases_without_running() -> None: + planned = CaseResult(case=_case(coverage=Coverage.PLANNED)) + not_applicable = CaseResult(case=_case(coverage=Coverage.NOT_APPLICABLE)) + + planned.set_initial_status() + not_applicable.set_initial_status() + + assert planned.status is RunStatus.PLANNED + assert not_applicable.status is RunStatus.NOT_APPLICABLE + + +def test_should_treat_an_all_planned_filtered_run_as_success(tmp_path: Path) -> None: + exit_code, run = run_pytest( + cases=(_case(coverage=Coverage.PLANNED),), + repo_root=tmp_path, + on_update=lambda _: None, + ) + + assert exit_code == 0 + assert next(iter(run.results.values())).status is RunStatus.PLANNED + + +def test_should_finalize_a_fully_passing_case() -> None: + result = CaseResult(case=_case(selectors=("tests/test_parity.py",))) + result.set_initial_status() + result.collected.update({"one", "two"}) + result.completed.update({"one", "two"}) + result.passed = 2 + + result.finalize() + + assert result.status is RunStatus.PASSED + + +def test_should_replace_a_pass_with_a_teardown_error() -> None: + result = CaseResult(case=_case(selectors=("tests/test_parity.py",))) + result.set_initial_status() + result.collected.add("one") + + result.record("one", RunStatus.PASSED, 0.1) + result.record("one", RunStatus.ERROR, 0.2) + + assert result.status is RunStatus.ERROR + assert result.passed == 0 + assert result.errors == 1 + assert result.duration == pytest.approx(0.3) + + +def test_should_filter_the_catalog_by_strategy_and_sdk_function() -> None: + strategies = load_catalog() + + cases = _select(strategies, {"e2e_fuzz_tests"}, {"messages"}) + + assert len(cases) == 1 + assert cases[0].key == "e2e_fuzz_tests:messages" + + +def test_should_reject_an_unknown_strategy() -> None: + with pytest.raises(ValueError, match="Unknown strategy"): + _select(load_catalog(), {"not-real"}, set()) + + +def test_should_pick_multiple_interactive_filters() -> None: + answers = iter(["nope", "1, 3"]) + + selected = _pick_values( + "Examples", + (("one", "One"), ("two", "Two"), ("three", "Three")), + input_fn=lambda _: next(answers), + ) + + assert selected == {"one", "three"} + + +def test_should_format_developer_facing_run_context() -> None: + run = HarnessRun.from_cases((_case(selectors=("tests/test_parity.py",)),)) + result = next(iter(run.results.values())) + result.collected.add("tests/test_parity.py::test_one") + result.record("tests/test_parity.py::test_one", RunStatus.PASSED, 1.25) + + assert _summary(run) == (1, 0, 0, 0) + assert _format_duration(1.25) == "1.2s" + assert _rerun_command("tests/test_parity.py::test_one") == ( + "poetry run pytest tests/test_parity.py::test_one -q" + ) + assert _rerun_command("tests/test_parity.py::test_one[value with spaces]") == ( + "poetry run pytest 'tests/test_parity.py::test_one[value with spaces]' -q" + ) + + +def test_should_build_python_coverage_reports_below_the_target_directory( + tmp_path: Path, +) -> None: + args = _coverage_pytest_args(tmp_path) + + assert tmp_path.is_dir() + assert "--cov=litellm" in args + assert "--cov-context=test" in args + assert f"--cov-report=json:{tmp_path / 'python.json'}" in args + assert f"--cov-report=xml:{tmp_path / 'python.xml'}" in args + assert f"--cov-report=html:{tmp_path / 'python-html'}" in args + + +def test_should_report_confidence_for_each_sdk_section() -> None: + strategies = load_catalog() + cases = tuple(case for strategy in strategies for case in strategy.cases) + run = HarnessRun.from_cases(cases) + passing = run.results["e2e_fuzz_tests:responses"] + passing.collected.add("tests/test_parity.py::test_one") + passing.record("tests/test_parity.py::test_one", RunStatus.PASSED) + + scores = { + score.sdk_function: score for score in section_confidence(run, strategies) + } + + assert scores["responses"].verified_strategies == 1 + assert scores["responses"].required_strategies == 3 + assert scores["responses"].percentage == 33 + assert scores["responses"].level.value == "MEDIUM" + assert scores["count_tokens"].percentage == 0 + assert scores["count_tokens"].level.value == "LOW" + + + +def test_should_report_no_ledger_for_a_function_without_one() -> None: + report = build_function_report("messages", repo_root=REPO_ROOT) + + assert report.has_ledger is False + assert report.is_clean is True + + +def test_should_report_ocr_ledger_stats_and_a_clean_audit() -> None: + ledger = load_ledger(ledger_path_for("ocr")) + + report = build_function_report("ocr", repo_root=REPO_ROOT) + + assert report.has_ledger is True + assert report.ledger.mapped_count == ledger.mapped_count + assert report.ledger.total_count == ledger.total_count + assert report.is_clean is True + + +def test_should_scope_validate_ledger_to_the_requested_function( + capsys: pytest.CaptureFixture[str], +) -> None: + exit_code = _validate_ledger({"messages"}) + + captured = capsys.readouterr() + assert exit_code == 0 + assert "messages" in captured.out + assert "no ledger yet" in captured.out + assert "ocr" not in captured.out + + +def test_should_have_every_python_and_rust_ocr_test_accounted_for_in_the_ledger() -> None: + ledger = load_ledger(ledger_path_for("ocr")) + + report = audit_ledger(ledger, repo_root=REPO_ROOT) + + assert report.is_clean, ( + "\nOCR test-parity ledger is out of sync with the live test files.\n" + f"Ledger references a Python test that no longer exists: {list(report.missing_python_tests)}\n" + f"Python test exists but is not tracked in the ledger: {list(report.stale_python_tests)}\n" + f"Ledger references a Rust test that no longer exists: {list(report.missing_rust_tests)}\n" + f"Rust test exists but is not tracked in the ledger: {list(report.stale_rust_tests)}\n" + ) diff --git a/tests/vector_store_tests/test_azure_ai_vector_store.py b/tests/vector_store_tests/test_azure_ai_vector_store.py index 58e45f259ab..d1fc8436fc9 100644 --- a/tests/vector_store_tests/test_azure_ai_vector_store.py +++ b/tests/vector_store_tests/test_azure_ai_vector_store.py @@ -1,10 +1,19 @@ -import pytest -import litellm import json import os +from unittest.mock import MagicMock + +import httpx +import pytest +import respx + +import litellm +from litellm.llms.azure_ai.vector_stores.transformation import AzureAIVectorStoreConfig +from litellm.types.utils import EmbeddingResponse +from litellm.vector_stores import ( + asearch as vector_store_asearch, +) from litellm.vector_stores import ( search as vector_store_search, - asearch as vector_store_asearch, ) @@ -30,10 +39,108 @@ async def test_basic_search_vector_store(sync_mode): if sync_mode: response = vector_store_search(query=default_query, **base_request_args) else: - response = await vector_store_asearch( - query=default_query, **base_request_args - ) + response = await vector_store_asearch(query=default_query, **base_request_args) except litellm.InternalServerError: pytest.skip("Skipping test due to litellm.InternalServerError") print("litellm response=", json.dumps(response, indent=4, default=str)) + + +class RecordingEmbeddingExecutor: + def __init__(self, response): + self.response = response + self.calls = [] + + def embed(self, model, query, configuration): + self.calls.append((model, query, dict(configuration))) + return self.response + + async def aembed(self, model, query, configuration): + self.calls.append((model, query, dict(configuration))) + return self.response + + +ALIAS_QUERY_VECTOR = [0.5, -0.25, 0.125] +ALIAS_EMBEDDING_RESPONSE = EmbeddingResponse( + data=[{"embedding": ALIAS_QUERY_VECTOR, "index": 0, "object": "embedding"}] +) +STORE_EMBEDDINGS_URL = "https://embedding.example/v1/embeddings" + + +def _transform_kwargs(executor): + logging_obj = MagicMock() + logging_obj.model_call_details = {} + return { + "vector_store_id": "my-vector-index", + "query": "what is azure search?", + "vector_store_search_optional_params": {"top_k": 2}, + "api_base": "https://azure-kb-search.search.windows.net", + "litellm_logging_obj": logging_obj, + "litellm_params": { + "litellm_embedding_model": "multilingual-e5-large", + "azure_search_vector_field": "embedding", + }, + "embedding_executor": executor, + } + + +@pytest.mark.asyncio +async def test_transform_uses_injected_executor_without_embedding_config(respx_mock: respx.MockRouter): + executor = RecordingEmbeddingExecutor(ALIAS_EMBEDDING_RESPONSE) + config = AzureAIVectorStoreConfig() + transform_kwargs = _transform_kwargs(executor) + + url, sync_body = config.transform_search_vector_store_request(**transform_kwargs) + _, async_body = await config.atransform_search_vector_store_request(**transform_kwargs) + + assert respx_mock.calls.call_count == 0 + assert executor.calls == [("multilingual-e5-large", "what is azure search?", {})] * 2 + assert ( + url == "https://azure-kb-search.search.windows.net/indexes/my-vector-index/docs/search?api-version=2024-07-01" + ) + assert sync_body == async_body + assert sync_body["vectorQueries"] == [ + {"vector": ALIAS_QUERY_VECTOR, "fields": "embedding", "kind": "vector", "k": 2} + ] + assert sync_body["top"] == 2 + logging_details = transform_kwargs["litellm_logging_obj"].model_call_details + assert logging_details["embedding_model"] == "multilingual-e5-large" + assert logging_details["top_k"] == 2 + + +def test_transform_falls_back_to_sdk_embedding_without_executor( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = respx_mock.post(STORE_EMBEDDINGS_URL).mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": ALIAS_QUERY_VECTOR}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + ) + transform_kwargs = _transform_kwargs(None) + transform_kwargs["litellm_params"] = { + "litellm_embedding_model": "openai/text-embedding-3-small", + "litellm_embedding_config": {"api_base": "https://embedding.example/v1", "api_key": "store-key"}, + } + + _, body = AzureAIVectorStoreConfig().transform_search_vector_store_request(**transform_kwargs) + + embedding_request = embedding_route.calls.last.request + assert embedding_request.headers["authorization"] == "Bearer store-key" + assert json.loads(embedding_request.read())["input"] == ["what is azure search?"] + assert body["vectorQueries"][0]["vector"] == ALIAS_QUERY_VECTOR + assert body["vectorQueries"][0]["fields"] == "contentVector" + + +def test_transform_requires_embedding_model(): + transform_kwargs = _transform_kwargs(RecordingEmbeddingExecutor(ALIAS_EMBEDDING_RESPONSE)) + transform_kwargs["litellm_params"] = {"litellm_embedding_config": {"api_key": "store-key"}} + + with pytest.raises(ValueError, match="litellm_embedding_model is required"): + AzureAIVectorStoreConfig().transform_search_vector_store_request(**transform_kwargs) diff --git a/tests/vector_store_tests/test_milvus_vector_store.py b/tests/vector_store_tests/test_milvus_vector_store.py index 6627f6006d1..2ba9168b49f 100644 --- a/tests/vector_store_tests/test_milvus_vector_store.py +++ b/tests/vector_store_tests/test_milvus_vector_store.py @@ -3,16 +3,19 @@ Tests for Milvus Vector Store """ import json -import os from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +import respx import litellm +from litellm import Router +from litellm.llms.milvus.vector_stores.transformation import MilvusVectorStoreConfig +from litellm.types.utils import EmbeddingResponse from litellm.vector_stores import asearch as vector_store_asearch from litellm.vector_stores import search as vector_store_search - # Mock response from actual Milvus API MOCK_MILVUS_SEARCH_RESPONSE = { "code": 0, @@ -98,7 +101,7 @@ class TestMilvusVectorStore: mock_response.json.return_value = MOCK_MILVUS_SEARCH_RESPONSE mock_response.text = json.dumps(MOCK_MILVUS_SEARCH_RESPONSE) - with patch("litellm.embedding") as mock_embedding: + with patch("litellm.aembedding", new_callable=AsyncMock) as mock_embedding: mock_embedding.return_value = MOCK_EMBEDDING_RESPONSE with patch( @@ -147,16 +150,10 @@ class TestMilvusVectorStore: else: # Fallback: check for json kwarg or in args request_data = call_args.kwargs.get("json") - if ( - request_data is None - and len(call_args.args) > 0 - and isinstance(call_args.args[0], dict) - ): + if request_data is None and len(call_args.args) > 0 and isinstance(call_args.args[0], dict): request_data = call_args.args[0] - assert ( - request_data is not None - ), f"Could not extract request data. Call args: {call_args}" + assert request_data is not None, f"Could not extract request data. Call args: {call_args}" print("Request data:", json.dumps(request_data, indent=2, default=str)) # Validate request structure @@ -213,9 +210,7 @@ class TestMilvusVectorStore: with patch("litellm.embedding") as mock_embedding: mock_embedding.return_value = MOCK_EMBEDDING_RESPONSE - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post" - ) as mock_post: + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: mock_post.return_value = mock_response # Make the search request @@ -252,16 +247,10 @@ class TestMilvusVectorStore: else: # Fallback: check for json kwarg or in args request_data = call_args.kwargs.get("json") - if ( - request_data is None - and len(call_args.args) > 0 - and isinstance(call_args.args[0], dict) - ): + if request_data is None and len(call_args.args) > 0 and isinstance(call_args.args[0], dict): request_data = call_args.args[0] - assert ( - request_data is not None - ), f"Could not extract request data. Call args: {call_args}" + assert request_data is not None, f"Could not extract request data. Call args: {call_args}" # Validate request structure assert "collectionName" in request_data @@ -316,11 +305,7 @@ class TestMilvusVectorStore: if request_data_str: return json.loads(request_data_str) request_data = call_args.kwargs.get("json") - if ( - request_data is None - and len(call_args.args) > 0 - and isinstance(call_args.args[0], dict) - ): + if request_data is None and len(call_args.args) > 0 and isinstance(call_args.args[0], dict): request_data = call_args.args[0] return request_data @@ -334,9 +319,7 @@ class TestMilvusVectorStore: with patch("litellm.embedding") as mock_embedding: mock_embedding.return_value = MOCK_EMBEDDING_RESPONSE - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post" - ) as mock_post: + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: mock_post.return_value = mock_response vector_store_search( @@ -375,9 +358,7 @@ class TestMilvusVectorStore: with patch("litellm.embedding") as mock_embedding: mock_embedding.return_value = MOCK_EMBEDDING_RESPONSE - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post" - ) as mock_post: + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: mock_post.return_value = mock_response vector_store_search( @@ -413,9 +394,7 @@ class TestMilvusVectorStore: with patch("litellm.embedding") as mock_embedding: mock_embedding.return_value = MOCK_EMBEDDING_RESPONSE - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post" - ) as mock_post: + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: mock_post.return_value = mock_response vector_store_search( @@ -492,3 +471,247 @@ if __name__ == "__main__": test.test_basic_search_with_mock_sync() print("\n✅ All mock tests passed!") + + +class RecordingEmbeddingExecutor: + def __init__(self, response): + self.response = response + self.calls = [] + + def embed(self, model, query, configuration): + self.calls.append((model, query, dict(configuration))) + return self.response + + async def aembed(self, model, query, configuration): + self.calls.append((model, query, dict(configuration))) + return self.response + + +ALIAS_QUERY_VECTOR = [0.5, -0.25, 0.125] +ALIAS_EMBEDDING_RESPONSE = EmbeddingResponse( + data=[{"embedding": ALIAS_QUERY_VECTOR, "index": 0, "object": "embedding"}] +) +OPENAI_EMBEDDINGS_URL = "https://api.openai.com/v1/embeddings" +MILVUS_SEARCH_URL = "https://milvus.example/v2/vectordb/entities/search" +ALIAS_SEARCH_KWARGS = { + "query": "what is machine learning?", + "vector_store_id": "book_2", + "custom_llm_provider": "milvus", + "api_base": "https://milvus.example", + "api_key": "mock_milvus_api_key", + "litellm_embedding_model": "multilingual-e5-large", + "milvus_text_field": "book_intro_text", +} + + +def _alias_router(): + return Router( + model_list=[ + { + "model_name": "multilingual-e5-large", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "deployment-key", + }, + } + ] + ) + + +def _mock_embedding_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(OPENAI_EMBEDDINGS_URL).mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": ALIAS_QUERY_VECTOR}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + ) + + +def _mock_search_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(MILVUS_SEARCH_URL).mock(return_value=httpx.Response(200, json=MOCK_MILVUS_SEARCH_RESPONSE)) + + +def _assert_alias_resolved(embedding_route: respx.Route, search_route: respx.Route, response): + embedding_request = embedding_route.calls.last.request + assert embedding_request.headers["authorization"] == "Bearer deployment-key" + embedding_body = json.loads(embedding_request.read()) + assert embedding_body["model"] == "text-embedding-3-small" + assert embedding_body["input"] == ["what is machine learning?"] + search_request = search_route.calls.last.request + assert search_request.headers["authorization"] == "Bearer mock_milvus_api_key" + assert json.loads(search_request.read())["data"] == [ALIAS_QUERY_VECTOR] + assert len(response["data"]) == len(MOCK_MILVUS_SEARCH_RESPONSE["data"]) + assert response["data"][0]["content"][0]["text"] == MOCK_MILVUS_SEARCH_RESPONSE["data"][0]["book_intro_text"] + + +def test_router_search_resolves_bare_embedding_alias_sync( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_search_route(respx_mock) + + response = _alias_router().vector_store_search(**ALIAS_SEARCH_KWARGS) + + _assert_alias_resolved(embedding_route, search_route, response) + + +@pytest.mark.asyncio +async def test_router_search_resolves_bare_embedding_alias_async( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_search_route(respx_mock) + + response = await _alias_router().avector_store_search(**ALIAS_SEARCH_KWARGS) + + _assert_alias_resolved(embedding_route, search_route, response) + + +def test_sdk_search_with_router_kwarg_resolves_bare_embedding_alias_sync( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_search_route(respx_mock) + + response = litellm.vector_stores.search(router=_alias_router(), **ALIAS_SEARCH_KWARGS) + + _assert_alias_resolved(embedding_route, search_route, response) + + +@pytest.mark.asyncio +async def test_sdk_search_with_router_kwarg_resolves_bare_embedding_alias_async( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_search_route(respx_mock) + + response = await litellm.vector_stores.asearch(router=_alias_router(), **ALIAS_SEARCH_KWARGS) + + _assert_alias_resolved(embedding_route, search_route, response) + + +def _team_alias_router(): + return Router( + model_list=[ + { + "model_name": "team-a-embedder", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "deployment-key", + }, + "model_info": {"team_id": "team-a", "team_public_model_name": "multilingual-e5-large"}, + } + ] + ) + + +@pytest.mark.asyncio +async def test_sdk_search_with_router_kwarg_resolves_team_alias_from_request_metadata( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_search_route(respx_mock) + + response = await litellm.vector_stores.asearch( + router=_team_alias_router(), metadata={"user_api_key_team_id": "team-a"}, **ALIAS_SEARCH_KWARGS + ) + + _assert_alias_resolved(embedding_route, search_route, response) + + +@pytest.mark.asyncio +async def test_sdk_search_with_router_kwarg_rejects_team_alias_without_team_metadata( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + _mock_search_route(respx_mock) + + with pytest.raises(litellm.APIConnectionError): + await litellm.vector_stores.asearch(router=_team_alias_router(), **ALIAS_SEARCH_KWARGS) + + assert embedding_route.call_count == 0 + + +@pytest.mark.asyncio +async def test_transform_uses_injected_executor_without_embedding_config(respx_mock: respx.MockRouter): + executor = RecordingEmbeddingExecutor(ALIAS_EMBEDDING_RESPONSE) + config = MilvusVectorStoreConfig() + logging_obj = MagicMock() + logging_obj.model_call_details = {} + transform_kwargs = { + "vector_store_id": "book_2", + "query": ["what is", "milvus?"], + "vector_store_search_optional_params": {"limit": 3}, + "api_base": "https://milvus.example", + "litellm_logging_obj": logging_obj, + "litellm_params": {"litellm_embedding_model": "multilingual-e5-large", "milvus_db_name": "docs"}, + "embedding_executor": executor, + } + + url, sync_body = config.transform_search_vector_store_request(**transform_kwargs) + _, async_body = await config.atransform_search_vector_store_request(**transform_kwargs) + + assert respx_mock.calls.call_count == 0 + assert executor.calls == [("multilingual-e5-large", "what is milvus?", {})] * 2 + assert url == MILVUS_SEARCH_URL + assert sync_body == async_body + assert sync_body == { + "collectionName": "book_2", + "data": [ALIAS_QUERY_VECTOR], + "annsField": "book_intro_vector", + "limit": 3, + "dbName": "docs", + } + assert logging_obj.model_call_details["input"] == "what is milvus?" + assert logging_obj.model_call_details["embedding_model"] == "multilingual-e5-large" + + +def test_transform_falls_back_to_sdk_embedding_without_executor_or_config( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("OPENAI_API_KEY", "env-key") + embedding_route = _mock_embedding_route(respx_mock) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + + _, body = MilvusVectorStoreConfig().transform_search_vector_store_request( + vector_store_id="book_2", + query="q", + vector_store_search_optional_params={}, + api_base="https://milvus.example", + litellm_logging_obj=logging_obj, + litellm_params={"litellm_embedding_model": "openai/text-embedding-3-small"}, + ) + + embedding_request = embedding_route.calls.last.request + assert embedding_request.headers["authorization"] == "Bearer env-key" + assert json.loads(embedding_request.read())["input"] == ["q"] + assert body["data"] == [ALIAS_QUERY_VECTOR] + + +def test_transform_requires_embedding_model(): + with pytest.raises(ValueError, match="litellm_embedding_model is required"): + MilvusVectorStoreConfig().transform_search_vector_store_request( + vector_store_id="book_2", + query="q", + vector_store_search_optional_params={}, + api_base="https://milvus.example", + litellm_logging_obj=MagicMock(), + litellm_params={"litellm_embedding_config": {"api_key": "store-key"}}, + embedding_executor=RecordingEmbeddingExecutor(ALIAS_EMBEDDING_RESPONSE), + ) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 52cb9628252..f3c4c7760c6 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,12 +1,12 @@ { "LIT001": { - "limit": 22364 + "limit": 22334 }, "LIT002": { - "limit": 26777 + "limit": 26763 }, "LIT003": { - "limit": 269 + "limit": 261 }, "LIT004": { "limit": 40 @@ -15,7 +15,7 @@ "limit": 0 }, "LIT006": { - "limit": 1039 + "limit": 1038 }, "LIT007": { "limit": 0 @@ -27,12 +27,12 @@ "limit": 0 }, "LIT010": { - "limit": 16507 + "limit": 16480 }, "LIT011": { - "limit": 5535 + "limit": 5520 }, "LIT012": { - "limit": 4495 + "limit": 4489 } } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx index 79bd2f6a21b..de1eb153f6f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx @@ -75,6 +75,40 @@ const langgraphInfo: AgentCreateInfo = { ], }; +const FULL_RUNTIME_ARN = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime/hosted_agent_4vm3i-BaTdfOELAs"; + +const BEDROCK_AGENTCORE_AGENT = { + agent_id: "agent-3", + agent_name: "bedrock-agent", + agent_card_params: { name: "bedrock-agent", description: "agentcore agent", url: "", version: "1.0.0", skills: [] }, + litellm_params: { + custom_llm_provider: "bedrock", + model: `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + }, +}; + +const bedrockAgentcoreInfo: AgentCreateInfo = { + agent_type: "bedrock_agentcore", + agent_type_display_name: "Bedrock AgentCore", + description: "Bedrock AgentCore runtimes", + logo_url: "/b.png", + use_a2a_form_fields: false, + litellm_params_template: { custom_llm_provider: "bedrock" }, + model_template: "bedrock/agentcore/{agent_runtime_arn}", + credential_fields: [ + { + key: "agent_runtime_arn", + label: "Agent Runtime ARN", + field_type: "text", + required: true, + include_in_litellm_params: false, + validation_pattern: "^arn:aws[a-zA-Z0-9-]*:bedrock-agentcore:[a-z0-9-]+:[0-9]{12}:runtime/.+$", + validation_message: + 'Enter the complete Bedrock AgentCore runtime ARN, including the runtime ID after "runtime/".', + }, + ], +}; + const setup = () => userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); const renderView = () => render(); @@ -247,6 +281,69 @@ describe("AgentInfoView update payload", () => { }); }); + it("preserves the full AgentCore runtime ARN (including the resource id after runtime/) across an unedited save", async () => { + vi.mocked(networking.getAgentCreateMetadata).mockResolvedValue([bedrockAgentcoreInfo]); + vi.mocked(networking.getAgentInfo).mockResolvedValue(BEDROCK_AGENTCORE_AGENT as never); + const user = setup(); + renderView(); + await openEditor(user); + + expect(await screen.findByLabelText("Agent Runtime ARN")).toHaveValue(FULL_RUNTIME_ARN); + + await save(user); + + expect((patchedPayload().litellm_params as Record).model).toBe( + `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + ); + }); + + it("blocks the save and shows a validation error when the Agent Runtime ARN is truncated", async () => { + vi.mocked(networking.getAgentCreateMetadata).mockResolvedValue([bedrockAgentcoreInfo]); + vi.mocked(networking.getAgentInfo).mockResolvedValue(BEDROCK_AGENTCORE_AGENT as never); + const user = setup(); + renderView(); + await openEditor(user); + + const arnField = await screen.findByLabelText("Agent Runtime ARN"); + await user.clear(arnField); + fireEvent.change(arnField, { + target: { value: "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime" }, + }); + await user.click(screen.getByRole("button", { name: "Save Changes" })); + + expect( + await screen.findByText( + 'Enter the complete Bedrock AgentCore runtime ARN, including the runtime ID after "runtime/".', + ), + ).toBeInTheDocument(); + expect(networking.patchAgentCall).not.toHaveBeenCalled(); + }); + + it("renders and saves normally when a field's validation_pattern is not a valid regex", async () => { + const infoWithBadPattern: AgentCreateInfo = { + ...bedrockAgentcoreInfo, + credential_fields: [ + { + ...bedrockAgentcoreInfo.credential_fields[0], + validation_pattern: "(unterminated", + }, + ], + }; + vi.mocked(networking.getAgentCreateMetadata).mockResolvedValue([infoWithBadPattern]); + vi.mocked(networking.getAgentInfo).mockResolvedValue(BEDROCK_AGENTCORE_AGENT as never); + const user = setup(); + renderView(); + await openEditor(user); + + expect(await screen.findByLabelText("Agent Runtime ARN")).toHaveValue(FULL_RUNTIME_ARN); + + await save(user); + + expect((patchedPayload().litellm_params as Record).model).toBe( + `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + ); + }); + it("reloads the agent and leaves edit mode when the edit is cancelled", async () => { const user = setup(); renderView(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.test.ts new file mode 100644 index 00000000000..0c2d2500776 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.test.ts @@ -0,0 +1,73 @@ +import { describe, it, expect } from "vitest"; +import { detectAgentType, extractModelTemplateValues, parseDynamicAgentForForm } from "./agent_type_utils"; +import type { AgentCreateInfo } from "@/components/networking"; +import type { Agent } from "@/components/agents/types"; + +const FULL_RUNTIME_ARN = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime/hosted_agent_4vm3i-BaTdfOELAs"; + +const bedrockAgentcoreInfo: AgentCreateInfo = { + agent_type: "bedrock_agentcore", + agent_type_display_name: "Bedrock AgentCore", + model_template: "bedrock/agentcore/{agent_runtime_arn}", + credential_fields: [ + { + key: "agent_runtime_arn", + label: "Agent Runtime ARN", + required: true, + include_in_litellm_params: false, + }, + ], +}; + +describe("extractModelTemplateValues", () => { + it("recovers a placeholder value that itself contains '/' (an AWS ARN resource path)", () => { + const values = extractModelTemplateValues( + "bedrock/agentcore/{agent_runtime_arn}", + `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + ); + + expect(values.agent_runtime_arn).toBe(FULL_RUNTIME_ARN); + }); + + it("recovers a placeholder value with no '/' (single path segment)", () => { + const values = extractModelTemplateValues("langgraph/{assistant_id}", "langgraph/asst_1"); + + expect(values.assistant_id).toBe("asst_1"); + }); + + it("returns no match when the model does not fit the template", () => { + const values = extractModelTemplateValues("langgraph/{assistant_id}", "azure_ai/agents/asst_1"); + + expect(values).toEqual({}); + }); +}); + +describe("parseDynamicAgentForForm", () => { + it("preserves the full runtime ARN, including the resource id after 'runtime/', when populating the edit form", () => { + const agent = { + agent_id: "agent-1", + agent_name: "bedrock-agent", + agent_card_params: { description: "" }, + litellm_params: { + custom_llm_provider: "bedrock", + model: `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + }, + } as unknown as Agent; + + const values = parseDynamicAgentForForm(agent, bedrockAgentcoreInfo); + + expect(values.agent_runtime_arn).toBe(FULL_RUNTIME_ARN); + }); +}); + +describe("detectAgentType", () => { + it("detects bedrock_agentcore agents from the model prefix", () => { + const agent = { + agent_id: "agent-1", + agent_name: "bedrock-agent", + litellm_params: { model: `bedrock/agentcore/${FULL_RUNTIME_ARN}` }, + } as unknown as Agent; + + expect(detectAgentType(agent)).toBe("bedrock_agentcore"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts index f91590c5732..506f355c519 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts @@ -25,6 +25,29 @@ export const detectAgentType = (agent: Agent): string => { return "a2a"; }; +const escapeRegExp = (segment: string): string => segment.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +/** + * Reverses a `model_template` (e.g. "bedrock/agentcore/{agent_runtime_arn}") against a stored + * `model` string to recover the placeholder values that produced it. Builds a regex from the + * template's literal segments rather than matching by split("/") position, because a + * placeholder's value can itself contain "/" (an AWS ARN's "runtime/" resource path, + * a Vertex AI reasoning engine's "projects/.../reasoningEngines/..." resource id) and would + * otherwise be cut off at the first one. + */ +export const extractModelTemplateValues = (template: string, model: string): Record => { + // Splitting on a regex with a capturing group interleaves the captured placeholder + // names between the surrounding literal segments, e.g. "a/{x}/b" -> ["a/", "x", "/b"]. + const parts = template.split(/\{([a-zA-Z0-9_]+)\}/g); + const fieldNames = parts.filter((_part, index) => index % 2 === 1); + const pattern = parts.map((part, index) => (index % 2 === 1 ? "(.+)" : escapeRegExp(part))).join(""); + + const match = model.match(new RegExp(`^${pattern}$`)); + if (!match) return {}; + + return Object.fromEntries(fieldNames.map((name, index) => [name, match[index + 1]])); +}; + /** * Parses agent data for dynamic form fields (non-A2A agents). * Extracts values from litellm_params based on the agent type metadata. @@ -35,24 +58,18 @@ export const parseDynamicAgentForForm = (agent: Agent, agentTypeInfo: AgentCreat description: agent.agent_card_params?.description || "", }; + const templateValues = + agentTypeInfo.model_template && agent.litellm_params?.model + ? extractModelTemplateValues(agentTypeInfo.model_template, agent.litellm_params.model) + : {}; + // Extract credential field values from litellm_params for (const field of agentTypeInfo.credential_fields) { if (field.include_in_litellm_params !== false) { values[field.key] = agent.litellm_params?.[field.key] || field.default_value || ""; - } else { - // For fields not in litellm_params (like agent_id), try to extract from model string - if (agentTypeInfo.model_template && agent.litellm_params?.model) { - const model = agent.litellm_params.model; - const templateParts = agentTypeInfo.model_template.split("/"); - const modelParts = model.split("/"); - - // Find the placeholder position and extract the value - templateParts.forEach((part, index) => { - if (part === `{${field.key}}` && modelParts[index]) { - values[field.key] = modelParts[index]; - } - }); - } + } else if (templateValues[field.key] !== undefined) { + // For fields not in litellm_params (like agent_runtime_arn), recover from the model string + values[field.key] = templateValues[field.key]; } } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx index 04a8b0df9d9..0f3355fe073 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx @@ -24,58 +24,80 @@ interface DynamicAgentFormFieldsProps { export const unmountedDynamicFieldNames = (mountedPanels: readonly string[]): readonly string[] => mountedPanels.includes(AGENT_FORM_CONFIG.cost.key) ? [] : COST_FIELD_NAMES; -const CredentialField = ({ field }: { field: AgentCredentialFieldMetadata }) => ( - - {({ value, onChange, ref, ...control }) => { - const text = typeof value === "string" ? value : ""; - if (field.field_type === "password") { - return ( - - ); - } - if (field.field_type === "textarea") { - return ( -