diff --git a/.circleci/config.yml b/.circleci/config.yml index 55fa9410845..dfc539fb80e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,10 +112,10 @@ commands: node --version npm --version install_rust: - description: "Install pinned rustup (1.28.2) and Rust toolchain (1.97.1) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself." + description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself." steps: - run: - name: Install Rust (rustup 1.28.2, toolchain 1.97.1) + name: Install Rust (rustup 1.28.2, toolchain 1.98.0) command: | case "$(uname -m)" in x86_64) @@ -135,7 +135,7 @@ commands: "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init" echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - chmod +x /tmp/rustup-init - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.97.1 + /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0 rm -f /tmp/rustup-init echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.cargo/bin:$PATH" @@ -300,7 +300,7 @@ jobs: if ($rustupActual -ne $rustupExpected) { throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual" } - & $rustupInit -y --profile minimal --default-toolchain stable + & $rustupInit -y --profile minimal --default-toolchain 1.98.0 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/.github/scripts/smoke_test_native_wheel.py b/.github/scripts/smoke_test_native_wheel.py new file mode 100644 index 00000000000..577bb32fcf0 --- /dev/null +++ b/.github/scripts/smoke_test_native_wheel.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path +from typing import Final + +CHILD_SCRIPT: Final = """ +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +import sys + +native_path = Path(sys.argv[1]) +spec = spec_from_file_location("litellm.rust_bridge._native", native_path) +if spec is None or spec.loader is None: + raise RuntimeError("cannot create native extension import specification") +module = module_from_spec(spec) +spec.loader.exec_module(module) + +before = module.gil_stats() +if not isinstance(before.get("releases"), int): + raise AssertionError(f"unexpected gil_stats result: {before!r}") + +try: + module._panic_for_test() +except BaseException as error: + if type(error).__name__ != "PanicException": + raise AssertionError(f"expected PanicException, got {type(error).__name__}") from error +else: + raise AssertionError("Rust panic returned without raising") + +after = module.gil_stats() +if not isinstance(after.get("releases"), int): + raise AssertionError(f"native module unusable after panic: {after!r}") +""" + + +def main() -> int: + if len(sys.argv) != 2: + sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n") + return 2 + + wheel: Final = Path(sys.argv[1]) + with tempfile.TemporaryDirectory() as temporary_directory, zipfile.ZipFile(wheel) as archive: + native_members: Final = tuple( + member + for member in archive.infolist() + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") + return 1 + + native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name + native_path.write_bytes(archive.read(native_members[0])) + result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False) + + if result.returncode != 0: + sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n") + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py new file mode 100644 index 00000000000..899e2a211c0 --- /dev/null +++ b/.github/scripts/verify_linux_native_wheel.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import importlib.util +import os +import re +import subprocess +import sys +import zipfile +from collections.abc import Callable, Mapping, Sequence +from itertools import product +from pathlib import Path, PurePosixPath +from types import MappingProxyType, ModuleType +from typing import Final, Protocol + +EXPECTED_PYTHON_TAG: Final = "cp310" +EXPECTED_ABI_TAG: Final = "abi3" +EXPECTED_PLATFORM_TAG: Final = "linux_x86_64" + + +class CommandRunner(Protocol): + def __call__( + self, + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +def _run_command( + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, check=check, capture_output=capture_output, text=text) + + +def _dist_info_directory(member: zipfile.ZipInfo) -> str | None: + parts: Final = PurePosixPath(member.filename).parts + if not parts or not parts[0].endswith(".dist-info"): + return None + return parts[0] + + +def _wheel_metadata_tags(archive: zipfile.ZipFile, members: tuple[zipfile.ZipInfo, ...]) -> tuple[str, ...]: + if len(members) != 1: + return () + lines: Final = archive.read(members[0]).splitlines() + return tuple(line.removeprefix(b"Tag:").strip().decode("ascii") for line in lines if line.startswith(b"Tag:")) + + +def _load_native_module(native_path: Path) -> ModuleType | None: + module_spec: Final = importlib.util.spec_from_file_location("litellm.rust_bridge._native", native_path) + if module_spec is None or module_spec.loader is None: + return None + try: + native_module: Final = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(native_module) + except Exception as error: # noqa: BLE001 # native module initialization can raise arbitrary exceptions + sys.stderr.write(f"native module load failed: {error}\n") + return None + return native_module + + +def main( + argv: Sequence[str] | None = None, + environment: Mapping[str, str] | None = None, + load_native_module: Callable[[Path], ModuleType | None] = _load_native_module, + run_command: CommandRunner = _run_command, +) -> int: + arguments: Final = tuple(sys.argv if argv is None else argv) + resolved_environment: Final = os.environ if environment is None else environment + if len(arguments) != 2: + sys.stderr.write(f"usage: {Path(arguments[0]).name} WHEEL\n") + return 2 + + wheel: Final = Path(arguments[1]) + wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3) + if len(wheel_tags) != 4: + sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n") + return 1 + + wheel_identity: Final = wheel_tags[0].split("-") + if len(wheel_identity) != 2 or wheel_identity[0] != "litellm" or not wheel_identity[1]: + sys.stderr.write(f"unexpected wheel identity: {wheel_tags[0]}\n") + return 1 + + expected_dist_info_directory: Final = f"{wheel_tags[0]}.dist-info" + expected_dist_info_directories: Final = frozenset((expected_dist_info_directory,)) + python_tag: Final = wheel_tags[1] + abi_tag: Final = wheel_tags[2] + platform_tag: Final = wheel_tags[3] + expanded_filename_tags: Final = frozenset( + "-".join(tag) for tag in product(python_tag.split("."), abi_tag.split("."), platform_tag.split(".")) + ) + + with zipfile.ZipFile(wheel) as archive: + wheel_members: Final = archive.infolist() + dist_info_directories: Final = frozenset( + directory for member in wheel_members if (directory := _dist_info_directory(member)) is not None + ) + required_dist_info_files: Final = ("METADATA", "RECORD", "WHEEL") + dist_info_file_counts: Final = MappingProxyType( + { + filename: sum( + member.filename == f"{expected_dist_info_directory}/{filename}" for member in wheel_members + ) + for filename in required_dist_info_files + } + ) + wheel_metadata_members: Final = tuple( + member for member in wheel_members if member.filename == f"{expected_dist_info_directory}/WHEEL" + ) + wheel_metadata_tags: Final = _wheel_metadata_tags(archive, wheel_metadata_members) + native_members: Final = tuple( + member + for member in wheel_members + if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so") + ) + if len(native_members) != 1: + sys.stderr.write(f"expected one native extension, found {len(native_members)}\n") + return 1 + + unexpected_members: Final = tuple( + member.filename + for member in wheel_members + if member.filename.endswith((".pdb", ".dwp", ".rlib", ".rmeta", "Cargo.toml", "Cargo.lock")) + or any(part.endswith(".dSYM") for part in PurePosixPath(member.filename).parts) + ) + native_member: Final = native_members[0] + uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members) + native_path: Final = wheel.parent / "native" / Path(native_member.filename).name + native_path.parent.mkdir(parents=True, exist_ok=True) + native_path.write_bytes(archive.read(native_member)) + + wheel_metadata_tags_match: Final = ( + len(wheel_metadata_tags) == len(expanded_filename_tags) + and frozenset(wheel_metadata_tags) == expanded_filename_tags + ) + commit_sha: Final = resolved_environment.get( + "RELEASE_WHEEL_COMMIT_SHA", resolved_environment.get("GITHUB_SHA", "unknown") + ) + rustc_version: Final = run_command( + ("rustc", "--version"), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text() + maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject) + if maturin_match is None: + sys.stderr.write("build-system does not pin an exact Maturin version\n") + return 1 + + maturin_version: Final = maturin_match.group(1) + native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100 + size_report: Final = "\n".join( + ( + "## Native wheel build report", + "", + "| Build | Value |", + "| --- | --- |", + f"| Commit | `{commit_sha}` |", + f"| Platform | `{platform_tag}` |", + f"| Python ABI | `{python_tag}-{abi_tag}` |", + f"| Rust compiler | `{rustc_version}` |", + f"| Maturin | `{maturin_version}` |", + "| Cargo profile | `release` |", + "", + "| Artifact | Size |", + "| --- | ---: |", + f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |", + f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |", + f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |", + f"| Native share | {native_percentage:.2f}% |", + "", + ) + ) + summary_path: Final = resolved_environment.get("GITHUB_STEP_SUMMARY") + if summary_path is None: + sys.stdout.write(size_report) + else: + Path(summary_path).write_text(size_report) + + sections: Final = run_command( + ("readelf", "--sections", "--wide", str(native_path)), + check=True, + capture_output=True, + text=True, + ).stdout + debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections) + debug_sections_absent: Final = not debug_sections + static_symbol_table_absent: Final = ".symtab" not in sections + + dynamic_symbols: Final = run_command( + ("readelf", "--dyn-syms", "--wide", str(native_path)), + check=True, + capture_output=True, + text=True, + ).stdout + extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols + native_module: Final = load_native_module(native_path) + native_module_loads: Final = native_module is not None + panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") + native_size_limit: Final = 20_000_000 + native_size_within_limit: Final = native_member.file_size <= native_size_limit + validations: Final = ( + (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), + (f"ABI tag is {EXPECTED_ABI_TAG}", abi_tag == EXPECTED_ABI_TAG), + (f"Platform tag is {EXPECTED_PLATFORM_TAG}", platform_tag == EXPECTED_PLATFORM_TAG), + ("Wheel dist-info directory matches the filename", dist_info_directories == expected_dist_info_directories), + ( + "Required dist-info files are present exactly once", + all(count == 1 for count in dist_info_file_counts.values()), + ), + ("Wheel metadata tags match the filename", wheel_metadata_tags_match), + ("Debug sections are absent", debug_sections_absent), + ("Static symbol table is absent", static_symbol_table_absent), + ("Python extension entry point is present", extension_entry_point_present), + ("Native module loads", native_module_loads), + ("Production module omits the panic test hook", panic_test_hook_absent), + ("Native extension does not exceed 20 MB", native_size_within_limit), + ("Wheel contents are valid", not unexpected_members), + ) + + verified_report: Final = size_report + "\n".join( + ("", "| Validation | Expected | Result |", "| --- | --- | :---: |") + + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations) + + ("",) + ) + if summary_path is not None: + Path(summary_path).write_text(verified_report) + + invalid_dist_info_files: Final = any(count != 1 for count in dist_info_file_counts.values()) + validation_errors: Final = tuple( + message + for failed, message in ( + (bool(debug_sections), f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}"), + (not static_symbol_table_absent, f"{native_member.filename} contains a static symbol table"), + (not extension_entry_point_present, "native extension does not export PyInit__native"), + ( + python_tag != EXPECTED_PYTHON_TAG, + f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}", + ), + (abi_tag != EXPECTED_ABI_TAG, f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}"), + ( + platform_tag != EXPECTED_PLATFORM_TAG, + f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}", + ), + ( + dist_info_directories != expected_dist_info_directories, + f"unexpected dist-info directories: expected {expected_dist_info_directory}, " + f"found {', '.join(sorted(dist_info_directories))}", + ), + (invalid_dist_info_files, f"required dist-info file counts are invalid: {dist_info_file_counts}"), + ( + not invalid_dist_info_files and not wheel_metadata_tags_match, + f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, " + f"found {', '.join(sorted(wheel_metadata_tags))}", + ), + ( + native_module is not None and not panic_test_hook_absent, + "production native module exposes _panic_for_test", + ), + ( + not native_size_within_limit, + f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB", + ), + (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"), + ) + if failed + ) + sys.stderr.write("".join(f"{message}\n" for message in validation_errors)) + + return 0 if all(passed for _, passed in validations) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/report-rust-release-wheel.yml b/.github/workflows/report-rust-release-wheel.yml new file mode 100644 index 00000000000..1d93b56f77f --- /dev/null +++ b/.github/workflows/report-rust-release-wheel.yml @@ -0,0 +1,130 @@ +name: Report LiteLLM Rust release wheel + +on: # zizmor: ignore[dangerous-triggers] reporter executes no PR code and consumes no PR artifacts or outputs + workflow_run: + workflows: + - LiteLLM Rust + types: + - completed + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || github.event.workflow_run.id }} + cancel-in-progress: false + +jobs: + report-release-wheel: + name: report release wheel + if: >- + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.path == '.github/workflows/test-rust.yml' && + github.event.workflow_run.head_repository.full_name == github.repository && + github.event.workflow_run.pull_requests[0].number != null + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write # PR comments use the issues API + pull-requests: read # Current-head validation rejects stale workflow runs + + steps: + - name: Link release wheel report on PR + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + env: + COMMENT_MARKER: "" + with: + script: | + const marker = process.env.COMMENT_MARKER; + const workflowRun = context.payload.workflow_run; + const allowedConclusions = new Set([ + "action_required", + "cancelled", + "failure", + "neutral", + "skipped", + "stale", + "startup_failure", + "success", + "timed_out", + ]); + if ( + !allowedConclusions.has(workflowRun.conclusion) || + workflowRun.event !== "pull_request" || + workflowRun.path !== ".github/workflows/test-rust.yml" || + workflowRun.head_repository?.full_name !== + `${context.repo.owner}/${context.repo.repo}` || + workflowRun.pull_requests?.length !== 1 + ) { + throw new Error("unexpected source workflow"); + } + const pullRequest = workflowRun.pull_requests[0]; + const pullRequestNumber = pullRequest.number; + const headSha = workflowRun.head_sha; + const runId = workflowRun.id; + if ( + !Number.isSafeInteger(pullRequestNumber) || + pullRequestNumber <= 0 || + !Number.isSafeInteger(runId) || + runId <= 0 || + !/^[0-9a-f]{40}$/.test(headSha) || + pullRequest.head?.sha !== headSha + ) { + throw new Error("invalid source workflow metadata"); + } + const runUrl = + `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + + `/actions/runs/${runId}`; + const result = + workflowRun.conclusion === "success" + ? "successfully" + : `with \`${workflowRun.conclusion}\``; + const body = [ + marker, + "## LiteLLM Rust workflow", + "", + `Workflow completed ${result} for \`${headSha}\``, + "", + `[View workflow run](${runUrl})`, + ].join("\n"); + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + per_page: 100, + }); + const existing = comments.find( + (comment) => + comment.user?.login === "github-actions[bot]" && + comment.body?.startsWith(marker), + ); + const currentPullRequest = ( + await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullRequestNumber, + }) + ).data; + if ( + currentPullRequest.state !== "open" || + currentPullRequest.head.repo?.full_name !== + `${context.repo.owner}/${context.repo.repo}` || + currentPullRequest.head.sha !== headSha + ) { + core.info("source workflow no longer matches the current pull request head"); + return; + } + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body, + }); + } diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 21e1bcb90c6..aada0fcf239 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -4,6 +4,11 @@ on: push: paths: - "litellm-rust/**" + - ".cargo/**" + - "pyproject.toml" + - "rust-toolchain.toml" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" pull_request: branches: @@ -13,6 +18,11 @@ on: - "litellm_**" paths: - "litellm-rust/**" + - ".cargo/**" + - "pyproject.toml" + - "rust-toolchain.toml" + - ".github/scripts/smoke_test_native_wheel.py" + - ".github/scripts/verify_linux_native_wheel.py" - ".github/workflows/test-rust.yml" permissions: @@ -40,9 +50,7 @@ jobs: persist-credentials: false - name: Set up Rust - run: | - rustup toolchain install stable --profile minimal --component clippy,rustfmt - rustup default stable + run: rustup toolchain install - name: Cache Cargo registry and target uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 @@ -51,7 +59,7 @@ jobs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-cargo-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-cargo-${{ hashFiles('rust-toolchain.toml', 'litellm-rust/Cargo.lock') }} restore-keys: | ${{ runner.os }}-cargo- @@ -69,3 +77,47 @@ jobs: - name: Run core tests with Bedrock auth run: cargo test -p litellm-core --features bedrock-auth --locked + + release-wheel: + name: release wheel + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + env: + CARGO_TERM_COLOR: always + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Set up Rust + run: rustup toolchain install + + - name: Build release wheel + run: uv build --wheel --out-dir dist + + - name: Build panic contract wheel + run: >- + uv build --wheel --out-dir panic-dist + --config-setting "maturin.build-args=--features panic-test,extension-module" + + - name: Smoke-test native panic unwinding + run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl + + - name: Verify stripped native extension + env: + RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index da788bf1ce3..d094c98f5ec 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 14076 + "limit": 14074 }, "reportArgumentType": { "limit": 2216 @@ -24,7 +24,7 @@ "limit": 19 }, "reportExplicitAny": { - "limit": 4128 + "limit": 4125 }, "reportFunctionMemberAccess": { "limit": 7 @@ -108,10 +108,10 @@ "limit": 38350 }, "reportUnknownParameterType": { - "limit": 19626 + "limit": 19625 }, "reportUnknownVariableType": { - "limit": 29890 + "limit": 29877 }, "reportUnnecessaryCast": { "limit": 111 @@ -138,7 +138,7 @@ "limit": 138 }, "reportUnusedImport": { - "limit": 543 + "limit": 542 }, "reportUnusedVariable": { "limit": 137 diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py index 1d3268da9a0..190141470df 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py @@ -39,9 +39,9 @@ async def available_enterprise_users( if not premium_user: # check if SSO is enabled - show 5 user limit - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso - if _has_user_setup_sso(): + if has_user_setup_sso(): premium_user_data = EnterpriseLicenseData( max_users=5, ) diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index 36a5ad5a8f4..b8b6291283d 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -1,6 +1,6 @@ # AGENTS.md -litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are MODULES inside the layers. +litellm-rust has four crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers. ## Crates @@ -8,9 +8,10 @@ litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes ( |-------|------| | litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. | | litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. ## Where a route lives @@ -28,7 +29,7 @@ core/src/messages/ Handlers never live in `ai-gateway`. `ocr`, `audio_transcription`, and `realtime` are still hosted there from before this rule; they move to `core` as they are touched. -Adding a crate: default to a MODULE. New crate ONLY on a real trigger — separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these. +Adding a crate: default to a module. A new crate requires a real trigger: separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these. Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional. diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md index fe6ceedbb86..3dcf1853efc 100644 --- a/litellm-rust/CLAUDE.md +++ b/litellm-rust/CLAUDE.md @@ -21,12 +21,13 @@ variants of it. The test for a good abstraction is that adding the next provider is a few declarative lines, not a new file of duplicated flow. Only diverge from the base when behavior is genuinely different, and say so explicitly in the PR. -## Crates (exactly three — see AGENTS.md) +## Crates (see AGENTS.md) `litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call. `litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and -`litellm-python-bridge` exposes it to the Python SDK. A crate is a **layer**, not -a route — add modules, not crates. +`litellm-python-bridge` exposes it to the Python SDK. `litellm-python-interop` +holds domain-neutral PyO3 primitives shared by Python-facing Rust code. A crate +is a layer or shared foundation, not a route; add modules, not crates. ## Core Boundary @@ -175,7 +176,7 @@ cd litellm-rust cargo fmt --check # the ai-gateway binary + server code is behind the `server` feature cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings -cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings +cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings cargo test --workspace ``` diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 4388e561026..dd41cf0e84b 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -919,6 +919,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.33" @@ -972,6 +978,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "h2" version = "0.3.27" @@ -1432,14 +1444,24 @@ dependencies = [ "criterion", "litellm-ai-gateway", "litellm-core", + "litellm-python-interop", "pyo3", "pyo3-async-runtimes", - "pythonize", - "serde", "serde_json", "tokio", ] +[[package]] +name = "litellm-python-interop" +version = "0.1.0" +dependencies = [ + "pyo3", + "pythonize", + "rstest", + "serde", + "serde_json", +] + [[package]] name = "litemap" version = "0.8.2" @@ -1627,6 +1649,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -1899,6 +1930,12 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.28" @@ -1956,6 +1993,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.119", + "unicode-ident", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -2488,6 +2554,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + [[package]] name = "tower" version = "0.5.3" @@ -2903,6 +2999,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index c17a0605fc7..c447d915abe 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/core", "crates/ai-gateway", + "crates/python-interop", "crates/python-bridge", ] resolver = "2" @@ -15,12 +16,14 @@ repository = "https://github.com/BerriAI/litellm" [workspace.dependencies] litellm-core = { path = "crates/core" } litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false } +litellm-python-interop = { path = "crates/python-interop" } axum = "0.7" pyo3 = "0.29.0" pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } pythonize = "0.29.0" rand = "0.8" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] } +rstest = "0.26.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10" diff --git a/litellm-rust/README.md b/litellm-rust/README.md index bcccf93300b..a0d79c6f0a5 100644 --- a/litellm-rust/README.md +++ b/litellm-rust/README.md @@ -26,9 +26,10 @@ coverage and production evidence. |-------|------| | litellm-core | The SDK. Per-route entrypoints (`messages::messages()`), types, provider transforms (modules under `providers/`), provider resolution, auth, the provider HTTP call, and the router. | | litellm-ai-gateway | The axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. ## Layout @@ -38,7 +39,8 @@ crates/ src/messages/ mod.rs (entrypoint), types, transformation, prepare, handler, client src/providers/anthropic/messages/transformation.rs ai-gateway/ Axum server + WebSocket hosts; calls core entrypoints. - python-bridge/ PyO3 bridge for Python LiteLLM. + python-interop/ Domain-neutral PyO3 conversion and GIL primitives. + python-bridge/ PyO3 API adapter for Python LiteLLM. ``` The folder shape follows the Python provider tree: diff --git a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md index a1860d8a9c9..4a689cb9579 100644 --- a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md +++ b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md @@ -54,6 +54,6 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages` cd litellm-rust cargo fmt --check cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings - cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings + cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings cargo test --workspace ``` diff --git a/litellm-rust/crates/ai-gateway/README.md b/litellm-rust/crates/ai-gateway/README.md index 7a6c620ee84..5cbb47220be 100644 --- a/litellm-rust/crates/ai-gateway/README.md +++ b/litellm-rust/crates/ai-gateway/README.md @@ -6,15 +6,16 @@ dials OpenAI upstream, and splices the two sockets frame-by-frame. ## Crates -`litellm-rust` is exactly three crates (a crate is a **layer**, not a route): +`litellm-rust` has four crates. A crate is a layer or shared foundation, not a route: | Crate | Role | |-------|------| | litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. | | litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | -| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. | +| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | +| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. | -Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge. +Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate. - **Client endpoint:** `wss:///v1/realtime?model=` (WebSocket) - **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset) diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs index 270d5c2d97a..140bc8aeea8 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs @@ -1,10 +1,8 @@ -use std::collections::BTreeMap; - -use litellm_core::CoreResult; use litellm_core::audio_transcription::transformation::AudioTranscriptionProviderConfig; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG; use serde_json::{Map, Value}; +use std::collections::BTreeMap; pub(super) fn audio_transcription_provider_config( provider: &str, @@ -17,7 +15,7 @@ pub(super) fn audio_transcription_provider_config( pub(super) fn string_headers( headers: Option>, -) -> CoreResult> { +) -> Result, Error> { headers .unwrap_or_default() .into_iter() @@ -26,7 +24,7 @@ pub(super) fn string_headers( .as_str() .map(|value| (key.clone(), value.to_string())) .ok_or_else(|| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "audio transcription extra_headers.{key} must be a string" )) }) diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs index 33c13550f58..1bdd4ae72a2 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs @@ -1,11 +1,9 @@ -use std::time::SystemTime; - -use litellm_core::CoreResult; use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::providers::bedrock::audio_transcription::aws_auth_config; use litellm_core::providers::bedrock::aws_base::{resolve_credentials, sign_bedrock_post}; use serde_json::Value; +use std::time::SystemTime; use super::common_utils::truncate_error_body; use super::types::ProviderAudioTranscriptionRequest; @@ -13,10 +11,9 @@ use crate::client::http_client; pub(crate) async fn execute_audio_transcription_provider_call( request: ProviderAudioTranscriptionRequest, -) -> CoreResult { - let body = serde_json::to_vec(&request.body).map_err(|error| { - CoreError::InvalidRequest(format!("invalid audio request body: {error}")) - })?; +) -> Result { + let body = serde_json::to_vec(&request.body) + .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?; let mut request_builder = http_client().post(&request.url).body(body.clone()); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -27,21 +24,20 @@ pub(crate) async fn execute_audio_transcription_provider_call( let response = request_builder .send() .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; let status = response.status(); let text = response .text() .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } - let response_json: Value = serde_json::from_str(&text).map_err(|error| { - CoreError::InvalidResponse(format!("invalid audio response JSON: {error}")) - })?; + let response_json: Value = serde_json::from_str(&text) + .map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?; Ok(request .config .transform_transcription_response(&request.model, response_json)? @@ -51,14 +47,13 @@ pub(crate) async fn execute_audio_transcription_provider_call( pub(crate) async fn sign_request( request: &ProviderAudioTranscriptionRequest, optional_params: &serde_json::Map, -) -> CoreResult { +) -> Result { let env_lookup = environment_lookup; let auth = request .config .auth_strategy(&request.model, optional_params, &env_lookup)?; - let body = serde_json::to_vec(&request.body).map_err(|error| { - CoreError::InvalidRequest(format!("invalid audio request body: {error}")) - })?; + let body = serde_json::to_vec(&request.body) + .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?; let mut headers = super::common_utils::string_headers(None)?; headers.insert("Content-Type".to_string(), "application/json".to_string()); headers.extend(request.upstream_headers.iter().cloned()); diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs index 0c9faeda6e7..5e1240de759 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs @@ -1,11 +1,9 @@ -use std::future::Future; -use std::pin::Pin; - -use litellm_core::CoreResult; use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth; use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use serde_json::{Map, Value, json}; +use std::future::Future; +use std::pin::Pin; use super::common_utils::{audio_transcription_provider_config, has_header, string_headers}; use super::handler::sign_request; @@ -26,7 +24,7 @@ pub(crate) struct AudioTranscriptionLifecycleHooks { request_metadata: RequestMetadata, } -type AudioFuture<'a, T> = Pin> + Send + 'a>>; +type AudioFuture<'a, T> = Pin> + Send + 'a>>; type AudioLogFuture<'a> = Pin + Send + 'a>>; impl AudioTranscriptionLifecycleHooks { @@ -45,7 +43,7 @@ impl AudioTranscriptionLifecycleHooks { async fn run_pre_call_guardrails( &self, request: PreparedAudioTranscriptionRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -63,17 +61,17 @@ impl AudioTranscriptionLifecycleHooks { .await .map_err(guardrail_error_to_core_error)?; let Value::Object(mut data) = guardrail_request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription pre_call guardrail must return an object".to_string(), )); }; let audio = data.remove("audio").ok_or_else(|| { - CoreError::InvalidRequest("audio transcription guardrail removed audio".to_string()) + Error::InvalidRequest("audio transcription guardrail removed audio".to_string()) })?; let optional_params = match data.remove("optional_params") { Some(Value::Object(value)) => value, Some(_) => { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription optional_params must be an object".to_string(), )); } @@ -89,9 +87,9 @@ impl AudioTranscriptionLifecycleHooks { async fn prepare_provider_request( &self, request: PreparedAudioTranscriptionRequest, - ) -> CoreResult { + ) -> Result { let config = audio_transcription_provider_config(&request.custom_llm_provider) - .ok_or_else(|| CoreError::InvalidProvider(request.custom_llm_provider.clone()))?; + .ok_or_else(|| Error::InvalidProvider(request.custom_llm_provider.clone()))?; let env_lookup = super::handler::environment_lookup; let headers = string_headers(request.extra_headers)?; let url = config.complete_url( @@ -135,7 +133,7 @@ impl AudioTranscriptionLifecycleHooks { async fn run_during_call_guardrails( &self, request: ProviderAudioTranscriptionRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -153,12 +151,12 @@ impl AudioTranscriptionLifecycleHooks { .await .map_err(guardrail_error_to_core_error)?; let Value::Object(mut data) = guardrail_request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "audio transcription during_call guardrail must return an object".to_string(), )); }; let body = data.remove("body").ok_or_else(|| { - CoreError::InvalidRequest("audio transcription guardrail removed body".to_string()) + Error::InvalidRequest("audio transcription guardrail removed body".to_string()) })?; Ok(ProviderAudioTranscriptionRequest { body, ..request }) } @@ -241,7 +239,7 @@ impl CallLifecycleHooks( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -281,22 +279,22 @@ fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { } } -fn guardrail_error_to_core_error(error: GuardrailError) -> CoreError { - CoreError::InvalidRequest(format!("{}: {}", error.kind, error.message)) +fn guardrail_error_to_core_error(error: GuardrailError) -> Error { + Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) } -fn core_error_kind(error: &CoreError) -> &'static str { +fn core_error_kind(error: &Error) -> &'static str { match error { - CoreError::Auth(_) => "AuthError", - CoreError::InvalidProvider(_) => "InvalidProvider", - CoreError::InvalidRequest(_) => "InvalidRequest", - CoreError::InvalidType { .. } => "InvalidType", - CoreError::MissingField(_) => "MissingField", - CoreError::Http { .. } => "HttpError", - CoreError::InvalidResponse(_) => "InvalidResponse", - CoreError::Network(_) => "NetworkError", - CoreError::Connect(_) => "ConnectError", - CoreError::Routing(_) => "RoutingError", - CoreError::Unsupported(_) => "UnsupportedRequest", + Error::Auth(_) => "AuthError", + Error::InvalidProvider(_) => "InvalidProvider", + Error::InvalidRequest(_) => "InvalidRequest", + Error::InvalidType { .. } => "InvalidType", + Error::MissingField(_) => "MissingField", + Error::Http { .. } => "HttpError", + Error::InvalidResponse(_) => "InvalidResponse", + Error::Network(_) => "NetworkError", + Error::Connect(_) => "ConnectError", + Error::Routing(_) => "RoutingError", + Error::Unsupported(_) => "UnsupportedRequest", } } diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs index 5d33d912c40..3983846d7b6 100644 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs @@ -1,4 +1,4 @@ -use litellm_core::CoreResult; +use litellm_core::Error; use litellm_core::call_lifecycle::CallLifecycle; use serde_json::Value; @@ -13,7 +13,7 @@ pub use types::AudioTranscriptionRequest; use handler::execute_audio_transcription_provider_call; use prepare::{PreparedAudioTranscriptionCall, prepare_audio_transcription_call}; -pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> CoreResult { +pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { let PreparedAudioTranscriptionCall { request, hooks } = prepare_audio_transcription_call(request); CallLifecycle::default() diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime.rs b/litellm-rust/crates/ai-gateway/src/io/realtime.rs index 845e7bf9527..662f7328982 100644 --- a/litellm-rust/crates/ai-gateway/src/io/realtime.rs +++ b/litellm-rust/crates/ai-gateway/src/io/realtime.rs @@ -15,8 +15,7 @@ use std::time::Duration; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{Sink, SinkExt, Stream, StreamExt}; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::realtime::transformation::RealtimeProviderConfig; use litellm_core::realtime::types::RealtimeEvent; use tokio::net::TcpStream; @@ -48,7 +47,7 @@ pub(crate) type UpstreamRx = SplitStream; /// Resolve the OpenAI API key from the explicit param or the environment. /// /// Blank/whitespace values are treated as absent (guard at resolution time). -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { +pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) @@ -58,7 +57,7 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { .ok() .filter(|key| !key.trim().is_empty()) }) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } /// Open the upstream WebSocket to OpenAI for `(model, api_key, api_base)`. @@ -70,24 +69,24 @@ pub(crate) async fn dial_upstream( model: &str, api_key: &str, api_base: Option<&str>, -) -> CoreResult { +) -> Result { let url = OPENAI_REALTIME_CONFIG.complete_url(api_base, model); let mut request = url .as_str() .into_client_request() - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; // GA realtime: only Authorization. The legacy OpenAI-Beta header triggers // beta_api_shape_disabled, so we do not send it. request.headers_mut().insert( AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|err| CoreError::Auth(err.to_string()))?, + .map_err(|err| Error::Auth(err.to_string()))?, ); let (upstream, _response) = connect_async(request) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; Ok(upstream) } @@ -96,22 +95,22 @@ pub(crate) async fn dial_upstream( /// Used by the pool to pre-read OpenAI's unprompted `session.created`. Returns an /// error on a non-text frame, a closed socket, or undecodable JSON so the pool can /// discard a misbehaving socket rather than warm it. -pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> CoreResult { +pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> Result { loop { let message = upstream_rx .next() .await - .ok_or_else(|| CoreError::Network("upstream closed before first event".to_string()))? - .map_err(|err| CoreError::Network(err.to_string()))?; + .ok_or_else(|| Error::Network("upstream closed before first event".to_string()))? + .map_err(|err| Error::Network(err.to_string()))?; match message { Message::Text(text) => { return serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(err.to_string())); + .map_err(|err| Error::InvalidResponse(err.to_string())); } // Ignore protocol frames (ping/pong) while waiting for the first event. Message::Ping(_) | Message::Pong(_) => continue, Message::Close(_) => { - return Err(CoreError::Network( + return Err(Error::Network( "upstream closed before first event".to_string(), )); } @@ -139,7 +138,7 @@ pub(crate) async fn splice( mut observe: impl FnMut(&RealtimeEvent) + Send, mut client_in: In, mut client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -154,7 +153,7 @@ where client_out .send(outbound) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } @@ -175,26 +174,26 @@ where // inflate its own spend log. Logging observes upstream events only. for outbound in config.transform_realtime_request(&event, model)?.events { let payload = serde_json::to_string(&outbound) - .map_err(|err| CoreError::InvalidResponse(err.to_string()))?; + .map_err(|err| Error::InvalidResponse(err.to_string()))?; upstream_tx .send(Message::Text(payload)) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } // upstream -> client upstream_message = upstream_rx.next() => { let Some(message) = upstream_message else { break }; // upstream closed - match message.map_err(|err| CoreError::Network(err.to_string()))? { + match message.map_err(|err| Error::Network(err.to_string()))? { Message::Text(text) => { let event: RealtimeEvent = serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(err.to_string()))?; + .map_err(|err| Error::InvalidResponse(err.to_string()))?; observe(&event); for outbound in config.transform_realtime_response(&event, model)?.events { client_out .send(outbound) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; } } Message::Close(_) => break, @@ -225,7 +224,7 @@ pub async fn realtime( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -258,7 +257,7 @@ pub async fn realtime_warm( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs b/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs index 4a1a3cd1166..49e9c459a88 100644 --- a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs +++ b/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs @@ -28,7 +28,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use futures_util::StreamExt; -use litellm_core::CoreResult; +use litellm_core::Error; use litellm_core::realtime::types::RealtimeEvent; use crate::io::realtime::{ @@ -438,7 +438,7 @@ impl RealtimePool { /// /// `key.api_key` is already resolved (non-blank). The first frame OpenAI sends /// unprompted is `session.created`; we buffer exactly that and read nothing more. -async fn warm_one(key: &UpstreamKey) -> CoreResult { +async fn warm_one(key: &UpstreamKey) -> Result { let upstream: UpstreamWs = dial_upstream(&key.model, &key.api_key, key.api_base.as_deref()).await?; let (tx, mut rx) = upstream.split(); diff --git a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs index 9b51019f4bc..0b01747b1a5 100644 --- a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs +++ b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs @@ -4,10 +4,10 @@ use std::time::Duration; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{Sink, SinkExt, Stream, StreamExt}; +use litellm_core::Error; use litellm_core::providers::openai::responses::transformation::OPENAI_RESPONSES_WS_CONFIG; use litellm_core::responses::types::ResponsesWsEvent; use litellm_core::responses::websocket::ResponsesWebSocketProviderConfig; -use litellm_core::{CoreError, CoreResult}; use tokio::net::TcpStream; use tokio::sync::Mutex; use tokio_tungstenite::tungstenite::Message; @@ -37,51 +37,49 @@ impl ResponsesWebSocketConnection { url: &str, headers: &HashMap, timeout: Option, - ) -> CoreResult { + ) -> Result { let mut request = url .into_client_request() - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; for (name, value) in headers { let header_name = name .parse::() - .map_err(|error| CoreError::InvalidRequest(error.to_string()))?; + .map_err(|error| Error::InvalidRequest(error.to_string()))?; let header_value = HeaderValue::from_str(value) - .map_err(|error| CoreError::InvalidRequest(error.to_string()))?; + .map_err(|error| Error::InvalidRequest(error.to_string()))?; request.headers_mut().insert(header_name, header_value); } let connect = connect_async(request); let result = match timeout { Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| { - CoreError::Network("Responses WebSocket connection timed out".to_string()) + Error::Network("Responses WebSocket connection timed out".to_string()) })?, None => connect.await, }; let (socket, _) = result.map_err(|error| match error { - tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http { + tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http { status: response.status().as_u16(), body: String::new(), }, - other => CoreError::Network(other.to_string()), + other => Error::Network(other.to_string()), })?; Ok(Self { socket: Arc::new(Mutex::new(Some(socket))), }) } - pub async fn send_text(&self, text: String) -> CoreResult<()> { + pub async fn send_text(&self, text: String) -> Result<(), Error> { let mut socket = self.socket.lock().await; let Some(socket) = socket.as_mut() else { - return Err(CoreError::Network( - "Responses WebSocket is closed".to_string(), - )); + return Err(Error::Network("Responses WebSocket is closed".to_string())); }; socket .send(Message::Text(text)) .await - .map_err(|error| CoreError::Network(error.to_string())) + .map_err(|error| Error::Network(error.to_string())) } - pub async fn recv_text(&self) -> CoreResult> { + pub async fn recv_text(&self) -> Result, Error> { let mut socket_guard = self.socket.lock().await; let Some(socket) = socket_guard.as_mut() else { return Ok(None); @@ -90,27 +88,27 @@ impl ResponsesWebSocketConnection { Some(Ok(Message::Text(text))) => Ok(Some(text)), Some(Ok(Message::Binary(bytes))) => String::from_utf8(bytes.to_vec()) .map(Some) - .map_err(|error| CoreError::InvalidResponse(error.to_string())), + .map_err(|error| Error::InvalidResponse(error.to_string())), Some(Ok(Message::Close(_))) | None => Ok(None), Some(Ok(_)) => Ok(None), - Some(Err(error)) => Err(CoreError::Network(error.to_string())), + Some(Err(error)) => Err(Error::Network(error.to_string())), } } - pub async fn close(&self) -> CoreResult<()> { + pub async fn close(&self) -> Result<(), Error> { let mut socket = self.socket.lock().await; if let Some(socket) = socket.as_mut() { socket .close(None) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } *socket = None; Ok(()) } } -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { +pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { api_key .map(str::trim) .filter(|value| !value.is_empty()) @@ -120,38 +118,38 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult { .ok() .filter(|value| !value.trim().is_empty()) }) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } async fn dial_upstream( model: &str, api_key: &str, api_base: Option<&str>, -) -> CoreResult { +) -> Result { let url = OPENAI_RESPONSES_WS_CONFIG.complete_websocket_url(api_base, model); let mut request = url .as_str() .into_client_request() - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; request.headers_mut().insert( AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|error| CoreError::Auth(error.to_string()))?, + .map_err(|error| Error::Auth(error.to_string()))?, ); let result = tokio::time::timeout( Duration::from_secs(DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS), connect_async(request), ) .await - .map_err(|_| CoreError::Network("Responses WebSocket connection timed out".to_string()))?; + .map_err(|_| Error::Network("Responses WebSocket connection timed out".to_string()))?; result .map(|(socket, _)| socket) .map_err(|error| match error { - tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http { + tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http { status: response.status().as_u16(), body: String::new(), }, - other => CoreError::Network(other.to_string()), + other => Error::Network(other.to_string()), }) } @@ -166,7 +164,7 @@ impl ResponsesWebSocketStreaming { observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, - ) -> CoreResult<()> + ) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -193,7 +191,7 @@ pub(crate) async fn splice( mut observe: impl FnMut(&ResponsesWsEvent) + Send, mut client_in: In, mut client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -210,18 +208,18 @@ where .events { let payload = serde_json::to_string(&outbound) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; upstream_tx.send(Message::Text(payload)) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } message = upstream_rx.next() => { let Some(message) = message else { break }; - match message.map_err(|error| CoreError::Network(error.to_string()))? { + match message.map_err(|error| Error::Network(error.to_string()))? { Message::Text(text) => { let event = serde_json::from_str::(&text) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; observe(&event); for outbound in OPENAI_RESPONSES_WS_CONFIG .transform_ws_response(&event, model)? @@ -229,7 +227,7 @@ where { client_out.send(outbound) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } Message::Close(_) => break, @@ -252,7 +250,7 @@ pub async fn async_responses_websocket( mut observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -267,11 +265,11 @@ where .events { let payload = serde_json::to_string(&outbound) - .map_err(|error| CoreError::InvalidResponse(error.to_string()))?; + .map_err(|error| Error::InvalidResponse(error.to_string()))?; upstream_tx .send(Message::Text(payload)) .await - .map_err(|error| CoreError::Network(error.to_string()))?; + .map_err(|error| Error::Network(error.to_string()))?; } } ResponsesWebSocketStreaming::bidirectional_forward( @@ -296,7 +294,7 @@ pub async fn responses_ws( observe: impl FnMut(&ResponsesWsEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, @@ -514,7 +512,7 @@ mod tests { ) .await .expect_err("status error"); - assert!(matches!(error, CoreError::Http { status: 401, .. })); + assert!(matches!(error, Error::Http { status: 401, .. })); server.await.expect("server task"); } @@ -543,7 +541,7 @@ mod tests { ) .await .expect_err("status error"); - assert!(matches!(error, CoreError::Http { status: 500, .. })); + assert!(matches!(error, Error::Http { status: 500, .. })); server.await.expect("server task"); } } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs b/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs index 9bc2818b6e7..e0ce165dc93 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs @@ -3,8 +3,7 @@ use std::time::{Duration, Instant}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrProviderConfig; use reqwest::Url; use serde_json::{Map, Value}; @@ -56,7 +55,7 @@ fn is_azure_document_intelligence_model(model: &str) -> bool { pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { extra_headers .unwrap_or_default() .into_iter() @@ -65,7 +64,7 @@ pub(super) fn string_headers( .as_str() .map(|value| (key.clone(), value.to_string())) .ok_or_else(|| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "OCR extra_headers.{key} must be a string, got {}", litellm_core::error::json_type_name(&value) )) @@ -80,7 +79,7 @@ pub(super) fn has_header(headers: &[(String, String)], name: &str) -> bool { .any(|(key, _)| key.eq_ignore_ascii_case(name)) } -fn document_url_field(document: &Value) -> CoreResult> { +fn document_url_field(document: &Value) -> Result, Error> { let Some(object) = document.as_object() else { return Ok(None); }; @@ -138,13 +137,13 @@ fn is_blocked_ip(ip: IpAddr) -> bool { } } -fn blocked_url_error(url: &Url) -> CoreError { - CoreError::InvalidRequest(format!( +fn blocked_url_error(url: &Url) -> Error { + Error::InvalidRequest(format!( "OCR document URL rejected by SSRF protection: {url}" )) } -async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { +async fn validate_safe_fetch_url(url: &Url) -> Result<(), Error> { if !matches!(url.scheme(), "http" | "https") { return Err(blocked_url_error(url)); } @@ -162,7 +161,7 @@ async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { .ok_or_else(|| blocked_url_error(url))?; let addresses = tokio::net::lookup_host((host, port)) .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let mut saw_address = false; for address in addresses { saw_address = true; @@ -176,25 +175,25 @@ async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> { Ok(()) } -fn redirect_location(response: &reqwest::Response, url: &Url) -> CoreResult { +fn redirect_location(response: &reqwest::Response, url: &Url) -> Result { let location = response .headers() .get(reqwest::header::LOCATION) .and_then(|value| value.to_str().ok()) .ok_or_else(|| { - CoreError::InvalidResponse("OCR document redirect missing Location header".to_string()) + Error::InvalidResponse("OCR document redirect missing Location header".to_string()) })?; url.join(location) - .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR document redirect: {err}"))) + .map_err(|err| Error::InvalidResponse(format!("invalid OCR document redirect: {err}"))) } -async fn safe_get_document_url(url: &str) -> CoreResult<(Url, reqwest::Response)> { +async fn safe_get_document_url(url: &str) -> Result<(Url, reqwest::Response), Error> { let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .build() - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let mut current_url = Url::parse(url) - .map_err(|err| CoreError::InvalidRequest(format!("invalid OCR document URL: {err}")))?; + .map_err(|err| Error::InvalidRequest(format!("invalid OCR document URL: {err}")))?; for _ in 0..MAX_SAFE_FETCH_REDIRECTS { validate_safe_fetch_url(¤t_url).await?; @@ -202,28 +201,28 @@ async fn safe_get_document_url(url: &str) -> CoreResult<(Url, reqwest::Response) .get(current_url.clone()) .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !response.status().is_redirection() { return Ok((current_url, response)); } current_url = redirect_location(&response, ¤t_url)?; } - Err(CoreError::InvalidRequest( + Err(Error::InvalidRequest( "Too many redirects while fetching OCR document URL".to_string(), )) } -fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> CoreResult<()> { +fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Result<(), Error> { if max_bytes == 0 { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "OCR document URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}" ))); } if content_length > max_bytes { let size_mb = content_length as f64 / (1024.0 * 1024.0); let max_size_mb = max_bytes as f64 / (1024.0 * 1024.0); - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "OCR document size ({size_mb:.2}MB) exceeds maximum allowed size ({max_size_mb:.2}MB). url={url}" ))); } @@ -233,7 +232,7 @@ fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Core async fn read_response_with_limit( mut response: reqwest::Response, url: &Url, -) -> CoreResult> { +) -> Result, Error> { let max_bytes = max_document_download_bytes(); if let Some(content_length) = response.content_length() { enforce_download_size(content_length, max_bytes, url)?; @@ -246,7 +245,7 @@ async fn read_response_with_limit( while let Some(chunk) = response .chunk() .await - .map_err(|err| CoreError::Network(err.to_string()))? + .map_err(|err| Error::Network(err.to_string()))? { bytes_downloaded += chunk.len() as u64; enforce_download_size(bytes_downloaded, max_bytes, url)?; @@ -255,7 +254,7 @@ async fn read_response_with_limit( Ok(bytes) } -pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreResult { +pub(super) async fn convert_document_url_to_data_uri(document: Value) -> Result { let Some((field, url)) = document_url_field(&document)? else { return Ok(document); }; @@ -267,7 +266,7 @@ pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreRes let status = response.status(); if !status.is_success() { let body = response.text().await.unwrap_or_default(); - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&body), }); @@ -290,7 +289,7 @@ pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreRes let mut transformed = document .as_object() .cloned() - .ok_or_else(|| CoreError::InvalidRequest("OCR document must be an object".to_string()))?; + .ok_or_else(|| Error::InvalidRequest("OCR document must be an object".to_string()))?; transformed.insert(field.to_string(), Value::String(data_uri)); Ok(Value::Object(transformed)) } @@ -316,11 +315,11 @@ fn retry_after_secs(response: &reqwest::Response) -> u64 { .unwrap_or(2) } -fn operation_status(response_json: &Value) -> CoreResult<&str> { +fn operation_status(response_json: &Value) -> Result<&str, Error> { let status = response_json .get("status") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("status"))?; + .ok_or(Error::MissingField("status"))?; match status { "succeeded" => Ok("succeeded"), "running" | "notStarted" => Ok("running"), @@ -330,11 +329,11 @@ fn operation_status(response_json: &Value) -> CoreResult<&str> { .and_then(|error| error.get("message")) .and_then(Value::as_str) .unwrap_or("Unknown error"); - Err(CoreError::InvalidResponse(format!( + Err(Error::InvalidResponse(format!( "Azure Document Intelligence analysis failed: {message}" ))) } - other => Err(CoreError::InvalidResponse(format!( + other => Err(Error::InvalidResponse(format!( "Unknown operation status: {other}" ))), } @@ -345,9 +344,9 @@ pub(super) async fn poll_document_intelligence( original_url: &str, headers: &[(String, String)], timeout: Option, -) -> CoreResult { +) -> Result { if !same_origin(operation_url, original_url) { - return Err(CoreError::InvalidResponse( + return Err(Error::InvalidResponse( "Azure Document Intelligence: rejected cross-origin polling URL".to_string(), )); } @@ -358,7 +357,7 @@ pub(super) async fn poll_document_intelligence( )); loop { if start.elapsed() > timeout { - return Err(CoreError::Network(format!( + return Err(Error::Network(format!( "Azure Document Intelligence operation polling timed out after {} seconds", timeout.as_secs() ))); @@ -373,21 +372,21 @@ pub(super) async fn poll_document_intelligence( let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let retry_after = retry_after_secs(&response); let status = response.status(); let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let response_json: Value = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}")) + Error::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}")) })?; if operation_status(&response_json)? == "succeeded" { return Ok(response_json); @@ -426,7 +425,7 @@ mod tests { assert!(matches!( error, - CoreError::InvalidRequest(message) + Error::InvalidRequest(message) if message.contains("SSRF protection") )); } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs b/litellm-rust/crates/ai-gateway/src/ocr/handler.rs index 1de34eb400e..815bc84363a 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/handler.rs @@ -1,5 +1,4 @@ -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrResponseHandling; use serde_json::Value; @@ -7,7 +6,7 @@ use super::common_utils::{poll_document_intelligence, truncate_error_body}; use super::types::ProviderOcrRequest; use crate::client::http_client; -pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> CoreResult { +pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Result { let mut request_builder = http_client().post(&request.url).json(&request.body); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -19,7 +18,7 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); if request.config.response_handling() == OcrResponseHandling::AzureDocumentIntelligencePoll @@ -31,7 +30,7 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co .and_then(|value| value.to_str().ok()) .map(str::to_string) .ok_or_else(|| { - CoreError::InvalidResponse( + Error::InvalidResponse( "Azure Document Intelligence returned 202 but no Operation-Location header found" .to_string(), ) @@ -52,17 +51,17 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let response_json: Value = serde_json::from_str(&text) - .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; + .map_err(|err| Error::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; Ok(request .config diff --git a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs index 95df566dc53..401e26d3b29 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs @@ -1,11 +1,9 @@ -use std::future::Future; -use std::pin::Pin; - -use litellm_core::CoreResult; use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrAuthStrategy; use serde_json::{Map, Value, json}; +use std::future::Future; +use std::pin::Pin; use super::common_utils::{ convert_document_url_to_data_uri, has_header, ocr_provider_config, string_headers, @@ -27,7 +25,7 @@ pub(crate) struct OcrLifecycleHooks { request_metadata: RequestMetadata, } -type OcrFuture<'a, T> = Pin> + Send + 'a>>; +type OcrFuture<'a, T> = Pin> + Send + 'a>>; type OcrLogFuture<'a> = Pin + Send + 'a>>; impl OcrLifecycleHooks { @@ -46,7 +44,7 @@ impl OcrLifecycleHooks { async fn run_pre_call_guardrails( &self, request: PreparedOcrRequest, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(request); } @@ -74,9 +72,9 @@ impl OcrLifecycleHooks { async fn prepare_provider_request( &self, request: PreparedOcrRequest, - ) -> CoreResult { + ) -> Result { let config = ocr_provider_config(&request.custom_llm_provider, &request.model) - .ok_or_else(|| CoreError::InvalidProvider(request.custom_llm_provider.clone()))?; + .ok_or_else(|| Error::InvalidProvider(request.custom_llm_provider.clone()))?; let env_lookup = |key: &str| std::env::var(key).ok(); let headers = string_headers(request.extra_headers)?; let auth_strategy = config.auth_strategy(); @@ -120,7 +118,7 @@ impl OcrLifecycleHooks { custom_llm_provider: &str, url: &str, body: Value, - ) -> CoreResult { + ) -> Result { if self.guardrail_runner.is_empty() { return Ok(body); } @@ -217,7 +215,7 @@ impl CallLifecycleHooks for OcrLi fn async_log_failure_event<'a>( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -278,19 +276,19 @@ fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { fn parse_ocr_pre_call_guardrail_request( request: GuardrailRequest, -) -> CoreResult<(Value, Map)> { +) -> Result<(Value, Map), Error> { let Value::Object(mut data) = request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR pre_call guardrail must return an object".to_string(), )); }; let document = data.remove("document").ok_or_else(|| { - CoreError::InvalidRequest("OCR pre_call guardrail removed document".to_string()) + Error::InvalidRequest("OCR pre_call guardrail removed document".to_string()) })?; let optional_params = match data.remove("optional_params") { Some(Value::Object(params)) => params, Some(_) => { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR pre_call guardrail optional_params must be an object".to_string(), )); } @@ -299,33 +297,32 @@ fn parse_ocr_pre_call_guardrail_request( Ok((document, optional_params)) } -fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> CoreResult { +fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> Result { let Value::Object(mut data) = request.data else { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "OCR during_call guardrail must return an object".to_string(), )); }; - data.remove("body").ok_or_else(|| { - CoreError::InvalidRequest("OCR during_call guardrail removed body".to_string()) - }) + data.remove("body") + .ok_or_else(|| Error::InvalidRequest("OCR during_call guardrail removed body".to_string())) } -fn guardrail_error_to_core_error(error: GuardrailError) -> CoreError { - CoreError::InvalidRequest(format!("{}: {}", error.kind, error.message)) +fn guardrail_error_to_core_error(error: GuardrailError) -> Error { + Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) } -fn core_error_kind(error: &CoreError) -> &'static str { +fn core_error_kind(error: &Error) -> &'static str { match error { - CoreError::Auth(_) => "AuthError", - CoreError::InvalidProvider(_) => "InvalidProvider", - CoreError::InvalidRequest(_) => "InvalidRequest", - CoreError::InvalidType { .. } => "InvalidType", - CoreError::MissingField(_) => "MissingField", - CoreError::Http { .. } => "HttpError", - CoreError::InvalidResponse(_) => "InvalidResponse", - CoreError::Network(_) => "NetworkError", - CoreError::Connect(_) => "ConnectError", - CoreError::Routing(_) => "RoutingError", - CoreError::Unsupported(_) => "UnsupportedRequest", + Error::Auth(_) => "AuthError", + Error::InvalidProvider(_) => "InvalidProvider", + Error::InvalidRequest(_) => "InvalidRequest", + Error::InvalidType { .. } => "InvalidType", + Error::MissingField(_) => "MissingField", + Error::Http { .. } => "HttpError", + Error::InvalidResponse(_) => "InvalidResponse", + Error::Network(_) => "NetworkError", + Error::Connect(_) => "ConnectError", + Error::Routing(_) => "RoutingError", + Error::Unsupported(_) => "UnsupportedRequest", } } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs index c4c13e2300c..b59ab626fd3 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs @@ -1,4 +1,4 @@ -use litellm_core::CoreResult; +use litellm_core::Error; use litellm_core::call_lifecycle::CallLifecycle; use serde_json::Value; @@ -13,7 +13,7 @@ pub use types::OcrRequest; use handler::execute_ocr_provider_call; use prepare::{PreparedOcrCall, prepare_ocr_call}; -pub async fn ocr(request: OcrRequest<'_>) -> CoreResult { +pub async fn ocr(request: OcrRequest<'_>) -> Result { let PreparedOcrCall { request, hooks } = prepare_ocr_call(request); CallLifecycle::default() .run_request(request, &hooks, execute_ocr_provider_call) diff --git a/litellm-rust/crates/ai-gateway/src/ocr/tests.rs b/litellm-rust/crates/ai-gateway/src/ocr/tests.rs index bb2a6b06501..8c3f0425149 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/tests.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/tests.rs @@ -1,7 +1,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::ocr::transformation::OcrResponseHandling; use serde_json::{Map, Value, json}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -395,7 +395,7 @@ async fn ocr_lifecycle_runs_failure_hook_on_provider_error() { .await .expect_err("provider error propagates"); - assert!(matches!(err, CoreError::Http { status: 500, .. })); + assert!(matches!(err, Error::Http { status: 500, .. })); server.await.expect("server task completes"); assert_eq!( logger.events(), @@ -439,7 +439,7 @@ async fn ocr_lifecycle_pre_call_block_skips_provider_socket() { .await .expect_err("guardrail blocks request"); - assert!(matches!(err, CoreError::InvalidRequest(_))); + assert!(matches!(err, Error::InvalidRequest(_))); assert_eq!(guardrail.events(), vec!["async_pre_call_hook"]); assert_eq!( logger.events(), @@ -607,7 +607,7 @@ fn string_headers_rejects_non_string_values() { let err = string_headers(Some(headers)).expect_err("non-string header rejected"); assert_eq!( err, - CoreError::InvalidRequest( + Error::InvalidRequest( "OCR extra_headers.x-retry-count must be a string, got number".to_string() ) ); diff --git a/litellm-rust/crates/ai-gateway/src/python/config.rs b/litellm-rust/crates/ai-gateway/src/python/config.rs index c028d3d6b51..d5a4dd69c8d 100644 --- a/litellm-rust/crates/ai-gateway/src/python/config.rs +++ b/litellm-rust/crates/ai-gateway/src/python/config.rs @@ -6,33 +6,31 @@ //! (and recorded in [`crate::gil`]); the realtime hot path never touches Python. //! //! Compiled only under the `python-config` feature. - -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::router::{Deployment, Router}; use pyo3::prelude::*; use crate::gil; /// Load the router's `model_list` from `config_path` via the Python reader. -pub fn load_router_from_config(config_path: &str) -> CoreResult { +pub fn load_router_from_config(config_path: &str) -> Result { gil::record_acquisition(); Python::attach(|py| { let model_list = py .import("litellm.proxy.read_model_list") .and_then(|module| module.getattr("read_model_list")) .and_then(|reader| reader.call1((config_path,))) - .map_err(|err| CoreError::Routing(format!("read_model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("read_model_list failed: {err}")))?; let model_list_json: String = py .import("json") .and_then(|json| json.getattr("dumps")) .and_then(|dumps| dumps.call1((model_list,))) .and_then(|encoded| encoded.extract()) - .map_err(|err| CoreError::Routing(format!("serializing model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("serializing model_list failed: {err}")))?; let deployments: Vec = serde_json::from_str(&model_list_json) - .map_err(|err| CoreError::Routing(format!("parsing model_list failed: {err}")))?; + .map_err(|err| Error::Routing(format!("parsing model_list failed: {err}")))?; Ok(Router::new(deployments)) }) diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs index 7e38d10c6ff..e9f8c477f36 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs @@ -9,7 +9,7 @@ use axum::http::StatusCode; use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE, HeaderMap, HeaderValue}; use axum::response::{IntoResponse, Response}; use axum::routing::post; -use litellm_core::CoreError; +use litellm_core::Error; use serde_json::{Map, Value}; use crate::auth::RequireMasterKey; @@ -46,7 +46,7 @@ fn stream_response(upstream: reqwest::Response) -> Result Result Result>, CoreError> { +fn forwarded_headers(headers: &HeaderMap) -> Result>, Error> { let forwarded = headers .iter() .filter(|(name, _)| { @@ -74,19 +74,19 @@ fn forwarded_headers(headers: &HeaderMap) -> Result>, }) .map(|(name, value)| { let value = value.to_str().map_err(|_| { - CoreError::InvalidRequest(format!("invalid value for header {}", name.as_str())) + Error::InvalidRequest(format!("invalid value for header {}", name.as_str())) })?; Ok((name.to_string(), Value::String(value.to_string()))) }) - .collect::, CoreError>>()?; + .collect::, Error>>()?; Ok((!forwarded.is_empty()).then_some(forwarded)) } #[derive(Debug)] -struct MessagesRouteError(CoreError); +struct MessagesRouteError(Error); -impl From for MessagesRouteError { - fn from(error: CoreError) -> Self { +impl From for MessagesRouteError { + fn from(error: Error) -> Self { Self(error) } } @@ -94,28 +94,28 @@ impl From for MessagesRouteError { impl IntoResponse for MessagesRouteError { fn into_response(self) -> Response { let (status, message) = match self.0 { - CoreError::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message), - CoreError::InvalidProvider(_) | CoreError::Routing(_) => ( + Error::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message), + Error::InvalidProvider(_) | Error::Routing(_) => ( StatusCode::NOT_FOUND, "no messages deployment is configured for this model".to_string(), ), - CoreError::Auth(_) => ( + Error::Auth(_) => ( StatusCode::BAD_GATEWAY, "messages provider authentication failed".to_string(), ), - CoreError::Http { .. } - | CoreError::Network(_) - | CoreError::Connect(_) - | CoreError::InvalidResponse(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) => ( + Error::Http { .. } + | Error::Network(_) + | Error::Connect(_) + | Error::InvalidResponse(_) + | Error::InvalidType { .. } + | Error::MissingField(_) => ( StatusCode::BAD_GATEWAY, "messages provider request failed".to_string(), ), // The gateway has no Python implementation to decline to, so a // request the core cannot serve is reported to the caller. The // reason is a fixed internal string, never provider content. - CoreError::Unsupported(reason) => ( + Error::Unsupported(reason) => ( StatusCode::BAD_REQUEST, format!("messages request is not supported: {reason}"), ), diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs index 5f4c5fe8de4..4fd29db05d6 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs @@ -1,10 +1,10 @@ use std::sync::Arc; +use litellm_core::Error; use litellm_core::constants::ANTHROPIC_MESSAGES_PROVIDER; use litellm_core::messages::types::MessagesRequest; use litellm_core::messages::{messages, messages_stream}; use litellm_core::router::Router; -use litellm_core::{CoreError, CoreResult}; use serde_json::{Map, Value}; pub(crate) enum MessagesResponse { @@ -16,16 +16,16 @@ pub async fn run( router: &Arc, body: Value, extra_headers: Option>, -) -> CoreResult { +) -> Result { let model = body .get("model") .and_then(Value::as_str) .map(str::trim) .filter(|model| !model.is_empty()) - .ok_or_else(|| CoreError::InvalidRequest("messages body requires a model".to_string()))?; - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + .ok_or_else(|| Error::InvalidRequest("messages body requires a model".to_string()))?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let provider_model = deployment.litellm_params.model.as_str(); let upstream_model = provider_model .split_once('/') @@ -37,7 +37,7 @@ pub async fn run( }; let mut body = body; body.as_object_mut() - .ok_or_else(|| CoreError::InvalidRequest("messages body must be an object".to_string()))? + .ok_or_else(|| Error::InvalidRequest("messages body must be an object".to_string()))? .insert( "model".to_string(), Value::String(upstream_model.to_string()), @@ -60,6 +60,6 @@ pub async fn run( serde_json::to_value(response) .map(MessagesResponse::Json) .map_err(|err| { - CoreError::InvalidResponse(format!("failed to serialize messages response: {err}")) + Error::InvalidResponse(format!("failed to serialize messages response: {err}")) }) } diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs index 4ae8cfe7379..b8ee77c4269 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs @@ -11,8 +11,7 @@ use std::time::Duration; use crate::io::realtime_pool::{RealtimePool, upstream_key}; use futures_util::{Sink, Stream}; -use litellm_core::CoreResult; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::realtime::types::RealtimeEvent; use litellm_core::router::Router; @@ -29,15 +28,15 @@ pub async fn run( observe: impl FnMut(&RealtimeEvent) + Send, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, >::Error: std::fmt::Display, { - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let params = &deployment.litellm_params; // Strip a leading `openai/` so the OpenAI-only realtime fn gets the bare model. let provider_model = params diff --git a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs b/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs index 165c95695d3..e8f840c0c8e 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs @@ -2,13 +2,13 @@ use std::sync::Arc; use std::time::Duration; use futures_util::{Sink, Stream}; +use litellm_core::Error; use litellm_core::call_lifecycle::{CallLifecycle, CallLifecycleContext}; use litellm_core::responses::instrumentation::{ ResponsesWsCallbackPayload, ResponsesWsInstrumentation, ResponsesWsLogOutcome, ResponsesWsMetadata, }; use litellm_core::responses::types::ResponsesWsEvent; -use litellm_core::{CoreError, CoreResult}; use crate::integrations::custom_logger::{ CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, ModelCallDetails, @@ -26,22 +26,22 @@ pub async fn run( metadata: RequestMetadata, client_in: In, client_out: Out, -) -> CoreResult<()> +) -> Result<(), Error> where In: Stream + Unpin + Send, Out: Sink + Unpin + Send, Out::Error: std::fmt::Display, { - let deployment = router.get_available_deployment(model).ok_or_else(|| { - CoreError::Routing(format!("no deployment available for model '{model}'")) - })?; + let deployment = router + .get_available_deployment(model) + .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; let params = &deployment.litellm_params; let provider_model = params .model .strip_prefix("openai/") .unwrap_or(¶ms.model); if params.model.contains('/') && !params.model.starts_with("openai/") { - return Err(CoreError::InvalidProvider( + return Err(Error::InvalidProvider( "Responses WebSocket route supports OpenAI deployments only".to_string(), )); } diff --git a/litellm-rust/crates/core/src/audio_transcription/transformation.rs b/litellm-rust/crates/core/src/audio_transcription/transformation.rs index eab34c13843..16a28fbcac0 100644 --- a/litellm-rust/crates/core/src/audio_transcription/transformation.rs +++ b/litellm-rust/crates/core/src/audio_transcription/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::CoreResult; - use super::types::{AudioTranscriptionRequestData, AudioTranscriptionResponseData}; #[derive(Clone, Debug, PartialEq, Eq)] @@ -32,13 +31,13 @@ pub trait AudioTranscriptionProviderConfig: Sync { model: &str, audio: Value, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_transcription_response( &self, model: &str, response_json: Value, - ) -> CoreResult; + ) -> Result; fn complete_url( &self, @@ -46,12 +45,12 @@ pub trait AudioTranscriptionProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth_strategy( &self, model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; } diff --git a/litellm-rust/crates/core/src/call_lifecycle/mod.rs b/litellm-rust/crates/core/src/call_lifecycle/mod.rs index d9b68a1b726..637c156e192 100644 --- a/litellm-rust/crates/core/src/call_lifecycle/mod.rs +++ b/litellm-rust/crates/core/src/call_lifecycle/mod.rs @@ -1,7 +1,7 @@ use std::future::Future; use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use crate::{CoreError, CoreResult}; +use crate::Error; pub mod types; @@ -11,14 +11,14 @@ pub use types::{ }; pub trait CallLifecycleHooks: Send + Sync { - type PreCallFuture<'a>: Future> + Send + 'a + type PreCallFuture<'a>: Future> + Send + 'a where Self: 'a, InitialReq: 'a, ProviderReq: 'a, Resp: 'a; - type DuringCallFuture<'a>: Future> + Send + 'a + type DuringCallFuture<'a>: Future> + Send + 'a where Self: 'a, InitialReq: 'a, @@ -56,7 +56,7 @@ pub trait CallLifecycleHooks: Send + Sync { fn async_log_failure_event<'a>( &'a self, context: &'a CallLifecycleContext, - error: &'a CoreError, + error: &'a Error, timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a>; } @@ -86,12 +86,12 @@ impl<'a> CallLifecycle<'a> { request: InitialReq, hooks: &Hooks, provider_call: ProviderCall, - ) -> CoreResult + ) -> Result where InitialReq: CallLifecycleRequest, Hooks: CallLifecycleHooks, ProviderCall: FnOnce(ProviderReq) -> ProviderFuture, - ProviderFuture: Future>, + ProviderFuture: Future>, { let context = request.lifecycle_context(); self.run(context, request, hooks, provider_call).await @@ -103,11 +103,11 @@ impl<'a> CallLifecycle<'a> { request: InitialReq, hooks: &Hooks, provider_call: ProviderCall, - ) -> CoreResult + ) -> Result where Hooks: CallLifecycleHooks, ProviderCall: FnOnce(ProviderReq) -> ProviderFuture, - ProviderFuture: Future>, + ProviderFuture: Future>, { let call_start = epoch_seconds(); let mut phases = Vec::new(); @@ -166,7 +166,7 @@ impl<'a> CallLifecycle<'a> { &self, context: &CallLifecycleContext, hooks: &Hooks, - error: &CoreError, + error: &Error, call_start: f64, phases: &mut Vec, ) where @@ -251,8 +251,8 @@ mod tests { } impl CallLifecycleHooks for RecordingHooks { - type PreCallFuture<'a> = BoxFuture<'a, CoreResult>; - type DuringCallFuture<'a> = BoxFuture<'a, CoreResult>; + type PreCallFuture<'a> = BoxFuture<'a, Result>; + type DuringCallFuture<'a> = BoxFuture<'a, Result>; type SuccessFuture<'a> = BoxFuture<'a, ()>; type FailureFuture<'a> = BoxFuture<'a, ()>; @@ -294,7 +294,7 @@ mod tests { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -304,8 +304,8 @@ mod tests { } impl CallLifecycleHooks for RecordingHooks { - type PreCallFuture<'a> = BoxFuture<'a, CoreResult>; - type DuringCallFuture<'a> = BoxFuture<'a, CoreResult>; + type PreCallFuture<'a> = BoxFuture<'a, Result>; + type DuringCallFuture<'a> = BoxFuture<'a, Result>; type SuccessFuture<'a> = BoxFuture<'a, ()>; type FailureFuture<'a> = BoxFuture<'a, ()>; @@ -345,7 +345,7 @@ mod tests { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -383,13 +383,13 @@ mod tests { "request".to_string(), &hooks, |_request| async move { - Err::(CoreError::Network("provider down".to_string())) + Err::(Error::Network("provider down".to_string())) }, ) .await .expect_err("call fails"); - assert_eq!(error, CoreError::Network("provider down".to_string())); + assert_eq!(error, Error::Network("provider down".to_string())); assert_eq!(hooks.events(), vec!["pre_call", "during_call", "failure"]); } diff --git a/litellm-rust/crates/core/src/chat_completions/common_utils.rs b/litellm-rust/crates/core/src/chat_completions/common_utils.rs index 36eaf242a5a..ca51471eb7c 100644 --- a/litellm-rust/crates/core/src/chat_completions/common_utils.rs +++ b/litellm-rust/crates/core/src/chat_completions/common_utils.rs @@ -1,8 +1,7 @@ -use serde_json::{Map, Value}; - -use crate::error::CoreResult; +use crate::Error; use crate::http_utils::string_headers as shared_string_headers; use crate::providers::anthropic::chat_completions::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG; +use serde_json::{Map, Value}; use super::transformation::ChatCompletionsProviderConfig; @@ -23,6 +22,6 @@ pub(super) fn chat_completions_provider_config( pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { shared_string_headers(HEADER_CONTEXT, extra_headers) } diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index afc4529fd26..7e2731442cc 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,6 +1,6 @@ use serde_json::Value; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::http_utils::truncate_error_body; use super::client::http_client; @@ -11,9 +11,9 @@ use super::types::{ pub(super) async fn execute_chat_completions_provider_call( request: ProviderChatCompletionsRequest, -) -> CoreResult { +) -> Result { let body = serde_json::to_vec(&request.body).map_err(|err| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "failed to serialize chat completions request: {err}" )) })?; @@ -32,9 +32,9 @@ pub(super) async fn execute_chat_completions_provider_call( // so the host can still serve it. Everything else here, a timeout // above all, may have reached the provider and been answered. if err.is_connect() || err.is_builder() { - CoreError::Connect(err.to_string()) + Error::Connect(err.to_string()) } else { - CoreError::Network(err.to_string()) + Error::Network(err.to_string()) } })?; @@ -42,17 +42,17 @@ pub(super) async fn execute_chat_completions_provider_call( let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } let body: Value = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid chat completions response JSON: {err}")) + Error::InvalidResponse(format!("invalid chat completions response JSON: {err}")) })?; request .config @@ -69,10 +69,10 @@ pub(super) async fn execute_chat_completions_provider_call( /// second kind has already been billed, and a host that keeps a reference /// implementation must not retry those, so collapse them to one variant that /// can only mean the provider was already called. -pub(super) fn as_response_error(err: CoreError) -> CoreError { +pub(super) fn as_response_error(err: Error) -> Error { match err { - already @ (CoreError::InvalidResponse(_) | CoreError::Http { .. }) => already, - other => CoreError::InvalidResponse(other.to_string()), + already @ (Error::InvalidResponse(_) | Error::Http { .. }) => already, + other => Error::InvalidResponse(other.to_string()), } } @@ -80,7 +80,7 @@ pub(super) fn as_response_error(err: CoreError) -> CoreError { pub(super) async fn signed_headers( request: &ProviderChatCompletionsRequest, body: &[u8], -) -> CoreResult> { +) -> Result, Error> { use std::collections::BTreeMap; use std::time::SystemTime; @@ -101,7 +101,7 @@ pub(super) async fn signed_headers( .iter() .any(|(name, _)| is_sigv4_computed_header(name)) { - return Err(CoreError::Unsupported( + return Err(Error::Unsupported( "request forwards a header AWS SigV4 computes", )); } @@ -137,9 +137,9 @@ pub(super) async fn signed_headers( pub(super) async fn signed_headers( request: &ProviderChatCompletionsRequest, _body: &[u8], -) -> CoreResult> { +) -> Result, Error> { match &request.auth { - ChatCompletionsAuth::AwsSigV4 { .. } => Err(CoreError::Unsupported( + ChatCompletionsAuth::AwsSigV4 { .. } => Err(Error::Unsupported( "AWS SigV4 requires the bedrock-auth feature", )), _ => Ok(request.upstream_headers.clone()), diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index f30ac1a24bf..0d009d36d16 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -6,6 +6,7 @@ //! credentials, and it resolves the provider, translates the conversation, //! calls the provider, and returns a typed OpenAI-shaped response. +use crate::Error; mod client; mod common_utils; pub mod conversation; @@ -17,15 +18,13 @@ pub mod types; use serde_json::{Map, Value}; -use crate::error::CoreResult; - use handler::execute_chat_completions_provider_call; use prepare::{parse_messages, prepare_chat_completions_call, resolve_provider_config}; use types::{ChatCompletionsRequest, ChatCompletionsResponse}; pub async fn chat_completions( request: ChatCompletionsRequest<'_>, -) -> CoreResult { +) -> Result { execute_chat_completions_provider_call(prepare_chat_completions_call(request)?).await } diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index 1e1c8d1bafd..142b2f2aaed 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -1,6 +1,6 @@ use serde_json::Value; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::http_utils::has_header; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; @@ -11,7 +11,7 @@ use super::types::{ChatCompletionsRequest, ChatMessage, ProviderChatCompletionsR pub(super) fn resolve_provider_config<'a>( model: &'a str, custom_llm_provider: Option<&'a str>, -) -> CoreResult<(String, &'static dyn ChatCompletionsProviderConfig)> { +) -> Result<(String, &'static dyn ChatCompletionsProviderConfig), Error> { let provider_info = get_custom_llm_provider(model, custom_llm_provider) .or_else(|| { custom_llm_provider.map(|provider| CustomLlmProvider { @@ -20,35 +20,34 @@ pub(super) fn resolve_provider_config<'a>( }) }) .ok_or_else(|| { - CoreError::InvalidProvider( + Error::InvalidProvider( "unable to resolve custom_llm_provider for chat completions request".to_string(), ) })?; let config = chat_completions_provider_config(provider_info.custom_llm_provider) - .ok_or_else(|| CoreError::InvalidProvider(provider_info.custom_llm_provider.to_string()))?; + .ok_or_else(|| Error::InvalidProvider(provider_info.custom_llm_provider.to_string()))?; Ok((provider_info.model.to_string(), config)) } -pub(super) fn parse_messages(messages: Value) -> CoreResult> { - serde_json::from_value(messages).map_err(|err| { - CoreError::InvalidRequest(format!("invalid chat completions messages: {err}")) - }) +pub(super) fn parse_messages(messages: Value) -> Result, Error> { + serde_json::from_value(messages) + .map_err(|err| Error::InvalidRequest(format!("invalid chat completions messages: {err}"))) } pub(super) fn prepare_chat_completions_call( request: ChatCompletionsRequest<'_>, -) -> CoreResult { +) -> Result { let (model, config) = resolve_provider_config(request.model, request.custom_llm_provider)?; let env_lookup = |key: &str| std::env::var(key).ok(); let messages = parse_messages(request.messages)?; if messages.is_empty() { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "chat completions requires at least one message".to_string(), )); } if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) { - return Err(CoreError::Unsupported(reason.0)); + return Err(Error::Unsupported(reason.0)); } let mut headers = string_headers(request.extra_headers)?; diff --git a/litellm-rust/crates/core/src/chat_completions/tests.rs b/litellm-rust/crates/core/src/chat_completions/tests.rs index e2383723cb0..2858d180e27 100644 --- a/litellm-rust/crates/core/src/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/chat_completions/tests.rs @@ -1,6 +1,6 @@ use serde_json::{Map, Value, json}; -use crate::error::CoreError; +use crate::error::Error; use super::prepare::prepare_chat_completions_call; use super::transformation::ChatCompletionsAuth; @@ -29,7 +29,7 @@ fn request<'a>( /// `ProviderChatCompletionsRequest` deliberately has no `Debug` (its headers /// carry resolved credentials), so unwrap the failure case by hand. -fn decline(request: ChatCompletionsRequest<'_>) -> CoreError { +fn decline(request: ChatCompletionsRequest<'_>) -> Error { match prepare_chat_completions_call(request) { Err(error) => error, Ok(prepared) => panic!("expected a decline, prepared a call to {}", prepared.url), @@ -196,7 +196,7 @@ fn declines_an_unsupported_request_before_resolving_credentials() { call.api_key = None; // No api_key is set and no env is consulted: the gate must run first, so the // error is the decline rather than a missing-credential error. - assert_eq!(decline(call), CoreError::Unsupported("streaming")); + assert_eq!(decline(call), Error::Unsupported("streaming")); } #[test] @@ -208,7 +208,7 @@ fn rejects_an_unknown_provider() { json!([{"role": "user", "content": "hi"}]), json!({}), )), - CoreError::InvalidProvider("openai".to_string()) + Error::InvalidProvider("openai".to_string()) ); } @@ -221,7 +221,7 @@ fn rejects_a_model_with_no_resolvable_provider() { json!([{"role": "user", "content": "hi"}]), json!({}), )), - CoreError::InvalidProvider(_) + Error::InvalidProvider(_) )); } @@ -234,7 +234,7 @@ fn rejects_an_empty_or_malformed_message_list() { json!([]), json!({}), )), - CoreError::InvalidRequest("chat completions requires at least one message".to_string()) + Error::InvalidRequest("chat completions requires at least one message".to_string()) ); assert!(matches!( decline(request( @@ -243,7 +243,7 @@ fn rejects_an_empty_or_malformed_message_list() { json!("not a list"), json!({}), )), - CoreError::InvalidRequest(_) + Error::InvalidRequest(_) )); } @@ -258,7 +258,7 @@ fn rejects_non_string_extra_headers() { call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))])); assert_eq!( decline(call), - CoreError::InvalidRequest( + Error::InvalidRequest( "chat completions extra_headers.x-trace must be a string, got number".to_string() ) ); @@ -374,7 +374,7 @@ async fn a_forwarded_header_the_signer_computes_declines_to_python() { .await .expect_err("{forwarded} should decline instead of being signed"); assert!( - matches!(error, CoreError::Unsupported(_)), + matches!(error, Error::Unsupported(_)), "{forwarded} declined as {error:?}, which the host would not fall back on" ); } @@ -727,7 +727,7 @@ mod round_trip { .expect_err("response cannot be normalized"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::InvalidResponse(_)), + matches!(err, Error::InvalidResponse(_)), "expected a post-send error, got {err:?}" ); } @@ -745,7 +745,7 @@ mod round_trip { .expect_err("response cannot be normalized"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::InvalidResponse(_)), + matches!(err, Error::InvalidResponse(_)), "expected a post-send error, got {err:?}" ); } @@ -763,7 +763,7 @@ mod round_trip { .expect_err("upstream rejects"); handle.await.expect("server task"); assert!( - matches!(err, CoreError::Http { status: 429, .. }), + matches!(err, Error::Http { status: 429, .. }), "expected a 429, got {err:?}" ); } @@ -787,7 +787,7 @@ mod round_trip { .await .expect_err("nothing is listening"); assert!( - matches!(err, CoreError::Connect(_)), + matches!(err, Error::Connect(_)), "expected a pre-send connect failure, got {err:?}" ); } @@ -797,24 +797,24 @@ mod round_trip { use crate::chat_completions::handler::as_response_error; for original in [ - CoreError::MissingField("usage"), - CoreError::Unsupported("non-text response content block"), - CoreError::InvalidRequest("whatever".to_string()), - CoreError::Auth("whatever".to_string()), + Error::MissingField("usage"), + Error::Unsupported("non-text response content block"), + Error::InvalidRequest("whatever".to_string()), + Error::Auth("whatever".to_string()), ] { let label = format!("{original:?}"); assert!( - matches!(as_response_error(original), CoreError::InvalidResponse(_)), + matches!(as_response_error(original), Error::InvalidResponse(_)), "{label} must not stay retryable once the provider has answered" ); } // An upstream status is already unambiguous, so it survives intact. assert!(matches!( - as_response_error(CoreError::Http { + as_response_error(Error::Http { status: 500, body: "boom".to_string() }), - CoreError::Http { status: 500, .. } + Error::Http { status: 500, .. } )); } } diff --git a/litellm-rust/crates/core/src/chat_completions/transformation.rs b/litellm-rust/crates/core/src/chat_completions/transformation.rs index a30ce9dc77c..a0868209305 100644 --- a/litellm-rust/crates/core/src/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/chat_completions/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::error::CoreResult; - use super::types::{ ChatCompletionsResponse, ChatMessage, ChatMessageContent, ProviderChatRequestData, ProviderChatResponseData, @@ -39,7 +38,7 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth( &self, @@ -47,7 +46,7 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn default_headers(&self) -> &'static [(&'static str, &'static str)] { &[("content-type", "application/json")] @@ -91,13 +90,13 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_response( &self, model: &str, response: ProviderChatResponseData, - ) -> CoreResult; + ) -> Result; } pub fn unsupported_param( diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index 739532f8cb5..db3fa2ec704 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -1,9 +1,7 @@ -use thiserror::Error; +use thiserror::Error as ThisError; -pub type CoreResult = Result; - -#[derive(Debug, Error, PartialEq, Eq)] -pub enum CoreError { +#[derive(Debug, ThisError, PartialEq, Eq)] +pub enum Error { #[error("expected {expected}, got {actual}")] InvalidType { expected: &'static str, diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index c541f50275b..10661fadf96 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -3,7 +3,7 @@ use serde_json::{Map, Value}; use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; /// Bound an upstream error body before it crosses a host boundary, so provider /// bodies stay data-minimized. @@ -18,7 +18,7 @@ pub fn truncate_error_body(body: &str) -> String { pub fn string_headers( context: &'static str, extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { extra_headers .unwrap_or_default() .into_iter() @@ -27,7 +27,7 @@ pub fn string_headers( .as_str() .map(|value| (key.clone(), value.to_string())) .ok_or_else(|| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "{context} extra_headers.{key} must be a string, got {}", json_type_name(&value) )) @@ -81,7 +81,7 @@ mod tests { let err = string_headers("chat completions", Some(headers)).expect_err("non-string value"); assert_eq!( err, - CoreError::InvalidRequest( + Error::InvalidRequest( "chat completions extra_headers.x-trace must be a string, got number".to_string() ) ); diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index dce4a425ea0..0e18d24e5d8 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -13,4 +13,4 @@ pub mod responses; pub mod router; pub mod routing_utils; -pub use error::{CoreError, CoreResult}; +pub use error::Error; diff --git a/litellm-rust/crates/core/src/messages/common_utils.rs b/litellm-rust/crates/core/src/messages/common_utils.rs index a14dffbc1fe..8dfdb2e361a 100644 --- a/litellm-rust/crates/core/src/messages/common_utils.rs +++ b/litellm-rust/crates/core/src/messages/common_utils.rs @@ -1,9 +1,8 @@ -use serde_json::{Map, Value}; - -use crate::error::CoreResult; +use crate::Error; use crate::http_utils::string_headers as shared_string_headers; use crate::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG; use crate::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG; +use serde_json::{Map, Value}; use super::transformation::AnthropicMessagesProviderConfig; @@ -23,6 +22,6 @@ pub(super) fn messages_provider_config( pub(super) fn string_headers( extra_headers: Option>, -) -> CoreResult> { +) -> Result, Error> { shared_string_headers(HEADER_CONTEXT, extra_headers) } diff --git a/litellm-rust/crates/core/src/messages/handler.rs b/litellm-rust/crates/core/src/messages/handler.rs index 1c895f66eba..13a65d86131 100644 --- a/litellm-rust/crates/core/src/messages/handler.rs +++ b/litellm-rust/crates/core/src/messages/handler.rs @@ -1,5 +1,5 @@ use crate::constants::ANTHROPIC_MESSAGES_PROVIDER; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use super::client::http_client; use super::common_utils::truncate_error_body; @@ -7,7 +7,7 @@ use super::types::{AnthropicMessagesResponse, ProviderMessagesRequest}; pub(super) async fn execute_messages_provider_call( request: ProviderMessagesRequest, -) -> CoreResult { +) -> Result { let mut request_builder = http_client().post(&request.url).json(&request.body); for (key, value) in &request.upstream_headers { request_builder = request_builder.header(key, value); @@ -19,32 +19,31 @@ pub(super) async fn execute_messages_provider_call( let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; if !status.is_success() { - return Err(CoreError::Http { + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); } - let response = serde_json::from_str(&text).map_err(|err| { - CoreError::InvalidResponse(format!("invalid messages response JSON: {err}")) - })?; + let response = serde_json::from_str(&text) + .map_err(|err| Error::InvalidResponse(format!("invalid messages response JSON: {err}")))?; request.config.transform_response(&request.model, response) } pub(super) async fn execute_messages_provider_stream( request: ProviderMessagesRequest, -) -> CoreResult { +) -> Result { if request.provider != ANTHROPIC_MESSAGES_PROVIDER { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "streaming messages is not supported for this provider".to_string(), )); } @@ -60,14 +59,14 @@ pub(super) async fn execute_messages_provider_stream( let response = request_builder .send() .await - .map_err(|err| CoreError::Network(err.to_string()))?; + .map_err(|err| Error::Network(err.to_string()))?; let status = response.status(); if !status.is_success() { let text = response .text() .await - .map_err(|err| CoreError::Network(err.to_string()))?; - return Err(CoreError::Http { + .map_err(|err| Error::Network(err.to_string()))?; + return Err(Error::Http { status: status.as_u16(), body: truncate_error_body(&text), }); diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index acb36d89daf..ee2877e61fc 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -7,6 +7,7 @@ //! is the streaming variant; it hands the raw upstream response back so a host //! can splice the event stream to its own caller. +use crate::Error; mod client; mod common_utils; mod handler; @@ -14,17 +15,15 @@ mod prepare; pub mod transformation; pub mod types; -use crate::error::CoreResult; - use handler::{execute_messages_provider_call, execute_messages_provider_stream}; use prepare::prepare_messages_call; use types::{AnthropicMessagesResponse, MessagesRequest}; -pub async fn messages(request: MessagesRequest<'_>) -> CoreResult { +pub async fn messages(request: MessagesRequest<'_>) -> Result { execute_messages_provider_call(prepare_messages_call(request)?).await } -pub async fn messages_stream(request: MessagesRequest<'_>) -> CoreResult { +pub async fn messages_stream(request: MessagesRequest<'_>) -> Result { execute_messages_provider_stream(prepare_messages_call(request)?).await } diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index 94b5b1eaed7..3b253ac3766 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}; @@ -7,7 +7,7 @@ use super::types::{MessagesRequest, ProviderMessagesRequest}; pub(super) fn prepare_messages_call( request: MessagesRequest<'_>, -) -> CoreResult { +) -> Result { let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider) .or_else(|| { request @@ -18,7 +18,7 @@ pub(super) fn prepare_messages_call( }) }) .ok_or_else(|| { - CoreError::InvalidProvider( + Error::InvalidProvider( "unable to resolve custom_llm_provider for messages request".to_string(), ) })?; @@ -26,7 +26,7 @@ pub(super) fn prepare_messages_call( let provider = provider_info.custom_llm_provider; let config = messages_provider_config(provider) - .ok_or_else(|| CoreError::InvalidProvider(provider.to_string()))?; + .ok_or_else(|| Error::InvalidProvider(provider.to_string()))?; let env_lookup = |key: &str| std::env::var(key).ok(); let mut headers = string_headers(request.extra_headers)?; @@ -53,11 +53,11 @@ pub(super) fn prepare_messages_call( let url = config.complete_url(request.api_base, &model, &env_lookup)?; let typed_request = serde_json::from_value(request.body).map_err(|err| { - CoreError::InvalidRequest(format!("invalid Anthropic messages request: {err}")) + Error::InvalidRequest(format!("invalid Anthropic messages request: {err}")) })?; let transformed = config.transform_request(typed_request)?; let body = serde_json::to_value(transformed).map_err(|err| { - CoreError::InvalidRequest(format!( + Error::InvalidRequest(format!( "failed to serialize Anthropic messages request: {err}" )) })?; diff --git a/litellm-rust/crates/core/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs index 9fc1763683b..df9f7051011 100644 --- a/litellm-rust/crates/core/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -4,7 +4,7 @@ use serde_json::{Map, Value, json}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; -use crate::error::CoreError; +use crate::error::Error; use super::common_utils::{ has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body, @@ -77,7 +77,7 @@ fn truncate_error_body_caps_long_payloads() { fn string_headers_rejects_non_string_values() { let headers = json!({"x-count": 3}).as_object().unwrap().clone(); let err = string_headers(Some(headers)).expect_err("non-string header rejected"); - assert!(matches!(err, CoreError::InvalidRequest(_))); + assert!(matches!(err, Error::InvalidRequest(_))); } #[test] @@ -341,7 +341,7 @@ async fn messages_requires_auth_when_no_key_and_no_header() { .await .expect_err("missing auth errors"); - assert!(matches!(err, CoreError::Auth(_))); + assert!(matches!(err, Error::Auth(_))); } #[tokio::test] @@ -420,7 +420,7 @@ async fn messages_maps_provider_error_status_to_http_error() { .await .expect_err("provider error propagates"); - assert!(matches!(err, CoreError::Http { status: 401, .. })); + assert!(matches!(err, Error::Http { status: 401, .. })); } #[tokio::test] @@ -437,5 +437,5 @@ async fn messages_rejects_unsupported_provider() { .await .expect_err("unsupported provider errors"); - assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "openai")); + assert!(matches!(err, Error::InvalidProvider(provider) if provider == "openai")); } diff --git a/litellm-rust/crates/core/src/messages/transformation.rs b/litellm-rust/crates/core/src/messages/transformation.rs index b478e20d24b..673a5728aca 100644 --- a/litellm-rust/crates/core/src/messages/transformation.rs +++ b/litellm-rust/crates/core/src/messages/transformation.rs @@ -1,6 +1,5 @@ -use crate::error::CoreResult; - use super::types::{AnthropicMessagesRequest, AnthropicMessagesResponse}; +use crate::Error; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MessagesAuthStrategy { @@ -23,13 +22,13 @@ pub trait AnthropicMessagesProviderConfig: Sync { api_base: Option<&str>, model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn resolve_api_key( &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth_strategy(&self) -> MessagesAuthStrategy { MessagesAuthStrategy::Header("x-api-key") @@ -49,7 +48,7 @@ pub trait AnthropicMessagesProviderConfig: Sync { fn transform_request( &self, request: AnthropicMessagesRequest, - ) -> CoreResult { + ) -> Result { Ok(request) } @@ -57,7 +56,7 @@ pub trait AnthropicMessagesProviderConfig: Sync { &self, _model: &str, response: AnthropicMessagesResponse, - ) -> CoreResult { + ) -> Result { Ok(response) } } diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs index cb3e735e533..3d3c16c8cb6 100644 --- a/litellm-rust/crates/core/src/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/transformation.rs @@ -1,7 +1,6 @@ +use crate::Error; use serde_json::{Map, Value}; -use crate::CoreResult; - use super::types::{OcrRequestData, OcrResponseData}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -43,13 +42,13 @@ pub trait OcrProviderConfig: Sync { model: &str, document: Value, optional_params: Map, - ) -> CoreResult; + ) -> Result; fn transform_ocr_response( &self, model: &str, response_json: Value, - ) -> CoreResult; + ) -> Result; fn complete_url( &self, @@ -57,13 +56,13 @@ pub trait OcrProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn resolve_api_key( &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult; + ) -> Result; fn auth_strategy(&self) -> OcrAuthStrategy { OcrAuthStrategy::Bearer diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs index 4534ac0182c..b22de6c47de 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::Error; use serde_json::json; fn messages(value: Value) -> Vec { @@ -19,7 +20,7 @@ fn transform(model: &str, msgs: Value, opts: Value) -> Value { .body } -fn transform_response(body: Value) -> CoreResult { +fn transform_response(body: Value) -> Result { ANTHROPIC_CHAT_COMPLETIONS_CONFIG .transform_response("claude-sonnet-4-5", ProviderChatResponseData { body }) } @@ -390,29 +391,26 @@ fn declines_a_response_carrying_a_non_text_block() { "usage": {"input_tokens": 1, "output_tokens": 1} })) .expect_err("non-text block"); - assert_eq!( - err, - CoreError::Unsupported("non-text response content block") - ); + assert_eq!(err, Error::Unsupported("non-text response content block")); } #[test] fn errors_on_a_response_missing_required_fields() { assert_eq!( transform_response(json!("nope")).expect_err("not an object"), - CoreError::InvalidResponse("messages response is not an object".to_string()) + Error::InvalidResponse("messages response is not an object".to_string()) ); assert_eq!( transform_response(json!({"model": "m", "usage": {}})).expect_err("no content"), - CoreError::MissingField("content") + Error::MissingField("content") ); assert_eq!( transform_response(json!({"model": "m", "content": []})).expect_err("no usage"), - CoreError::MissingField("usage") + Error::MissingField("usage") ); assert_eq!( transform_response(json!({"content": [], "usage": {}})).expect_err("no model"), - CoreError::MissingField("model") + Error::MissingField("model") ); } diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs index 3658642b539..97cc48aa6f2 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs @@ -10,7 +10,7 @@ use crate::chat_completions::types::{ ProviderChatRequestData, ProviderChatResponseData, }; use crate::constants::ANTHROPIC_OAUTH_TOKEN_PREFIX; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::providers::anthropic::messages::transformation::{ complete_anthropic_url, resolve_anthropic_api_key, }; @@ -74,7 +74,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_anthropic_url(api_base, env_lookup)) } @@ -84,7 +84,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(ChatCompletionsAuth::Header { name: "x-api-key", value: resolve_anthropic_api_key(api_key, env_lookup)?, @@ -137,7 +137,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult { + ) -> Result { Ok(ProviderChatRequestData { body: anthropic_body(model, &build_conversation(&messages), optional_params), }) @@ -147,15 +147,16 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { &self, _model: &str, response: ProviderChatResponseData, - ) -> CoreResult { - let body = response.body.as_object().ok_or_else(|| { - CoreError::InvalidResponse("messages response is not an object".into()) - })?; + ) -> Result { + let body = response + .body + .as_object() + .ok_or_else(|| Error::InvalidResponse("messages response is not an object".into()))?; let content = body .get("content") .and_then(Value::as_array) - .ok_or(CoreError::MissingField("content"))?; + .ok_or(Error::MissingField("content"))?; // The route declines tool and thinking requests, so a non-text block // means the response carries something this path never asked for. // Decline rather than silently dropping it; the host falls back. @@ -163,7 +164,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { .iter() .any(|block| block.get("type").and_then(Value::as_str) != Some("text")) { - return Err(CoreError::Unsupported("non-text response content block")); + return Err(Error::Unsupported("non-text response content block")); } let text: String = content .iter() @@ -173,7 +174,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { let usage = body .get("usage") .and_then(Value::as_object) - .ok_or(CoreError::MissingField("usage"))?; + .ok_or(Error::MissingField("usage"))?; let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0); Ok(ChatCompletionsResponse { @@ -181,7 +182,7 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { model: body .get("model") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("model"))? + .ok_or(Error::MissingField("model"))? .to_string(), choices: vec![ChatCompletionsChoice { index: 0, diff --git a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs index 829f2260d3c..8fcc0f36c7d 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; @@ -17,12 +17,12 @@ pub fn non_empty(value: Option<&str>) -> Option<&str> { pub fn resolve_anthropic_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(ANTHROPIC_API_KEY_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY \ environment variable" .to_string(), @@ -52,7 +52,7 @@ impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { api_base: Option<&str>, _model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_anthropic_url(api_base, env_lookup)) } @@ -60,7 +60,7 @@ impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_anthropic_api_key(api_key, env_lookup) } @@ -121,7 +121,7 @@ mod tests { ); assert!(matches!( resolve_anthropic_api_key(None, &|_| None).expect_err("missing key"), - CoreError::Auth(_) + Error::Auth(_) )); } diff --git a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs index 7b958c77ba3..70dad0300f1 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; use crate::messages::types::{ AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock, @@ -28,12 +28,12 @@ pub const AZURE_ANTHROPIC_MESSAGES_CONFIG: AzureAnthropicMessagesConfig = pub fn resolve_azure_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { non_empty(api_key) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_KEY_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Azure API Key - Set `api_key` or the AZURE_API_KEY environment variable" .to_string(), ) @@ -43,12 +43,12 @@ pub fn resolve_azure_api_key( pub fn complete_azure_anthropic_url( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let api_base = non_empty(api_base) .map(str::to_string) .or_else(|| env_lookup(AZURE_API_BASE_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Azure API Base - Set `api_base` or the AZURE_API_BASE environment variable. \ Expected format: https://.services.ai.azure.com/anthropic" .to_string(), @@ -147,7 +147,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { api_base: Option<&str>, _model: &str, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_azure_anthropic_url(api_base, env_lookup) } @@ -155,7 +155,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_azure_api_key(api_key, env_lookup) } @@ -174,7 +174,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { fn transform_request( &self, request: AnthropicMessagesRequest, - ) -> CoreResult { + ) -> Result { let mut request = fold_system_role_messages(request); if let Some(system) = request.system.as_mut() { strip_scope_from_system(system); @@ -190,7 +190,7 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { &self, model: &str, response: AnthropicMessagesResponse, - ) -> CoreResult { + ) -> Result { self.anthropic.transform_response(model, response) } } @@ -268,7 +268,7 @@ mod tests { "https://env.services.ai.azure.com/anthropic/v1/messages" ); let err = complete_azure_anthropic_url(Some(" "), &|_| None).expect_err("missing base"); - assert!(matches!(err, CoreError::Auth(_))); + assert!(matches!(err, Error::Auth(_))); } #[test] @@ -284,7 +284,7 @@ mod tests { ); assert!(matches!( resolve_azure_api_key(None, &|_| None).expect_err("missing key"), - CoreError::Auth(_) + Error::Auth(_) )); } diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs index eabd15677cc..b26a7925e8a 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs @@ -1,6 +1,6 @@ use std::collections::BTreeSet; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::{OcrAuthStrategy, OcrProviderConfig, OcrResponseHandling}; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value, json}; @@ -32,17 +32,17 @@ fn resolve_value( env_name: &str, env_lookup: &dyn Fn(&str) -> Option, missing_message: &str, -) -> CoreResult { +) -> Result { non_empty(explicit) .map(str::to_string) .or_else(|| env_lookup(env_name).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| CoreError::Auth(missing_message.to_string())) + .ok_or_else(|| Error::Auth(missing_message.to_string())) } pub fn resolve_azure_ai_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_key, AZURE_AI_API_KEY_ENV, @@ -54,7 +54,7 @@ pub fn resolve_azure_ai_api_key( pub fn resolve_azure_ai_api_base( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_base, AZURE_AI_API_BASE_ENV, @@ -66,7 +66,7 @@ pub fn resolve_azure_ai_api_base( pub fn complete_azure_ai_url( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let base = resolve_azure_ai_api_base(api_base, env_lookup)?; Ok(format!( "{}/providers/mistral/azure/ocr", @@ -77,7 +77,7 @@ pub fn complete_azure_ai_url( pub fn resolve_document_intelligence_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_key, AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV, @@ -89,7 +89,7 @@ pub fn resolve_document_intelligence_api_key( pub fn resolve_document_intelligence_endpoint( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { resolve_value( api_base, AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV, @@ -127,7 +127,7 @@ fn pages_token_is_valid(token: &str) -> bool { } } -fn normalize_pages_param(pages: &Value) -> CoreResult> { +fn normalize_pages_param(pages: &Value) -> Result, Error> { match pages { Value::String(value) => { let normalized = value @@ -138,7 +138,7 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { if normalized.split(',').all(pages_token_is_valid) { Ok(Some(normalized)) } else { - Err(CoreError::InvalidRequest(format!( + Err(Error::InvalidRequest(format!( "Invalid `pages` string for Azure Document Intelligence: {value:?}. Expected format like '1-3,5,7-9'." ))) } @@ -152,7 +152,7 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { for value in values { let page = value.as_i64().expect("checked is_i64"); if page < 0 { - return Err(CoreError::InvalidRequest( + return Err(Error::InvalidRequest( "`pages` integers must be >= 0 (Mistral 0-based indices)".to_string(), )); } @@ -176,16 +176,16 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { if normalized.split(',').all(pages_token_is_valid) { return Ok(Some(normalized)); } - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Invalid `pages` list for Azure Document Intelligence: {values:?}. Expected tokens like '1' or '3-5'." ))); } - Err(CoreError::InvalidRequest( + Err(Error::InvalidRequest( "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." .to_string(), )) } - _ => Err(CoreError::InvalidRequest( + _ => Err(Error::InvalidRequest( "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." .to_string(), )), @@ -197,7 +197,7 @@ pub fn complete_document_intelligence_url( model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let endpoint = resolve_document_intelligence_endpoint(api_base, env_lookup)?; let mut url = format!( "{}/documentintelligence/documentModels/{}:analyze?api-version={}", @@ -216,20 +216,20 @@ pub fn complete_document_intelligence_url( Ok(url) } -fn document_url_from_mistral_document(document: &Value) -> CoreResult<&str> { - let object = document.as_object().ok_or_else(|| CoreError::InvalidType { +fn document_url_from_mistral_document(document: &Value) -> Result<&str, Error> { + let object = document.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(document), })?; let doc_type = object .get("type") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("document.type"))?; + .ok_or(Error::MissingField("document.type"))?; let field_name = match doc_type { "document_url" => "document_url", "image_url" => "image_url", other => { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Invalid document type: {other}. Must be 'document_url' or 'image_url'" ))); } @@ -238,7 +238,7 @@ fn document_url_from_mistral_document(document: &Value) -> CoreResult<&str> { .get(field_name) .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField(field_name)) + .ok_or(Error::MissingField(field_name)) } fn extract_base64_from_data_uri(data_uri: &str) -> &str { @@ -290,7 +290,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } @@ -298,7 +298,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -308,7 +308,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_azure_ai_url(api_base, env_lookup) } @@ -316,7 +316,7 @@ impl OcrProviderConfig for AzureAiOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_azure_ai_api_key(api_key, env_lookup) } @@ -335,7 +335,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { _model: &str, document: Value, _optional_params: Map, - ) -> CoreResult { + ) -> Result { let document_url = document_url_from_mistral_document(&document)?; let mut data = Map::new(); if document_url.starts_with("data:") { @@ -359,19 +359,19 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; let status = response .get("status") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("status"))?; + .ok_or(Error::MissingField("status"))?; if status != "succeeded" { - return Err(CoreError::InvalidResponse(format!( + return Err(Error::InvalidResponse(format!( "Azure Document Intelligence analysis failed with status: {status}" ))); } @@ -414,7 +414,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_document_intelligence_url(api_base, model, optional_params, env_lookup) } @@ -422,7 +422,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_document_intelligence_api_key(api_key, env_lookup) } diff --git a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs index 5e885734182..bb4f6afe5f9 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs @@ -6,7 +6,7 @@ use crate::audio_transcription::transformation::{ use crate::audio_transcription::types::{ AudioTranscriptionRequestData, AudioTranscriptionResponseData, }; -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; pub use super::aws_base::{aws_auth_config, bedrock_model_id_and_region, resolve_bedrock_region}; use super::constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}; @@ -18,8 +18,8 @@ pub static BEDROCK_AUDIO_TRANSCRIPTION_CONFIG: BedrockAudioTranscriptionConfig = pub struct BedrockAudioTranscriptionConfig; -fn audio_fields(audio: Value) -> CoreResult<(String, String)> { - let object = audio.as_object().ok_or_else(|| CoreError::InvalidType { +fn audio_fields(audio: Value) -> Result<(String, String), Error> { + let object = audio.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&audio), })?; @@ -27,13 +27,13 @@ fn audio_fields(audio: Value) -> CoreResult<(String, String)> { .get("data") .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField("audio.data"))?; + .ok_or(Error::MissingField("audio.data"))?; let format = object .get("format") .and_then(Value::as_str) .filter(|value| matches!(*value, "wav" | "mp3" | "flac" | "ogg")) .ok_or_else(|| { - CoreError::InvalidRequest("audio.format must be wav, mp3, flac, or ogg".to_string()) + Error::InvalidRequest("audio.format must be wav, mp3, flac, or ogg".to_string()) })?; Ok((data.to_string(), format.to_string())) } @@ -55,7 +55,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { _model: &str, audio: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { let (data, format) = audio_fields(audio)?; let mut instruction = "Transcribe the audio. Respond with only the transcript.".to_string(); if let Some(language) = optional_string(&optional_params, "language") { @@ -87,14 +87,14 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { &self, _model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let content = response_json .get("output") .and_then(|value| value.get("message")) .and_then(|value| value.get("content")) .and_then(Value::as_array) .ok_or_else(|| { - CoreError::InvalidResponse("Bedrock response has no output content".to_string()) + Error::InvalidResponse("Bedrock response has no output content".to_string()) })?; let mut text = String::new(); for block in content { @@ -111,7 +111,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (model_id, model_region) = bedrock_model_id_and_region(model); let region = resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup); let endpoint = optional_params @@ -133,7 +133,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (_, model_region) = bedrock_model_id_and_region(model); Ok(AudioTranscriptionAuth::AwsSigV4 { region: resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup), diff --git a/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs b/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs index b11639aa09b..e5e52bfce95 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/aws_base.rs @@ -4,7 +4,7 @@ use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; use crate::caching::in_memory_cache::InMemoryCache; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use aws_credential_types::Credentials; use aws_credential_types::provider::ProvideCredentials; use aws_sigv4::http_request::{ @@ -197,7 +197,7 @@ pub fn classify_auth( pub async fn resolve_credentials( config: AwsAuthConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), -) -> CoreResult { +) -> Result { let resolved = config.clone().with_environment(env_lookup); let flow = classify_auth(config, env_lookup); match flow { @@ -244,9 +244,10 @@ pub async fn resolve_credentials( let provider = aws_config::profile::ProfileFileCredentialsProvider::builder() .profile_name(name) .build(); - provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS profile credentials failed: {error}")) - }) + provider + .provide_credentials() + .await + .map_err(|error| Error::Auth(format!("AWS profile credentials failed: {error}"))) } AwsAuthFlow::AssumeRole { role, session_name } => { if is_already_running_as_role(&role, &resolved).await? { @@ -260,7 +261,7 @@ pub async fn resolve_credentials( .build() .await; let credentials = provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS default credentials failed: {error}")) + Error::Auth(format!("AWS default credentials failed: {error}")) })?; set_cached_credentials( key, @@ -301,7 +302,7 @@ pub async fn resolve_credentials( provider .provide_credentials() .await - .map_err(|error| CoreError::Auth(format!("AWS role credentials failed: {error}"))) + .map_err(|error| Error::Auth(format!("AWS role credentials failed: {error}"))) } AwsAuthFlow::WebIdentity { token, @@ -325,13 +326,13 @@ pub async fn resolve_credentials( .send() .await .map_err(|error| { - CoreError::Auth(format!("AWS web identity credentials failed: {error}")) + Error::Auth(format!("AWS web identity credentials failed: {error}")) })?; let credentials = response.credentials().ok_or_else(|| { - CoreError::Auth("AWS web identity response had no credentials".to_string()) + Error::Auth("AWS web identity response had no credentials".to_string()) })?; let expiration = SystemTime::try_from(*credentials.expiration()).map_err(|error| { - CoreError::Auth(format!("AWS web identity expiration was invalid: {error}")) + Error::Auth(format!("AWS web identity expiration was invalid: {error}")) })?; Ok(Credentials::new( credentials.access_key_id(), @@ -350,9 +351,10 @@ pub async fn resolve_credentials( aws_config::default_provider::credentials::DefaultCredentialsChain::builder() .build() .await; - let credentials = provider.provide_credentials().await.map_err(|error| { - CoreError::Auth(format!("AWS default credentials failed: {error}")) - })?; + let credentials = provider + .provide_credentials() + .await + .map_err(|error| Error::Auth(format!("AWS default credentials failed: {error}")))?; set_cached_credentials( key, credentials.clone(), @@ -363,7 +365,7 @@ pub async fn resolve_credentials( } } -async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> CoreResult { +async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> Result { if role_identity(role).is_none() { return Ok(false); } @@ -437,7 +439,7 @@ pub fn sign_bedrock_post( region: &str, credentials: &Credentials, signing_time: SystemTime, -) -> CoreResult> { +) -> Result, Error> { let identity: Identity = credentials.clone().into(); let params = v4::SigningParams::builder() .identity(&identity) @@ -447,14 +449,14 @@ pub fn sign_bedrock_post( .settings(SigningSettings::default()) .build() .map(SigningParams::from) - .map_err(|error| CoreError::Auth(format!("AWS signing parameters failed: {error}")))?; + .map_err(|error| Error::Auth(format!("AWS signing parameters failed: {error}")))?; let header_refs = headers .iter() .map(|(name, value)| (name.as_str(), value.as_str())); let request = SignableRequest::new("POST", url, header_refs, SignableBody::Bytes(body)) - .map_err(|error| CoreError::Auth(format!("AWS signable request failed: {error}")))?; + .map_err(|error| Error::Auth(format!("AWS signable request failed: {error}")))?; let (instructions, _) = sign(request, ¶ms) - .map_err(|error| CoreError::Auth(format!("AWS request signing failed: {error}")))? + .map_err(|error| Error::Auth(format!("AWS request signing failed: {error}")))? .into_parts(); Ok(instructions .headers() diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs index 4b75dcb8e9d..c86f061b9ca 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::Error; use serde_json::json; fn messages(value: Value) -> Vec { @@ -23,7 +24,7 @@ fn transform(msgs: Value, opts: Value) -> Value { .body } -fn transform_response(body: Value) -> CoreResult { +fn transform_response(body: Value) -> Result { BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response( "anthropic.claude-sonnet-4-5-v1:0", ProviderChatResponseData { body }, @@ -478,25 +479,22 @@ fn declines_a_response_carrying_a_tool_use_block() { "usage": {"inputTokens": 1, "outputTokens": 1} })) .expect_err("tool use block"); - assert_eq!( - err, - CoreError::Unsupported("non-text response content block") - ); + assert_eq!(err, Error::Unsupported("non-text response content block")); } #[test] fn errors_on_a_response_missing_required_fields() { assert_eq!( transform_response(json!("nope")).expect_err("not an object"), - CoreError::InvalidResponse("converse response is not an object".to_string()) + Error::InvalidResponse("converse response is not an object".to_string()) ); assert_eq!( transform_response(json!({"usage": {}})).expect_err("no output"), - CoreError::MissingField("output.message.content") + Error::MissingField("output.message.content") ); assert_eq!( transform_response(json!({"output": {"message": {"content": []}}})).expect_err("no usage"), - CoreError::MissingField("usage") + Error::MissingField("usage") ); } diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs index b107950748e..ef5f44b4a14 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs @@ -11,7 +11,7 @@ use crate::chat_completions::types::{ ChatCompletionsUsage, ChatMessage, ChatMessageContent, ProviderChatRequestData, ProviderChatResponseData, }; -use crate::error::{CoreError, CoreResult}; +use crate::error::Error; use super::super::aws_base::{bedrock_model_id_and_region, resolve_bedrock_region}; use super::super::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE}; @@ -110,7 +110,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { let (model_id, model_region) = bedrock_model_id_and_region(model); let region = resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup); let endpoint = optional_params @@ -137,7 +137,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { // Python reads `api_key` as the Bedrock bearer token and consults the // env only when the caller passed none, so a caller-supplied empty key // falls through to SigV4 without reaching for the environment. An @@ -208,7 +208,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { _model: &str, messages: Vec, optional_params: Map, - ) -> CoreResult { + ) -> Result { Ok(ProviderChatRequestData { body: converse_body(&build_conversation(&messages), &optional_params), }) @@ -218,17 +218,18 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { &self, model: &str, response: ProviderChatResponseData, - ) -> CoreResult { - let body = response.body.as_object().ok_or_else(|| { - CoreError::InvalidResponse("converse response is not an object".into()) - })?; + ) -> Result { + let body = response + .body + .as_object() + .ok_or_else(|| Error::InvalidResponse("converse response is not an object".into()))?; let content = body .get("output") .and_then(|output| output.get("message")) .and_then(|message| message.get("content")) .and_then(Value::as_array) - .ok_or(CoreError::MissingField("output.message.content"))?; + .ok_or(Error::MissingField("output.message.content"))?; // The route declines tool requests, so anything other than a text block // is something this path never asked for. Decline; the host falls back. if content.iter().any(|block| { @@ -236,7 +237,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { .as_object() .is_none_or(|block| block.len() != 1 || !block.contains_key("text")) }) { - return Err(CoreError::Unsupported("non-text response content block")); + return Err(Error::Unsupported("non-text response content block")); } let text: String = content .iter() @@ -246,7 +247,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { let usage = body .get("usage") .and_then(Value::as_object) - .ok_or(CoreError::MissingField("usage"))?; + .ok_or(Error::MissingField("usage"))?; let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0); let computed = usage_from_parts( field("inputTokens"), diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs index dc720cc4244..6a8a38204a9 100644 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value}; @@ -47,7 +47,7 @@ pub fn complete_url(api_base: Option<&str>) -> String { /// Resolve the Mistral API key from the explicit param or the environment. /// -/// Blank/whitespace values are treated as absent. Returns `CoreError::Auth` +/// Blank/whitespace values are treated as absent. Returns `Error::Auth` /// when no usable key is available. /// /// Note: the env fallback only reads the process environment. Secret-manager @@ -56,13 +56,13 @@ pub fn complete_url(api_base: Option<&str>) -> String { pub fn resolve_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) .map(str::to_string) .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty())) - .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string())) + .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) } pub struct MistralOcrConfig; @@ -79,9 +79,9 @@ impl OcrProviderConfig for MistralOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { if !document.is_object() { - return Err(CoreError::InvalidType { + return Err(Error::InvalidType { expected: "object", actual: json_type_name(&document), }); @@ -104,10 +104,10 @@ impl OcrProviderConfig for MistralOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response_object = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; @@ -140,7 +140,7 @@ impl OcrProviderConfig for MistralOcrConfig { _model: &str, _optional_params: &Map, _env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { Ok(complete_url(api_base)) } @@ -148,7 +148,7 @@ impl OcrProviderConfig for MistralOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_api_key(api_key, env_lookup) } } @@ -165,11 +165,11 @@ pub fn transform_ocr_request( model: &str, document: Value, optional_params: Map, -) -> CoreResult { +) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } -pub fn transform_ocr_response(model: &str, response_json: Value) -> CoreResult { +pub fn transform_ocr_response(model: &str, response_json: Value) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -250,7 +250,7 @@ mod tests { assert_eq!( err, - CoreError::InvalidType { + Error::InvalidType { expected: "object", actual: "string", } @@ -307,6 +307,6 @@ mod tests { #[test] fn resolve_api_key_errors_when_absent() { let err = resolve_api_key(None, &|_| None).expect_err("missing key should error"); - assert_eq!(err, CoreError::Auth(MISSING_KEY_MESSAGE.to_string())); + assert_eq!(err, Error::Auth(MISSING_KEY_MESSAGE.to_string())); } } diff --git a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs index b3f6b03b28a..f1985f81b7d 100644 --- a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs +++ b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::realtime::transformation::RealtimeProviderConfig; use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; @@ -72,7 +72,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { &self, event: &RealtimeEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(RealtimeTransformResult::passthrough(event.clone())) } @@ -80,7 +80,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { &self, event: &RealtimeEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(RealtimeTransformResult::passthrough(event.clone())) } } @@ -88,14 +88,14 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { pub fn transform_realtime_request( event: &RealtimeEvent, model: &str, -) -> CoreResult { +) -> Result { OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model) } pub fn transform_realtime_response( event: &RealtimeEvent, model: &str, -) -> CoreResult { +) -> Result { OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model) } diff --git a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs index e15197c468c..be86bb90311 100644 --- a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs +++ b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::responses::types::{ResponsesWsEvent, ResponsesWsTransformResult}; use crate::responses::websocket::{ResponsesWebSocketProviderConfig, enforce_model}; @@ -15,7 +15,7 @@ impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig { &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult { + ) -> Result { Ok(ResponsesWsTransformResult::passthrough(enforce_model( event, model, ))) @@ -25,7 +25,7 @@ impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig { &self, event: &ResponsesWsEvent, _model: &str, - ) -> CoreResult { + ) -> Result { Ok(ResponsesWsTransformResult::passthrough(event.clone())) } } diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs index 6300149c237..ee095447028 100644 --- a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs @@ -1,4 +1,4 @@ -use crate::error::{CoreError, CoreResult, json_type_name}; +use crate::error::{Error, json_type_name}; use crate::ocr::transformation::OcrProviderConfig; use crate::ocr::types::{OcrRequestData, OcrResponseData}; use serde_json::{Map, Value, json}; @@ -43,7 +43,7 @@ pub fn is_deepseek_model(model: &str) -> bool { pub fn resolve_vertex_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { api_key .map(str::trim) .filter(|key| !key.is_empty()) @@ -51,7 +51,7 @@ pub fn resolve_vertex_api_key( .or_else(|| env_lookup(VERTEX_AI_API_KEY_ENV).filter(|key| !key.trim().is_empty())) .or_else(|| env_lookup(VERTEXAI_API_KEY_ENV).filter(|key| !key.trim().is_empty())) .ok_or_else(|| { - CoreError::Auth( + Error::Auth( "Missing Vertex AI access token - pass api_key or provide Authorization via extra_headers" .to_string(), ) @@ -61,12 +61,12 @@ pub fn resolve_vertex_api_key( fn vertex_project( params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { string_param(params, &["vertex_project", "vertex_ai_project"]) .map(str::to_string) .or_else(|| env_lookup(VERTEXAI_PROJECT_ENV).filter(|value| !value.trim().is_empty())) .ok_or_else(|| { - CoreError::InvalidRequest( + Error::InvalidRequest( "Missing vertex_project - Set VERTEXAI_PROJECT environment variable or pass vertex_project parameter" .to_string(), ) @@ -99,7 +99,7 @@ pub fn complete_vertex_mistral_url( model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let project = vertex_project(optional_params, env_lookup)?; let location = vertex_location(optional_params, env_lookup); let base = vertex_mistral_api_base(api_base, &location); @@ -112,7 +112,7 @@ pub fn complete_vertex_deepseek_url( api_base: Option<&str>, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, -) -> CoreResult { +) -> Result { let project = vertex_project(optional_params, env_lookup)?; let location = vertex_location(optional_params, env_lookup); let base = api_base @@ -125,20 +125,20 @@ pub fn complete_vertex_deepseek_url( )) } -fn document_content_item(document: &Value) -> CoreResult { - let object = document.as_object().ok_or_else(|| CoreError::InvalidType { +fn document_content_item(document: &Value) -> Result { + let object = document.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(document), })?; let doc_type = object .get("type") .and_then(Value::as_str) - .ok_or(CoreError::MissingField("document.type"))?; + .ok_or(Error::MissingField("document.type"))?; let url_field = match doc_type { "image_url" => "image_url", "document_url" => "document_url", other => { - return Err(CoreError::InvalidRequest(format!( + return Err(Error::InvalidRequest(format!( "Unsupported document type: {other}. Expected 'image_url' or 'document_url'" ))); } @@ -147,7 +147,7 @@ fn document_content_item(document: &Value) -> CoreResult { .get(url_field) .and_then(Value::as_str) .filter(|value| !value.is_empty()) - .ok_or(CoreError::MissingField(url_field))?; + .ok_or(Error::MissingField(url_field))?; Ok(json!({ "type": "image_url", @@ -163,7 +163,7 @@ fn deepseek_model_name(model: &str) -> String { } } -fn first_choice_content(response: &Value) -> CoreResult { +fn first_choice_content(response: &Value) -> Result { response .get("choices") .and_then(Value::as_array) @@ -176,9 +176,7 @@ fn first_choice_content(response: &Value) -> CoreResult { Value::Object(_) => true, _ => false, }) - .ok_or_else(|| { - CoreError::InvalidResponse("No content in DeepSeek OCR response".to_string()) - }) + .ok_or_else(|| Error::InvalidResponse("No content in DeepSeek OCR response".to_string())) } fn ocr_data_from_content(content: Value, usage: Option, model: &str) -> Value { @@ -219,7 +217,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) } @@ -227,7 +225,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) } @@ -237,7 +235,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_vertex_mistral_url(api_base, model, optional_params, env_lookup) } @@ -245,7 +243,7 @@ impl OcrProviderConfig for VertexAiOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_vertex_api_key(api_key, env_lookup) } @@ -264,7 +262,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { model: &str, document: Value, optional_params: Map, - ) -> CoreResult { + ) -> Result { let mut data = Map::new(); data.insert( "model".to_string(), @@ -289,10 +287,10 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { &self, model: &str, response_json: Value, - ) -> CoreResult { + ) -> Result { let response = response_json .as_object() - .ok_or_else(|| CoreError::InvalidType { + .ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&response_json), })?; @@ -314,7 +312,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { }); } - let object = ocr_data.as_object().ok_or_else(|| CoreError::InvalidType { + let object = ocr_data.as_object().ok_or_else(|| Error::InvalidType { expected: "object", actual: json_type_name(&ocr_data), })?; @@ -346,7 +344,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { _model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { complete_vertex_deepseek_url(api_base, optional_params, env_lookup) } @@ -354,7 +352,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, - ) -> CoreResult { + ) -> Result { resolve_vertex_api_key(api_key, env_lookup) } } diff --git a/litellm-rust/crates/core/src/realtime/transformation.rs b/litellm-rust/crates/core/src/realtime/transformation.rs index 69b88687000..b08084514ef 100644 --- a/litellm-rust/crates/core/src/realtime/transformation.rs +++ b/litellm-rust/crates/core/src/realtime/transformation.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; pub trait RealtimeProviderConfig { @@ -11,12 +11,12 @@ pub trait RealtimeProviderConfig { &self, event: &RealtimeEvent, model: &str, - ) -> CoreResult; + ) -> Result; /// Transform a backend → client event before it is forwarded downstream. fn transform_realtime_response( &self, event: &RealtimeEvent, model: &str, - ) -> CoreResult; + ) -> Result; } diff --git a/litellm-rust/crates/core/src/responses/instrumentation.rs b/litellm-rust/crates/core/src/responses/instrumentation.rs index ec04571da14..b1098f4d386 100644 --- a/litellm-rust/crates/core/src/responses/instrumentation.rs +++ b/litellm-rust/crates/core/src/responses/instrumentation.rs @@ -5,9 +5,9 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde_json::Value; +use crate::Error; use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType}; -use crate::{CoreError, CoreResult}; #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ResponsesWsUsage { @@ -205,7 +205,7 @@ impl ResponsesWsInstrumentation { } } -type LifecycleFuture<'a, T> = Pin> + Send + 'a>>; +type LifecycleFuture<'a, T> = Pin> + Send + 'a>>; impl CallLifecycleHooks<(), (), ()> for ResponsesWsInstrumentation { type PreCallFuture<'a> = LifecycleFuture<'a, ()>; @@ -246,7 +246,7 @@ impl CallLifecycleHooks<(), (), ()> for ResponsesWsInstrumentation { fn async_log_failure_event<'a>( &'a self, _context: &'a CallLifecycleContext, - _error: &'a CoreError, + _error: &'a Error, _timing: &'a CallLifecycleTiming, ) -> Self::FailureFuture<'a> { Box::pin(async move { @@ -342,7 +342,7 @@ mod tests { ), (), &instrumentation, - |_| async { Ok::<(), CoreError>(()) }, + |_| async { Ok::<(), Error>(()) }, ) .await; diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/core/src/responses/websocket.rs index 92dc19627a0..5d037e9cf1b 100644 --- a/litellm-rust/crates/core/src/responses/websocket.rs +++ b/litellm-rust/crates/core/src/responses/websocket.rs @@ -1,4 +1,4 @@ -use crate::CoreResult; +use crate::Error; use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}; use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult}; @@ -19,13 +19,13 @@ pub trait ResponsesWebSocketProviderConfig: Sync { &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult; + ) -> Result; fn transform_ws_response( &self, event: &ResponsesWsEvent, model: &str, - ) -> CoreResult; + ) -> Result; } pub fn complete_websocket_url( diff --git a/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs b/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs index 656ba033b62..8a8a5ea263a 100644 --- a/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs +++ b/litellm-rust/crates/core/tests/workspace_crate_allowlist.rs @@ -1,7 +1,8 @@ -//! Enforcement: the litellm-rust workspace has exactly three crates. +//! Enforcement: the litellm-rust workspace has exactly four crates. //! -//! `core` (pure translation), `ai-gateway` (routes + all network I/O), and -//! `python-bridge` (the PyO3 cdylib). Adding or removing a crate must be a +//! `core` (the Rust SDK), `ai-gateway` (the HTTP/WebSocket host), +//! `python-interop` (domain-neutral PyO3 primitives), and `python-bridge` (the +//! PyO3 cdylib). Adding or removing a crate must be a //! deliberate act: this test fails until the allowlist here is updated, forcing //! whoever changes the crate set to justify the new crate per the rule that a //! crate is a layer needing independent compilation / its own deps / a separate @@ -16,10 +17,15 @@ use std::path::{Path, PathBuf}; /// The one true crate set. Update BOTH this and `litellm-rust/AGENTS.md` when the /// workspace legitimately gains or loses a crate. -const EXPECTED_MEMBERS: &[&str] = &["crates/core", "crates/ai-gateway", "crates/python-bridge"]; +const EXPECTED_MEMBERS: &[&str] = &[ + "crates/core", + "crates/ai-gateway", + "crates/python-interop", + "crates/python-bridge", +]; /// The crate subdirectory names that must exist under `crates/`. -const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-bridge"]; +const EXPECTED_CRATE_DIRS: &[&str] = &["core", "ai-gateway", "python-interop", "python-bridge"]; const MISMATCH: &str = "litellm-rust crate set changed — update this allowlist AND litellm-rust/AGENTS.md, and justify the crate per the rule (crate = layer needing independent compilation / its own deps / a separate artifact)."; diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index ad3cddfa5fd..42282ca4da4 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -1,3 +1,3 @@ -litellm-python-bridge is the PyO3 cdylib that exposes Rust to the litellm Python SDK — a thin adapter (Python objects → Rust calls → Python results) over the litellm-core route entrypoints (e.g. `litellm_core::messages::messages`). +litellm-python-bridge is the PyO3 cdylib that exposes LiteLLM Rust APIs to the Python SDK. Keep API registration, domain dependency wiring, request assembly, and Python exception mapping here. Put domain-neutral Python/Serde conversion and GIL primitives in litellm-python-interop. Keep it thin: no business logic, no transforms, no I/O orchestration — just marshal in/out and call the core entrypoint. diff --git a/litellm-rust/crates/python-bridge/CLAUDE.md b/litellm-rust/crates/python-bridge/CLAUDE.md index 3ce8b8c639a..d25ae5a8130 100644 --- a/litellm-rust/crates/python-bridge/CLAUDE.md +++ b/litellm-rust/crates/python-bridge/CLAUDE.md @@ -5,8 +5,9 @@ Rules for `litellm-rust/crates/python-bridge`. ## Responsibility `python-bridge` is the PyO3 boundary between Python LiteLLM and Rust transforms. -Keep this crate thin. It adapts Python objects to Rust payloads and returns -Python-compatible dictionaries. +Keep this crate thin. It exposes LiteLLM Rust APIs, assembles domain requests, +maps domain errors to Python exceptions, and delegates generic conversion and +GIL handling to `litellm-python-interop`. ## Bridge Shape diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index d461a483ae0..498003de149 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -13,14 +13,14 @@ crate-type = ["cdylib"] default = ["abi3"] abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] +panic-test = [] [dependencies] litellm-core = { workspace = true, features = ["bedrock-auth"] } litellm-ai-gateway = { workspace = true, default-features = false } +litellm-python-interop.workspace = true pyo3.workspace = true pyo3-async-runtimes.workspace = true -pythonize.workspace = true -serde.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/python-bridge/benches/serialization.rs b/litellm-rust/crates/python-bridge/benches/serialization.rs index 8a90cf667d0..0b9436d0cb7 100644 --- a/litellm-rust/crates/python-bridge/benches/serialization.rs +++ b/litellm-rust/crates/python-bridge/benches/serialization.rs @@ -2,6 +2,7 @@ use std::hint::black_box; use std::time::Duration; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use litellm_python_interop::{from_py, to_py}; use pyo3::prelude::*; use pyo3::types::PyDict; use serde_json::{Value, json}; @@ -25,7 +26,7 @@ fn former_json_roundtrip_from_py(py: Python<'_>, value: &Bound<'_, PyAny>) -> Va } fn pythonize_from_py(value: &Bound<'_, PyAny>) -> Value { - pythonize::depythonize(value).expect("payload should depythonize") + from_py(value).expect("payload should depythonize") } fn former_json_roundtrip_to_py(py: Python<'_>, value: &Value) -> Py { @@ -37,12 +38,10 @@ fn former_json_roundtrip_to_py(py: Python<'_>, value: &Value) -> Py { } fn pythonize_to_py(py: Python<'_>, value: &Value) -> Py { - pythonize::pythonize(py, value) - .expect("response should pythonize") - .unbind() + to_py(py, value).expect("response should pythonize") } -fn serialization(c: &mut Criterion) { +fn bridge_serialization(c: &mut Criterion) { Python::initialize(); Python::attach(|py| { for &(label, payload_bytes) in PAYLOAD_SIZES { @@ -98,6 +97,6 @@ criterion_group! { .sample_size(20) .warm_up_time(Duration::from_secs(1)) .measurement_time(Duration::from_secs(4)); - targets = serialization + targets = bridge_serialization } criterion_main!(benches); diff --git a/litellm-rust/crates/python-bridge/src/gil.rs b/litellm-rust/crates/python-bridge/src/gil.rs deleted file mode 100644 index e887c8ec1e3..00000000000 --- a/litellm-rust/crates/python-bridge/src/gil.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! GIL accounting. -//! -//! A single chokepoint for releasing the GIL around blocking work. Every -//! blocking call in the bridge goes through [`release_gil`] instead of calling -//! `Python::detach` directly, so the release count stays accurate and we -//! have one place to extend later (timing histograms, per-call labels, etc.). - -use std::sync::atomic::{AtomicU64, Ordering}; - -use pyo3::prelude::*; - -/// Number of times the bridge has released the GIL since process start. -static GIL_RELEASES: AtomicU64 = AtomicU64::new(0); - -/// Release the GIL around `f`, recording the release. -/// -/// `f` must not touch any Python state — that is what makes releasing the GIL -/// safe. Returning the value back to Python re-acquires the GIL at the call -/// site, after `f` has finished. -pub fn release_gil(py: Python<'_>, f: F) -> T -where - F: FnOnce() -> T + Send, - T: Send, -{ - GIL_RELEASES.fetch_add(1, Ordering::Relaxed); - py.detach(f) -} - -/// Total GIL releases performed by the bridge so far. -pub fn release_count() -> u64 { - GIL_RELEASES.load(Ordering::Relaxed) -} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f9e75f45f75..68aa9436b15 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -10,19 +10,15 @@ use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompleti use litellm_core::chat_completions::{ chat_completions as run_chat_completions, chat_completions_decline_reason, }; -use litellm_core::error::CoreError; +use litellm_core::error::Error; use litellm_core::messages::messages as run_messages; use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; +use litellm_python_interop::{from_py, release_count, release_gil, to_py}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyAny, PyDict}; use serde_json::{Map, Value}; -mod gil; -mod marshal; - -use marshal::{from_py, to_py}; - pyo3::create_exception!( _native, RustBridgeDeclined, @@ -58,13 +54,13 @@ fn chat_completions_response_to_py( to_py(py, &response) } -fn core_error_to_pyerr(err: CoreError) -> PyErr { +fn core_error_to_pyerr(err: Error) -> PyErr { match err { - CoreError::Auth(message) => PyValueError::new_err(message), - CoreError::InvalidProvider(_) - | CoreError::InvalidRequest(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) => PyValueError::new_err(err.to_string()), + Error::Auth(message) => PyValueError::new_err(message), + Error::InvalidProvider(_) + | Error::InvalidRequest(_) + | Error::InvalidType { .. } + | Error::MissingField(_) => PyValueError::new_err(err.to_string()), other => PyRuntimeError::new_err(other.to_string()), } } @@ -75,22 +71,22 @@ fn core_error_to_pyerr(err: CoreError) -> PyErr { /// Everything raised before the request goes out is safe for the host to retry /// on its own path; anything after it is not, because the provider has already /// done the work and billed for it. -fn chat_completions_error_to_pyerr(err: CoreError) -> PyErr { +fn chat_completions_error_to_pyerr(err: Error) -> PyErr { match err { - CoreError::Unsupported(_) - | CoreError::Auth(_) - | CoreError::InvalidProvider(_) - | CoreError::InvalidRequest(_) - | CoreError::InvalidType { .. } - | CoreError::MissingField(_) - | CoreError::Routing(_) + Error::Unsupported(_) + | Error::Auth(_) + | Error::InvalidProvider(_) + | Error::InvalidRequest(_) + | Error::InvalidType { .. } + | Error::MissingField(_) + | Error::Routing(_) // Nothing reached the provider, so serving it on Python cannot double // bill and is the only way the caller gets an answer at all. - | CoreError::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), - CoreError::Http { status, body } => { + | Error::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), + Error::Http { status, body } => { RustUpstreamError::new_err((status, format!("{status}: {body}"))) } - CoreError::Network(message) | CoreError::InvalidResponse(message) => { + Error::Network(message) | Error::InvalidResponse(message) => { RustUpstreamError::new_err((0u16, message)) } } @@ -230,7 +226,7 @@ fn ocr( timeout_seconds, )?; - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_ocr(OcrRequest { model: &model, document, @@ -318,7 +314,7 @@ fn transcription( }; let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; let timeout = optional_timeout(timeout_seconds); - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_audio_transcription( AudioTranscriptionRequest { model: &model, @@ -419,7 +415,7 @@ fn messages( let (body, extra_headers, timeout) = marshal_messages_inputs(py, body, extra_headers, timeout_seconds)?; - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_messages(MessagesRequest { model: &model, body, @@ -546,7 +542,7 @@ fn chat_completions( timeout_seconds, )?; - let result = gil::release_gil(py, || { + let result = release_gil(py, || { pyo3_async_runtimes::tokio::get_runtime().block_on(run_chat_completions( ChatCompletionsRequest { model: &model, @@ -610,10 +606,16 @@ fn achat_completions( #[pyfunction] fn gil_stats(py: Python<'_>) -> PyResult> { let stats = PyDict::new(py); - stats.set_item("releases", gil::release_count())?; + stats.set_item("releases", release_count())?; Ok(stats.into_any().unbind()) } +#[cfg(feature = "panic-test")] +#[pyfunction] +fn _panic_for_test() { + panic!("intentional PyO3 panic smoke test"); +} + #[pymodule] fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { let py = module.py(); @@ -630,5 +632,7 @@ fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(achat_completions, module)?)?; module.add_class::()?; module.add_function(wrap_pyfunction!(gil_stats, module)?)?; + #[cfg(feature = "panic-test")] + module.add_function(wrap_pyfunction!(_panic_for_test, module)?)?; Ok(()) } diff --git a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs index 6a6ede22e85..d397d20b9fd 100644 --- a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs +++ b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs @@ -1,7 +1,7 @@ use std::fs; use std::path::{Path, PathBuf}; -const DISALLOWED_OUTSIDE_MARSHAL: &[&str] = &[ +const DISALLOWED_OUTSIDE_INTEROP: &[&str] = &[ "py.import(\"json\")", "pythonize::", "serde_json::to_string", @@ -33,18 +33,15 @@ fn rust_sources(directory: &Path) -> Vec { } #[test] -fn serialization_is_centralized_in_marshal_module() { +fn serialization_uses_the_interop_boundary() { let root = source_root(); for path in rust_sources(&root) { - if path == root.join("marshal.rs") { - continue; - } let source = fs::read_to_string(&path).expect("bridge source should be readable"); - for disallowed in DISALLOWED_OUTSIDE_MARSHAL { + for disallowed in DISALLOWED_OUTSIDE_INTEROP { assert!( !source.contains(disallowed), - "{} bypasses the typed marshal module with `{disallowed}`", + "{} bypasses litellm-python-interop with `{disallowed}`", path.display() ); } diff --git a/litellm-rust/crates/python-interop/AGENTS.md b/litellm-rust/crates/python-interop/AGENTS.md new file mode 100644 index 00000000000..d1d61e5dfa0 --- /dev/null +++ b/litellm-rust/crates/python-interop/AGENTS.md @@ -0,0 +1 @@ +litellm-python-interop is the domain-neutral PyO3 foundation. Keep generic Python/Serde conversion and interpreter primitives here. Do not add LiteLLM domain crates, route types, API registration, or cdylib build features. diff --git a/litellm-rust/crates/python-interop/Cargo.toml b/litellm-rust/crates/python-interop/Cargo.toml new file mode 100644 index 00000000000..9da6af6e2e2 --- /dev/null +++ b/litellm-rust/crates/python-interop/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "litellm-python-interop" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +pyo3.workspace = true +pythonize.workspace = true +serde.workspace = true + +[dev-dependencies] +rstest.workspace = true +serde_json.workspace = true diff --git a/litellm-rust/crates/python-interop/src/gil.rs b/litellm-rust/crates/python-interop/src/gil.rs new file mode 100644 index 00000000000..04b966a6002 --- /dev/null +++ b/litellm-rust/crates/python-interop/src/gil.rs @@ -0,0 +1,21 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use pyo3::prelude::*; + +static GIL_RELEASES: AtomicU64 = AtomicU64::new(0); + +/// Runs work detached from the interpreter and records the release. +/// +/// `f` must not access Python state while the interpreter is detached. +pub fn release_gil(py: Python<'_>, f: F) -> T +where + F: FnOnce() -> T + Send, + T: Send, +{ + GIL_RELEASES.fetch_add(1, Ordering::Relaxed); + py.detach(f) +} + +pub fn release_count() -> u64 { + GIL_RELEASES.load(Ordering::Relaxed) +} diff --git a/litellm-rust/crates/python-interop/src/lib.rs b/litellm-rust/crates/python-interop/src/lib.rs new file mode 100644 index 00000000000..df2bd260fdb --- /dev/null +++ b/litellm-rust/crates/python-interop/src/lib.rs @@ -0,0 +1,5 @@ +mod gil; +mod marshal; + +pub use gil::{release_count, release_gil}; +pub use marshal::{from_py, to_py}; diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-interop/src/marshal.rs similarity index 100% rename from litellm-rust/crates/python-bridge/src/marshal.rs rename to litellm-rust/crates/python-interop/src/marshal.rs diff --git a/litellm-rust/crates/python-interop/tests/interop.rs b/litellm-rust/crates/python-interop/tests/interop.rs new file mode 100644 index 00000000000..9c456dcb938 --- /dev/null +++ b/litellm-rust/crates/python-interop/tests/interop.rs @@ -0,0 +1,44 @@ +use pyo3::Python; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; + +use litellm_python_interop::{from_py, release_count, release_gil, to_py}; + +struct InitializedPython; + +impl InitializedPython { + fn attach(&self, f: F) -> R + where + F: for<'py> FnOnce(Python<'py>) -> R, + { + Python::attach(f) + } +} + +#[fixture] +#[once] +fn initialized_python() -> InitializedPython { + Python::initialize(); + InitializedPython +} + +#[rstest] +fn serde_values_round_trip_through_python(#[from(initialized_python)] python: &InitializedPython) { + python.attach(|py| { + let expected = json!({"model": "test", "items": [1, true, null]}); + let python_value = to_py(py, &expected).expect("value should convert to Python"); + let actual: Value = + from_py(python_value.bind(py)).expect("Python value should convert to serde"); + + assert_eq!(actual, expected); + }); +} + +#[rstest] +fn release_gil_runs_work_and_records_it(#[from(initialized_python)] python: &InitializedPython) { + let before = release_count(); + let result = python.attach(|py| release_gil(py, || 42)); + + assert_eq!(result, 42); + assert_eq!(release_count(), before + 1); +} diff --git a/litellm/__init__.py b/litellm/__init__.py index 4eeececdb7e..61794dabddc 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -29,7 +29,7 @@ def _dev_env_hot_reload_enabled() -> bool: if os.getenv("LITELLM_MODE", "DEV") == "DEV": _dotenv.load_dotenv(override=_dev_env_hot_reload_enabled()) -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import ( Any, Callable, @@ -490,6 +490,7 @@ public_mcp_hub_strict_whitelist: bool = True public_model_groups: Optional[List[str]] = None public_agent_groups: Optional[List[str]] = None agent_search_embedding_model: Optional[str] = None +mcp_tool_search: Optional[Mapping[str, object]] = None # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } # Old format: { "displayName": "url" } (for backward compatibility) diff --git a/litellm/_logging.py b/litellm/_logging.py index 9435562f890..14cda772234 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -6,6 +6,7 @@ import sys from datetime import datetime from logging import Formatter from typing import Any, Final, TextIO +from urllib.parse import unquote import litellm from litellm.constants import ( @@ -146,6 +147,72 @@ class SecretRedactionFilter(logging.Filter): _secret_filter: Final = SecretRedactionFilter() +_MAX_SCRUBBED_ACCESS_ARG: Final = 512 + +_REDACTION_PLACEHOLDER: Final = "REDACTED" + + +def _hides_a_credential(value: str) -> bool: + """Whether *value* only looks clean until it is percent-decoded.""" + decoded: Final = unquote(value) + return _redact_string(decoded) != decoded + + +def _drop_encoded_credential(scrubbed: str) -> str: + """Drop the part of a request target that only decoding shows to be a secret. + + The request parser decodes query names and values, so `?k%65y=sk%2D...` is a + working credential that the patterns, which match literal text, do not see. + The decoded text is never logged back: it can carry a newline, and forging + log lines is not a trade worth making for a readable request target. + """ + path, separator, _query = scrubbed.partition("?") + if _hides_a_credential(path): + return _REDACTION_PLACEHOLDER + if separator and _hides_a_credential(scrubbed): + return f"{path}?{_REDACTION_PLACEHOLDER}" + return scrubbed + + +def _scrub_access_arg(value: str) -> str: + """Redact one access-log positional arg, bounding the scanned length. + + The request target is the only input to the secret regex an unauthenticated + caller controls end to end, so it is cut back to a whole query parameter + before it is scanned; a half-parameter would be too short to match its + pattern and would then be logged raw. + """ + if len(value) <= _MAX_SCRUBBED_ACCESS_ARG: + return _drop_encoded_credential(_redact_string(value)) + head: Final = value[:_MAX_SCRUBBED_ACCESS_ARG] + kept: Final = head[: max(head.rfind("?"), head.rfind("&"))] if "?" in head else head + scrubbed: Final = _drop_encoded_credential(_redact_string(kept)) + return f"{scrubbed}... ({len(value) - len(kept)} more chars truncated) ..." + + +class AccessLogRedactionFilter(logging.Filter): + """Scrubs known secret/credential patterns from HTTP access-log records. + + uvicorn's AccessFormatter unpacks ``record.args`` as a five-element tuple at + emit time, so SecretRedactionFilter cannot be reused here: it collapses the + record into ``record.msg`` and clears the args, and the formatter then raises. + """ + + def filter(self, record: logging.LogRecord) -> bool: + if not _ENABLE_SECRET_REDACTION: + return True + if isinstance(record.args, tuple) and record.args: + record.args = tuple( # rebind-ok: a Filter scrubs records in place + _scrub_access_arg(arg) if isinstance(arg, str) else arg for arg in record.args + ) + return True + # No positional args means everything is in msg, where collapsing is correct. + return _secret_filter.filter(record) + + +_access_log_filter: Final = AccessLogRedactionFilter() + + def _get_max_string_length_stdout_log() -> int: """Read the limit per record so a value loaded later via proxy config environment_variables is honored.""" @@ -553,6 +620,14 @@ _REDACTED_THIRD_PARTY_LOGGERS: Final[tuple[str, ...]] = ( "uvicorn.error", ) +# Access loggers, which emit the full request target, so a credential passed as a +# query parameter (e.g. `/key/info?key=`) lands on stdout verbatim. uvicorn.access +# covers uvicorn.run, --run_gunicorn (its worker_class is UvicornWorker, so the +# access line is still uvicorn's) and an embedding host app. --run_hypercorn and +# --run_granian log through their own loggers in their own record shapes, and +# both ship with access logging off. +_REDACTED_ACCESS_LOGGERS: Final[tuple[str, ...]] = ("uvicorn.access",) + def _redact_third_party_loggers() -> None: """Extend secret redaction to records litellm does not emit directly. @@ -575,6 +650,8 @@ def _redact_third_party_loggers() -> None: """ for name in _REDACTED_THIRD_PARTY_LOGGERS: logging.getLogger(name).addFilter(_secret_filter) + for name in _REDACTED_ACCESS_LOGGERS: + logging.getLogger(name).addFilter(_access_log_filter) # Call the suppression function diff --git a/litellm/constants.py b/litellm/constants.py index 1c1939bd350..c7b74e176db 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1742,6 +1742,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "anthropic_prompt_caching_ttl", "max_ui_session_budget", "budget_rollover", + "mcp_tool_search", ] SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"] DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) diff --git a/litellm/containers/main.py b/litellm/containers/main.py index 97ca11872c1..90d59af009f 100644 --- a/litellm/containers/main.py +++ b/litellm/containers/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars import json -from collections.abc import Coroutine, Mapping +from collections.abc import Callable, Coroutine, Mapping from functools import partial from typing import Final, Literal, overload @@ -47,6 +47,13 @@ __all__ = [ ##### Container Create ####################### +async def _encode_created_container_id( + pending: Coroutine[object, object, ContainerObject], + encode: Callable[[ContainerObject], ContainerObject], +) -> ContainerObject: + return encode(await pending) + + @client async def acreate_container( name: str, @@ -256,16 +263,16 @@ def create_container( _is_async=_is_async, ) - # Encode container_id with provider/model metadata for routing + encode: Final = partial( + ContainerRequestUtils.encode_container_id_in_response, + custom_llm_provider=custom_llm_provider, + litellm_metadata=kwargs.get("litellm_metadata"), + extra_body=extra_body, + ) if isinstance(container_obj, ContainerObject): - container_obj = ContainerRequestUtils.encode_container_id_in_response( - response_obj=container_obj, - custom_llm_provider=custom_llm_provider, - litellm_metadata=kwargs.get("litellm_metadata"), - extra_body=extra_body, - ) + return encode(container_obj) - return container_obj + return _encode_created_container_id(pending=container_obj, encode=encode) except Exception as e: raise litellm.exception_type( diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 545b0f40018..3519240dda9 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -755,6 +755,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): def record_gateway_injection( request_kwargs: Mapping[str, object], added: int, + injected_for_every_deployment: bool = False, ) -> None: """Name the deployment whose payload the gateway, not the client, put breakpoints on. @@ -771,7 +772,16 @@ class AnthropicCacheControlHook(CustomPromptManagement): A pass that runs before a deployment is chosen, which is what the proxy does for prompt templates, injects into the payload every leg goes on to send, so it marks - the request for all of them rather than for one. + the request for all of them rather than for one. Such a pass says so with + ``injected_for_every_deployment`` instead of relying on the shape of + ``request_kwargs``: the router's prompt-management factory stamps a provisional + deployment's ``model_info`` into kwargs before the prompt pass runs, and billing + the request through any other deployment would silently drop the credit. An + every-deployment mark, once written, also never narrows: a later per-leg stamp + (the Bedrock converse tool_config one included) describes one leg of a payload + every leg sends, so narrowing to it would uncredit whichever leg gets billed + after a failover. Both losses are fail-closed under-crediting, which is why the + guard only protects the sentinel and per-leg marks still overwrite each other. Only what this pass actually placed counts. A ``tool_config`` point is placed by the Bedrock converse transform, and only when the request carries tools, so the @@ -801,13 +811,19 @@ class AnthropicCacheControlHook(CustomPromptManagement): ), None, ) - if bucket is not None: - model_info: Final = request_kwargs.get("model_info") - bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = ( - model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT) - if isinstance(model_info, dict) - else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT - ) + if bucket is None: + return + if bucket.get(GATEWAY_INJECTED_CACHE_METADATA_KEY) == GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT: + return + if injected_for_every_deployment: + bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT + return + model_info: Final = request_kwargs.get("model_info") + bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = ( + model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT) + if isinstance(model_info, dict) + else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT + ) @staticmethod def maybe_inject_cache_control( diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py new file mode 100644 index 00000000000..ed47533e700 --- /dev/null +++ b/litellm/integrations/otel/langfuse_logger.py @@ -0,0 +1,60 @@ +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_logger +from litellm.integrations.otel.logger import OpenTelemetryV2 +from litellm.integrations.otel.mappers.langfuse import LANGFUSE_OBSERVATION_INPUT, LANGFUSE_OBSERVATION_OUTPUT +from litellm.integrations.otel.model.request_io import request_input, response_output, stream_output +from litellm.integrations.otel.plumbing.context import request_root_span + +if TYPE_CHECKING: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.utils import ModelResponseStream + + +class LangfuseOpenTelemetryV2(OpenTelemetryV2): + """Stamps the request's input and output on the root observation while it is still recording. + + Langfuse shows a trace's input and output from its root observation. The proxy's root span ends + when the response is sent, before the success callback runs, so both stamps come from the + post-call hooks in the request task: the request as it stands after the pre-call chain and the + response as it is returned, for the call types whose response renders as a message. + """ + + async def async_post_call_success_hook( + self, + data: Mapping[str, object], + user_api_key_dict: "UserAPIKeyAuth", + response: object, + ) -> None: + self._stamp_root_io(data, lambda: response_output(response)) + + async def async_post_call_streaming_iterator_hook( + self, + user_api_key_dict: "UserAPIKeyAuth", + response: "AsyncIterator[ModelResponseStream]", + request_data: Mapping[str, object], + ) -> "AsyncGenerator[ModelResponseStream, None]": + relayed: Final[list[ModelResponseStream]] = [] # mutable-ok: relayed as they arrive, assembled at end of stream + async for chunk in response: + relayed.append(chunk) + yield chunk + self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data)) + + def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: + root: Final = request_root_span() + if root is None or not root.is_recording(): + return + try: + output: Final = render_output() + if output is None: + return + root.set_attribute(LANGFUSE_OBSERVATION_OUTPUT, output) + rendered_input: Final = request_input(data) + except Exception: # noqa: BLE001 # telemetry must never fail the request it describes + verbose_logger.debug( + "otel v2 langfuse: could not render the root observation input or output", exc_info=True + ) + return + if rendered_input is not None: + root.set_attribute(LANGFUSE_OBSERVATION_INPUT, rendered_input) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index d2a32ef73b6..a550dca6cc8 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -4,6 +4,7 @@ from collections import OrderedDict from collections.abc import Callable, Iterator, Mapping, Sequence from contextlib import contextmanager from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast from opentelemetry.context import Context, attach, get_current @@ -909,3 +910,29 @@ def phase_span(name: str) -> "Iterator[Span | None]": return with logger.start_phase_span(name) as span: yield span + + +def build_otel_v2_logger( + config: OpenTelemetryV2Config, + callback_name: str | None = None, + tracer_provider: TracerProvider | None = None, + logger_provider: LoggerProvider | None = None, + meter_provider: "MeterProvider | None" = None, + settings: Mapping[str, object] = MappingProxyType({}), +) -> OpenTelemetryV2: + return _logger_class(config)( + config=config, + callback_name=callback_name, + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + **settings, + ) + + +def _logger_class(config: OpenTelemetryV2Config) -> type[OpenTelemetryV2]: + if "langfuse" not in config.mapper_names or not config.capture_span_content: + return OpenTelemetryV2 + from litellm.integrations.otel.langfuse_logger import LangfuseOpenTelemetryV2 + + return LangfuseOpenTelemetryV2 diff --git a/litellm/integrations/otel/mappers/langfuse.py b/litellm/integrations/otel/mappers/langfuse.py index 6d4f1b4fd0a..01063d85355 100644 --- a/litellm/integrations/otel/mappers/langfuse.py +++ b/litellm/integrations/otel/mappers/langfuse.py @@ -11,6 +11,7 @@ the JSON-serialized payloads. ``_llm_call`` just applies both tables. import json from collections.abc import Callable +from typing import Final from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData from litellm.integrations.otel.mappers.utils import ( @@ -25,6 +26,9 @@ from litellm.integrations.otel.model.payloads import ( LLMUsage, ) +LANGFUSE_OBSERVATION_INPUT: Final = "langfuse.observation.input" +LANGFUSE_OBSERVATION_OUTPUT: Final = "langfuse.observation.output" + class LangfuseMapper: _LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = { @@ -56,8 +60,8 @@ class LangfuseMapper: "langfuse.observation.model.parameters": lambda d: json_if( collect(LangfuseMapper._MODEL_PARAMS, d.request_params) ), - "langfuse.observation.input": lambda d: serialize_messages(d.messages_in), - "langfuse.observation.output": lambda d: serialize_messages(output_messages(d)), + LANGFUSE_OBSERVATION_INPUT: lambda d: serialize_messages(d.messages_in), + LANGFUSE_OBSERVATION_OUTPUT: lambda d: serialize_messages(output_messages(d)), "langfuse.observation.usage_details": lambda d: json_if(collect(LangfuseMapper._USAGE_FIELDS, d.usage)), "langfuse.observation.cost_details": lambda d: ( json.dumps({"total": d.response_cost}) if d.response_cost is not None else None diff --git a/litellm/integrations/otel/model/request_io.py b/litellm/integrations/otel/model/request_io.py new file mode 100644 index 00000000000..4e80fb91993 --- /dev/null +++ b/litellm/integrations/otel/model/request_io.py @@ -0,0 +1,90 @@ +from collections.abc import Mapping, Sequence +from typing import Final, Literal + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError +from typing_extensions import ReadOnly, TypedDict + +import litellm +from litellm.integrations.otel.mappers.utils import json_or_none +from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_stream, is_raw_sse_stream +from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse +from litellm.types.utils import ModelResponse, ModelResponseStream + +_SYSTEM_KEYS: Final = ("system", "instructions") +_TURNS: Final = TypeAdapter(tuple[object, ...]) +_MESSAGES: Final = TypeAdapter(list[object] | None) + + +class _Turn(TypedDict): + role: ReadOnly[str] + content: ReadOnly[object] + + +class _AnthropicMessage(BaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["message"] = Field(exclude=True) + role: str = "assistant" + content: object = None + + +def request_input(data: Mapping[str, object]) -> str | None: + turns: Final = data.get("messages", data.get("input")) + if turns is None: + return None + return json_or_none((*_system_turns(data), *_user_turns(turns))) + + +def _system_turns(data: Mapping[str, object]) -> tuple[_Turn, ...]: + return tuple(_Turn(role="system", content=data[key]) for key in _SYSTEM_KEYS if data.get(key) is not None) + + +def _user_turns(turns: object) -> tuple[object, ...]: + if isinstance(turns, str): + return (_Turn(role="user", content=turns),) + try: + return _TURNS.validate_python(turns) + except ValidationError: + return (_Turn(role="user", content=turns),) + + +def response_output(response: object) -> str | None: + match response: + case ModelResponse(): + return json_or_none(tuple(choice.message.model_dump(exclude_none=True) for choice in response.choices)) + case ResponsesAPIResponse(): + return json_or_none(response.model_dump(exclude_none=True).get("output")) + case _: + return _anthropic_message_output(response) + + +def _anthropic_message_output(message: object) -> str | None: + try: + parsed: Final = _AnthropicMessage.model_validate(message) + except ValidationError: + return None + return json_or_none((parsed.model_dump(),)) + + +def stream_output(chunks: Sequence[object], data: Mapping[str, object]) -> str | None: + if not chunks: + return None + if is_raw_sse_stream(chunks): + return response_output(assemble_anthropic_sse_stream(chunks)) + if all(isinstance(chunk, ModelResponseStream) for chunk in chunks): + return response_output(_assembled_chat_stream(chunks, data)) + return response_output(_completed_response(chunks)) + + +def _assembled_chat_stream(chunks: Sequence[object], data: Mapping[str, object]) -> object: + try: + return litellm.stream_chunk_builder( # pyright: ignore[reportUnknownMemberType] # upstream types chunks as a bare list + chunks=list(chunks), # mutable-ok: stream_chunk_builder takes a list + messages=_MESSAGES.validate_python(data.get("messages")), + ) + except (litellm.APIError, ValidationError): + return None + + +def _completed_response(chunks: Sequence[object]) -> ResponsesAPIResponse | None: + return next((chunk.response for chunk in reversed(chunks) if isinstance(chunk, ResponseCompletedEvent)), None) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9a6fb11f978..f54eeca5178 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -901,6 +901,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_label: str | None = None, prompt_version: int | None = None, request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs + injected_for_every_deployment: bool = False, ) -> tuple[str, list[AllMessageValues], dict]: from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook @@ -933,6 +934,7 @@ class Logging(LiteLLMLoggingBaseClass): AnthropicCacheControlHook.record_gateway_injection( request_kwargs, AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before, + injected_for_every_deployment=injected_for_every_deployment, ) self.messages = messages return model, messages, non_default_params @@ -950,6 +952,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_label: str | None = None, prompt_version: int | None = None, request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs + injected_for_every_deployment: bool = False, ) -> tuple[str, list[AllMessageValues], dict]: from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook @@ -985,6 +988,7 @@ class Logging(LiteLLMLoggingBaseClass): AnthropicCacheControlHook.record_gateway_injection( request_kwargs, AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before, + injected_for_every_deployment=injected_for_every_deployment, ) self.messages = messages return model, messages, non_default_params @@ -4390,13 +4394,15 @@ def _init_custom_logger_compatible_class( from litellm.integrations.otel.model.config import is_otel_v2_enabled if is_otel_v2_enabled(): - from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger + from litellm.integrations.otel.model.config import OpenTelemetryV2Config for callback in _in_memory_loggers: - if type(callback) is OpenTelemetryV2: + if isinstance(callback, OpenTelemetryV2): return callback - otel_logger_v2: Final = OpenTelemetryV2( - **_get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) + otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) + otel_logger_v2: Final = build_otel_v2_logger( + config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings ) _in_memory_loggers.append(otel_logger_v2) _maybe_auto_initialize_arize_phoenix(_in_memory_loggers) @@ -4759,7 +4765,7 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom if not is_otel_v2_enabled(): return None - from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger from litellm.integrations.otel.presets import PRESET_BY_CALLBACK preset_fn: Final = PRESET_BY_CALLBACK.get(callback_name) @@ -4774,7 +4780,7 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None - v2_logger: Final = OpenTelemetryV2(config=config, callback_name=callback_name) + v2_logger: Final = build_otel_v2_logger(config=config, callback_name=callback_name) _in_memory_loggers.append(v2_logger) return v2_logger diff --git a/litellm/litellm_core_utils/secret_redaction.py b/litellm/litellm_core_utils/secret_redaction.py index 5d5bd547d22..e93ab155786 100644 --- a/litellm/litellm_core_utils/secret_redaction.py +++ b/litellm/litellm_core_utils/secret_redaction.py @@ -30,6 +30,11 @@ def _build_secret_patterns() -> "re.Pattern[str]": r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}", # OpenAI / Anthropic sk- prefixed keys rf"sk-[A-Za-z0-9\-_]{{{MINIMUM_CUSTOM_KEY_LENGTH - len('sk-')},}}", + # Credentials passed as URL query params. Terminated by "&" like the key= + # and sig= patterns below, so the rest of the request line survives in an + # access log. Must precede the generic patterns to win at the same position. + r"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))" + r"=[^\s&'\"]+", # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr) r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}", # x-api-key / api-key header values (handles 'key': 'value' dict repr) @@ -45,8 +50,10 @@ def _build_secret_patterns() -> "re.Pattern[str]": # Word boundary prevents O(n^2) backtracking on long word-char runs. r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)" r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", - # Database connection string credentials (scheme://user:pass@host) - r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)", + # Database connection string credentials (scheme://user:pass@host). + # The user half stops at the ":" separator and both halves are length-capped, + # so a long attacker-supplied URL cannot backtrack quadratically. + r"(?<=://)[^\s'\":]{0,4096}:[^\s'\"]{1,4096}(?=@)", # Databricks personal access tokens r"dapi[0-9a-f]{32}", # Module-level provider keys logged as litellm._key= @@ -67,8 +74,10 @@ def _build_secret_patterns() -> "re.Pattern[str]": r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""", # Raw JWTs (without Bearer prefix) r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*", - # Azure SAS tokens in URLs - r"[?&]sig=[A-Za-z0-9%+/=]+", + # Azure SAS tokens in URLs. The delimiter is a lookbehind, like the + # `key=` pattern above, so the `?` or `&` survives and the redacted URL + # stays well formed (this string is often a request line in a log). + r"(?<=[?&])sig=[A-Za-z0-9%+/=]+", # Full JSON service-account blobs (single-line and multi-line) r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', ] diff --git a/litellm/llms/base_llm/files/azure_blob_storage_backend.py b/litellm/llms/base_llm/files/azure_blob_storage_backend.py index e22cf528856..a192226110c 100644 --- a/litellm/llms/base_llm/files/azure_blob_storage_backend.py +++ b/litellm/llms/base_llm/files/azure_blob_storage_backend.py @@ -7,16 +7,36 @@ to reuse all authentication and Azure Storage operations. """ import time +from pathlib import Path from typing import Final from urllib.parse import quote, urlparse from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger +from litellm.proxy.common_utils.path_utils import safe_filename from .storage_backend import BaseFileStorageBackend +def _safe_basename(original_filename: str) -> str: + try: + return safe_filename(original_filename) + except ValueError: + return "file" + + +def _safe_extension(original_filename: str) -> str: + """The extension off a basename, with no path separators or traversal sequences. + + original_filename.split(".")[-1] does not parse path structure, so a filename + like "a.jsonl/../../etc/cron.d/x" would put "../../etc/cron.d/x" straight into + the blob path built below. Path.suffix only ever looks at the last path + component, so routing through safe_filename() first closes that off. + """ + return Path(_safe_basename(original_filename)).suffix.lstrip(".") + + class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): """ Azure Blob Storage backend implementation. @@ -81,16 +101,15 @@ class AzureBlobStorageBackend(BaseFileStorageBackend, AzureBlobStorageLogger): def _generate_file_name(self, original_filename: str, file_naming_strategy: str) -> str: """Generate file name based on naming strategy.""" if file_naming_strategy == "original_filename": - # Use original filename, but sanitize it - return quote(original_filename, safe="") + return quote(_safe_basename(original_filename), safe="") elif file_naming_strategy == "timestamp": # Use timestamp - extension = original_filename.split(".")[-1] if "." in original_filename else "" + extension = _safe_extension(original_filename) timestamp: Final = int(time.time() * 1000) # milliseconds return f"{timestamp}.{extension}" if extension else str(timestamp) else: # default to "uuid" # Use UUID - extension = original_filename.split(".")[-1] if "." in original_filename else "" + extension = _safe_extension(original_filename) file_uuid: Final = str(uuid.uuid4()) return f"{file_uuid}.{extension}" if extension else file_uuid diff --git a/litellm/main.py b/litellm/main.py index 01c106adc7c..0128e4defe5 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -141,7 +141,6 @@ from litellm.utils import ( convert_to_model_response_object, create_pretrained_tokenizer, create_tokenizer, - get_api_key, get_llm_provider, get_model_info, get_non_default_completion_params, diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index ce539258fd2..609d24b0ffb 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -1149,10 +1149,11 @@ class MCPRequestHandler: would miss a real outage wrapped inside it.""" from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler - if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e): + outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(e) + if outage is not None: raise HTTPException( status_code=503, - detail="Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + detail=PrismaDBExceptionHandler.database_unavailable_message(outage), ) from None @staticmethod diff --git a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py index 09a3703e904..35a30127e27 100644 --- a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py +++ b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py @@ -101,18 +101,29 @@ class _ResolvedKey: key: "UserAPIKeyAuth" -_KeyResolutionFailure = Literal["no_active_key", "unavailable", "unresolvable"] +_KeyResolutionFailure = Literal["no_active_key", "unavailable", "faulted", "unresolvable"] """Why a token request yielded no active litellm key, kept distinct so a caller statuses each truthfully instead of blaming the client for a gateway problem: - ``no_active_key``: none was presented, or the presented key is unknown / blocked / expired (the caller's request is at fault) - ``unavailable``: the auth database was transiently unreachable while resolving (retryable) +- ``faulted``: the auth database's query engine reported a fault that retrying will not clear (still a + 503, but the wording must not tell the operator to wait) - ``unresolvable``: the gateway cannot resolve identity right now (no DB connection, or an unexpected error) -- a gateway fault, not the caller's The classification mirrors admission's ``_reload_admitted_key`` so the mint (ingress) and admission (egress) never disagree on the status of the same outage.""" +def _database_failure(exc: Exception) -> Literal["unavailable", "faulted"]: + from litellm.proxy.db.exception_handler import ( # noqa: PLC0415 # inline import avoids a module-load circular import + PrismaDBExceptionHandler, + ) + + fault: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(exc) or exc + return "faulted" if PrismaDBExceptionHandler.is_permanent_database_fault(fault) else "unavailable" + + async def _resolve_active_litellm_key(request: Request) -> "_ResolvedKey | _KeyResolutionFailure": """Resolve the presented litellm key to an active key record, or say precisely why not. @@ -170,7 +181,7 @@ async def _reload_active_key_by_hash(key_hash: str) -> "_ResolvedKey | _KeyResol return "no_active_key" except Exception as exc: # noqa: BLE001 # classify: a DB outage is retryable, anything else is an opaque gateway fault if PrismaDBExceptionHandler.is_database_service_unavailable_error(exc): - return "unavailable" + return _database_failure(exc) verbose_logger.debug( "_reload_active_key_by_hash: unexpected key-resolution error (%s)", type(exc).__name__, @@ -225,8 +236,9 @@ async def load_active_user_by_id(user_id: str) -> "LiteLLM_UserTable | _KeyResol except (ProxyException, HTTPException): return "no_active_key" except Exception as exc: # noqa: BLE001 # a DB outage is retryable; a missing user (get_user_object's wrapped ValueError) or any other resolution failure fails closed as no_active_key, never a 500 - if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(exc): - return "unavailable" + outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(exc) + if outage is not None: + return _database_failure(outage) verbose_logger.debug("_reload_active_user_by_id: user-resolution error (%s)", type(exc).__name__) return "no_active_key" if user_object is None: @@ -383,6 +395,7 @@ _BridgeMintError = Literal[ "no_identity", "invalid_refresh", "identity_unavailable", + "identity_faulted", "identity_unresolvable", "not_configured", "no_upstream_token", @@ -433,6 +446,13 @@ def _bridge_mint_error_response(error: _BridgeMintError) -> JSONResponse: "temporarily_unavailable", "the authentication database is temporarily unreachable; retry shortly", ) + case "identity_faulted": + status, code, desc = ( + 503, + "temporarily_unavailable", + "the authentication database reported a fault that is not a transient outage; " + "retrying will not help until the gateway deployment is repaired", + ) case "identity_unresolvable": status, code, desc = ( 500, @@ -485,6 +505,8 @@ def _key_resolution_failure_to_mint_error(failure: _KeyResolutionFailure) -> _Br return "no_identity" case "unavailable": return "identity_unavailable" + case "faulted": + return "identity_faulted" case "unresolvable": return "identity_unresolvable" case _: @@ -569,6 +591,8 @@ def _refresh_key_failure_to_mint_error(failure: _KeyResolutionFailure) -> _Bridg return "invalid_refresh" case "unavailable": return "identity_unavailable" + case "faulted": + return "identity_faulted" case "unresolvable": return "identity_unresolvable" case _: diff --git a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py b/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py index a43e762a456..c7b0045dde5 100644 --- a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py +++ b/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py @@ -150,11 +150,18 @@ _CLIENT_RECORD_DEBUG_KEY: Final = "gateway_dcr_client" _CONNECT_FLOW_DEBUG_KEY: Final = "gateway_connect_flow" _AUTH_CODE_DEBUG_KEY: Final = "gateway_authorization_code" -ReloadUserFailure = Literal["unresolvable", "unavailable", "no_active_key"] +ReloadUserFailure = Literal["unresolvable", "unavailable", "faulted", "no_active_key"] ReloadUser = Callable[[str], Awaitable[ReloadUserFailure | None]] """Injected live-user revalidation (the token endpoint's mirror of admission): -``None`` means the user is active; ``unavailable`` is a retryable DB outage; anything -else fails the grant closed.""" +``None`` means the user is active; ``unavailable`` is a retryable DB outage; ``faulted`` is +a DB fault retrying will not clear (still 503, worded so nobody just waits); anything else +fails the grant closed.""" + +_DB_UNAVAILABLE_DESCRIPTION: Final = "the gateway database is unavailable; retry" +_DB_FAULTED_DESCRIPTION: Final = ( + "the gateway database reported a fault that is not a transient outage; " + "retrying will not help until the gateway deployment is repaired" +) PROXY_API_AUDIENCE: Final[SessionAudience] = "proxy_api" """The audience a native client (``lite login --pkce``, a Go CLI) asks for by sending the @@ -659,7 +666,9 @@ def _set_flow_cookie(response: Response, request: Request, handle: str, flow: _C def _consent_lookup_failure_response(failure: ReloadUserFailure) -> Response: match failure: case "unavailable": - return _oauth_error(503, "temporarily_unavailable", "the gateway database is unavailable; retry") + return _oauth_error(503, "temporarily_unavailable", _DB_UNAVAILABLE_DESCRIPTION) + case "faulted": + return _oauth_error(503, "temporarily_unavailable", _DB_FAULTED_DESCRIPTION) case "unresolvable": return _oauth_error(500, "server_error", "the gateway is not configured to resolve users") case "no_active_key": @@ -962,7 +971,9 @@ def _reload_failure_response(failure: ReloadUserFailure) -> Response: ``ReloadUserFailure`` member is a type error here rather than silently 400ing.""" match failure: case "unavailable": - return _oauth_error(503, "temporarily_unavailable", "the gateway database is unavailable; retry") + return _oauth_error(503, "temporarily_unavailable", _DB_UNAVAILABLE_DESCRIPTION) + case "faulted": + return _oauth_error(503, "temporarily_unavailable", _DB_FAULTED_DESCRIPTION) case "unresolvable": return _oauth_error(500, "server_error", "the gateway is not configured to resolve users") case "no_active_key": @@ -981,7 +992,7 @@ def _mint_failure_response(failure: ProxyCredentialMintFailure) -> Response: return _oauth_error( 400, "invalid_grant", "this user belongs to a team; sign in again and pick the team for this credential" ) - case "unavailable" | "unresolvable" | "no_active_key": + case "unavailable" | "faulted" | "unresolvable" | "no_active_key": return _reload_failure_response(failure) case _: assert_never(failure) @@ -1297,8 +1308,8 @@ async def introspect_gateway_token( if peeked == "claimed": return _inactive_introspection_response() failure: Final = await reload_user(opened.principal.user_id) - if failure == "unavailable": - return _oauth_error(503, "temporarily_unavailable", "the gateway database is unavailable; retry") + if failure == "unavailable" or failure == "faulted": + return _reload_failure_response(failure) if failure is not None: return _inactive_introspection_response() return _active_introspection_response(opened) diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py index f79765f6d01..4f6305d88cf 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_search.py +++ b/litellm/proxy/_experimental/mcp_server/tool_search.py @@ -2,20 +2,31 @@ from __future__ import annotations import json from collections.abc import Mapping, Sequence +from dataclasses import dataclass from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, TypedDict, assert_never +from pydantic import ValidationError from typing_extensions import ReadOnly, Required import litellm from litellm.proxy.agent_endpoints.agent_search import DEFAULT_AGENT_SEARCH_TOP_K +from litellm.proxy.common_utils.semantic_text_index import ( + Embedder, + EmbeddingFailed, + SemanticTextIndex, + router_embedder, +) +from litellm.types.mcp import MCPToolSearchSettings if TYPE_CHECKING: - from mcp.types import CallToolResult + from mcp.types import CallToolResult, Tool from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import UserAPIKeyAuth +MCP_TOOL_SEARCH_SETTINGS_KEY: Final[str] = "mcp_tool_search" MCP_TOOL_SEARCH_TOOL_NAME: Final[str] = "mcp_tool_search" MCP_TOOL_CALL_TOOL_NAME: Final[str] = "mcp_tool_call" AGENT_SEARCH_TOOL_NAME: Final[str] = "agent_search" @@ -29,17 +40,91 @@ def coerce_top_k(value: Any, default: int = 5) -> int: return default -def search_tools(query: str, tools: list[dict[str, Any]], top_k: int = 5) -> list[dict[str, Any]]: +class ToolSearchResult(TypedDict, total=False): + name: Required[ReadOnly[str]] + description: Required[ReadOnly[str]] + inputSchema: Required[ReadOnly[Mapping[str, object]]] + score: ReadOnly[float] + + +@dataclass(frozen=True, slots=True) +class SemanticToolRanker: + embed: Embedder + embedding_model: str + index: SemanticTextIndex + + +global_mcp_tool_search_index: Final = SemanticTextIndex() + + +def mcp_tool_search_settings() -> MCPToolSearchSettings | ValidationError: + try: + return MCPToolSearchSettings.model_validate(litellm.mcp_tool_search or {}) + except ValidationError as exc: + return exc + + +def _tool_result(tool: Tool) -> ToolSearchResult: + return {"name": tool.name, "description": tool.description or "", "inputSchema": tool.inputSchema} + + +def _scored_result(tool: Tool, score: float) -> ToolSearchResult: + return {"name": tool.name, "description": tool.description or "", "inputSchema": tool.inputSchema, "score": score} + + +def _tool_text(tool: Tool) -> str: + return "\n".join(part for part in (tool.name, tool.description or "") if part) + + +def _keyword_score(query: str, tool: Tool) -> float: + haystack: Final = _tool_text(tool).lower() + return float(sum(1 for token in query.lower().split() if token in haystack)) + + +def _split_core_tools(tools: Sequence[Tool], core_tools: Sequence[str]) -> tuple[tuple[Tool, ...], tuple[Tool, ...]]: + by_name: Final = MappingProxyType({tool.name: tool for tool in tools}) + core: Final = tuple(by_name[name] for name in dict.fromkeys(core_tools) if name in by_name) + rest: Final = tuple(tool for tool in tools if tool.name not in frozenset(core_tools)) + return core, rest + + +def _top_hits( + tools: Sequence[Tool], scores: Sequence[float], minimum: float, limit: int +) -> tuple[tuple[float, Tool], ...]: + hits: Final = ((score, tool) for score, tool in zip(scores, tools, strict=True) if score >= minimum) + return tuple(sorted(hits, key=lambda hit: hit[0], reverse=True)[:limit]) + + +def search_tools(query: str, tools: Sequence[Tool], top_k: int = 5) -> tuple[ToolSearchResult, ...]: + """Keyword fallback used when no embedding model is configured: one point per query token found in the tool.""" if not query: - return [] - tokens: Final = query.lower().split() + return () + scores: Final = tuple(_keyword_score(query, tool) for tool in tools) + return tuple(_tool_result(tool) for _, tool in _top_hits(tools, scores, minimum=1.0, limit=top_k)) - def _score(tool: dict[str, Any]) -> int: - haystack: Final = (tool.get("name", "") + " " + tool.get("description", "")).lower() - return sum(1 for t in tokens if t in haystack) - scored: Final = ((s, tool) for tool in tools if (s := _score(tool)) > 0) - return [tool for _, tool in sorted(scored, key=lambda x: x[0], reverse=True)[:top_k]] +async def search_mcp_tools( + query: str, + tools: Sequence[Tool], + top_k: int, + settings: MCPToolSearchSettings, + ranker: SemanticToolRanker | None, +) -> tuple[ToolSearchResult, ...] | EmbeddingFailed: + """Core tools the caller can access come first, then up to `top_k` ranked matches from the remaining tools.""" + core, rest = _split_core_tools(tools, settings.core_tools) + limit: Final = min(top_k, settings.top_k) + core_results: Final = tuple(_tool_result(tool) for tool in core) + if ranker is None: + return (*core_results, *search_tools(query, rest, limit)) + if not query: + return core_results + scores: Final = await ranker.index.scores( + query, tuple(_tool_text(tool) for tool in rest), ranker.embed, ranker.embedding_model + ) + if isinstance(scores, EmbeddingFailed): + return scores + hits: Final = _top_hits(rest, scores, minimum=settings.similarity_threshold, limit=limit) + return (*core_results, *(_scored_result(tool, score) for score, tool in hits)) class _ToolParamSchema(TypedDict, total=False): @@ -66,11 +151,17 @@ def _json_array(*items: str) -> Sequence[str]: _MCP_TOOL_SEARCH_DEFINITION: Final[VirtualToolDefinition] = { "name": MCP_TOOL_SEARCH_TOOL_NAME, - "description": "Search for MCP tools by keyword. Returns top matching tools with names, descriptions, and input schemas.", + "description": ( + "Search for MCP tools by describing what you need. " + "Returns top matching tools with names, descriptions, and input schemas." + ), "inputSchema": { "type": "object", "properties": { - "query": {"type": "string", "description": "Keywords to search for in tool names and descriptions."}, + "query": { + "type": "string", + "description": "What the tool should do, matched against names and descriptions.", + }, "top_k": {"type": "integer", "description": "Maximum number of results to return.", "default": 5}, }, "required": _json_array("query"), @@ -165,10 +256,28 @@ async def handle_mcp_tool_search( oauth2_headers: dict[str, str] | None = None, raw_headers: dict[str, str] | None = None, ) -> CallToolResult: - from mcp.types import CallToolResult, TextContent - from litellm.proxy._experimental.mcp_server.server import _list_mcp_tools + from litellm.proxy.proxy_server import llm_router + settings: Final = mcp_tool_search_settings() + if isinstance(settings, ValidationError): + return _text_tool_result( + f"litellm_settings.{MCP_TOOL_SEARCH_SETTINGS_KEY} is invalid: {settings}", is_error=True + ) + if settings.embedding_model is not None and llm_router is None: + return _text_tool_result( + f"litellm_settings.{MCP_TOOL_SEARCH_SETTINGS_KEY}.embedding_model needs a model_list so it can be called", + is_error=True, + ) + ranker: Final = ( + SemanticToolRanker( + embed=router_embedder(llm_router, settings.embedding_model, user_api_key_dict), + embedding_model=settings.embedding_model, + index=global_mcp_tool_search_index, + ) + if settings.embedding_model is not None and llm_router is not None + else None + ) mcp_listing: Final = await _list_mcp_tools( user_api_key_auth=user_api_key_dict, mcp_servers=mcp_servers, @@ -178,17 +287,10 @@ async def handle_mcp_tool_search( oauth2_headers=oauth2_headers, raw_headers=raw_headers, ) - mcp_tools: Final = mcp_listing.tools - tools: Final = [ - { - "name": t.name, - "description": t.description or "", - "inputSchema": t.inputSchema, - } - for t in mcp_tools - ] - results: Final = search_tools(query, tools, top_k) - return CallToolResult(content=[TextContent(type="text", text=json.dumps(results))], isError=False) + results: Final = await search_mcp_tools(query, mcp_listing.tools, top_k, settings, ranker) + if isinstance(results, EmbeddingFailed): + return _text_tool_result(results.reason, is_error=True) + return _text_tool_result(json.dumps(results), is_error=False) async def handle_mcp_tool_call( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 18714256a8f..2da7ceb2d50 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2508,6 +2508,14 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="max batch input file size in MB for /v1/files uploads with purpose=batch, if a file is larger than this size it will be rejected before being forwarded to the provider", ) + max_file_size_mb: int | None = Field( + None, + description="max file size in MB for /v1/files uploads, for any purpose, if a file is larger than this size it will be rejected before being forwarded to the provider", + ) + blocked_file_extensions: tuple[str, ...] | None = Field( + None, + description="file extensions (e.g. ['.exe', '.sh']) rejected on /v1/files uploads, for any purpose, matched case-insensitively against the uploaded filename", + ) max_response_size_mb: int | None = Field( None, description="max response size in MB, if a response is larger than this size it will be rejected", @@ -2696,6 +2704,40 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="List of MCP server fields that must be filled in for a submission to pass standards checks (e.g. ['description', 'source_url', 'alias']).", ) + password_policy_min_length: int | None = Field( + None, + description=( + "Minimum length required for a locally-managed user's password. Default is 12; " + "a value below 8 is floored to 8 rather than weakening the requirement further." + ), + ) + password_policy_require_uppercase: bool | None = Field( + None, + description="If True (default), a locally-managed user's password must contain an uppercase letter.", + ) + password_policy_require_lowercase: bool | None = Field( + None, + description="If True (default), a locally-managed user's password must contain a lowercase letter.", + ) + password_policy_require_numbers: bool | None = Field( + None, + description="If True (default), a locally-managed user's password must contain a number.", + ) + password_policy_require_special_characters: bool | None = Field( + None, + description="If True (default), a locally-managed user's password must contain a special (non-alphanumeric) character.", + ) + disable_password_login_when_sso_enabled: bool | None = Field( + None, + description=( + "If True and SSO is configured (MICROSOFT_CLIENT_ID, GOOGLE_CLIENT_ID, " + "GENERIC_CLIENT_ID, or SAML_IDP_METADATA_URL/XML), disables username/password " + "login on /login, /v2/login, and /v3/login so SSO is the only way to reach the " + "Admin UI. An admin locked out of the UI can still administer the proxy over the " + "API with the master key; unset this setting and restart the proxy to restore " + "UI username/password login. Default is False." + ), + ) disable_budget_reservation: bool | None = Field( None, description=( diff --git a/litellm/proxy/agent_endpoints/agent_registry.py b/litellm/proxy/agent_endpoints/agent_registry.py index 144de52d0d2..c7b6bca72cf 100644 --- a/litellm/proxy/agent_endpoints/agent_registry.py +++ b/litellm/proxy/agent_endpoints/agent_registry.py @@ -6,8 +6,12 @@ from datetime import datetime, timezone from types import MappingProxyType from typing import TYPE_CHECKING, Final, NamedTuple, Protocol, TypedDict +from pydantic import TypeAdapter, ValidationError + import litellm +from litellm.constants import REDACTED_BY_LITELM_STRING from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.proxy.management_helpers.object_permission_utils import ( handle_update_object_permission_common, ) @@ -52,6 +56,9 @@ class AgentRecord(Protocol): @property def agent_name(self) -> str: ... + @property + def litellm_params(self) -> Mapping[str, object] | None: ... + @property def object_permission_id(self) -> str | None: ... @@ -121,6 +128,188 @@ def _dump_agent_params(raw: Mapping[str, object]) -> dict[str, object]: return dict(raw) if raw else {} +_AGENT_PARAMS_MASKER: Final = SensitiveDataMasker() +_REDACT_AGENT_PARAMS_MAX_DEPTH: Final = 10 +_AGENT_PARAMS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter( + dict[str, object] +) # mutable-ok: safe_dumps() and AgentResponse.litellm_params both require a real dict, not a Mapping +_AGENT_PARAMS_SEQUENCE_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) +_EMPTY_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) + + +def redact_sensitive_agent_litellm_params(litellm_params: object, _depth: int = 0) -> object: + """ + Replace credential-bearing values in an agent's litellm_params with + ``REDACTED_BY_LITELM_STRING`` while preserving non-secret keys (``model``, + ``is_public``, rate-limit config). Used so list/get/create/update + responses never echo a stored provider credential back to the caller. + + Handles a plain dict, a JSON-serialized string (some callers hold the + in-memory registry's params that way), and ``None`` at the top level; + anything else is passed through. Recursion depth is bounded to match the + convention documented in ``tests/code_coverage_tests/recursive_detector.py``. + """ + if litellm_params is None: + return None + if isinstance(litellm_params, str): + if _depth >= _REDACT_AGENT_PARAMS_MAX_DEPTH: + return REDACTED_BY_LITELM_STRING + try: + parsed_params: Final = _AGENT_PARAMS_ADAPTER.validate_json(litellm_params) + except ValidationError: + return REDACTED_BY_LITELM_STRING + return json.dumps(_redact_agent_params_tree(parsed_params, _depth + 1)) + return _redact_agent_params_tree(litellm_params, _depth) + + +def _redact_agent_params_tree(value: object, _depth: int) -> object: + """Structural recursion over an already-parsed litellm_params value: a + dict redacts sensitive keys and recurses into the rest, a list redacts + each element (so a secret nested inside a list of provider configs is + still caught), and anything else -- including a plain string leaf, which + must never be re-interpreted as a JSON blob -- passes through unchanged. + """ + if _depth >= _REDACT_AGENT_PARAMS_MAX_DEPTH: + return REDACTED_BY_LITELM_STRING + if isinstance(value, list): + typed_items: Final = _AGENT_PARAMS_SEQUENCE_ADAPTER.validate_python(value) + return tuple(_redact_agent_params_tree(item, _depth + 1) for item in typed_items) + if not isinstance(value, dict): + return value + typed_params: Final = _AGENT_PARAMS_ADAPTER.validate_python(value) + return { + key: ( + REDACTED_BY_LITELM_STRING + if _AGENT_PARAMS_MASKER.is_sensitive_key(key) + else _redact_agent_params_tree(nested_value, _depth + 1) + ) + for key, nested_value in typed_params.items() + } # mutable-ok: consumed by json.dumps()/AgentResponse.litellm_params, both of which require a real dict + + +def parse_agent_litellm_params(value: object) -> Mapping[str, object]: + """Normalize a stored litellm_params column to a read-only mapping. + + The prisma Json column comes back as either an already-parsed dict or a + JSON string depending on the read path, so handle both rather than + assuming one. Only ever read from (merge-source lookups), never mutated + or re-serialized directly, so a read-only view is enough here. + """ + if isinstance(value, str): + try: + return _AGENT_PARAMS_ADAPTER.validate_json(value) + except ValidationError: + return _EMPTY_LITELLM_PARAMS + if isinstance(value, Mapping): + try: + return _AGENT_PARAMS_ADAPTER.validate_python(value) + except ValidationError: + return _EMPTY_LITELLM_PARAMS + return _EMPTY_LITELLM_PARAMS + + +_MISSING_AGENT_PARAM: Final = object() +_RESTORE_AGENT_PARAMS_MAX_DEPTH: Final = 10 + + +def _restore_redacted_nested_value(incoming_value: object, existing_value: object, _depth: int) -> object: + """Recurse into a non-sensitively-named dict/list value so a secret + nested underneath it (e.g. inside a list of per-provider configs) is + still restored, not just top-level keys. Mirrors the shapes + ``redact_sensitive_agent_litellm_params`` recurses into on read, so + restore and redact stay symmetric. + + List elements are paired with the existing list by position: with no + stable per-element identity in an arbitrary ``dict[str, object]`` schema, + index is the same correspondence every other part of this restore (and + the endpoints' existing full-replace-on-PUT semantics) already assumes. + This correctly preserves a masked secret across an ordinary edit of that + same entry's other fields; it does not protect against a caller who both + reorders/resizes the list AND echoes back a masked marker in the same + request, which is a known, narrow limitation (see LIT-6736 PR discussion) + rather than a cross-entry credential leak in the common case. + + A value collapsed to the flat marker by the read side's depth cap is + recovered wholesale from ``existing_value`` (rather than the marker + string itself getting persisted) whenever ``existing_value`` isn't + already that same flat marker. Depth-bounded like its read-side + counterpart; a value at the cap is returned unchanged rather than + corrupted. + """ + if incoming_value == REDACTED_BY_LITELM_STRING and existing_value != REDACTED_BY_LITELM_STRING: + return existing_value + if _depth >= _RESTORE_AGENT_PARAMS_MAX_DEPTH: + return incoming_value + if isinstance(incoming_value, Mapping): + typed_incoming_map: Final = _AGENT_PARAMS_ADAPTER.validate_python(incoming_value) + existing_map: Final = ( + _AGENT_PARAMS_ADAPTER.validate_python(existing_value) + if isinstance(existing_value, Mapping) + else _EMPTY_LITELLM_PARAMS + ) + return _restore_redacted_litellm_params(typed_incoming_map, existing_map, _depth + 1) + if isinstance(incoming_value, (list, tuple)): + typed_incoming_seq: Final = _AGENT_PARAMS_SEQUENCE_ADAPTER.validate_python(incoming_value) + existing_seq: Final = ( + _AGENT_PARAMS_SEQUENCE_ADAPTER.validate_python(existing_value) + if isinstance(existing_value, (list, tuple)) + else () + ) + return tuple( + _restore_redacted_nested_value( + item, + existing_seq[index] if index < len(existing_seq) else None, + _depth + 1, + ) + for index, item in enumerate(typed_incoming_seq) + ) + return incoming_value + + +def _resolved_agent_param_value( + key: str, + incoming: Mapping[str, object], + existing: Mapping[str, object], + _depth: int, +) -> object: + """The value ``key`` should end up with in a restored litellm_params, or + ``_MISSING_AGENT_PARAM`` when it should be dropped entirely.""" + if key in incoming: + value: Final = incoming[key] + if _AGENT_PARAMS_MASKER.is_sensitive_key(key): + return existing.get(key, _MISSING_AGENT_PARAM) if value == REDACTED_BY_LITELM_STRING else value + return _restore_redacted_nested_value(value, existing.get(key), _depth) + if _AGENT_PARAMS_MASKER.is_sensitive_key(key): + return existing.get(key, _MISSING_AGENT_PARAM) + return _MISSING_AGENT_PARAM + + +def _restore_redacted_litellm_params( + incoming: Mapping[str, object], + existing: Mapping[str, object], + _depth: int = 0, +) -> dict[str, object]: + """Restore the real credential behind any litellm_params value the caller + echoed back as ``REDACTED_BY_LITELM_STRING``, and behind any sensitive key + omitted entirely, so an edit to an unrelated field never overwrites (or + silently drops) a stored provider credential -- the UI never has to + read-and-resend a secret to keep it. Recurses into nested dicts and lists + so a secret nested under a non-sensitively-named key is restored too. + + A sensitive key given a real (non-marker) value, including an explicit + empty string, is treated as a deliberate update -- that's how a caller + clears a credential. Non-sensitive keys always take the incoming value + (recursed into), matching the endpoints' existing full-replace-on-PUT / + merge-on-PATCH semantics for everything that isn't a secret. + """ + all_keys: Final = frozenset(incoming) | frozenset(existing) + return { + key: value + for key in all_keys + if (value := _resolved_agent_param_value(key, incoming, existing, _depth)) is not _MISSING_AGENT_PARAM + } # mutable-ok: fed to safe_dumps() for JSON-column storage, which requires a real dict + + class GrantMigrationResult(NamedTuple): rewritten: int missed: int @@ -301,9 +490,14 @@ class AgentRegistry: try: agent_name: Final = agent.get("agent_name") - # Serialize litellm_params + # Serialize litellm_params. A create has no stored row to restore a + # secret behind, so a sensitive key submitted as the redaction + # marker (e.g. a stray client re-post) is dropped rather than + # persisted as the literal placeholder string. litellm_params_obj: Final = agent.get("litellm_params", {}) - litellm_params_dict: Final[dict[str, object]] = _dump_agent_params(litellm_params_obj) + litellm_params_dict: Final = _restore_redacted_litellm_params( + _dump_agent_params(litellm_params_obj), _EMPTY_LITELLM_PARAMS + ) litellm_params: Final[str] = safe_dumps(litellm_params_dict) # Serialize agent_card_params @@ -410,8 +604,14 @@ class AgentRegistry: update_data: Final[dict[str, object]] = {} if augment_agent.get("agent_name"): update_data["agent_name"] = augment_agent.get("agent_name") - if augment_agent.get("litellm_params"): - update_data["litellm_params"] = safe_dumps(augment_agent.get("litellm_params")) + if "litellm_params" in agent: + existing_litellm_params: Final = parse_agent_litellm_params(existing_agent.get("litellm_params")) + update_data["litellm_params"] = safe_dumps( + _restore_redacted_litellm_params( + _dump_agent_params(agent.get("litellm_params") or _EMPTY_LITELLM_PARAMS), + existing_litellm_params, + ) + ) if augment_agent.get("agent_card_params"): update_data["agent_card_params"] = safe_dumps(augment_agent.get("agent_card_params")) @@ -474,9 +674,22 @@ class AgentRegistry: try: agent_name: Final = agent.get("agent_name") + # A PUT fully replaces litellm_params from the request body, so the + # existing row is read up front to restore any sensitive key the + # caller echoed back redacted (or omitted) rather than persisting + # the marker -- or nothing -- over the real stored credential. + existing_row: Final = await agents_table(prisma_client).find_unique( + where={"agent_id": agent_id} # mutable-ok: prisma's query builder rejects a Mapping/MappingProxyType + ) + existing_litellm_params: Final = parse_agent_litellm_params( + existing_row.litellm_params if existing_row is not None else None + ) + # Serialize litellm_params litellm_params_obj: Final = agent.get("litellm_params", {}) - litellm_params_dict: Final[dict[str, object]] = _dump_agent_params(litellm_params_obj) + litellm_params_dict: Final = _restore_redacted_litellm_params( + _dump_agent_params(litellm_params_obj), existing_litellm_params + ) litellm_params: Final[str] = safe_dumps(litellm_params_dict) # Serialize agent_card_params @@ -512,9 +725,8 @@ class AgentRegistry: update_data[rate_field] = _val if agent.get("object_permission") is not None: - existing_agent: Final = await agents_table(prisma_client).find_unique(where={"agent_id": agent_id}) existing_object_permission_id: Final = ( - existing_agent.object_permission_id if existing_agent is not None else None + existing_row.object_permission_id if existing_row is not None else None ) agent_copy: Final = dict(agent) object_permission_id: Final = await handle_update_object_permission_common( diff --git a/litellm/proxy/agent_endpoints/agent_search.py b/litellm/proxy/agent_endpoints/agent_search.py index 46ab36d7b72..76e3fe6c5ad 100644 --- a/litellm/proxy/agent_endpoints/agent_search.py +++ b/litellm/proxy/agent_endpoints/agent_search.py @@ -2,17 +2,18 @@ from __future__ import annotations -import math -from collections.abc import Awaitable, Mapping, Sequence +from collections.abc import Sequence from dataclasses import dataclass -from itertools import chain -from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Protocol, TypeAlias +from typing import TYPE_CHECKING, Final, TypeAlias -from openai import OpenAIError from pydantic import BaseModel, ConfigDict, ValidationError -from litellm.exceptions import BudgetExceededError +from litellm.proxy.common_utils.semantic_text_index import ( + Embedder, + EmbeddingFailed, + SemanticTextIndex, + router_embedder, +) from litellm.types.agents import AgentResponse if TYPE_CHECKING: @@ -21,12 +22,6 @@ if TYPE_CHECKING: DEFAULT_AGENT_SEARCH_TOP_K: Final = 5 -Vector: TypeAlias = tuple[float, ...] - - -class Embedder(Protocol): - def __call__(self, texts: Sequence[str]) -> Awaitable[Sequence[Vector]]: ... - @dataclass(frozen=True, slots=True) class AgentSearchHit: @@ -67,18 +62,6 @@ class _SearchableCard(BaseModel): skills: tuple[_SearchableSkill, ...] = () -class _EmbeddingItem(BaseModel): - model_config = ConfigDict(frozen=True, extra="ignore") - - embedding: tuple[float, ...] - - -class _EmbeddingData(BaseModel): - model_config = ConfigDict(frozen=True, extra="ignore") - - data: tuple[_EmbeddingItem, ...] - - class AgentSearchResult(BaseModel): model_config = ConfigDict(frozen=True) @@ -117,110 +100,21 @@ def agent_search_result(hit: AgentSearchHit) -> AgentSearchResult: ) -def cosine_similarity(left: Vector, right: Vector) -> float: - dot: Final = sum(a * b for a, b in zip(left, right, strict=True)) - norms: Final = math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right)) - return dot / norms if norms else 0.0 - - -def embedding_spend_metadata(user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: - from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup - - return { # mutable-ok: the router mutates the metadata dict it is handed - **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict), - "user_api_key": user_api_key_dict.api_key, - } - - -def router_embedder(router: Router, embedding_model: str, user_api_key_dict: UserAPIKeyAuth) -> Embedder: - async def embed(texts: Sequence[str]) -> Sequence[Vector]: - batch: Final = list(texts) # mutable-ok: Router.aembedding accepts only str | list input - response: Final = await router.aembedding( - model=embedding_model, input=batch, metadata=embedding_spend_metadata(user_api_key_dict) - ) - return tuple(item.embedding for item in _EmbeddingData.model_validate(response.model_dump()).data) - - return embed - - -_NO_VECTORS: Final[Mapping[str, Vector]] = MappingProxyType({}) - - -async def _embed_all(embed: Embedder, texts: Sequence[str]) -> tuple[Vector, ...] | AgentSearchEmbeddingFailed: - try: - vectors: Final = tuple(await embed(texts)) - except (OpenAIError, ValueError, BudgetExceededError) as exc: - return AgentSearchEmbeddingFailed(reason=f"embedding the search query failed: {exc}") - if len(vectors) != len(texts): - return AgentSearchEmbeddingFailed( - reason=f"embedding model returned {len(vectors)} vectors for {len(texts)} inputs" - ) - return vectors - - -@dataclass(frozen=True, slots=True) -class _Embedded: - query_vector: Vector - vectors: Mapping[str, Vector] - - -def _same_dimension(query_vector: Vector, vectors: Mapping[str, Vector], texts: Sequence[str]) -> bool: - return all(len(vectors[text]) == len(query_vector) for text in texts) - - -async def _embed_query_and_agents( - embed: Embedder, query: str, texts: Sequence[str], cached: Mapping[str, Vector] -) -> _Embedded | AgentSearchEmbeddingFailed: - missing: Final = tuple(dict.fromkeys(text for text in texts if text not in cached)) - embedded: Final = await _embed_all(embed, (query, *missing)) - if isinstance(embedded, AgentSearchEmbeddingFailed): - return embedded - vectors: Final = MappingProxyType(dict(chain(cached.items(), zip(missing, embedded[1:], strict=True)))) - if _same_dimension(embedded[0], vectors, texts): - return _Embedded(query_vector=embedded[0], vectors=vectors) - unique: Final = tuple(dict.fromkeys(texts)) - reembedded: Final = await _embed_all(embed, (query, *unique)) - if isinstance(reembedded, AgentSearchEmbeddingFailed): - return reembedded - return _Embedded( - query_vector=reembedded[0], vectors=MappingProxyType(dict(zip(unique, reembedded[1:], strict=True))) - ) - - class AgentSearchIndex: """Caches one vector per distinct agent text per embedding model, so repeat searches only embed the query.""" def __init__(self) -> None: - self._vectors: Mapping[str, Mapping[str, Vector]] = MappingProxyType({}) - - def _merged(self, embedding_model: str, embedded: _Embedded) -> Mapping[str, Vector]: - kept: Final = { - text: vector - for text, vector in self._vectors.get(embedding_model, _NO_VECTORS).items() - if len(vector) == len(embedded.query_vector) - } - return MappingProxyType({**kept, **embedded.vectors}) + self._index: Final = SemanticTextIndex() async def search( self, query: str, agents: Sequence[AgentResponse], top_k: int, embed: Embedder, embedding_model: str ) -> AgentSearchHits | AgentSearchEmbeddingFailed: - if not agents: - return AgentSearchHits(hits=()) texts: Final = tuple(agent_search_text(agent) for agent in agents) - cached: Final = self._vectors.get(embedding_model, _NO_VECTORS) - embedded: Final = await _embed_query_and_agents(embed, query, texts, cached) - if isinstance(embedded, AgentSearchEmbeddingFailed): - return embedded - if not _same_dimension(embedded.query_vector, embedded.vectors, texts): - return AgentSearchEmbeddingFailed( - reason=f"embedding model {embedding_model} returned vectors of mixed dimensions" - ) - self._vectors = MappingProxyType({**self._vectors, embedding_model: self._merged(embedding_model, embedded)}) + scores: Final = await self._index.scores(query, texts, embed, embedding_model) + if isinstance(scores, EmbeddingFailed): + return AgentSearchEmbeddingFailed(reason=scores.reason) ranked: Final = sorted( - ( - AgentSearchHit(agent=agent, score=cosine_similarity(embedded.query_vector, embedded.vectors[text])) - for agent, text in zip(agents, texts, strict=True) - ), + (AgentSearchHit(agent=agent, score=score) for agent, score in zip(agents, scores, strict=True)), key=lambda hit: hit.score, reverse=True, ) diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index b6c41a17503..3e4dc07a521 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -20,7 +20,6 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.litellm_logging import _get_masked_values from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._types import ( CommonProxyErrors, @@ -33,6 +32,10 @@ from litellm.proxy.a2a.agent_card import ( merge_agent_card, normalize_protocol_version, ) +from litellm.proxy.agent_endpoints.agent_registry import ( + parse_agent_litellm_params, + redact_sensitive_agent_litellm_params, +) from litellm.proxy.agent_endpoints.agent_search import ( DEFAULT_AGENT_SEARCH_TOP_K, AgentSearchEmbeddingFailed, @@ -139,25 +142,37 @@ async def _attach_keys_to_agents(agents: Sequence[AgentResponse], prisma_client) agent.keys = matched_keys or None +def _redact_agent_litellm_params_dict( + litellm_params: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: AgentResponse.litellm_params is declared as a plain dict, not Mapping + """Type-narrowing wrapper: a dict in always yields a dict back from + ``redact_sensitive_agent_litellm_params``, which the function's general + (possible-JSON-string, possibly-None) signature can't express.""" + return dict( # mutable-ok: AgentResponse.litellm_params is declared as a plain dict, not Mapping + parse_agent_litellm_params(redact_sensitive_agent_litellm_params(litellm_params)) + ) + + def _redact_sensitive_agent_fields( agents: Sequence[AgentResponse], + *, + is_admin: bool, ) -> list[AgentResponse]: """ - Return copies of the given agents with sensitive configuration fields - redacted. The original objects are not modified. + Return copies of the given agents with credential-bearing litellm_params + values replaced by a fixed marker (never returned to ANY caller, + admin included) and, for non-admin callers, virtual-key and header + fields stripped entirely. The original objects are not modified. """ redacted: Final[list[AgentResponse]] = [] for agent in agents: copy = agent.model_copy(deep=True) - copy.static_headers = None - copy.extra_headers = None - copy.keys = None + if not is_admin: + copy.static_headers = None + copy.extra_headers = None + copy.keys = None if copy.litellm_params: - copy.litellm_params = _get_masked_values( - copy.litellm_params, - unmasked_length=4, - number_of_asterisks=4, - ) + copy.litellm_params = _redact_agent_litellm_params_dict(copy.litellm_params) redacted.append(copy) return redacted @@ -345,13 +360,13 @@ async def get_agents( global_agent_registry.ids_for_agent(agent.agent_id).isdisjoint(litellm.public_agent_groups) ) - # Redact sensitive fields for non-admin users + # litellm_params secrets are always redacted; keys/headers stay + # admin-only. is_admin: Final = ( user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value ) - if not is_admin: - returned_agents = _redact_sensitive_agent_fields(returned_agents) + returned_agents = _redact_sensitive_agent_fields(returned_agents, is_admin=is_admin) if health_check: agents_with_url: Final = [agent for agent in returned_agents if (agent.agent_card_params or {}).get("url")] @@ -505,7 +520,9 @@ async def create_agent( "Failed to register agent '%s' (ID: %s) in memory: %s", agent_name, agent_id, reg_error ) - return result + # The caller is a proxy admin (enforced above); litellm_params + # secrets are still never echoed back in the response. + return _redact_sensitive_agent_fields((result,), is_admin=True)[0] except HTTPException: raise @@ -578,13 +595,13 @@ async def get_agent_by_id( await _attach_keys_to_agents([agent], prisma_client) - # Redact sensitive fields for non-admin users + # litellm_params secrets are always redacted; keys/headers stay + # admin-only. is_admin = ( user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value ) - if not is_admin: - agent = _redact_sensitive_agent_fields([agent])[0] + agent = _redact_sensitive_agent_fields((agent,), is_admin=is_admin)[0] return agent except HTTPException: @@ -688,7 +705,7 @@ async def update_agent( "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id ) - return result + return _redact_sensitive_agent_fields((result,), is_admin=True)[0] except HTTPException: raise except Exception as e: @@ -791,7 +808,7 @@ async def patch_agent( "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id ) - return result + return _redact_sensitive_agent_fields((result,), is_admin=True)[0] except HTTPException: raise except Exception as e: diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 5703c6cd5e8..0db4cbc3bf2 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -4705,6 +4705,13 @@ async def is_valid_fallback_model( return True +# The shape abbreviate_api_key writes into LiteLLM_VerificationToken.key_name. The +# last four characters are only barred from being whitespace or a control code, +# because a custom key's can be anything else, punctuation and non-ASCII included; +# a real key is at least MINIMUM_CUSTOM_KEY_LENGTH long, so it never fullmatches. +_MASKED_KEY_NAME_RE: Final = re.compile(r"sk-\.\.\.(?:[^\s\x00-\x1f\x7f-\x9f]{4})?") + + def _apply_budget_exceeded_throttle(valid_token: UserAPIKeyAuth) -> bool: """ Throttle an over-budget key instead of blocking it, when the key opted in @@ -4785,10 +4792,15 @@ async def _virtual_key_max_budget_check( if math.isfinite(valid_token.max_budget) and spend >= valid_token.max_budget: if _apply_budget_exceeded_throttle(valid_token): return - # name the key in the error so operators don't have to reverse-map - # spend back to a key; key_name is the masked form (last 4 chars) + # This message is returned to the caller, and key_name has no enforced + # shape (a direct DB write bypasses abbreviate_api_key), so echo it only + # when it still looks masked and fall back to the alias otherwise. key_label: Final = valid_token.key_alias or "key" - key_descriptor: Final = f"{key_label} ({valid_token.key_name})" if valid_token.key_name else key_label + key_descriptor: Final = ( + f"{key_label} ({valid_token.key_name})" + if valid_token.key_name and _MASKED_KEY_NAME_RE.fullmatch(valid_token.key_name) + else key_label + ) raise litellm.BudgetExceededError( current_cost=spend, max_budget=valid_token.max_budget, diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 64878a480a7..b36c8a038fc 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -61,9 +61,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException: return e if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): return ProxyException( - message=( - "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." - ), + message=PrismaDBExceptionHandler.database_unavailable_message(e), type=ProxyErrorTypes.no_db_connection, param="None", code=status.HTTP_503_SERVICE_UNAVAILABLE, diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 89b2c92cdfd..d6007a2d56e 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1,3 +1,4 @@ +import importlib.util import os import re import sys @@ -1402,7 +1403,7 @@ def is_pass_through_provider_route(route: str) -> bool: return False -def _has_user_setup_sso() -> bool: +def has_user_setup_sso() -> bool: """ Check if the user has set up single sign-on (SSO). @@ -1425,6 +1426,63 @@ def _has_user_setup_sso() -> bool: ) +def _is_google_ready() -> bool: + return bool(os.getenv("GOOGLE_CLIENT_ID")) and bool(os.getenv("GOOGLE_CLIENT_SECRET")) + + +def _is_microsoft_ready() -> bool: + return ( + bool(os.getenv("MICROSOFT_CLIENT_ID")) + and bool(os.getenv("MICROSOFT_CLIENT_SECRET")) + and bool(os.getenv("MICROSOFT_TENANT")) + ) + + +def _is_generic_oauth_ready() -> bool: + return ( + bool(os.getenv("GENERIC_CLIENT_ID")) + and bool(os.getenv("GENERIC_CLIENT_SECRET")) + and bool(os.getenv("GENERIC_AUTHORIZATION_ENDPOINT")) + and bool(os.getenv("GENERIC_TOKEN_ENDPOINT")) + and bool(os.getenv("GENERIC_USERINFO_ENDPOINT")) + ) + + +def _is_saml_ready() -> bool: + if not (os.getenv("SAML_IDP_METADATA_URL") or os.getenv("SAML_IDP_METADATA_XML")): + return False + # SAML's runtime (python3-saml) is an optional dependency; the SAML + # handler itself fails closed on every request when it is missing + # (SAMLAuthHandler raises before touching the IdP), so metadata alone + # is not "ready" either. find_spec raises ModuleNotFoundError (rather + # than returning None) when the top-level package is absent entirely, + # so this must not be a bare boolean expression or every password + # login would 500 on a deployment that configured SAML metadata + # without installing the optional extra. + try: + return importlib.util.find_spec("onelogin.saml2.auth") is not None + except ModuleNotFoundError: + return False + + +def is_sso_provider_fully_configured() -> bool: + """Whether ANY configured SSO provider has every companion setting it + needs to actually authenticate a user, not merely a client id. + + A lone ``MICROSOFT_CLIENT_ID`` with no secret or tenant makes + ``has_user_setup_sso()`` return True while every real sign-in attempt + fails, so a gate that BLOCKS the password fallback (unlike the UI + discovery use of ``has_user_setup_sso()``, where a dead login button is + merely confusing) must check readiness here, or it can lock every admin + out with no way to sign in at all. Checks every provider independently + (mirroring ``/sso/readiness``'s per-provider requirements) rather than + stopping at the first one with a client id set, so a stray leftover + client id for an unused provider can never mask a different, fully + configured provider that would otherwise satisfy this gate. + """ + return _is_google_ready() or _is_microsoft_ready() or _is_generic_oauth_ready() or _is_saml_ready() + + def get_customer_user_header_from_mapping(user_id_mapping) -> list | None: """Return the header_name mapped to CUSTOMER role, if any (dict-based).""" if not user_id_mapping: diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index fba95972944..8d4f6f81363 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -7,7 +7,9 @@ login endpoints (e.g., /login and /v2/login). import os import secrets +from collections.abc import Mapping from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import Final, Literal, cast import jwt @@ -24,6 +26,7 @@ from litellm.proxy._types import ( UpdateUserRequest, UserAPIKeyAuth, ) +from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, @@ -111,6 +114,7 @@ async def authenticate_user( password: str, master_key: str | None, prisma_client: PrismaClient | None, + general_settings: Mapping[str, object] = MappingProxyType({}), ) -> LoginResult: """ Authenticate a user and generate an API key for UI access. @@ -124,13 +128,40 @@ async def authenticate_user( password: Password from the login form master_key: Master key for the proxy (required) prisma_client: Prisma database client (optional) + general_settings: Proxy general_settings, checked for + `disable_password_login_when_sso_enabled` Returns: LoginResult: Object containing authentication data Raises: - ProxyException: If authentication fails or required configuration is missing + ProxyException: If authentication fails or required configuration is missing, + or if username/password login is disabled while SSO is configured + + Recovery: an admin locked out of the UI by + `disable_password_login_when_sso_enabled` can still administer the proxy over + the API with the master key (Authorization: Bearer ), which never + goes through this function. To restore UI username/password login, unset the + setting in config.yaml (or the DB-persisted general_settings) and restart the + proxy; this is a deliberate, auditable config change rather than a hidden + bypass. + + The gate below requires the SSO provider to be FULLY configured (every + companion secret/endpoint an actual sign-in needs), not merely that a + client id is present, so an incomplete SSO setup can never disable the + only working login path. """ + if general_settings.get("disable_password_login_when_sso_enabled") is True and is_sso_provider_fully_configured(): + raise ProxyException( + message=( + "Username/password login is disabled because SSO is configured " + "and 'disable_password_login_when_sso_enabled' is set. Sign in via SSO." + ), + type=ProxyErrorTypes.auth_error, + param="disable_password_login_when_sso_enabled", + code=403, + ) + if master_key is None: raise ProxyException( message="Master Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml. https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.", diff --git a/litellm/proxy/auth/password_policy.py b/litellm/proxy/auth/password_policy.py new file mode 100644 index 00000000000..ab7a565894a --- /dev/null +++ b/litellm/proxy/auth/password_policy.py @@ -0,0 +1,92 @@ +"""Password-strength policy enforcement for locally-managed proxy users. + +Applied at every path that persists a new or changed password for a DB-backed +user (``/user/update``, ``/user/bulk_update``, and the invitation onboarding +claim flow), so the strength bar is configured in one place instead of +per-endpoint. +""" + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from litellm.proxy._types import ProxyErrorTypes, ProxyException + +DEFAULT_MIN_LENGTH: Final = 12 +MIN_ALLOWED_LENGTH: Final = 8 + + +def _has_uppercase(password: str) -> bool: + return any(ch.isupper() for ch in password) + + +def _has_lowercase(password: str) -> bool: + return any(ch.islower() for ch in password) + + +def _has_digit(password: str) -> bool: + return any(ch.isdigit() for ch in password) + + +def _has_special_character(password: str) -> bool: + """Unicode-aware: a letter or digit from ANY script counts as + alphanumeric, not just ASCII, so an accented letter (e.g. the second + character of "Passwörd1234") cannot be miscounted as the required + special character the way an ASCII-only `[^A-Za-z0-9]` regex would.""" + return any(not ch.isalnum() for ch in password) + + +@dataclass(frozen=True, slots=True) +class PasswordPolicy: + min_length: int + require_uppercase: bool + require_lowercase: bool + require_numbers: bool + require_special_characters: bool + + +def _configured_min_length(general_settings: Mapping[str, object]) -> int: + """The configured minimum, floored at MIN_ALLOWED_LENGTH so a nonpositive + or too-low override (a typo, or `0`/`false` coercing through) cannot + silently disable the length requirement rather than merely relaxing it.""" + min_length_setting: Final = general_settings.get("password_policy_min_length") + if isinstance(min_length_setting, bool) or not isinstance(min_length_setting, (int, float)): + return DEFAULT_MIN_LENGTH + return max(MIN_ALLOWED_LENGTH, int(min_length_setting)) + + +def get_password_policy(general_settings: Mapping[str, object]) -> PasswordPolicy: + return PasswordPolicy( + min_length=_configured_min_length(general_settings), + require_uppercase=general_settings.get("password_policy_require_uppercase", True) is not False, + require_lowercase=general_settings.get("password_policy_require_lowercase", True) is not False, + require_numbers=general_settings.get("password_policy_require_numbers", True) is not False, + require_special_characters=( + general_settings.get("password_policy_require_special_characters", True) is not False + ), + ) + + +def _policy_violations(password: str, policy: PasswordPolicy) -> tuple[str, ...]: + checks: Final = ( + (len(password) < policy.min_length, f"be at least {policy.min_length} characters long"), + (policy.require_uppercase and not _has_uppercase(password), "include an uppercase letter"), + (policy.require_lowercase and not _has_lowercase(password), "include a lowercase letter"), + (policy.require_numbers and not _has_digit(password), "include a number"), + (policy.require_special_characters and not _has_special_character(password), "include a special character"), + ) + return tuple(message for failed, message in checks if failed) + + +def validate_password_policy(password: str, general_settings: Mapping[str, object]) -> None: + """Raise ``ProxyException`` (400) if ``password`` fails the configured policy.""" + policy: Final = get_password_policy(general_settings) + violations: Final = _policy_violations(password, policy) + if not violations: + return + raise ProxyException( + message="Password does not meet the required policy: must " + ", ".join(violations) + ".", + type=ProxyErrorTypes.validation_error, + param="password", + code=400, + ) diff --git a/litellm/proxy/common_utils/semantic_text_index.py b/litellm/proxy/common_utils/semantic_text_index.py new file mode 100644 index 00000000000..0820459af49 --- /dev/null +++ b/litellm/proxy/common_utils/semantic_text_index.py @@ -0,0 +1,142 @@ +"""Embedding-similarity ranking over short texts with a per-model vector cache, shared by agent search and MCP tool search.""" + +from __future__ import annotations + +import math +from collections.abc import Awaitable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias + +from openai import OpenAIError +from pydantic import BaseModel, ConfigDict + +from litellm.exceptions import BudgetExceededError + +if TYPE_CHECKING: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.router import Router + +Vector: TypeAlias = tuple[float, ...] + + +class Embedder(Protocol): + def __call__(self, texts: Sequence[str]) -> Awaitable[Sequence[Vector]]: ... + + +@dataclass(frozen=True, slots=True) +class EmbeddingFailed: + reason: str + + +class _EmbeddingItem(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + embedding: tuple[float, ...] + + +class _EmbeddingData(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + data: tuple[_EmbeddingItem, ...] + + +def cosine_similarity(left: Vector, right: Vector) -> float: + dot: Final = sum(a * b for a, b in zip(left, right, strict=True)) + norms: Final = math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right)) + return dot / norms if norms else 0.0 + + +def embedding_spend_metadata(user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: # mutable-ok: router mutates it + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + return { # mutable-ok: the router mutates the metadata dict it is handed + **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict), + "user_api_key": user_api_key_dict.api_key, + } + + +def router_embedder(router: Router, embedding_model: str, user_api_key_dict: UserAPIKeyAuth) -> Embedder: + async def embed(texts: Sequence[str]) -> Sequence[Vector]: + batch: Final = list(texts) # mutable-ok: Router.aembedding accepts only str | list input + response: Final = await router.aembedding( + model=embedding_model, input=batch, metadata=embedding_spend_metadata(user_api_key_dict) + ) + return tuple(item.embedding for item in _EmbeddingData.model_validate(response.model_dump()).data) + + return embed + + +_NO_VECTORS: Final[Mapping[str, Vector]] = MappingProxyType({}) + + +async def _embed_all(embed: Embedder, texts: Sequence[str]) -> tuple[Vector, ...] | EmbeddingFailed: + try: + vectors: Final = tuple(await embed(texts)) + except (OpenAIError, ValueError, BudgetExceededError) as exc: + return EmbeddingFailed(reason=f"embedding the search query failed: {exc}") + if len(vectors) != len(texts): + return EmbeddingFailed(reason=f"embedding model returned {len(vectors)} vectors for {len(texts)} inputs") + return vectors + + +@dataclass(frozen=True, slots=True) +class _Embedded: + query_vector: Vector + vectors: Mapping[str, Vector] + + +def _same_dimension(query_vector: Vector, vectors: Mapping[str, Vector], texts: Sequence[str]) -> bool: + return all(len(vectors[text]) == len(query_vector) for text in texts) + + +async def _embed_query_and_texts( + embed: Embedder, query: str, texts: Sequence[str], cached: Mapping[str, Vector] +) -> _Embedded | EmbeddingFailed: + missing: Final = tuple(dict.fromkeys(text for text in texts if text not in cached)) + embedded: Final = await _embed_all(embed, (query, *missing)) + if isinstance(embedded, EmbeddingFailed): + return embedded + vectors: Final = MappingProxyType(dict(chain(cached.items(), zip(missing, embedded[1:], strict=True)))) + if _same_dimension(embedded[0], vectors, texts): + return _Embedded(query_vector=embedded[0], vectors=vectors) + unique: Final = tuple(dict.fromkeys(texts)) + reembedded: Final = await _embed_all(embed, (query, *unique)) + if isinstance(reembedded, EmbeddingFailed): + return reembedded + return _Embedded( + query_vector=reembedded[0], vectors=MappingProxyType(dict(zip(unique, reembedded[1:], strict=True))) + ) + + +class SemanticTextIndex: + """Caches one vector per distinct text per embedding model, so repeat searches only embed the query.""" + + def __init__(self) -> None: + self._vectors: Mapping[str, Mapping[str, Vector]] = MappingProxyType({}) + + def _merged(self, embedding_model: str, embedded: _Embedded) -> Mapping[str, Vector]: + kept: Final = MappingProxyType( + { + text: vector + for text, vector in self._vectors.get(embedding_model, _NO_VECTORS).items() + if len(vector) == len(embedded.query_vector) + } + ) + return MappingProxyType({**kept, **embedded.vectors}) + + async def scores( + self, query: str, texts: Sequence[str], embed: Embedder, embedding_model: str + ) -> tuple[float, ...] | EmbeddingFailed: + """Cosine similarity of `query` to each entry of `texts`, in the same order.""" + if not texts: + return () + cached: Final = self._vectors.get(embedding_model, _NO_VECTORS) + embedded: Final = await _embed_query_and_texts(embed, query, texts, cached) + if isinstance(embedded, EmbeddingFailed): + return embedded + if not _same_dimension(embedded.query_vector, embedded.vectors, texts): + return EmbeddingFailed(reason=f"embedding model {embedding_model} returned vectors of mixed dimensions") + self._vectors = MappingProxyType({**self._vectors, embedding_model: self._merged(embedding_model, embedded)}) + return tuple(cosine_similarity(embedded.query_vector, embedded.vectors[text]) for text in texts) diff --git a/litellm/proxy/container_endpoints/endpoints.py b/litellm/proxy/container_endpoints/endpoints.py index 4a088140725..eaa3db336a9 100644 --- a/litellm/proxy/container_endpoints/endpoints.py +++ b/litellm/proxy/container_endpoints/endpoints.py @@ -208,7 +208,7 @@ async def list_containers( # Read query parameters query_params: Final = dict(request.query_params) - data: Final[dict[str, Any]] = {"query_params": query_params} + data: Final[dict[str, Any]] = {"query_params": query_params, "model": query_params.get("model")} # Extract custom_llm_provider using priority chain custom_llm_provider: Final = ( diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 5502543b926..ef1c4a66203 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -1,4 +1,4 @@ -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Iterator from typing import Any, Final, TypeVar from litellm._logging import verbose_proxy_logger @@ -9,10 +9,43 @@ from litellm.proxy._types import ( ) from litellm.secret_managers.main import str_to_bool -# Bounds the __cause__/__context__ walk in is_database_service_unavailable_error_in_chain. +# Bounds the __cause__/__context__ walk in find_database_service_unavailable_error_in_chain. # Real exception chains are a few links deep; the cap also makes the walk cycle-safe. _MAX_EXCEPTION_CHAIN_DEPTH: Final = 20 +_TRANSIENT_DB_UNAVAILABLE_MESSAGE: Final = ( + "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." +) + + +def _exception_chain(e: BaseException) -> Iterator[BaseException]: + current = e # rebind-ok: advances one link per iteration of the bounded walk + for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH): + yield current + following = current.__cause__ or current.__context__ + if following is None: + return + current = following + + +def _database_service_unavailable_errors(e: BaseException) -> tuple[Exception, ...]: + return tuple( + link + for link in _exception_chain(e) + if isinstance(link, Exception) and PrismaDBExceptionHandler.is_database_service_unavailable_error(link) + ) + + +def _exception_types(*candidates: object) -> tuple[type[BaseException], ...]: + """Keep only the real exception classes among ``candidates``. + + The predicates below resolve prisma's error classes at call time, so a test + that swaps ``sys.modules["prisma"]`` for a ``MagicMock`` hands them mocks, + and ``isinstance`` against a mock raises ``TypeError`` instead of answering + False. Dropping the non-types lets the call fall through to the other checks. + """ + return tuple(c for c in candidates if isinstance(c, type) and issubclass(c, BaseException)) + class PrismaDBExceptionHandler: """ @@ -59,7 +92,7 @@ class PrismaDBExceptionHandler: if isinstance(e, DB_CONNECTION_ERROR_TYPES): return True - if isinstance(e, prisma.engine.errors.EngineConnectionError): + if isinstance(e, _exception_types(prisma.engine.errors.EngineConnectionError)): return True return isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection @@ -81,7 +114,7 @@ class PrismaDBExceptionHandler: """ import prisma - data_layer_errors: Final = ( + data_layer_errors: Final = _exception_types( prisma.errors.DataError, prisma.errors.UniqueViolationError, prisma.errors.ForeignKeyViolationError, @@ -94,7 +127,7 @@ class PrismaDBExceptionHandler: return False if isinstance(e, DB_CONNECTION_ERROR_TYPES): return True - if isinstance(e, prisma.errors.PrismaError): + if isinstance(e, _exception_types(prisma.errors.PrismaError)): return True if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection: return True @@ -138,13 +171,13 @@ class PrismaDBExceptionHandler: return True if isinstance( e, - ( + _exception_types( prisma.errors.ClientNotConnectedError, prisma.errors.HTTPClientClosedError, ), ): return True - if isinstance(e, prisma.errors.PrismaError): + if isinstance(e, _exception_types(prisma.errors.PrismaError)): error_message: Final = str(e).lower() connection_keywords: Final = ( "can't reach database server", @@ -171,7 +204,7 @@ class PrismaDBExceptionHandler: """True iff ``e`` is a Postgres deadlock (P2034 / 40P01) surfaced through prisma.""" import prisma - if not isinstance(e, prisma.errors.PrismaError): + if not isinstance(e, _exception_types(prisma.errors.PrismaError)): return False if getattr(e, "code", None) == "P2034": return True @@ -202,7 +235,7 @@ class PrismaDBExceptionHandler: """ import prisma - if isinstance(e, prisma.errors.PrismaError): + if isinstance(e, _exception_types(prisma.errors.PrismaError)): return False tb = getattr(e, "__traceback__", None) while tb is not None: @@ -268,6 +301,51 @@ class PrismaDBExceptionHandler: ), ) + @staticmethod + def is_permanent_database_fault(e: Exception) -> bool: + """True for a service-unavailable failure that will not clear on its + own: an engine-layer ``PrismaError`` (missing or version-skewed engine + binary, engine error status, misused transaction) that is neither the + transient ``EngineConnectionError`` nor a reconnectable transport failure. + + Picks only the wording of a 503, never whether one is sent; + ``is_database_service_unavailable_error`` stays the status gate. + """ + if PrismaDBExceptionHandler.is_database_connection_error(e): + return False + if PrismaDBExceptionHandler.is_database_transport_error(e): + return False + return PrismaDBExceptionHandler.is_database_infrastructure_error(e) + + @staticmethod + def database_unavailable_message(e: Exception) -> str: + """The 503 detail for a service-unavailable database failure: retry + guidance for a transient outage, a pointer at the deployment for a + fault that retrying cannot fix. A permanent fault anywhere in the + exception chain wins, since the transport error that surfaced it is + not what blocks recovery.""" + fault: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(e) or e + if not PrismaDBExceptionHandler.is_permanent_database_fault(fault): + return _TRANSIENT_DB_UNAVAILABLE_MESSAGE + return ( + "Service Unavailable, the authentication database query engine reported " + f"{type(fault).__name__}, which is not a transient outage and will not clear by retrying. " + "The proxy deployment needs attention." + ) + + @staticmethod + def find_database_service_unavailable_error_in_chain(e: BaseException) -> Exception | None: + """The exception in the ``__cause__`` / ``__context__`` chain that + ``is_database_service_unavailable_error`` accepts, or ``None``. Callers + that word a response by the kind of outage need the wrapped database + error itself, not just the fact that one is present. A permanent fault + outranks a transient one wherever it sits in the chain: a reconnect that + dies on a missing engine binary raises the transport error last, but the + binary is what keeps the database down.""" + outages: Final = _database_service_unavailable_errors(e) + permanent: Final = next(filter(PrismaDBExceptionHandler.is_permanent_database_fault, outages), None) + return permanent if permanent is not None else next(iter(outages), None) + @staticmethod def is_database_service_unavailable_error_in_chain(e: BaseException) -> bool: """Like ``is_database_service_unavailable_error`` but also walks the @@ -285,14 +363,7 @@ class PrismaDBExceptionHandler: The walk is depth-bounded, which also makes it cycle-safe. """ - current: BaseException | None = e - for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH): - if not isinstance(current, Exception): - return False - if PrismaDBExceptionHandler.is_database_service_unavailable_error(current): - return True - current = current.__cause__ or current.__context__ - return False + return PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(e) is not None @staticmethod def handle_db_exception(e: Exception): diff --git a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py index 64414b90ae7..c2053693f2e 100644 --- a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -14,7 +14,7 @@ router: Final = APIRouter() @router.get("/.well-known/litellm-ui-config", response_model=UiDiscoveryEndpoints) @router.get("/litellm/.well-known/litellm-ui-config", response_model=UiDiscoveryEndpoints) # if mounted at root path async def get_ui_config(): - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso from litellm.proxy.proxy_server import general_settings from litellm.proxy.utils import get_proxy_base_url, get_server_root_path @@ -28,7 +28,7 @@ async def get_ui_config(): or general_settings.get("hide_default_credentials_hint", False) is True ) - sso_configured: Final = _has_user_setup_sso() + sso_configured: Final = has_user_setup_sso() from litellm.proxy.proxy_server import proxy_config diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index da51a905ae3..70ea21320ee 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -1633,6 +1633,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): Update the guardrails litellm params in memory """ super().update_in_memory_litellm_params(litellm_params) + if self.apply_to_output: + self.output_parse_pii = False if litellm_params.pii_entities_config: self.pii_entities_config = litellm_params.pii_entities_config if litellm_params.presidio_score_thresholds: diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 76dea1b7784..16369abbfb0 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -2,6 +2,7 @@ from typing import Any, Final import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import CommonProxyErrors from litellm.types.guardrails import * @@ -85,7 +86,7 @@ def initialize_lakera_v2(litellm_params: LitellmParams, guardrail: Guardrail): return _lakera_v2_callback -def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail): +def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail) -> tuple[CustomGuardrail, ...]: from litellm.proxy.guardrails.guardrail_hooks.presidio import ( _OPTIONAL_PresidioPIIMasking, ) @@ -94,7 +95,7 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail): run_input: Final = filter_scope in ("input", "both") run_output: Final = filter_scope in ("output", "both") - def _make_presidio_callback(**overrides): + def _make_presidio_callback(**overrides) -> CustomGuardrail: params: Final = dict( guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, @@ -120,27 +121,27 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail): litellm.logging_callback_manager.add_litellm_callback(callback) return callback - primary_callback = None - - if run_input: - primary_callback = _make_presidio_callback() - - if litellm_params.output_parse_pii: - _make_presidio_callback( - output_parse_pii=True, - event_hook=GuardrailEventHooks.post_call.value, - ) - - if run_output: - output_callback: Final = _make_presidio_callback( + input_callback: Final = _make_presidio_callback() if run_input else None + unmask_output_callback: Final = ( + _make_presidio_callback( + output_parse_pii=True, + event_hook=GuardrailEventHooks.post_call.value, + ) + if run_input and litellm_params.output_parse_pii + else None + ) + mask_output_callback: Final = ( + _make_presidio_callback( apply_to_output=True, event_hook=GuardrailEventHooks.post_call.value, output_parse_pii=False, ) - if primary_callback is None: - primary_callback = output_callback - - return primary_callback + if run_output + else None + ) + return tuple( + callback for callback in (input_callback, unmask_output_callback, mask_output_callback) if callback is not None + ) def initialize_hide_secrets(litellm_params: LitellmParams, guardrail: Guardrail): diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index dc13c09dd38..bd35782444b 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -3,10 +3,10 @@ import asyncio import importlib import os -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence from datetime import datetime, timezone from itertools import chain, count -from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, cast +from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, TypeAlias, cast from pydantic import ValidationError @@ -90,6 +90,8 @@ guardrail_initializer_registry: Final = { CONFIG_GUARDRAIL_ID_NAMESPACE: Final = uuid.UUID("625f63f4-935a-50e5-98b5-fbe77babc74a") +GuardrailCallbacks: TypeAlias = tuple[CustomGuardrail, ...] + guardrail_class_registry: Final[dict[str, type[CustomGuardrail]]] = { SupportedGuardrailIntegrations.BEDROCK.value: BedrockGuardrail, SupportedGuardrailIntegrations.GRAYSWAN.value: GraySwanGuardrail, @@ -424,6 +426,41 @@ def _apply_configured_bool_overrides(instance: CustomGuardrail, litellm_params: instance.scan_raw_request = bool(litellm_params.scan_raw_request) +def _as_callback_tuple( + initialized: CustomGuardrail | Sequence[CustomGuardrail] | None, +) -> GuardrailCallbacks: + if initialized is None: + return () + if isinstance(initialized, (list, tuple)): + return tuple(initialized) + return (initialized,) + + +def _configure_callback_scoping( + custom_guardrail_callback: CustomGuardrail, guardrail_name: str, litellm_params: LitellmParams +) -> None: + for scoping_param in ( + "skip_system_message_in_guardrail", + "skip_tool_message_in_guardrail", + "scan_only_tool_results", + ): + setattr(custom_guardrail_callback, scoping_param, getattr(litellm_params, scoping_param, None)) + scan_only_tool_results_enabled: Final = effective_scan_only_tool_results_for_guardrail(custom_guardrail_callback) + if scan_only_tool_results_enabled and not custom_guardrail_callback.supports_scan_only_tool_results(): + raise ValueError( + f"Guardrail {guardrail_name}: scan_only_tool_results is enabled, but this " + "guardrail's role filtering never scans tool results, so no request content would ever " + "be scanned. Remove scan_only_tool_results or the guardrail's role-filtering option." + ) + if scan_only_tool_results_enabled and effective_skip_tool_message_for_guardrail(custom_guardrail_callback): + raise ValueError( + f"Guardrail {guardrail_name}: scan_only_tool_results and " + "skip_tool_message_in_guardrail are enabled together, which excludes every message from " + "scanning, so no request content would ever be scanned. Remove one of the two." + ) + _apply_configured_bool_overrides(custom_guardrail_callback, litellm_params) + + class InMemoryGuardrailHandler: """ Class that handles initializing guardrails and adding them to the CallbackManager @@ -440,6 +477,8 @@ class InMemoryGuardrailHandler: Guardrail id to CustomGuardrail object mapping """ + self.guardrail_id_to_sibling_callbacks: dict[str, GuardrailCallbacks] = {} # mutable-ok: per-id registry + self._sources: dict[str, Literal["db", "config"]] = {} """ Guardrail id to provenance marker. "db" entries are reconciled against @@ -474,7 +513,6 @@ class InMemoryGuardrailHandler: self._sources[guardrail_id] = source return self.IN_MEMORY_GUARDRAILS[guardrail_id] - custom_guardrail_callback: CustomGuardrail | None = None litellm_params_data: Final = guardrail["litellm_params"] verbose_proxy_logger.debug("litellm_params= %s", litellm_params_data) @@ -498,54 +536,15 @@ class InMemoryGuardrailHandler: if guardrail_type is None: raise ValueError("guardrail_type is required") - initializer: Final = guardrail_initializer_registry.get(guardrail_type) - - if initializer: - # Try to call with llm_router first, fall back to without if it fails - import inspect - - sig: Final = inspect.signature(initializer) - if "llm_router" in sig.parameters: - custom_guardrail_callback = initializer( - litellm_params, - guardrail, - llm_router, - ) - else: - custom_guardrail_callback = initializer(litellm_params, guardrail) - elif isinstance(guardrail_type, str) and "." in guardrail_type: - custom_guardrail_callback = self.initialize_custom_guardrail( - guardrail=guardrail, - guardrail_type=guardrail_type, - litellm_params=litellm_params, - config_file_path=config_file_path, - ) - else: - raise ValueError(f"Unsupported guardrail: {guardrail_type}") - - if custom_guardrail_callback is not None: - for scoping_param in ( - "skip_system_message_in_guardrail", - "skip_tool_message_in_guardrail", - "scan_only_tool_results", - ): - setattr(custom_guardrail_callback, scoping_param, getattr(litellm_params, scoping_param, None)) - scan_only_tool_results_enabled: Final = effective_scan_only_tool_results_for_guardrail( - custom_guardrail_callback - ) - if scan_only_tool_results_enabled and not custom_guardrail_callback.supports_scan_only_tool_results(): - raise ValueError( - f"Guardrail {guardrail['guardrail_name']}: scan_only_tool_results is enabled, but this " - "guardrail's role filtering never scans tool results, so no request content would ever " - "be scanned. Remove scan_only_tool_results or the guardrail's role-filtering option." - ) - if scan_only_tool_results_enabled and effective_skip_tool_message_for_guardrail(custom_guardrail_callback): - raise ValueError( - f"Guardrail {guardrail['guardrail_name']}: scan_only_tool_results and " - "skip_tool_message_in_guardrail are enabled together, which excludes every message from " - "scanning, so no request content would ever be scanned. Remove one of the two." - ) - _apply_configured_bool_overrides(custom_guardrail_callback, litellm_params) + created_callbacks: Final = self._create_callbacks( + guardrail=guardrail, + guardrail_type=guardrail_type, + litellm_params=litellm_params, + config_file_path=config_file_path, + llm_router=llm_router, + ) + for custom_guardrail_callback in created_callbacks: + _configure_callback_scoping(custom_guardrail_callback, guardrail["guardrail_name"], litellm_params) parsed_guardrail: Final = Guardrail( guardrail_id=guardrail.get("guardrail_id"), @@ -556,11 +555,44 @@ class InMemoryGuardrailHandler: # store references to the guardrail in memory self.IN_MEMORY_GUARDRAILS[guardrail_id] = parsed_guardrail - self.guardrail_id_to_custom_guardrail[guardrail_id] = custom_guardrail_callback + self.guardrail_id_to_custom_guardrail[guardrail_id] = created_callbacks[0] if created_callbacks else None + self.guardrail_id_to_sibling_callbacks[guardrail_id] = created_callbacks[1:] self._sources[guardrail_id] = source return parsed_guardrail + def _create_callbacks( + self, + guardrail: Guardrail, + guardrail_type: str, + litellm_params: LitellmParams, + config_file_path: str | None, + llm_router: Optional["Router"], + ) -> GuardrailCallbacks: + initializer: Final = guardrail_initializer_registry.get(guardrail_type) + if initializer: + import inspect + + sig: Final = inspect.signature(initializer) + if "llm_router" in sig.parameters: + return _as_callback_tuple(initializer(litellm_params, guardrail, llm_router)) + return _as_callback_tuple(initializer(litellm_params, guardrail)) + if isinstance(guardrail_type, str) and "." in guardrail_type: + return _as_callback_tuple( + self.initialize_custom_guardrail( + guardrail=guardrail, + guardrail_type=guardrail_type, + litellm_params=litellm_params, + config_file_path=config_file_path, + ) + ) + raise ValueError(f"Unsupported guardrail: {guardrail_type}") + + def _tracked_callbacks(self, guardrail_id: str) -> GuardrailCallbacks: + primary: Final = self.guardrail_id_to_custom_guardrail.get(guardrail_id) + siblings: Final = self.guardrail_id_to_sibling_callbacks.get(guardrail_id, ()) + return (() if primary is None else (primary,)) + siblings + def initialize_custom_guardrail( self, guardrail: Guardrail, @@ -630,10 +662,15 @@ class InMemoryGuardrailHandler: self.IN_MEMORY_GUARDRAILS[guardrail_id] = guardrail self._sources[guardrail_id] = source - custom_guardrail_callback: Final = self.guardrail_id_to_custom_guardrail.get(guardrail_id) - if custom_guardrail_callback: - updated_litellm_params: Final = cast(LitellmParams, guardrail.get("litellm_params", {})) - custom_guardrail_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params) + tracked_callbacks: Final = self._tracked_callbacks(guardrail_id) + if not tracked_callbacks: + return + updated_litellm_params: Final = cast(LitellmParams, guardrail.get("litellm_params", {})) + tracked_callbacks[0].update_in_memory_litellm_params(litellm_params=updated_litellm_params) + for sibling_callback in tracked_callbacks[1:]: + sibling_stage = sibling_callback.event_hook + sibling_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params) + sibling_callback.event_hook = sibling_stage def delete_in_memory_guardrail(self, guardrail_id: str) -> None: """ @@ -648,11 +685,11 @@ class InMemoryGuardrailHandler: self.IN_MEMORY_GUARDRAILS.pop(guardrail_id, None) self._sources.pop(guardrail_id, None) - custom_guardrail_callback: Final = self.guardrail_id_to_custom_guardrail.pop(guardrail_id, None) - if custom_guardrail_callback is None: - return - - litellm.logging_callback_manager.remove_callback_from_all_lists(custom_guardrail_callback) + tracked_callbacks: Final = self._tracked_callbacks(guardrail_id) + self.guardrail_id_to_custom_guardrail.pop(guardrail_id, None) + self.guardrail_id_to_sibling_callbacks.pop(guardrail_id, None) + for custom_guardrail_callback in tracked_callbacks: + litellm.logging_callback_manager.remove_callback_from_all_lists(custom_guardrail_callback) def list_in_memory_guardrails(self) -> list[Guardrail]: """ diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index c08ca5b7783..5326074ad3c 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -27,6 +27,7 @@ from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.proxy._types import * from litellm.proxy.auth.auth_checks import get_team_object, get_user_object +from litellm.proxy.auth.password_policy import validate_password_policy from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.user_api_key_cache import ( object_permission_cache_key, @@ -154,9 +155,10 @@ def _team_membership_table( return team_membership_table -def _hash_password_in_dict(data: dict) -> None: - """Hash password field in-place if present.""" +def _hash_password_in_dict(data: dict, general_settings: Mapping[str, object]) -> None: + """Validate and hash password field in-place if present.""" if "password" in data and data["password"] is not None: + validate_password_policy(data["password"], general_settings) data["password"] = hash_password(data["password"]) @@ -500,7 +502,7 @@ async def new_user( ``` """ try: - from litellm.proxy.proxy_server import _license_check, prisma_client + from litellm.proxy.proxy_server import _license_check, general_settings, prisma_client if prisma_client is None: raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) @@ -548,7 +550,7 @@ async def new_user( # generate_key_helper_fn only forwards object_permission_id, so without this the entitlement # the caller sent would be dropped on the floor. data_json = await _set_object_permission(data_json=data_json, prisma_client=prisma_client) - _hash_password_in_dict(data_json) + _hash_password_in_dict(data_json, general_settings) teams = data.teams if teams is None: teams = check_if_default_team_set() @@ -1405,7 +1407,7 @@ async def _update_single_user_helper( Returns the updated user data or raises an exception on failure. """ - from litellm.proxy.proxy_server import litellm_proxy_admin_name, prisma_client + from litellm.proxy.proxy_server import general_settings, litellm_proxy_admin_name, prisma_client if prisma_client is None: raise Exception("Not connected to DB!") @@ -1420,7 +1422,7 @@ async def _update_single_user_helper( data_json: Final[dict] = user_request.model_dump(exclude_unset=True) non_default_values = _update_internal_user_params(data_json=data_json, data=user_request) - _hash_password_in_dict(non_default_values) + _hash_password_in_dict(non_default_values, general_settings) existing_user_row: BaseModel | None = None if user_request.user_id: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c3403cf477c..d7d20d168b5 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -3785,15 +3785,22 @@ async def info_key_fn_v2( @router.get("/key/info", tags=["key management"], dependencies=[Depends(user_api_key_auth)]) @management_endpoint_wrapper async def info_key_fn( - key: str | None = fastapi.Query(default=None, description="Key in the request parameters"), + key: str | None = fastapi.Query( + default=None, + description=( + "Key to look up. Pass the key's sha256 hash so the raw key stays out of URLs and access " + "logs. Example key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa'" + ), + ), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ Retrieve information about a key. Parameters: - - key: str | None (query parameter) - The key to look up. Accepts the plaintext key or its hash. - Defaults to the key in the Authorization header. + - key: str | None (query parameter) - The key to look up. Accepts the plaintext key or its hash; + prefer the hash, since a query parameter is recorded verbatim by any HTTP access log in front + of the proxy. Defaults to the key in the Authorization header. Returns: - key: str - The key that was looked up, echoed back as it was passed in @@ -3825,7 +3832,7 @@ async def info_key_fn( Example Curl: ``` - curl -X GET "http://0.0.0.0:4000/key/info?key=sk-test-example-key-123" \ + curl -X GET "http://0.0.0.0:4000/key/info?key=d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa" \ -H "Authorization: Bearer sk-1234" ``` diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 606569c5b8b..6b98d9f9a26 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -89,7 +89,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken, get_user_object from litellm.proxy.auth.auth_utils import ( _get_request_ip_address, - _has_user_setup_sso, + has_user_setup_sso, ) from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -2617,7 +2617,7 @@ async def get_ui_settings(request: Request): _proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None) _logout_url: Final = os.getenv("PROXY_LOGOUT_URL", None) _api_doc_base_url: Final = os.getenv("LITELLM_UI_API_DOC_BASE_URL", None) - _is_sso_enabled: Final = _has_user_setup_sso() + _is_sso_enabled: Final = has_user_setup_sso() disable_expensive_db_queries: Final = ( proxy_state.get_proxy_state_variable("spend_logs_row_count") > MAX_SPENDLOG_ROWS_TO_QUERY ) diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 9bc90260de1..bf07f4748ef 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -70,6 +70,15 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( validate_managed_files_requirement, validate_managed_id_requirement, ) +from litellm.proxy.openai_files_endpoints.general_upload_validation import ( + MB, + check_blocked_extension, + check_unsafe_filename, + check_upload_file_size, + coerce_optional_int_setting, + coerce_optional_str_list_setting, + raise_upload_validation_failure, +) from litellm.proxy.utils import ProxyLogging, is_known_model from litellm.repositories.table_repositories import ManagedFileRepository from litellm.router import Router @@ -397,13 +406,23 @@ async def create_file( # descriptor and its disk blocks until the collector runs. spools: Final[list[BinaryIO]] = [] # mutable-ok: filled as the scan opens handles try: + unsafe_filename_failure: Final = check_unsafe_filename(file.filename) + if unsafe_filename_failure is not None: + raise_upload_validation_failure(unsafe_filename_failure) + + max_file_size_mb: Final = coerce_optional_int_setting(general_settings.get("max_file_size_mb")) + # Batch uploads can be gigabytes. Starlette has already spooled the upload # to disk, so stream from that handle instead of reading it into memory. - # Other uploads are small and stay in-memory bytes. + # Other uploads stay in-memory bytes, bounded to max_file_size_mb (plus one + # byte, to still tell "exactly at the limit" from "over it") when it is set, + # so an oversized upload cannot be read to completion before it is rejected. file_source: bytes | BinaryIO if purpose == "batch": await file.seek(0) file_source = file.file + elif max_file_size_mb is not None and max_file_size_mb > 0: + file_source = await file.read(max_file_size_mb * MB + 1) else: file_source = await file.read() custom_llm_provider = ( @@ -442,6 +461,15 @@ async def create_file( # Cast purpose to OpenAIFilesPurpose type purpose = cast(OpenAIFilesPurpose, purpose) + general_size_failure: Final = check_upload_file_size(file_source, max_file_size_mb) + if general_size_failure is not None: + raise_upload_validation_failure(general_size_failure) + + blocked_extensions: Final = coerce_optional_str_list_setting(general_settings.get("blocked_file_extensions")) + blocked_extension_failure: Final = check_blocked_extension(file.filename, blocked_extensions) + if blocked_extension_failure is not None: + raise_upload_validation_failure(blocked_extension_failure) + if purpose == "batch": batch_file_failure: Final = await asyncio.to_thread( check_batch_file_upload, diff --git a/litellm/proxy/openai_files_endpoints/general_upload_validation.py b/litellm/proxy/openai_files_endpoints/general_upload_validation.py new file mode 100644 index 00000000000..9d450cb5b8d --- /dev/null +++ b/litellm/proxy/openai_files_endpoints/general_upload_validation.py @@ -0,0 +1,150 @@ +""" +Upload validation applied to every purpose at POST /v1/files. + +batch_file_validation.py checks the JSONL shape of purpose="batch" uploads; this +module applies the same fast-fail-before-forwarding shape (size cap, blocked +extensions, path-traversal filenames) regardless of purpose. +""" + +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO, Final, NoReturn, assert_never + +from litellm.proxy._types import ProxyException +from litellm.proxy.common_utils.path_utils import safe_filename + +MB: Final = 1024 * 1024 + + +def coerce_optional_int_setting(raw: object) -> int | None: + """A general_settings value declared as an optional integer, e.g. max_file_size_mb. + + bool is an int subclass, so an explicit isinstance(raw, bool) exclusion is needed + or a YAML `true`/`false` would silently pass as 1/0. + """ + if raw is None: + return None + if isinstance(raw, int) and not isinstance(raw, bool): + return raw + raise TypeError(f"expected an integer, got {raw!r}") + + +def coerce_optional_str_list_setting(raw: object) -> tuple[str, ...]: + """A general_settings value declared as an optional list of strings, e.g. blocked_file_extensions.""" + if raw is None: + return () + if not isinstance(raw, list) or not all(isinstance(item, str) for item in raw): + raise TypeError(f"expected a list of strings, got {raw!r}") + return tuple(raw) + + +@dataclass(frozen=True, slots=True) +class UploadedFileTooLarge: + size_bytes: int + limit_mb: int + + +@dataclass(frozen=True, slots=True) +class UploadedFileBlockedExtension: + extension: str + + +@dataclass(frozen=True, slots=True) +class UploadedFileUnsafeFilename: + filename: str + + +UploadValidationFailure = UploadedFileTooLarge | UploadedFileBlockedExtension | UploadedFileUnsafeFilename + + +def _file_size_bytes(file_source: bytes | BinaryIO) -> int: + if isinstance(file_source, bytes): + return len(file_source) + original_position: Final = file_source.tell() + file_source.seek(0, 2) + size: Final = file_source.tell() + file_source.seek(original_position) + return size + + +def check_upload_file_size( + file_source: bytes | BinaryIO, + max_file_size_mb: int | None, +) -> UploadedFileTooLarge | None: + if max_file_size_mb is None or max_file_size_mb <= 0: + return None + size_bytes: Final = _file_size_bytes(file_source) + if size_bytes > max_file_size_mb * MB: + return UploadedFileTooLarge(size_bytes=size_bytes, limit_mb=max_file_size_mb) + return None + + +def check_blocked_extension( + filename: str | None, + blocked_extensions: tuple[str, ...], +) -> UploadedFileBlockedExtension | None: + if not blocked_extensions or not filename: + return None + try: + extension: Final = Path(safe_filename(filename)).suffix.lower() + except ValueError: + return None + # The uploaded name's extension is normalized above; blocked_extensions comes + # straight from config.yaml or the DB and is normalized here too, so a + # differently-cased entry (".EXE") still catches a lowercase upload. + normalized_blocked: Final = frozenset(item.lower() for item in blocked_extensions) + if extension and extension in normalized_blocked: + return UploadedFileBlockedExtension(extension=extension) + return None + + +def check_unsafe_filename(filename: str | None) -> UploadedFileUnsafeFilename | None: + """Reject a filename before it can influence any storage path or backend call. + + Only flags a genuine traversal component ("..") or a null byte, so an ordinary + name like "report.v2.pdf" or ".env" is never rejected. + """ + if not filename: + return None + if "\x00" in filename: + return UploadedFileUnsafeFilename(filename=filename) + normalized: Final = filename.replace("\\", "/") + if any(part == ".." for part in normalized.split("/")): + return UploadedFileUnsafeFilename(filename=filename) + return None + + +def raise_upload_validation_failure(failure: UploadValidationFailure) -> NoReturn: + match failure: + case UploadedFileTooLarge(size_bytes=size_bytes, limit_mb=limit_mb): + raise ProxyException( + message=( + f"Uploaded file exceeds the configured max_file_size_mb of {limit_mb} MB " + f"(read stopped at {size_bytes / MB:.1f} MB). The file was not forwarded to the provider." + ), + type="invalid_request_error", + param="file", + code=413, + ) + case UploadedFileBlockedExtension(extension=extension): + raise ProxyException( + message=( + f"File extension '{extension}' is blocked by this proxy's blocked_file_extensions " + "setting. The file was not forwarded to the provider." + ), + type="invalid_request_error", + param="file", + code=400, + ) + case UploadedFileUnsafeFilename(filename=filename): + raise ProxyException( + message=( + f"Filename '{filename}' is not allowed: directory traversal sequences are not " + "permitted in uploaded file names. The file was not forwarded to the provider." + ), + type="invalid_request_error", + param="file", + code=400, + ) + case _: + assert_never(failure) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d52f05f6166..38488c03922 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -310,6 +310,7 @@ from litellm.proxy.auth.model_checks import ( get_mcp_server_ids, get_team_models, ) +from litellm.proxy.auth.password_policy import validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( _fetch_global_spend_with_event_coordination, user_api_key_auth, @@ -6644,6 +6645,12 @@ class ProxyConfig: if "max_batch_file_size_mb" not in self._yaml_general_settings_keys: general_settings["max_batch_file_size_mb"] = _general_settings.get("max_batch_file_size_mb") + if "max_file_size_mb" not in self._yaml_general_settings_keys: + general_settings["max_file_size_mb"] = _general_settings.get("max_file_size_mb") + + if "blocked_file_extensions" not in self._yaml_general_settings_keys: + general_settings["blocked_file_extensions"] = _general_settings.get("blocked_file_extensions") + ## ALERTING ARGS ## if "alerting_args" in _general_settings: general_settings["alerting_args"] = _general_settings["alerting_args"] @@ -15236,6 +15243,7 @@ async def login(request: Request): password=password, master_key=master_key, prisma_client=prisma_client, + general_settings=general_settings, ) # Create UI token object @@ -15310,6 +15318,7 @@ async def login_v2(request: Request): password=password, master_key=master_key, prisma_client=prisma_client, + general_settings=general_settings, ) returned_ui_token_object: Final = create_ui_token_object( @@ -15380,6 +15389,7 @@ async def login_v3(request: Request): password=password, master_key=master_key, prisma_client=prisma_client, + general_settings=general_settings, ) returned_ui_token_object: Final = create_ui_token_object( @@ -15749,6 +15759,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request): detail={"error": "Invalid onboarding session for invitation link."}, ) + validate_password_policy(data.password, general_settings) hashed_pw: Final = hash_password(data.password) current_time = litellm.utils.get_utc_datetime() async with prisma_client.db.tx() as tx: @@ -16412,6 +16423,8 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro "global_max_parallel_requests": "Integer", "max_request_size_mb": "Integer", "max_batch_file_size_mb": "Integer", + "max_file_size_mb": "Integer", + "blocked_file_extensions": "List", "max_response_size_mb": "Integer", "proxy_config_reload_interval_seconds": "Integer", "pass_through_endpoints": "PydanticModel", diff --git a/litellm/proxy/public_endpoints/agent_create_fields.json b/litellm/proxy/public_endpoints/agent_create_fields.json index 36484cc1065..cc2fc17d759 100644 --- a/litellm/proxy/public_endpoints/agent_create_fields.json +++ b/litellm/proxy/public_endpoints/agent_create_fields.json @@ -107,7 +107,9 @@ "required": true, "field_type": "text", "default_value": null, - "include_in_litellm_params": false + "include_in_litellm_params": false, + "validation_pattern": "^arn:aws[a-zA-Z0-9-]*:bedrock-agentcore:[a-z0-9-]+:[0-9]{12}:runtime/.+$", + "validation_message": "Enter the complete Bedrock AgentCore runtime ARN, including the runtime ID after \"runtime/\" (e.g. arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime)." } ], "litellm_params_template": { diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 41c65b1d5c5..c48750cea72 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1204,7 +1204,10 @@ async def get_global_spend_report( ), api_key: str | None = fastapi.Query( default=None, - description="View spend for a specific api_key. Example api_key='sk-1234", + description=( + "View spend for a specific api_key. Pass the key's sha256 hash so the raw key stays " + "out of URLs and access logs. Example api_key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa'" + ), ), internal_user_id: str | None = fastapi.Query( default=None, @@ -1685,7 +1688,11 @@ async def get_key_spend_report( api_key: Annotated[ str | None, fastapi.Query( - description="View spend for a specific api_key. Proxy admin only; other callers are scoped to their own key." + description=( + "View spend for a specific api_key. Proxy admin only; other callers are scoped to their " + "own key. Pass the key's sha256 hash so the raw key stays out of URLs and access logs. " + "Example api_key='d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa'" + ) ), ] = None, ) -> Sequence[Mapping[str, object]]: @@ -2945,7 +2952,7 @@ async def view_spend_logs( Example Request for specific api_key ``` - curl -X GET "http://0.0.0.0:8000/spend/logs?api_key=sk-test-example-key-123" \ + curl -X GET "http://0.0.0.0:8000/spend/logs?api_key=d5345c0ecc68ae6295c69f91926b2bd379e25481a40c34b5884d157a9f65d8fa" \ -H "Authorization: Bearer sk-1234" ``` diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 52258602581..c12d071dd36 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -21,6 +21,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_keys +from litellm.proxy._experimental.mcp_server.tool_search import MCP_TOOL_SEARCH_SETTINGS_KEY from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.config_resolvers.sso import ( @@ -38,6 +39,7 @@ from litellm.repositories.table_repositories import ( UISettingsRepository, ) from litellm.repositories.team_repository import TeamRepository +from litellm.types.mcp import MCPToolSearchSettings from litellm.types.proxy.management_endpoints.ui_sso import ( DefaultTeamSSOParams, SSOConfig, @@ -448,6 +450,10 @@ class MCPSemanticFilterSettingsResponse(SettingsResponse): """Response model for MCP semantic filter settings""" +class MCPToolSearchSettingsResponse(SettingsResponse): + """Response model for native MCP tool search settings""" + + @router.get( "/get/allowed_ips", tags=["Budget & Spend Tracking"], @@ -835,7 +841,7 @@ async def update_default_team_member_budget(teams: list[NewUserRequestTeam], use async def _update_litellm_setting( - settings: DefaultInternalUserParams | DefaultTeamSSOParams | MCPSemanticFilterSettings, + settings: DefaultInternalUserParams | DefaultTeamSSOParams | MCPSemanticFilterSettings | MCPToolSearchSettings, settings_key: str, success_message: str, user_api_key_dict: UserAPIKeyAuth, @@ -861,7 +867,7 @@ async def _update_litellm_setting( detail={"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."}, ) - in_memory_var: Final = settings.model_dump(exclude_none=True) + in_memory_var: Final = settings.model_dump(mode="json", exclude_none=True) # Load existing config first, then set in-memory value after, # because get_config() may overwrite litellm. with stale DB values @@ -1359,6 +1365,59 @@ async def update_mcp_semantic_filter_settings( return result +@router.get( + "/get/mcp_tool_search_settings", + tags=["Settings"], # mutable-ok: FastAPI's route decorator only accepts a list + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI's route decorator only accepts a list + response_model=MCPToolSearchSettingsResponse, +) +async def get_mcp_tool_search_settings( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> Mapping[str, object]: + """ + Get the `litellm_settings.mcp_tool_search` configuration used by the native `mcp_tool_search` virtual tool. + """ + from litellm.proxy.proxy_server import prisma_client, proxy_config + + if prisma_client is None: + raise HTTPException(status_code=500, detail="Database not connected. Please connect a database.") + + config: Final = await proxy_config.get_config() + + return await _get_settings_with_schema( + settings_key=MCP_TOOL_SEARCH_SETTINGS_KEY, + settings_class=MCPToolSearchSettings, + config=config, + ) + + +@router.patch( + "/update/mcp_tool_search_settings", + tags=["Settings"], # mutable-ok: FastAPI's route decorator only accepts a list + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI's route decorator only accepts a list +) +async def update_mcp_tool_search_settings( + settings: MCPToolSearchSettings, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> Mapping[str, object]: + """ + Update `litellm_settings.mcp_tool_search` in the database. + Settings will be picked up by all pods within approximately 10 seconds via background polling. + """ + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail="Only proxy admins can update MCP tool search settings.", + ) + + return await _update_litellm_setting( + settings=settings, + settings_key=MCP_TOOL_SEARCH_SETTINGS_KEY, + success_message="MCP tool search settings updated successfully. Changes will be applied across all pods within 10 seconds.", + user_api_key_dict=user_api_key_dict, + ) + + UI_SETTINGS_CACHE_KEY: Final = "ui_settings:settings_dict" UI_SETTINGS_CACHE_TTL: Final = 600 # 10 minutes @@ -1594,7 +1653,10 @@ async def update_ui_settings( tags=["UI Theme Settings"], dependencies=[Depends(user_api_key_auth)], ) -async def upload_logo(file: UploadFile = File(...)): +async def upload_logo( + file: UploadFile = File(...), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ Upload a custom logo for the admin UI. Accepts image files (PNG, JPG, JPEG, SVG) and stores them for use in the UI. @@ -1602,6 +1664,12 @@ async def upload_logo(file: UploadFile = File(...)): import os from pathlib import Path + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail="Only proxy admins can upload a UI logo.", + ) + # Validate file type allowed_extensions: Final = {".png", ".jpg", ".jpeg", ".svg"} file_extension: Final = Path(file.filename or "").suffix.lower() @@ -1612,9 +1680,11 @@ async def upload_logo(file: UploadFile = File(...)): detail=f"Invalid file type. Allowed types: {', '.join(allowed_extensions)}", ) - # Validate file size (max 5MB) - file_content: Final = await file.read() - if len(file_content) > 5 * 1024 * 1024: # 5MB + # Read bounded to one byte past the limit, so an oversized upload is never + # fully buffered in memory before being rejected. + max_logo_size_bytes: Final = 5 * 1024 * 1024 + file_content: Final = await file.read(max_logo_size_bytes + 1) + if len(file_content) > max_logo_size_bytes: raise HTTPException(status_code=400, detail="File size too large. Maximum size is 5MB.") # Create uploads directory if it doesn't exist diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 051d36c4d0f..cab2bd6d9db 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1524,6 +1524,7 @@ class ProxyLogging: prompt_label=data.pop("prompt_label", None) or {}, prompt_version=data.pop("prompt_version", None) or {}, request_kwargs=data, + injected_for_every_deployment=True, ) data.update(optional_params) diff --git a/litellm/router.py b/litellm/router.py index 6b48e7b5c32..9353e391e76 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -30,7 +30,7 @@ import anyio import httpx import openai from openai import AsyncOpenAI -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import overload import litellm @@ -383,6 +383,28 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) return False +_NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({}) +_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +def _with_router_resolved_session_model(session: object, model_name: str) -> Mapping[str, Mapping[str, object]]: + """ + Realtime client-secret requests carry the model inside ``session`` as well, and the caller's copy of it still + holds the pre-routing model group name, so it has to follow the deployment the router just picked. + + Returns kwargs to merge into the downstream call, empty when there is no session model to resolve. + """ + try: + typed_session: Final = _SESSION_ADAPTER.validate_python(session) + except ValidationError: + return _NO_SESSION_KWARGS + if "model" not in typed_session: + return _NO_SESSION_KWARGS + return MappingProxyType( + {"session": {**typed_session, "model": model_name}} # mutable-ok: callees deepcopy and JSON-dump session + ) + + # Router._aanthropic_messages_streaming_iterator buffers lifecycle chunks # until real content commits the primary stream; a hostile or slow-starting # upstream that never emits content or an error could otherwise grow that @@ -4043,6 +4065,7 @@ class Router: prompt_variables=prompt_variables, prompt_label=prompt_label, request_kwargs=kwargs, + injected_for_every_deployment=True, ) # Filter out prompt management specific parameters from data before merging @@ -4930,6 +4953,7 @@ class Router: "caching": self.cache_responses, **kwargs, "model": model_name, + **_with_router_resolved_session_model(kwargs.get("session"), model_name), } # Only set custom_llm_provider if it's not None if custom_llm_provider is not None: @@ -6714,7 +6738,10 @@ class Router: metadata. When present, decode the ID, replace ``container_id`` with the upstream value, and route through ``_ageneric_api_call_with_fallbacks`` so deployment credentials (e.g. regional ``api_base`` for Azure) match - :meth:`_init_responses_api_endpoints`. Otherwise call the handler directly. + :meth:`_init_responses_api_endpoints`. Create/list calls carry no container ID, so + they route through the deployment named by ``model`` when the caller passes one, + falling back to the direct call when no deployment matches. Otherwise call the + handler directly with global provider credentials. """ if custom_llm_provider and "custom_llm_provider" not in kwargs: kwargs["custom_llm_provider"] = custom_llm_provider @@ -6746,6 +6773,14 @@ class Router: **kwargs, ) + requested_model: Final = kwargs.get("model") + if isinstance(requested_model, str) and requested_model.strip(): + return await self._ageneric_api_call_with_fallbacks( + original_function=original_function, + passthrough_on_no_deployment=True, + **kwargs, + ) + return await original_function(**kwargs) async def _init_responses_api_endpoints( diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index 1b8baf2da09..a59fcb1bcb5 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal from urllib.parse import urlsplit import httpx -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, Field from typing_extensions import TypedDict from litellm.types.llms.base import HiddenParams @@ -91,6 +91,33 @@ class MCPPublicServer(BaseModel): mcp_info: dict[str, Any] | None = None +class MCPToolSearchSettings(BaseModel): + """`litellm_settings.mcp_tool_search`: how the native `mcp_tool_search` virtual tool ranks the caller's tools.""" + + model_config = ConfigDict(frozen=True) + + embedding_model: str | None = Field( + default=None, + description="Embedding model from model_list used to rank tools by meaning. Unset keeps keyword matching.", + ) + top_k: int = Field( + default=5, + ge=1, + le=100, + description="Most ranked tools a search returns. A smaller top_k in the tool call wins. Core tools do not count.", + ) + similarity_threshold: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Lowest cosine similarity a tool needs to appear in semantic results (0.0 = no cutoff).", + ) + core_tools: tuple[str, ...] = Field( + default=(), + description="Tool names always returned first when the caller can access them, e.g. `my_server-get_rates`.", + ) + + # OAuth 2.0 token-endpoint client authentication method (RFC 6749 section 2.3.1). MCPTokenEndpointAuthMethod = Literal["client_secret_basic", "client_secret_post"] diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index f6ee054ceaa..c7f80a61e0f 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -43,6 +43,8 @@ class AgentCredentialField(BaseModel): options: list[str] | None = None default_value: str | None = None include_in_litellm_params: bool | None = None + validation_pattern: str | None = None + validation_message: str | None = None class AgentCreateInfo(BaseModel): diff --git a/litellm/utils.py b/litellm/utils.py index 252b6756937..ba456fc353b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5106,49 +5106,6 @@ def get_response_string(response_obj: ModelResponse | ModelResponseStream) -> st return "".join(response_parts) -def get_api_key(llm_provider: str, dynamic_api_key: str | None): - api_key = dynamic_api_key or litellm.api_key - # openai - if llm_provider == "openai" or llm_provider == "text-completion-openai": - api_key = api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") - # anthropic - elif llm_provider == "anthropic" or llm_provider == "anthropic_text": - api_key = api_key or litellm.anthropic_key or get_secret("ANTHROPIC_API_KEY") - # ai21 - elif llm_provider == "ai21": - api_key = api_key or litellm.ai21_key or get_secret("AI21_API_KEY") - # aleph_alpha - elif llm_provider == "aleph_alpha": - api_key = api_key or litellm.aleph_alpha_key or get_secret("ALEPH_ALPHA_API_KEY") - # baseten - elif llm_provider == "baseten": - api_key = api_key or litellm.baseten_key or get_secret("BASETEN_API_KEY") - # cohere - elif llm_provider == "cohere" or llm_provider == "cohere_chat": - api_key = api_key or litellm.cohere_key or get_secret("COHERE_API_KEY") - # huggingface - elif llm_provider == "huggingface": - api_key = api_key or litellm.huggingface_key or get_secret("HUGGINGFACE_API_KEY") - # nlp_cloud - elif llm_provider == "nlp_cloud": - api_key = api_key or litellm.nlp_cloud_key or get_secret("NLP_CLOUD_API_KEY") - # replicate - elif llm_provider == "replicate": - api_key = api_key or litellm.replicate_key or get_secret("REPLICATE_API_KEY") - # together_ai - elif llm_provider == "together_ai": - api_key = ( - api_key or litellm.togetherai_api_key or get_secret("TOGETHERAI_API_KEY") or get_secret("TOGETHER_AI_TOKEN") - ) - # nebius - elif llm_provider == "nebius": - api_key = api_key or litellm.nebius_key or get_secret("NEBIUS_API_KEY") - # wandb - elif llm_provider == "wandb": - api_key = api_key or litellm.wandb_key or get_secret("WANDB_API_KEY") - return api_key - - def get_utc_datetime(): import datetime as dt from datetime import datetime diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 9b1cc977a64..ae91b711e13 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -9,7 +9,7 @@ "limit": 809 }, "ANN201": { - "limit": 2001 + "limit": 2000 }, "ANN202": { "limit": 835 @@ -108,7 +108,7 @@ "limit": 3 }, "F401": { - "limit": 13 + "limit": 12 }, "LOG015": { "limit": 5 @@ -147,7 +147,7 @@ "limit": 3 }, "PLR1714": { - "limit": 256 + "limit": 253 }, "PLW0127": { "limit": 57 diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000000..a1598ccbb34 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.98.0" +profile = "minimal" +components = ["rustfmt", "clippy"] diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 790956156b0..0578dc60119 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -64,6 +64,8 @@ IGNORE_FUNCTIONS = [ "_flatten_form_field", # bounded by the nesting depth of the already-parsed request body (a finite JSON tree, no cycles possible). "_flatten_form_data_field", # bounded by the nesting depth of the already-parsed request body (a finite JSON tree, no cycles possible). "_json_safe", # max depth set (_MAX_DEPTH) plus a seen-ids cycle guard for self-referential input. + "_redact_agent_params_tree", # max depth set (default 10), same shape as _redact_sensitive_litellm_params. + "_restore_redacted_nested_value", # max depth set (default 10), mirrors _redact_agent_params_tree on the write side. ] diff --git a/tests/router_unit_tests/test_router_endpoints.py b/tests/router_unit_tests/test_router_endpoints.py index d37af5b456a..a06aaa363ad 100644 --- a/tests/router_unit_tests/test_router_endpoints.py +++ b/tests/router_unit_tests/test_router_endpoints.py @@ -1324,6 +1324,98 @@ async def test_init_containers_api_endpoints_managed_id_without_model_id_applies assert call_kw["custom_llm_provider"] == "azure" +@pytest.mark.asyncio +async def test_init_containers_api_endpoints_create_with_model_uses_deployment_credentials(monkeypatch): + """ + ``POST /v1/containers`` carries no container ID, so a ``model`` in the request + body is the only way to pick a deployment. The upstream call must receive that + deployment's ``api_key``/``api_base`` instead of falling back to the global + ``OPENAI_API_KEY`` (which may be unset on the proxy). + """ + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + router = Router( + model_list=[ + { + "model_name": "gpt-5.4", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_key": "sk-model-list-key", + "api_base": "https://custom.openai.example/v1", + }, + } + ] + ) + mock_original_function = AsyncMock(return_value={"id": "cntr_test", "name": "Test Container"}) + + await router._init_containers_api_endpoints( + original_function=mock_original_function, + custom_llm_provider="openai", + name="Test Container", + model="gpt-5.4", + ) + + mock_original_function.assert_called_once() + call_kw = mock_original_function.call_args.kwargs + assert call_kw["api_key"] == "sk-model-list-key" + assert call_kw["api_base"] == "https://custom.openai.example/v1" + assert call_kw["model"] == "openai/gpt-5.4" + assert call_kw["name"] == "Test Container" + + +@pytest.mark.asyncio +async def test_init_containers_api_endpoints_create_without_model_calls_directly(): + """ + Without ``model`` (or with ``model=None`` as the proxy forwards it), create/list + must keep calling the handler directly with global provider credentials. + """ + router = Router(model_list=[]) + router._ageneric_api_call_with_fallbacks = AsyncMock() + mock_original_function = AsyncMock(return_value={"id": "cntr_test"}) + + await router._init_containers_api_endpoints( + original_function=mock_original_function, + custom_llm_provider="openai", + name="Test Container", + model=None, + ) + + router._ageneric_api_call_with_fallbacks.assert_not_called() + mock_original_function.assert_called_once_with(custom_llm_provider="openai", name="Test Container", model=None) + + +@pytest.mark.asyncio +async def test_init_containers_api_endpoints_create_with_unknown_model_passes_through(monkeypatch): + """ + A ``model`` that names no configured deployment must not turn into a 400. The call + falls through to the handler with the caller's model and no injected deployment + credentials, matching the behaviour before model-based routing existed. + """ + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + router = Router( + model_list=[ + { + "model_name": "gpt-5.4", + "litellm_params": {"model": "openai/gpt-5.4", "api_key": "sk-model-list-key"}, + } + ] + ) + mock_original_function = AsyncMock(return_value={"id": "cntr_test"}) + + await router._init_containers_api_endpoints( + original_function=mock_original_function, + custom_llm_provider="openai", + name="Test Container", + model="does-not-exist", + ) + + mock_original_function.assert_called_once() + call_kw = mock_original_function.call_args.kwargs + assert call_kw["model"] == "does-not-exist" + assert call_kw["name"] == "Test Container" + assert "api_key" not in call_kw + assert "api_base" not in call_kw + + def test_router_model_group_encrypted_content_affinity_callback_registration(): from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, diff --git a/tests/test_litellm/containers/test_container_api.py b/tests/test_litellm/containers/test_container_api.py index 885c4cd294a..16a3844431e 100644 --- a/tests/test_litellm/containers/test_container_api.py +++ b/tests/test_litellm/containers/test_container_api.py @@ -152,6 +152,42 @@ class TestContainerAPI: assert response.id == "cntr_async_123" assert response.name == "Async Test Container" + @pytest.mark.asyncio + async def test_acreate_container_encodes_router_model_id(self): + """ + The async handler returns a coroutine, so the managed-ID encoding must run + after it resolves. Otherwise follow-up calls (retrieve/delete/files) lose the + deployment and fall back to global provider credentials. + """ + upstream_response = ContainerObject( + id="cntr_upstream_123", + object="container", + created_at=1747857508, + status="running", + expires_after={"anchor": "last_active_at", "minutes": 20}, + last_active_at=1747857508, + name="Routed Container", + ) + + async def _resolve_upstream(): + return upstream_response + + with patch.object( # test-quality-ok: create_container exposes no client seam, only the handler + base_llm_http_handler, + "container_create_handler", + side_effect=lambda **kwargs: _resolve_upstream() if kwargs["_is_async"] else upstream_response, + ): + response = await acreate_container( + name="Routed Container", + custom_llm_provider="openai", + litellm_metadata={"model_info": {"id": "deployment-abc"}}, + ) + + decoded = ResponsesAPIRequestUtils._decode_container_id(response.id) + assert decoded["model_id"] == "deployment-abc" + assert decoded["custom_llm_provider"] == "openai" + assert decoded["response_id"] == "cntr_upstream_123" + @pytest.mark.asyncio async def test_alist_containers_basic(self): """Test basic async container listing functionality.""" diff --git a/tests/test_litellm/integrations/otel/test_langfuse_logger.py b/tests/test_litellm/integrations/otel/test_langfuse_logger.py new file mode 100644 index 00000000000..8f93a9a564f --- /dev/null +++ b/tests/test_litellm/integrations/otel/test_langfuse_logger.py @@ -0,0 +1,359 @@ +"""Tests for ``LangfuseOpenTelemetryV2``: the root observation's input and output are stamped from the +request-task hooks, while the root span is still recording, so Langfuse can show them on the trace.""" + +import asyncio +import json +from collections.abc import AsyncIterator, Sequence +from typing import Final + +import pytest + +pytest.importorskip("opentelemetry") + +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 + +import litellm # noqa: E402 +from litellm.caching.dual_cache import DualCache # noqa: E402 +from litellm.integrations.otel.logger import build_otel_v2_logger # noqa: E402 +from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402 +from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402 +from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402 +from litellm.integrations.otel.plumbing import providers # noqa: E402 +from litellm.integrations.otel.plumbing.context import set_request_root_span # noqa: E402 +from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 # noqa: E402 +from litellm.proxy._types import UserAPIKeyAuth # noqa: E402 +from litellm.proxy.utils import ProxyLogging # noqa: E402 +from litellm.types.llms.openai import ( # noqa: E402 + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) +from litellm.types.utils import ( # noqa: E402 + Choices, + Delta, + Embedding, + EmbeddingResponse, + Message, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) + +INPUT_ATTR: Final = "langfuse.observation.input" +OUTPUT_ATTR: Final = "langfuse.observation.output" +CHAT_DATA: Final = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "ping"}]} + + +@pytest.fixture(autouse=True) +def _reset_request_root_span(): + otel_context._request_root_span.set(None) + yield + otel_context._request_root_span.set(None) + + +def _logger(*, capture: str = "span_only", mappers: Sequence[str] = ("genai", "langfuse")): + cfg = OpenTelemetryV2Config(exporter="in_memory", mapper_names=list(mappers), capture_message_content=capture) + exporter = InMemorySpanExporter() + tracer_provider = providers.build_tracer_provider(cfg, exporter=exporter) + return build_otel_v2_logger(config=cfg, tracer_provider=tracer_provider), exporter + + +def _start_root(logger): + root = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + set_request_root_span(root) + return root + + +def _root_attrs(exporter): + by_name = {span.name: span for span in exporter.get_finished_spans()} + return dict(by_name[LITELLM_PROXY_REQUEST_SPAN_NAME].attributes or {}) + + +def _run_request(logger, data: dict, call_type: str, response: object): + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), data, call_type)) + asyncio.run(logger.async_post_call_success_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), response=response)) + root.end() + + +async def _relay(logger, chunks: Sequence[object], data: dict) -> list[object]: + async def source() -> AsyncIterator[object]: + for chunk in chunks: + yield chunk + + return [chunk async for chunk in logger.async_post_call_streaming_iterator_hook(UserAPIKeyAuth(), source(), data)] + + +def _run_stream(logger, data: dict, chunks: Sequence[object]) -> list[object]: + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), data, "acompletion")) + relayed = asyncio.run(_relay(logger, chunks, data)) + root.end() + return relayed + + +def _chat_chunk(content: str | None, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-1", + created=1, + model="gpt-5.4-mini", + choices=[StreamingChoices(index=0, delta=Delta(content=content), finish_reason=finish_reason)], + ) + + +def _responses_api_response() -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_1", + created_at=1, + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "pong", "annotations": []}], + } + ], + ) + + +def _anthropic_sse_frames() -> tuple[bytes, ...]: + events = ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "po"}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ng"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 2}}, + {"type": "message_stop"}, + ) + return tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events) + + +def test_chat_request_stamps_root_observation_input_and_output(): + logger, exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + + _run_request(logger, CHAT_DATA, "acompletion", response) + + attrs = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [{"role": "user", "content": "ping"}] + output = json.loads(attrs[OUTPUT_ATTR]) + assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] + + +def test_responses_request_folds_instructions_into_input_and_stamps_output_items(): + logger, exporter = _logger() + data = {"model": "gpt-5.4-mini", "instructions": "be terse", "input": "ping"} + + _run_request(logger, data, "aresponses", _responses_api_response()) + + attrs = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "ping"}, + ] + output = json.loads(attrs[OUTPUT_ATTR]) + assert output[0]["role"] == "assistant" + assert output[0]["content"][0]["text"] == "pong" + + +def test_anthropic_messages_request_folds_system_into_input_and_stamps_content_blocks(): + logger, exporter = _logger() + data = {"model": "claude-sonnet-4-5", "system": "be terse", "messages": [{"role": "user", "content": "ping"}]} + response = {"type": "message", "role": "assistant", "content": [{"type": "text", "text": "pong"}]} + + _run_request(logger, data, "aanthropic_messages", response) + + attrs = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "ping"}, + ] + assert json.loads(attrs[OUTPUT_ATTR]) == [{"role": "assistant", "content": [{"type": "text", "text": "pong"}]}] + + +def test_chat_stream_relays_chunks_untouched_and_stamps_assembled_output(): + logger, exporter = _logger() + chunks = (_chat_chunk("po"), _chat_chunk("ng"), _chat_chunk(None, finish_reason="stop")) + + relayed = _run_stream(logger, CHAT_DATA, chunks) + + assert [id(chunk) for chunk in relayed] == [id(chunk) for chunk in chunks] + output = json.loads(_root_attrs(exporter)[OUTPUT_ATTR]) + assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] + + +def test_responses_stream_stamps_output_from_the_completed_event(): + logger, exporter = _logger() + completed = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_responses_api_response() + ) + chunks = ({"type": "response.created"}, {"type": "response.output_text.delta", "delta": "pong"}, completed) + + relayed = _run_stream(logger, {"model": "gpt-5.4-mini", "input": "ping"}, chunks) + + assert relayed == list(chunks) + output = json.loads(_root_attrs(exporter)[OUTPUT_ATTR]) + assert output[0]["content"][0]["text"] == "pong" + + +def test_anthropic_sse_stream_stamps_output_from_the_assembled_frames(): + logger, exporter = _logger() + frames = _anthropic_sse_frames() + + relayed = _run_stream( + logger, {"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "ping"}]}, frames + ) + + assert relayed == list(frames) + output = json.loads(_root_attrs(exporter)[OUTPUT_ATTR]) + assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] + + +def test_root_observation_io_survives_the_root_ending_before_the_success_callback(): + logger, exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) + logger.log_pre_api_call( + model="gpt-5.4-mini", + messages=[], + kwargs={"litellm_call_id": "call_1", "litellm_params": {"metadata": {}}}, + ) + asyncio.run( + logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + root.end() + + payload = { + "call_type": "acompletion", + "custom_llm_provider": "openai", + "model": "gpt-5.4-mini", + "messages": CHAT_DATA["messages"], + "response": response.model_dump(), + "status": "success", + "litellm_call_id": "call_1", + "metadata": {}, + "hidden_params": {}, + } + asyncio.run( + logger.async_log_success_event( + {"standard_logging_object": payload, "litellm_params": {"metadata": {}}}, response, None, None + ) + ) + + attrs = _root_attrs(exporter) + assert INPUT_ATTR in attrs and OUTPUT_ATTR in attrs + generation = next(span for span in exporter.get_finished_spans() if span.name != LITELLM_PROXY_REQUEST_SPAN_NAME) + assert OUTPUT_ATTR in dict(generation.attributes or {}) + + +def test_root_input_is_the_request_as_the_pre_call_chain_left_it(): + logger, exporter = _logger() + raw = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "my ssn is 123-45-6789"}]} + masked = {"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "my ssn is [REDACTED]"}]} + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="noted"))]) + root = _start_root(logger) + asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), raw, "acompletion")) + asyncio.run(logger.async_post_call_success_hook(data=masked, user_api_key_dict=UserAPIKeyAuth(), response=response)) + root.end() + + assert json.loads(_root_attrs(exporter)[INPUT_ATTR]) == masked["messages"] + + +def test_root_already_ended_is_left_alone(): + logger, exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + root = _start_root(logger) + root.end() + + asyncio.run( + logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs + + +def test_responses_without_a_message_body_stamp_neither_input_nor_output(): + logger, exporter = _logger() + embedding = EmbeddingResponse(model="e", data=[Embedding(embedding=[0.1], index=0, object="embedding")]) + + _run_request(logger, {"model": "e", "input": "ping"}, "aembedding", embedding) + + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs + + +def test_unrenderable_output_never_raises_into_the_request(): + logger, exporter = _logger() + + _run_request(logger, CHAT_DATA, "acompletion", object()) + + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs + + +@pytest.mark.parametrize( + ("capture", "mappers"), + [("no_content", ("genai", "langfuse")), ("span_only", ("genai",))], +) +def test_factory_keeps_the_base_logger_unless_langfuse_content_capture_is_on(capture, mappers): + logger, exporter = _logger(capture=capture, mappers=mappers) + + _run_request(logger, CHAT_DATA, "acompletion", ModelResponse()) + attrs = _root_attrs(exporter) + assert INPUT_ATTR not in attrs and OUTPUT_ATTR not in attrs + + +@pytest.mark.parametrize( + ("capture", "mappers", "relays_streams"), + [ + ("span_only", ("genai", "langfuse"), True), + ("no_content", ("genai", "langfuse"), False), + ("span_only", ("genai",), False), + ], +) +def test_only_langfuse_content_capture_takes_proxy_streams_off_the_fast_path( + monkeypatch, capture, mappers, relays_streams +): + logger, _ = _logger(capture=capture, mappers=mappers) + monkeypatch.setattr(litellm, "callbacks", [logger]) + + assert ProxyLogging._callback_capabilities().has_iterator_override is relays_streams + + +def test_langfuse_otel_preset_builds_a_logger_that_stamps_the_root(monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk") + monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") + monkeypatch.setenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "span_only") + is_otel_v2_enabled.cache_clear() + + loggers: list = [] + try: + built = _maybe_construct_otel_v2("langfuse_otel", loggers) + assert built is not None + assert _maybe_construct_otel_v2("langfuse_otel", loggers) is built + root = _start_root(built) + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + asyncio.run( + built.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + attrs = dict(root.attributes or {}) + assert INPUT_ATTR in attrs and OUTPUT_ATTR in attrs + finally: + is_otel_v2_enabled.cache_clear() diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index e995cbae782..de8b654987b 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -2858,6 +2858,27 @@ class TestRecordGatewayInjection: AnthropicCacheControlHook.record_gateway_injection(kwargs, 0) assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT + def test_an_every_deployment_mark_survives_a_later_per_deployment_stamp(self): + """A per-leg stamp like the Bedrock converse tool_config one describes one leg of + a payload every leg sends, so narrowing an every-deployment mark to that leg's + deployment would uncredit whichever leg gets billed after a failover.""" + kwargs: dict = {"litellm_metadata": {self.KEY: ""}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 1) + assert kwargs["litellm_metadata"][self.KEY] == "" + + def test_a_pre_choice_pass_stamps_the_sentinel_over_a_provisional_deployment(self): + """The router's prompt-management factory stamps a provisional deployment's + model_info into kwargs before the prompt pass runs, and any other deployment can + end up billed, so the pass declares every-deployment scope explicitly.""" + kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 1, injected_for_every_deployment=True) + assert kwargs["litellm_metadata"][self.KEY] == "" + + def test_a_per_deployment_mark_still_follows_the_latest_leg(self): + kwargs: dict = {"litellm_metadata": {self.KEY: "dep-old"}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 1) + assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT + def test_v1_messages_auto_injection_stamps_the_marker(self, monkeypatch): monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}} diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 366f61ded49..f1de7390b5b 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6002,7 +6002,9 @@ async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_o """The savings gate reads litellm_gateway_injected_cache from the request's metadata bucket. Recording lives in the shared prompt-hook wrappers, so chat, /v1/responses, router prompt deployments, and proxy prompt templates all mark - injected requests the same way; a hook that injects nothing leaves no marker.""" + injected requests the same way; a hook that injects nothing leaves no marker. + A pass that runs before deployment choice declares it and gets the every-deployment + sentinel, which a later per-deployment pass never narrows.""" from litellm.integrations.custom_prompt_management import CustomPromptManagement class _InjectingHook(CustomPromptManagement): @@ -6086,6 +6088,28 @@ async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_o ) assert "litellm_gateway_injected_cache" not in untouched["metadata"] + pre_choice = {"metadata": {}, "model_info": {"id": "dep-of-this-attempt"}} + logging_obj.get_chat_completion_prompt( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + non_default_params={}, + prompt_variables=None, + prompt_management_logger=_InjectingHook(), + request_kwargs=pre_choice, + injected_for_every_deployment=True, + ) + assert pre_choice["metadata"]["litellm_gateway_injected_cache"] == "" + + await logging_obj.async_get_chat_completion_prompt( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "a fresh turn"}], + non_default_params={}, + prompt_variables=None, + prompt_management_logger=_InjectingHook(), + request_kwargs=pre_choice, + ) + assert pre_choice["metadata"]["litellm_gateway_injected_cache"] == "" + def test_get_standard_logging_object_payload_reads_overhead_from_logging_obj_for_dict_results(logging_obj): """LIT-5466: /v1/messages returns a plain dict with no _hidden_params, so the overhead diff --git a/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py b/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py index b924ea8f93f..7e1139ca79a 100644 --- a/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py +++ b/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py @@ -184,6 +184,50 @@ async def test_download_file_drops_query_string_from_the_stored_url(mock_env_var ) +@pytest.mark.parametrize( + "malicious_filename", + [ + "report.jsonl/../../etc/cron.d/evil", + "a.b/../../../root/.ssh/authorized_keys", + ], +) +@pytest.mark.parametrize("strategy", ["uuid", "timestamp"]) +@pytest.mark.asyncio +async def test_generate_file_name_strips_path_traversal_from_extension(mock_env_vars, malicious_filename, strategy): + """ + original_filename.split(".")[-1] does not parse path structure, so a filename whose + last "." is followed by a directory traversal sequence used to put that sequence + straight into the blob path built from this name. The mutant this pins is reverting + _safe_extension() back to that bare split. + """ + backend = _make_backend() + generated = backend._generate_file_name(malicious_filename, strategy) + assert "/" not in generated + assert ".." not in generated + + +@pytest.mark.asyncio +async def test_generate_file_name_uuid_strategy_preserves_ordinary_extension(mock_env_vars): + backend = _make_backend() + generated = backend._generate_file_name("data.jsonl", "uuid") + assert generated.endswith(".jsonl") + + +@pytest.mark.asyncio +async def test_generate_file_name_original_filename_strategy_strips_directory_components(mock_env_vars): + """The blob name must never carry a directory the caller supplied, traversal or not.""" + backend = _make_backend() + generated = backend._generate_file_name("../../etc/passwd", "original_filename") + assert generated == "passwd" + + +@pytest.mark.asyncio +async def test_generate_file_name_null_byte_filename_falls_back_to_safe_default(mock_env_vars): + backend = _make_backend() + generated = backend._generate_file_name("report.pdf\x00.exe", "uuid") + assert "\x00" not in generated + + @pytest.mark.parametrize( "env_fixture, expected_suffix", [("mock_env_vars", "core.windows.net"), ("mock_gov_env_vars", GOV_SUFFIX)], diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 7caf31ef6f1..a48de45b69b 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -6147,6 +6147,44 @@ class TestMCPDcrBridgeDelegateAdmission: await MCPRequestHandler.process_mcp_request(scope) assert exc_info.value.status_code == 503 + assert exc_info.value.detail == ( + "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." + ) + + async def test_user_subject_envelope_permanent_db_fault_is_503_not_worded_as_transient(self): + """A query engine fault that never heals (a missing engine binary) still fails admission with 503, + but the detail must not call the database "temporarily unreachable" or ask the client to retry: the + DCR client would loop on a retry that can never succeed. The fault reaches the handler wrapped in + get_user_object's bare ValueError, so the wording has to be picked off the wrapped cause.""" + from prisma.engine.errors import BinaryNotFoundError + + envelope = self._mint_bridge_envelope(user_id="sso-user-7") + scope = { + "type": "http", + "method": "POST", + "path": "/mcp/bridge_delegate_server", + "headers": [(b"authorization", f"Bearer {envelope}".encode("latin-1"))], + } + with ( + patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling admission tests + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" + ) as mock_mgr, + patch( # test-quality-ok: the envelope opener reads master_key off the proxy module, no injection seam + "litellm.proxy.proxy_server.master_key", self._MASTER_KEY + ), + self._patch_user_reload( + side_effect=self._wrapped_user_lookup_error(BinaryNotFoundError("query engine binary not found")) + ), + ): + mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server() + with pytest.raises(HTTPException) as exc_info: + await MCPRequestHandler.process_mcp_request(scope) + + assert exc_info.value.status_code == 503 + assert "temporarily unreachable" not in exc_info.value.detail + assert "retry shortly" not in exc_info.value.detail.lower() + assert "BinaryNotFoundError" in exc_info.value.detail + assert "will not clear by retrying" in exc_info.value.detail async def test_user_subject_envelope_scim_deactivated_user_fails_closed_401(self): """SCIM-deactivating the envelope's user revokes it immediately: the reloaded user carries diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 598e9276423..588eba4adb7 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -6432,6 +6432,23 @@ async def test_bridge_mint_db_outage_is_503_before_upstream(): response, post = await _prepare_only_bridge_exchange("unavailable") assert response.status_code == 503 assert json.loads(response.body)["error"] == "temporarily_unavailable" + assert "retry shortly" in json.loads(response.body)["error_description"] + post.assert_not_called() + + +@pytest.mark.asyncio +async def test_bridge_mint_permanent_db_fault_is_503_without_retry_advice(): + """A query engine fault that never heals is still a 503 (the gateway is at fault, not the client), but + the description must not tell the client the database is temporarily unreachable and to retry: that + sends an operator to wait out an outage that is not one. The code stays temporarily_unavailable, the + only RFC 6749 error a client treats as a server-side 503.""" + response, post = await _prepare_only_bridge_exchange("faulted") + assert response.status_code == 503 + body = json.loads(response.body) + assert body["error"] == "temporarily_unavailable" + assert "temporarily unreachable" not in body["error_description"] + assert "retry shortly" not in body["error_description"] + assert "not a transient outage" in body["error_description"] post.assert_not_called() @@ -7143,6 +7160,56 @@ async def test_resolve_active_litellm_key_db_outage_is_unavailable(proxy_globals assert await _resolve_active_litellm_key(request) == "unavailable" +@pytest.mark.asyncio +async def test_resolve_active_litellm_key_permanent_engine_fault_is_faulted(proxy_globals): + """A query engine that is missing or version-skewed cannot resolve any key until the deployment is + repaired, so the resolver reports "faulted" (still statused 503 by the mint) rather than "unavailable", + whose wording promises the outage is transient and asks the client to retry.""" + from prisma.engine.errors import BinaryNotFoundError + + from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + _resolve_active_litellm_key, + ) + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + class _FaultedPrisma: + async def get_data(self, token, table_name, parent_otel_span=None, proxy_logging_obj=None): + raise BinaryNotFoundError("query engine binary not found") + + proxy_globals.user_api_key_cache = UserApiKeyCache() + proxy_globals.prisma_client = _FaultedPrisma() + + request = _token_request({"x-litellm-api-key": "sk-during-engine-fault"}) + assert await _resolve_active_litellm_key(request) == "faulted" + + +@pytest.mark.asyncio +async def test_resolve_active_litellm_key_transport_error_over_permanent_fault_is_faulted(proxy_globals): + """A reconnect that dies on a missing engine binary raises the transport error last, with the + BinaryNotFoundError as __context__. The binary is what blocks recovery, so the key read is "faulted", + not the "unavailable" that the outer ConnectError alone would suggest.""" + import httpx + from prisma.engine.errors import BinaryNotFoundError + + from litellm.proxy._experimental.mcp_server.bridge_token_flow import ( + _resolve_active_litellm_key, + ) + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + class _ReconnectFailedPrisma: + async def get_data(self, token, table_name, parent_otel_span=None, proxy_logging_obj=None): + try: + raise BinaryNotFoundError("query engine binary not found") + except BinaryNotFoundError: + raise httpx.ConnectError("All connection attempts failed") + + proxy_globals.user_api_key_cache = UserApiKeyCache() + proxy_globals.prisma_client = _ReconnectFailedPrisma() + + request = _token_request({"x-litellm-api-key": "sk-during-failed-reconnect"}) + assert await _resolve_active_litellm_key(request) == "faulted" + + @pytest.mark.asyncio async def test_resolve_active_litellm_key_no_database_is_unresolvable(proxy_globals): """With no database connection configured the gateway cannot verify the presented key at all, so @@ -7214,6 +7281,26 @@ async def test_reload_active_user_by_id_db_outage_is_unavailable(proxy_globals): assert await _reload_active_user_by_id("sso-user-7") == "unavailable" +@pytest.mark.asyncio +async def test_reload_active_user_by_id_permanent_engine_fault_is_faulted(proxy_globals): + """A permanent query engine fault while re-validating the user on refresh is "faulted", not + "unavailable": both are 503s, but only the transient one may tell the client to retry. get_user_object + wraps the fault in a bare ValueError, so the classification has to read the wrapped cause.""" + from prisma.engine.errors import MismatchedVersionsError + + from litellm.proxy._experimental.mcp_server.bridge_token_flow import _reload_active_user_by_id + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + proxy_globals.user_api_key_cache = UserApiKeyCache() + proxy_globals.prisma_client = object() + + with patch( # test-quality-ok: get_user_object is the DB seam that wraps the fault; same patch as the outage sibling + "litellm.proxy.auth.auth_checks.get_user_object", + new=AsyncMock(side_effect=_wrapped_user_lookup_error(MismatchedVersionsError(expected="1", got="2"))), + ): + assert await _reload_active_user_by_id("sso-user-7") == "faulted" + + @pytest.mark.asyncio async def test_token_endpoint_uses_client_secret_basic_when_configured(): """LIT-4091: a server with token_endpoint_auth_method=client_secret_basic must send the diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py index 32a3f70c357..1670370f082 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py @@ -426,6 +426,7 @@ async def test_token_rejects_expired_code_and_missing_configuration(): [ ("no_active_key", 400, "invalid_grant"), ("unavailable", 503, "temporarily_unavailable"), + ("faulted", 503, "temporarily_unavailable"), ("unresolvable", 500, "server_error"), ], ) @@ -460,6 +461,25 @@ async def test_token_gates_on_live_user_revalidation(failure, expected_status, e assert json.loads(response.body)["error"] == expected_error +def test_permanent_db_fault_503_does_not_promise_a_retry_will_help(): + """Both DB failures are 503 temporarily_unavailable (the only OAuth error a client reads as a + server-side outage), so the description is the one place the two are told apart: a transient outage + says retry, a fault that never heals must say retrying will not help and point at the deployment.""" + from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import ( + _consent_lookup_failure_response, + _mint_failure_response, + _reload_failure_response, + ) + + for render in (_reload_failure_response, _consent_lookup_failure_response, _mint_failure_response): + transient = json.loads(render("unavailable").body)["error_description"] + faulted = json.loads(render("faulted").body)["error_description"] + assert transient == "the gateway database is unavailable; retry" + assert "retry" not in faulted.replace("retrying will not help", "") + assert "not a transient outage" in faulted + assert "retrying will not help" in faulted + + @pytest.mark.asyncio async def test_flow_is_single_use_shared_cache_rejects_second_complete(): """A double-submit of the finish step mints only ONE code: the second complete over the @@ -1250,6 +1270,7 @@ async def test_native_authorize_refuses_a_hosted_redirect_for_the_proxy_api(): "failure, status, error", [ ("unavailable", 503, "temporarily_unavailable"), + ("faulted", 503, "temporarily_unavailable"), ("unresolvable", 500, "server_error"), ("no_active_key", 403, "access_denied"), ], @@ -1424,6 +1445,7 @@ async def test_native_code_without_a_minter_is_refused_server_side(): ("team_required", 400, "invalid_grant"), ("no_active_key", 400, "invalid_grant"), ("unavailable", 503, "temporarily_unavailable"), + ("faulted", 503, "temporarily_unavailable"), ("unresolvable", 500, "server_error"), ], ) @@ -1805,5 +1827,12 @@ async def test_introspect_fails_closed_on_dead_user_and_503s_on_outage(): status, body = await _introspect(minted.token.get_secret_value(), reload_user=_reload_user_outage) assert (status, body["error"]) == (503, "temporarily_unavailable") + async def _reload_user_faulted(user_id: str): + return "faulted" + + status, body = await _introspect(minted.token.get_secret_value(), reload_user=_reload_user_faulted) + assert (status, body["error"]) == (503, "temporarily_unavailable") + assert "not a transient outage" in body["error_description"] + status, body = await _introspect(minted.token.get_secret_value(), master_key=None) assert (status, body["error"]) == (500, "server_error") diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py index 16221f44efe..239f89ebd90 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py @@ -10,33 +10,36 @@ Covers: """ import json +from collections.abc import Sequence from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest +from mcp.types import Tool +import litellm from litellm.models.object_permission import LiteLLM_ObjectPermissionTable from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.proxy._experimental.mcp_server.tool_search import ( AGENT_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME, MCP_TOOL_SEARCH_TOOL_NAME, + SemanticToolRanker, + ToolSearchResult, coerce_top_k, get_virtual_tool_definitions, + search_mcp_tools, search_tools, ) from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.common_utils.semantic_text_index import EmbeddingFailed, SemanticTextIndex, Vector +from litellm.types.mcp import MCPToolSearchSettings -def _make_tools(specs: list[tuple[str, str]]) -> list[dict[str, Any]]: - return [ - { - "name": name, - "description": desc, - "inputSchema": {"type": "object", "properties": {}}, - } - for name, desc in specs - ] +def _make_tools(specs: list[tuple[str, str]]) -> tuple[Tool, ...]: + return tuple( + Tool(name=name, description=desc, inputSchema={"type": "object", "properties": {}}) for name, desc in specs + ) def _make_perm(**kwargs: Any) -> LiteLLM_ObjectPermissionTable: @@ -54,6 +57,160 @@ SAMPLE_TOOLS = _make_tools( ) +FX_TOOL = Tool( + name="treasury-get_rates", + description="Get foreign exchange rates for a currency pair", + inputSchema={"type": "object", "properties": {}}, +) +WEATHER_TOOL = Tool( + name="weather-forecast", + description="Get the weather forecast for a city", + inputSchema={"type": "object", "properties": {}}, +) +CALENDAR_TOOL = Tool( + name="calendar-create_event", + description="Create a calendar event", + inputSchema={"type": "object", "properties": {}}, +) +CATALOG = (FX_TOOL, WEATHER_TOOL, CALENDAR_TOOL) + +# A stand-in embedding space: "FX" sits next to the foreign-exchange tool and far from the rest. +FAKE_VECTORS: dict[str, Vector] = { + "FX": (1.0, 0.0), + f"{FX_TOOL.name}\n{FX_TOOL.description}": (0.9, 0.1), + f"{WEATHER_TOOL.name}\n{WEATHER_TOOL.description}": (0.3, 1.0), + f"{CALENDAR_TOOL.name}\n{CALENDAR_TOOL.description}": (0.0, 1.0), +} + + +class RecordingEmbedder: + def __init__(self) -> None: + self.calls: list[tuple[str, ...]] = [] + + async def __call__(self, texts: Sequence[str]) -> Sequence[Vector]: + self.calls.append(tuple(texts)) + return tuple(FAKE_VECTORS[text] for text in texts) + + +def _ranker(embedder: RecordingEmbedder | None = None) -> SemanticToolRanker: + return SemanticToolRanker(embed=embedder or RecordingEmbedder(), embedding_model="emb", index=SemanticTextIndex()) + + +def _names(results: Sequence[ToolSearchResult] | EmbeddingFailed) -> list[str]: + assert not isinstance(results, EmbeddingFailed) + return [tool["name"] for tool in results] + + +class TestSearchMcpTools: + @pytest.mark.asyncio + async def test_semantic_mode_finds_foreign_exchange_tool_for_fx(self) -> None: + keyword_only = await search_mcp_tools("FX", CATALOG, 5, MCPToolSearchSettings(), ranker=None) + assert _names(keyword_only) == [] + + results = await search_mcp_tools("FX", CATALOG, 5, MCPToolSearchSettings(embedding_model="emb"), _ranker()) + assert _names(results) == [FX_TOOL.name, WEATHER_TOOL.name, CALENDAR_TOOL.name] + assert not isinstance(results, EmbeddingFailed) + assert results[0]["score"] > results[1]["score"] > results[2]["score"] + assert results[0]["inputSchema"] == FX_TOOL.inputSchema + + @pytest.mark.asyncio + async def test_similarity_threshold_drops_weak_matches(self) -> None: + settings = MCPToolSearchSettings(embedding_model="emb", similarity_threshold=0.5) + results = await search_mcp_tools("FX", CATALOG, 5, settings, _ranker()) + assert _names(results) == [FX_TOOL.name] + + @pytest.mark.asyncio + async def test_request_top_k_limits_semantic_results(self) -> None: + results = await search_mcp_tools("FX", CATALOG, 2, MCPToolSearchSettings(embedding_model="emb"), _ranker()) + assert _names(results) == [FX_TOOL.name, WEATHER_TOOL.name] + + @pytest.mark.asyncio + async def test_configured_top_k_caps_request_top_k(self) -> None: + settings = MCPToolSearchSettings(embedding_model="emb", top_k=1) + assert _names(await search_mcp_tools("FX", CATALOG, 50, settings, _ranker())) == [FX_TOOL.name] + assert _names(await search_mcp_tools("weather", CATALOG, 50, MCPToolSearchSettings(top_k=1), None)) == [ + WEATHER_TOOL.name + ] + + @pytest.mark.asyncio + async def test_core_tools_lead_and_do_not_consume_top_k(self) -> None: + settings = MCPToolSearchSettings(embedding_model="emb", top_k=1, core_tools=(CALENDAR_TOOL.name,)) + results = await search_mcp_tools("FX", CATALOG, 1, settings, _ranker()) + assert _names(results) == [CALENDAR_TOOL.name, FX_TOOL.name] + assert not isinstance(results, EmbeddingFailed) + assert "score" not in results[0] + + @pytest.mark.asyncio + async def test_core_tools_apply_in_keyword_mode_too(self) -> None: + settings = MCPToolSearchSettings(core_tools=(CALENDAR_TOOL.name,)) + assert _names(await search_mcp_tools("weather", CATALOG, 5, settings, None)) == [ + CALENDAR_TOOL.name, + WEATHER_TOOL.name, + ] + + @pytest.mark.asyncio + async def test_core_tools_outside_the_callers_catalog_are_not_returned(self) -> None: + settings = MCPToolSearchSettings(embedding_model="emb", core_tools=("payroll-run", CALENDAR_TOOL.name)) + results = await search_mcp_tools("FX", (FX_TOOL, WEATHER_TOOL), 5, settings, _ranker()) + assert _names(results) == [FX_TOOL.name, WEATHER_TOOL.name] + + @pytest.mark.asyncio + async def test_core_tools_are_listed_once_and_never_embedded(self) -> None: + embedder = RecordingEmbedder() + settings = MCPToolSearchSettings(embedding_model="emb", core_tools=(FX_TOOL.name, FX_TOOL.name)) + results = await search_mcp_tools("FX", CATALOG, 5, settings, _ranker(embedder)) + assert _names(results) == [FX_TOOL.name, WEATHER_TOOL.name, CALENDAR_TOOL.name] + assert all(FX_TOOL.description not in text for call in embedder.calls for text in call) + + @pytest.mark.asyncio + async def test_empty_query_returns_only_core_tools_without_embedding(self) -> None: + embedder = RecordingEmbedder() + settings = MCPToolSearchSettings(embedding_model="emb", core_tools=(CALENDAR_TOOL.name,)) + assert _names(await search_mcp_tools("", CATALOG, 5, settings, _ranker(embedder))) == [CALENDAR_TOOL.name] + assert embedder.calls == [] + + @pytest.mark.asyncio + async def test_repeat_searches_only_embed_the_query(self) -> None: + embedder = RecordingEmbedder() + ranker = _ranker(embedder) + settings = MCPToolSearchSettings(embedding_model="emb") + await search_mcp_tools("FX", CATALOG, 5, settings, ranker) + await search_mcp_tools("FX", CATALOG, 5, settings, ranker) + assert [len(call) for call in embedder.calls] == [4, 1] + + @pytest.mark.asyncio + async def test_embedding_failure_is_reported_not_raised(self) -> None: + async def failing(texts: Sequence[str]) -> Sequence[Vector]: + raise ValueError("embedding model is down") + + ranker = SemanticToolRanker(embed=failing, embedding_model="emb", index=SemanticTextIndex()) + result = await search_mcp_tools("FX", CATALOG, 5, MCPToolSearchSettings(embedding_model="emb"), ranker) + assert isinstance(result, EmbeddingFailed) + assert "embedding model is down" in result.reason + + +class TestMcpToolSearchSettings: + def test_rejects_out_of_range_values(self) -> None: + from pydantic import ValidationError + + with pytest.raises(ValidationError): + MCPToolSearchSettings(top_k=0) + with pytest.raises(ValidationError): + MCPToolSearchSettings(similarity_threshold=1.5) + + def test_yaml_shape_round_trips(self) -> None: + settings = MCPToolSearchSettings.model_validate( + {"embedding_model": "emb", "top_k": 3, "similarity_threshold": 0.2, "core_tools": ["a", "b"]} + ) + assert settings.core_tools == ("a", "b") + assert settings.model_dump() == { + "embedding_model": "emb", + "top_k": 3, + "similarity_threshold": 0.2, + "core_tools": ("a", "b"), + } + + class TestCoerceTopK: def test_int_passthrough(self) -> None: assert coerce_top_k(3) == 3 @@ -92,10 +249,10 @@ class TestSearchTools: assert len(results) <= 2 def test_empty_query_returns_empty(self) -> None: - assert search_tools("", SAMPLE_TOOLS) == [] + assert search_tools("", SAMPLE_TOOLS) == () def test_no_match_returns_empty(self) -> None: - assert search_tools("xyzzy_nonexistent_zzz", SAMPLE_TOOLS) == [] + assert search_tools("xyzzy_nonexistent_zzz", SAMPLE_TOOLS) == () def test_matches_description_not_just_name(self) -> None: results = search_tools("channel", SAMPLE_TOOLS) @@ -603,6 +760,63 @@ class TestCallToolRestApiVirtualTools: assert result.isError is True assert result.content[0].text == "set agent_search_embedding_model" + def _semantic_request(self, query: str = "FX") -> MagicMock: + return self._make_request({"name": MCP_TOOL_SEARCH_TOOL_NAME, "arguments": {"query": query}}) + + @pytest.mark.asyncio + async def test_mcp_tool_search_ranks_the_callers_catalog_with_the_configured_embedding_model( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "mcp_tool_search", {"embedding_model": "emb", "similarity_threshold": 0.5}) + user_api_key_dict = UserAPIKeyAuth( + api_key="k", team_id="team-1", object_permission=_make_perm(mcp_tool_search_enabled=True) + ) + + async def fake_aembedding(model: str, input: list[str], metadata: dict[str, Any]) -> MagicMock: + assert model == "emb" + assert metadata["user_api_key"] == "k" + assert metadata["user_api_key_team_id"] == "team-1" + response = MagicMock() + response.model_dump.return_value = {"data": [{"embedding": list(FAKE_VECTORS[t])} for t in input]} + return response + + router = MagicMock() + router.aembedding = AsyncMock(side_effect=fake_aembedding) + with ( + patch( # test-quality-ok: the proxy's router is a module global; the handler reaches it the way production does + "litellm.proxy.proxy_server.llm_router", router + ), + patch( # test-quality-ok: the authorized catalog is the seam every virtual tool shares; the ranking under test stays real + "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + new_callable=AsyncMock, + return_value=AggregateToolListing(tools=list(CATALOG), outcomes={}), + ) as mock_list, + ): + result = await self._get_call_fn()(request=self._semantic_request(), user_api_key_dict=user_api_key_dict) + + assert mock_list.await_args.kwargs["user_api_key_auth"] is user_api_key_dict + assert result.isError is False + assert [t["name"] for t in json.loads(result.content[0].text)] == [FX_TOOL.name] + + @pytest.mark.asyncio + async def test_mcp_tool_search_reports_missing_router_as_tool_error(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "mcp_tool_search", {"embedding_model": "emb"}) + user_api_key_dict = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + with patch( # test-quality-ok: the proxy's router is a module global; the handler reaches it the way production does + "litellm.proxy.proxy_server.llm_router", None + ): + result = await self._get_call_fn()(request=self._semantic_request(), user_api_key_dict=user_api_key_dict) + assert result.isError is True + assert "mcp_tool_search.embedding_model" in result.content[0].text + + @pytest.mark.asyncio + async def test_mcp_tool_search_reports_invalid_settings_as_tool_error(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "mcp_tool_search", {"top_k": 0}) + user_api_key_dict = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) + result = await self._get_call_fn()(request=self._semantic_request(), user_api_key_dict=user_api_key_dict) + assert result.isError is True + assert "top_k" in result.content[0].text + @pytest.mark.asyncio async def test_agent_search_requires_flag_enabled(self) -> None: from fastapi import HTTPException diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py index 7ce62fdf648..231626c7eb5 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py @@ -8,7 +8,18 @@ from unittest.mock import AsyncMock, MagicMock import pytest -from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry, GrantMigrationResult +from litellm.constants import REDACTED_BY_LITELM_STRING +from litellm.proxy.agent_endpoints.agent_registry import ( + AgentRegistry, + GrantMigrationResult, + _restore_redacted_litellm_params, + redact_sensitive_agent_litellm_params, +) + +# Obviously-fake stand-ins for a real AWS credential pair (LIT-6736 regression +# fixtures) -- never a real key shape, and must never appear in any response. +SENTINEL_AWS_ACCESS_KEY_ID: Final = "AKIATESTSENTINEL0000" +SENTINEL_AWS_SECRET_ACCESS_KEY: Final = "test-sentinel-do-not-use-secret-value" def _sample_agent_card_params() -> dict: @@ -49,6 +60,7 @@ async def test_update_agent_in_db_clears_static_headers_and_extra_headers_when_o mock_update = AsyncMock(return_value=updated_agent) mock_prisma.db.litellm_agentstable.update = mock_update + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock(return_value=None) # Agent config WITHOUT static_headers or extra_headers (omitted) agent_config = { @@ -95,6 +107,7 @@ async def test_update_agent_in_db_preserves_explicit_static_headers_and_extra_he mock_update = AsyncMock(return_value=updated_agent) mock_prisma.db.litellm_agentstable.update = mock_update + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock(return_value=None) agent_config = { "agent_name": "Updated Agent", @@ -436,6 +449,9 @@ async def test_update_agent_in_db_raises_when_row_deleted_mid_update(): guard the code dereferences None and reports an opaque AttributeError instead of the id.""" registry: Final = AgentRegistry() mock_prisma: Final = MagicMock() + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=SimpleNamespace(litellm_params={}, object_permission_id=None) + ) mock_prisma.db.litellm_agentstable.update = AsyncMock(return_value=None) with pytest.raises(Exception, match="Error updating agent in DB") as exc_info: @@ -485,3 +501,492 @@ async def test_delete_agent_from_db_raises_when_row_already_gone(): await registry.delete_agent_from_db(agent_id="agent-123", prisma_client=mock_prisma) assert str(exc_info.value) == "Error deleting agent from DB: Agent not found, passed agent_id=agent-123" + + +# ---------- LIT-6736: agent litellm_params secret redaction ---------- + + +def test_redact_sensitive_agent_litellm_params_masks_secrets_keeps_the_rest(): + """The sentinel secret must never appear in the redacted output; non-secret + keys (model reference, is_public) must survive untouched.""" + redacted = redact_sensitive_agent_litellm_params( + { + "aws_access_key_id": SENTINEL_AWS_ACCESS_KEY_ID, + "aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, + "model": "bedrock/agentcore/my-agent", + "is_public": True, + } + ) + + assert SENTINEL_AWS_ACCESS_KEY_ID not in json.dumps(redacted) + assert SENTINEL_AWS_SECRET_ACCESS_KEY not in json.dumps(redacted) + assert redacted["aws_access_key_id"] == REDACTED_BY_LITELM_STRING + assert redacted["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert redacted["model"] == "bedrock/agentcore/my-agent" + assert redacted["is_public"] is True + + +def test_redact_sensitive_agent_litellm_params_recurses_into_nested_dicts(): + """A secret nested one level down (e.g. a per-provider sub-config) must + also be redacted, not just top-level keys.""" + redacted = redact_sensitive_agent_litellm_params( + {"provider_config": {"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "region": "us-east-1"}} + ) + + assert SENTINEL_AWS_SECRET_ACCESS_KEY not in json.dumps(redacted) + assert redacted["provider_config"]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert redacted["provider_config"]["region"] == "us-east-1" + + +def test_redact_sensitive_agent_litellm_params_handles_none_and_json_string(): + assert redact_sensitive_agent_litellm_params(None) is None + + serialized = json.dumps({"api_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "model": "gpt-4"}) + redacted = redact_sensitive_agent_litellm_params(serialized) + + assert SENTINEL_AWS_SECRET_ACCESS_KEY not in redacted + assert json.loads(redacted)["api_key"] == REDACTED_BY_LITELM_STRING + assert json.loads(redacted)["model"] == "gpt-4" + + +def test_redact_sensitive_agent_litellm_params_recurses_into_lists_of_dicts(): + """A secret nested inside a list of provider sub-configs (a shape a + non-sensitively-named key can legitimately hold) must also be redacted, + not silently returned as-is.""" + redacted = redact_sensitive_agent_litellm_params( + { + "provider_configs": [ + {"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "region": "us-east-1"}, + {"aws_secret_access_key": "other-" + SENTINEL_AWS_SECRET_ACCESS_KEY, "region": "us-west-2"}, + ] + } + ) + + assert SENTINEL_AWS_SECRET_ACCESS_KEY not in json.dumps(redacted) + assert redacted["provider_configs"][0]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert redacted["provider_configs"][0]["region"] == "us-east-1" + assert redacted["provider_configs"][1]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert redacted["provider_configs"][1]["region"] == "us-west-2" + + +def test_redact_sensitive_agent_litellm_params_redacts_secrets_inside_model_list(): + """The exact shape flagged in review: litellm_params.model_list, where each + entry carries its own nested litellm_params with a provider credential.""" + redacted = redact_sensitive_agent_litellm_params( + { + "model_list": [ + { + "model_name": "gpt-4", + "litellm_params": {"api_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "model": "gpt-4"}, + }, + { + "model_name": "claude", + "litellm_params": { + "aws_secret_access_key": "other-" + SENTINEL_AWS_SECRET_ACCESS_KEY, + "model": "bedrock/claude", + }, + }, + ] + } + ) + + assert SENTINEL_AWS_SECRET_ACCESS_KEY not in json.dumps(redacted) + assert redacted["model_list"][0]["litellm_params"]["api_key"] == REDACTED_BY_LITELM_STRING + assert redacted["model_list"][0]["litellm_params"]["model"] == "gpt-4" + assert redacted["model_list"][1]["litellm_params"]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert redacted["model_list"][1]["litellm_params"]["model"] == "bedrock/claude" + + +def test_restore_redacted_litellm_params_preserves_secret_inside_model_list(): + """The write-side counterpart: a caller editing a model_list entry's own + non-secret field (renaming it) while leaving that same entry's nested + secret masked must not corrupt the stored per-deployment credential. + List entries correspond by position (see the module docstring on + ``_restore_redacted_nested_value``), so this -- the common "edit this + entry, keep its secret" pattern -- must keep working.""" + existing = { + "agent_name": "my-agent", + "model_list": [ + { + "model_name": "gpt-4", + "litellm_params": {"api_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "model": "gpt-4"}, + }, + ], + } + incoming = { + "agent_name": "my-agent-renamed", + "model_list": [ + { + "model_name": "gpt-4-renamed", + "litellm_params": {"api_key": REDACTED_BY_LITELM_STRING, "model": "gpt-4"}, + }, + ], + } + + restored = _restore_redacted_litellm_params(incoming, existing) + + assert SENTINEL_AWS_SECRET_ACCESS_KEY == restored["model_list"][0]["litellm_params"]["api_key"] + assert restored["model_list"][0]["model_name"] == "gpt-4-renamed" + assert restored["agent_name"] == "my-agent-renamed" + + +def test_restore_redacted_litellm_params_matches_list_entries_by_position(): + """Documents the accepted trade-off: a list has no stable per-element + identity in a plain ``dict[str, object]`` schema, so restoration matches + entries by index, the same correspondence every other part of this merge + (and the endpoints' full-replace-on-PUT semantics) already assumes. If a + caller both reorders the list AND echoes back a masked marker in the same + request, a credential can end up attached to a different logical entry. + That is a known, narrow limitation -- not a leak between different + agents or tenants, since it only reshuffles one agent's own stored + values -- and this test pins the current, deliberate behavior rather + than asserting it away.""" + existing = { + "model_list": [ + {"model_name": "gpt-4", "litellm_params": {"api_key": SENTINEL_AWS_SECRET_ACCESS_KEY}}, + {"model_name": "claude", "litellm_params": {"api_key": "other-" + SENTINEL_AWS_SECRET_ACCESS_KEY}}, + ], + } + incoming = { + "model_list": [ + # Same index (0) now holds what used to be at index 1's entry. + {"model_name": "claude", "litellm_params": {"api_key": REDACTED_BY_LITELM_STRING}}, + ], + } + + restored = _restore_redacted_litellm_params(incoming, existing) + + assert restored["model_list"][0]["litellm_params"]["api_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY + + +def test_restore_redacted_litellm_params_recovers_a_whole_subtree_collapsed_by_the_depth_cap(): + """Past the read-side recursion depth cap, a whole nested subtree is + collapsed to the flat REDACTED_BY_LITELM marker rather than a dict/list. + If the caller echoes that flat marker back unchanged, the whole + subtree -- not just the literal marker string -- must be restored.""" + existing_subtree = {"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, "region": "us-east-1"} + incoming = {"provider_config": REDACTED_BY_LITELM_STRING} + existing = {"provider_config": existing_subtree} + + restored = _restore_redacted_litellm_params(incoming, existing) + + assert restored["provider_config"] == existing_subtree + + +def test_redact_sensitive_agent_litellm_params_does_not_reinterpret_plain_string_values_as_json(): + """A plain non-JSON string value (most string leaves) must pass through + unchanged rather than failing to parse and getting redacted.""" + redacted = redact_sensitive_agent_litellm_params({"model": "bedrock/agentcore/my-agent", "is_public": True}) + + assert redacted["model"] == "bedrock/agentcore/my-agent" + assert redacted["is_public"] is True + + +@pytest.mark.asyncio +async def test_add_agent_to_db_drops_a_sentinel_value_instead_of_storing_the_placeholder(): + """A create has nothing stored to restore behind a redaction marker, so a + sensitive key submitted as the literal marker is dropped rather than + persisted as the placeholder string itself.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + created_agent = MagicMock() + created_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + created_agent.object_permission = None + mock_create = AsyncMock(return_value=created_agent) + mock_prisma.db.litellm_agentstable.create = mock_create + + await registry.add_agent_to_db( + agent={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": { + "aws_secret_access_key": REDACTED_BY_LITELM_STRING, + "model": "bedrock/agentcore/my-agent", + }, + }, + prisma_client=mock_prisma, + created_by="test-user", + ) + + stored_params: Final = json.loads(mock_create.call_args.kwargs["data"]["litellm_params"]) + assert "aws_secret_access_key" not in stored_params + assert stored_params["model"] == "bedrock/agentcore/my-agent" + + +@pytest.mark.asyncio +async def test_update_agent_in_db_preserves_secret_when_echoed_back_redacted(): + """PUT round-trips the GET response, which shows the secret redacted. Saving + an unrelated field change must not overwrite the real stored credential + with the redaction marker.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=SimpleNamespace( + litellm_params={ + "aws_access_key_id": SENTINEL_AWS_ACCESS_KEY_ID, + "aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, + "model": "bedrock/agentcore/my-agent", + }, + object_permission_id=None, + ) + ) + updated_agent = MagicMock() + updated_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Renamed Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + updated_agent.object_permission = None + mock_update = AsyncMock(return_value=updated_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.update_agent_in_db( + agent_id="agent-123", + agent={ + "agent_name": "Renamed Agent", + "agent_card_params": _sample_agent_card_params(), + # The UI round-tripped the redacted secret and the untouched + # access key id verbatim; only agent_name actually changed. + "litellm_params": { + "aws_access_key_id": SENTINEL_AWS_ACCESS_KEY_ID, + "aws_secret_access_key": REDACTED_BY_LITELM_STRING, + "model": "bedrock/agentcore/my-agent", + }, + }, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"]) + assert stored_params["aws_secret_access_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY + assert stored_params["aws_access_key_id"] == SENTINEL_AWS_ACCESS_KEY_ID + assert stored_params["model"] == "bedrock/agentcore/my-agent" + + +@pytest.mark.asyncio +async def test_update_agent_in_db_preserves_secret_when_key_omitted_entirely(): + """Omitting the sensitive key altogether must fall back to the stored + value too, not just an explicit redaction-marker round-trip.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=SimpleNamespace( + litellm_params={"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY}, + object_permission_id=None, + ) + ) + updated_agent = MagicMock() + updated_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + updated_agent.object_permission = None + mock_update = AsyncMock(return_value=updated_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.update_agent_in_db( + agent_id="agent-123", + agent={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {"model": "bedrock/agentcore/my-agent"}, + }, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"]) + assert stored_params["aws_secret_access_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY + + +@pytest.mark.asyncio +async def test_update_agent_in_db_preserves_secret_nested_under_a_non_sensitive_key(): + """A secret nested inside a dict held by a non-sensitively-named key + (e.g. a per-provider sub-config) must also survive an echoed-back + redaction marker, not just top-level secret keys.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=SimpleNamespace( + litellm_params={ + "provider_config": { + "aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, + "region": "us-east-1", + } + }, + object_permission_id=None, + ) + ) + updated_agent = MagicMock() + updated_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + updated_agent.object_permission = None + mock_update = AsyncMock(return_value=updated_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.update_agent_in_db( + agent_id="agent-123", + agent={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": { + # The GET response redacted the nested secret; the caller + # round-trips it verbatim while changing nothing. + "provider_config": { + "aws_secret_access_key": REDACTED_BY_LITELM_STRING, + "region": "us-west-2", + } + }, + }, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"]) + assert stored_params["provider_config"]["aws_secret_access_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY + assert stored_params["provider_config"]["region"] == "us-west-2" + + +@pytest.mark.asyncio +async def test_update_agent_in_db_clears_secret_on_explicit_empty_value(): + """An explicit empty string is a deliberate clear, distinct from an omitted + key or the redaction marker, and must actually clear the stored secret.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=SimpleNamespace( + litellm_params={"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY}, + object_permission_id=None, + ) + ) + updated_agent = MagicMock() + updated_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + updated_agent.object_permission = None + mock_update = AsyncMock(return_value=updated_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.update_agent_in_db( + agent_id="agent-123", + agent={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {"aws_secret_access_key": ""}, + }, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"]) + assert stored_params["aws_secret_access_key"] == "" + + +@pytest.mark.asyncio +async def test_patch_agent_in_db_preserves_secret_when_litellm_params_omitted(): + """A PATCH that only renames the agent must not touch (let alone drop) the + stored litellm_params secret.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value={ + "agent_id": "agent-123", + "agent_name": "Old Name", + "litellm_params": {"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY}, + "object_permission_id": None, + } + ) + patched_agent = MagicMock() + patched_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "New Name", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {"aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY}, + "object_permission": None, + } + patched_agent.object_permission = None + mock_update = AsyncMock(return_value=patched_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.patch_agent_in_db( + agent_id="agent-123", + agent={"agent_name": "New Name"}, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + update_data: Final = mock_update.call_args.kwargs["data"] + assert "litellm_params" not in update_data + + +@pytest.mark.asyncio +async def test_patch_agent_in_db_preserves_secret_when_echoed_back_redacted(): + """A PATCH that includes litellm_params (e.g. to flip an unrelated flag) + with the secret round-tripped as the redaction marker must not clobber + the stored credential.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value={ + "agent_id": "agent-123", + "agent_name": "Test Agent", + "litellm_params": { + "aws_secret_access_key": SENTINEL_AWS_SECRET_ACCESS_KEY, + "is_public": False, + }, + "object_permission_id": None, + } + ) + patched_agent = MagicMock() + patched_agent.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + } + patched_agent.object_permission = None + mock_update = AsyncMock(return_value=patched_agent) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.patch_agent_in_db( + agent_id="agent-123", + agent={ + "litellm_params": { + "aws_secret_access_key": REDACTED_BY_LITELM_STRING, + "is_public": True, + } + }, + prisma_client=mock_prisma, + updated_by="test-user", + ) + + stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"]) + assert stored_params["aws_secret_access_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY + assert stored_params["is_public"] is True diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py index 3fb09076e5f..daca244c0a1 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_search.py @@ -16,13 +16,12 @@ from litellm.proxy.agent_endpoints.agent_search import ( AgentSearchHits, AgentSearchIndex, AgentSearchNotConfigured, - Vector, agent_search_text, - cosine_similarity, search_agents, ) from litellm.proxy.agent_endpoints.auth.agent_permission_handler import RestrictedAgentAccess from litellm.proxy.agent_endpoints.endpoints import router, user_api_key_auth +from litellm.proxy.common_utils.semantic_text_index import Vector, cosine_similarity from litellm.types.agents import AgentResponse CALLER: Final = UserAPIKeyAuth(api_key="hashed-caller-key", team_id="team-1", user_id="user-1") diff --git a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py index ea196bda529..a78b3238a9a 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py @@ -4,6 +4,7 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient +from litellm.constants import REDACTED_BY_LITELM_STRING from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.agent_endpoints import endpoints as agent_endpoints from litellm.proxy.agent_endpoints.auth.agent_permission_handler import ( @@ -484,11 +485,15 @@ class TestAgentRBACInternalUserViewOnly: assert resp.status_code == 403 +SENTINEL_AGENT_API_KEY = "sk-test-sentinel-do-not-use" + + class TestAgentRBACProxyAdminViewOnly: """Read-only proxy admins go through the object-permission scoped branch on GET /v1/agents (the admin fast path stays full PROXY_ADMIN only, so viewers - cannot fan out health checks beyond their allowlist), and secret unredaction - also stays gated on full PROXY_ADMIN.""" + cannot fan out health checks beyond their allowlist). litellm_params + secrets are redacted for every caller, admin included (LIT-6736); only the + virtual-key/header visibility stays gated on full PROXY_ADMIN.""" @pytest.fixture(autouse=True) def _setup(self, monkeypatch): @@ -501,7 +506,7 @@ class TestAgentRBACProxyAdminViewOnly: agent_id=f"agent-{index}", agent_name=f"Agent {index}", agent_card_params=_sample_agent_card_params(), - litellm_params={"api_key": "sk-super-secret-agent-key"}, + litellm_params={"api_key": SENTINEL_AGENT_API_KEY}, ) for index in (1, 2) ] @@ -544,7 +549,7 @@ class TestAgentRBACProxyAdminViewOnly: def test_should_still_redact_secrets_for_view_only_admin(self): """An unrestricted viewer sees the same agents as an admin but with keys - stripped and litellm_params masked.""" + stripped; litellm_params secrets never appear in either response.""" self.allowed_agents_spy.return_value = UnrestrictedAgentAccess() viewer_resp = self._list_agents(self.viewer_client) admin_resp = self._list_agents(self.admin_client) @@ -553,14 +558,12 @@ class TestAgentRBACProxyAdminViewOnly: viewer_by_id = {agent["agent_id"]: agent for agent in viewer_resp.json()} assert set(viewer_by_id) == {"agent-1", "agent-2"} assert viewer_by_id["agent-1"]["keys"] is None - assert "sk-super-secret-agent-key" not in viewer_resp.text + assert SENTINEL_AGENT_API_KEY not in viewer_resp.text admin_by_id = {agent["agent_id"]: agent for agent in admin_resp.json()} assert admin_by_id["agent-1"]["keys"][0]["token"] == "hash-aaa" - assert ( - admin_by_id["agent-1"]["litellm_params"]["api_key"] - == "sk-super-secret-agent-key" - ) + assert SENTINEL_AGENT_API_KEY not in admin_resp.text + assert admin_by_id["agent-1"]["litellm_params"]["api_key"] == REDACTED_BY_LITELM_STRING class TestAgentRBACProxyAdmin: @@ -616,6 +619,109 @@ class TestAgentRBACProxyAdmin: # Security scheme is the LiteLLM scheme. assert "LiteLLMKey" in stored_card["securitySchemes"] + def test_create_agent_response_never_echoes_secret(self): + """LIT-6736: POST /v1/agents must not echo the stored secret back, even + though it's the caller's own value and even for a proxy admin.""" + with patch("litellm.proxy.proxy_server.prisma_client"): # test-quality-ok: proxy_server module global is the endpoint's only injection point + self.mock_registry.get_agent_by_name = MagicMock(return_value=None) + self.mock_registry.add_agent_to_db = AsyncMock( + return_value=AgentResponse( + agent_id="agent-123", + agent_name="Test Agent", + agent_card_params=_sample_agent_card_params(), + litellm_params={ + "aws_secret_access_key": SENTINEL_AGENT_API_KEY, + "model": "bedrock/agentcore/my-agent", + }, + ) + ) + self.mock_registry.register_agent = MagicMock() + + resp = self.admin_client.post( + "/v1/agents", + json={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": { + "aws_secret_access_key": SENTINEL_AGENT_API_KEY, + "model": "bedrock/agentcore/my-agent", + }, + }, + headers={"Authorization": "Bearer k"}, + ) + + assert resp.status_code == 200 + assert SENTINEL_AGENT_API_KEY not in resp.text + body = resp.json() + assert body["litellm_params"]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + assert body["litellm_params"]["model"] == "bedrock/agentcore/my-agent" + + def test_update_agent_response_never_echoes_secret(self): + """LIT-6736: PUT /v1/agents/{id} must not echo the stored secret back.""" + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: # test-quality-ok: proxy_server module global is the endpoint's only injection point + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value={ + "agent_id": "agent-123", + "agent_name": "Existing Agent", + "agent_card_params": _sample_agent_card_params(), + } + ) + self.mock_registry.update_agent_in_db = AsyncMock( + return_value=AgentResponse( + agent_id="agent-123", + agent_name="Test Agent", + agent_card_params=_sample_agent_card_params(), + litellm_params={"aws_secret_access_key": SENTINEL_AGENT_API_KEY}, + ) + ) + self.mock_registry.deregister_agent = MagicMock() + self.mock_registry.register_agent = MagicMock() + + resp = self.admin_client.put( + "/v1/agents/agent-123", + json={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {"aws_secret_access_key": REDACTED_BY_LITELM_STRING}, + }, + headers={"Authorization": "Bearer k"}, + ) + + assert resp.status_code == 200 + assert SENTINEL_AGENT_API_KEY not in resp.text + assert resp.json()["litellm_params"]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + + def test_patch_agent_response_never_echoes_secret(self): + """LIT-6736: PATCH /v1/agents/{id} must not echo the stored secret back.""" + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: # test-quality-ok: proxy_server module global is the endpoint's only injection point + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value={ + "agent_id": "agent-123", + "agent_name": "Existing Agent", + "agent_card_params": _sample_agent_card_params(), + } + ) + self.mock_registry.patch_agent_in_db = AsyncMock( + return_value=AgentResponse( + agent_id="agent-123", + agent_name="Renamed Agent", + agent_card_params=_sample_agent_card_params(), + litellm_params={"aws_secret_access_key": SENTINEL_AGENT_API_KEY}, + ) + ) + self.mock_registry.deregister_agent = MagicMock() + self.mock_registry.register_agent = MagicMock() + + resp = self.admin_client.patch( + "/v1/agents/agent-123", + json={"agent_name": "Renamed Agent"}, + headers={"Authorization": "Bearer k"}, + ) + + assert resp.status_code == 200 + assert SENTINEL_AGENT_API_KEY not in resp.text + assert resp.json()["litellm_params"]["aws_secret_access_key"] == REDACTED_BY_LITELM_STRING + def test_should_allow_admin_to_delete_agent(self): existing = { "agent_id": "agent-123", diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 3dea89ed67b..ffe3e0fee12 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -7448,3 +7448,64 @@ async def test_delete_cache_key_object_is_best_effort_when_the_cache_backend_fai healthy_cache.delete_cache.assert_called_once_with(key=hashed_token) healthy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache.assert_awaited_once_with(key=hashed_token) assert caplog.records == [], "a healthy eviction must stay silent, and must still reach both caches" + + +# --------------------------------------------------------------------------- +# Budget-exceeded error text must not carry a raw virtual key (LIT-5909) +# --------------------------------------------------------------------------- + + +class _BudgetAlertRecorder: + async def budget_alerts(self, type, user_info): + return None + + +async def _run_key_budget_check(key_name: str) -> str: + """Drive the over-budget key path and return the raised message.""" + valid_token = UserAPIKeyAuth( + token="hashed-token", + key_name=key_name, + key_alias="prod-key", + spend=10.0, + max_budget=1.0, + ) + with pytest.raises(litellm.BudgetExceededError, match="Budget has been exceeded") as exc_info: + await _virtual_key_max_budget_check( + valid_token=valid_token, + proxy_logging_obj=_BudgetAlertRecorder(), + ) + await asyncio.sleep(0) + return exc_info.value.message + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_name", + [ + "sk-mx5ous1o9Iezz5fj3pkLuA", + "my-company-key-2026", + "sk-...5LuA-but-longer", + # /key/generate takes a custom key ending in an escape sequence, and this + # message reaches a terminal and a log viewer + "sk-...\x1b[2J", + "sk-...a\x9bm", + ], +) +async def test_key_budget_error_does_not_carry_a_raw_key_name(key_name): + """key_name is written masked, but the column has no enforced shape (a direct DB + write bypasses abbreviate_api_key) and this message is returned to the caller.""" + message = await _run_key_budget_check(key_name) + assert key_name not in message + assert "Key=prod-key Current cost" in message + + +@pytest.mark.asyncio +@pytest.mark.parametrize("key_name", ["sk-...5LuA", "sk-...", "sk-...ke.!", "sk-...café"]) +async def test_key_budget_error_keeps_the_masked_key_name(key_name): + """The masked form is the whole point of naming the key, so it must survive. + + abbreviate_api_key takes the last four characters of the key verbatim, and a + custom key may end in punctuation or a non-ASCII character, so those masked + names are just as valid as the alphanumeric ones.""" + message = await _run_key_budget_check(key_name) + assert f"Key=prod-key ({key_name}) Current cost" in message diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 90be51cfa5b..21e0b83791f 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -9,6 +9,7 @@ from prisma import errors as prisma_errors from prisma.engine.errors import ( BinaryNotFoundError, EngineConnectionError, + EngineRequestError, MismatchedVersionsError, ) from prisma.errors import ( @@ -32,6 +33,12 @@ from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_exception_handler import UserAPIKeyAuthExceptionHandler +class _EngineHttp500: + """The response half of an EngineRequestError: the query engine answered a request with HTTP 500.""" + + status = 500 + + @pytest.mark.asyncio @pytest.mark.parametrize( "db_error", @@ -113,6 +120,90 @@ async def test_handle_authentication_error_permanent_fault_gets_no_fallback_iden assert exc_info.value.code == str(status.HTTP_503_SERVICE_UNAVAILABLE) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "prisma_error", + [ + pytest.param(BinaryNotFoundError("query engine binary not found"), id="BinaryNotFoundError"), + pytest.param(MismatchedVersionsError(expected="1", got="2"), id="MismatchedVersionsError"), + pytest.param(EngineRequestError(_EngineHttp500(), "query engine crashed"), id="EngineRequestError"), + pytest.param(PrismaError(), id="bare_PrismaError"), + ], +) +async def test_handle_authentication_error_permanent_fault_503_is_not_worded_as_transient(prisma_error): + """The 503 for a fault that never heals must not say the database is + "temporarily unreachable" and ask the caller to retry. The status is right + (the service is at fault) but that wording sends the operator to wait out an + outage that is not one, so the message has to say retrying will not help and + name the engine fault.""" + handler = UserAPIKeyAuthExceptionHandler() + + with patch( # test-quality-ok: the handler reads general_settings off the proxy module, no injection seam + "litellm.proxy.proxy_server.general_settings", {"allow_requests_on_db_unavailable": False} + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error(prisma_error, MagicMock(), {}, "/test", None, "test-key") + + assert exc_info.value.code == str(status.HTTP_503_SERVICE_UNAVAILABLE) + assert exc_info.value.type == ProxyErrorTypes.no_db_connection + assert "temporarily unreachable" not in exc_info.value.message + assert "retry shortly" not in exc_info.value.message.lower() + assert "will not clear by retrying" in exc_info.value.message + assert type(prisma_error).__name__ in exc_info.value.message + + +@pytest.mark.asyncio +async def test_handle_authentication_error_transport_error_raised_over_a_permanent_fault_names_the_fault(): + """A reconnect attempt that fails because the engine binary is missing surfaces as a transport + error with the BinaryNotFoundError as __context__. The response must describe the binary, which is + what keeps the database down, rather than promise the connection will come back.""" + try: + raise BinaryNotFoundError("query engine binary not found") + except BinaryNotFoundError: + try: + raise httpx.ConnectError("All connection attempts failed") + except httpx.ConnectError as surfaced: + transport_over_fault = surfaced + handler = UserAPIKeyAuthExceptionHandler() + + with patch( # test-quality-ok: the handler reads general_settings off the proxy module, no injection seam + "litellm.proxy.proxy_server.general_settings", {"allow_requests_on_db_unavailable": False} + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error(transport_over_fault, MagicMock(), {}, "/test", None, "k") + + assert exc_info.value.code == str(status.HTTP_503_SERVICE_UNAVAILABLE) + assert "temporarily unreachable" not in exc_info.value.message + assert "BinaryNotFoundError" in exc_info.value.message + assert "will not clear by retrying" in exc_info.value.message + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "db_error", + [ + pytest.param(httpx.ConnectError("All connection attempts failed"), id="ConnectError"), + pytest.param(EngineConnectionError(), id="EngineConnectionError"), + pytest.param(PrismaError("can't reach database server"), id="P1001_text"), + ], +) +async def test_handle_authentication_error_transient_outage_503_keeps_retry_wording(db_error): + """A genuine outage is expected to come back, so its 503 keeps telling the + caller the database is temporarily unreachable and to retry.""" + handler = UserAPIKeyAuthExceptionHandler() + + with patch( # test-quality-ok: the handler reads general_settings off the proxy module, no injection seam + "litellm.proxy.proxy_server.general_settings", {"allow_requests_on_db_unavailable": False} + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error(db_error, MagicMock(), {}, "/test", None, "test-key") + + assert exc_info.value.code == str(status.HTTP_503_SERVICE_UNAVAILABLE) + assert exc_info.value.message == ( + "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( "prisma_error", diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 9301176f3ed..f513f397b64 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -3169,7 +3169,7 @@ class TestIsRequestBodySafeChecksBracketNotationMetadata: class TestHasUserSetupSso: - """_has_user_setup_sso must treat SAML IdP metadata as SSO configured. + """has_user_setup_sso must treat SAML IdP metadata as SSO configured. Regression: UI discovery used this helper for sso_configured, but it only checked OAuth client IDs, so SAML-only setups left the login button gray. @@ -3187,29 +3187,167 @@ class TestHasUserSetupSso: monkeypatch.delenv(key, raising=False) def test_false_when_no_sso_env(self): - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso - assert _has_user_setup_sso() is False + assert has_user_setup_sso() is False def test_true_for_oauth_client_ids(self, monkeypatch): - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso monkeypatch.setenv("GOOGLE_CLIENT_ID", "google-client") - assert _has_user_setup_sso() is True + assert has_user_setup_sso() is True def test_true_for_saml_metadata_url(self, monkeypatch): - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso monkeypatch.setenv( "SAML_IDP_METADATA_URL", "https://idp.example.com/metadata.xml" ) - assert _has_user_setup_sso() is True + assert has_user_setup_sso() is True def test_true_for_saml_metadata_xml(self, monkeypatch): - from litellm.proxy.auth.auth_utils import _has_user_setup_sso + from litellm.proxy.auth.auth_utils import has_user_setup_sso monkeypatch.setenv("SAML_IDP_METADATA_XML", "") - assert _has_user_setup_sso() is True + assert has_user_setup_sso() is True + + +class TestIsSsoProviderFullyConfigured: + """A lone client id must not read as ready: `has_user_setup_sso()` only + checks the client id (correct for a UI-discovery "show the login button" + decision), but a gate that BLOCKS the password fallback needs every + companion setting the provider requires, or an incomplete setup locks + every admin out with no working login path at all.""" + + @pytest.fixture(autouse=True) + def _clear_sso_env(self, monkeypatch): + for key in ( + "GOOGLE_CLIENT_ID", + "GOOGLE_CLIENT_SECRET", + "MICROSOFT_CLIENT_ID", + "MICROSOFT_CLIENT_SECRET", + "MICROSOFT_TENANT", + "GENERIC_CLIENT_ID", + "GENERIC_CLIENT_SECRET", + "GENERIC_AUTHORIZATION_ENDPOINT", + "GENERIC_TOKEN_ENDPOINT", + "GENERIC_USERINFO_ENDPOINT", + "SAML_IDP_METADATA_URL", + "SAML_IDP_METADATA_XML", + ): + monkeypatch.delenv(key, raising=False) + + def test_false_when_nothing_configured(self): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + assert is_sso_provider_fully_configured() is False + + def test_google_client_id_alone_is_not_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GOOGLE_CLIENT_ID", "google-client") + assert is_sso_provider_fully_configured() is False + + def test_google_with_secret_is_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GOOGLE_CLIENT_ID", "google-client") + monkeypatch.setenv("GOOGLE_CLIENT_SECRET", "google-secret") + assert is_sso_provider_fully_configured() is True + + def test_microsoft_client_id_alone_is_not_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("MICROSOFT_CLIENT_ID", "ms-client") + assert is_sso_provider_fully_configured() is False + + def test_microsoft_missing_tenant_is_not_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("MICROSOFT_CLIENT_ID", "ms-client") + monkeypatch.setenv("MICROSOFT_CLIENT_SECRET", "ms-secret") + assert is_sso_provider_fully_configured() is False + + def test_microsoft_with_secret_and_tenant_is_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("MICROSOFT_CLIENT_ID", "ms-client") + monkeypatch.setenv("MICROSOFT_CLIENT_SECRET", "ms-secret") + monkeypatch.setenv("MICROSOFT_TENANT", "ms-tenant") + assert is_sso_provider_fully_configured() is True + + def test_generic_client_id_alone_is_not_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GENERIC_CLIENT_ID", "generic-client") + assert is_sso_provider_fully_configured() is False + + def test_generic_missing_one_endpoint_is_not_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GENERIC_CLIENT_ID", "generic-client") + monkeypatch.setenv("GENERIC_CLIENT_SECRET", "generic-secret") + monkeypatch.setenv("GENERIC_AUTHORIZATION_ENDPOINT", "https://idp.example.com/authorize") + monkeypatch.setenv("GENERIC_TOKEN_ENDPOINT", "https://idp.example.com/token") + # GENERIC_USERINFO_ENDPOINT deliberately left unset. + assert is_sso_provider_fully_configured() is False + + def test_generic_with_every_endpoint_is_ready(self, monkeypatch): + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GENERIC_CLIENT_ID", "generic-client") + monkeypatch.setenv("GENERIC_CLIENT_SECRET", "generic-secret") + monkeypatch.setenv("GENERIC_AUTHORIZATION_ENDPOINT", "https://idp.example.com/authorize") + monkeypatch.setenv("GENERIC_TOKEN_ENDPOINT", "https://idp.example.com/token") + monkeypatch.setenv("GENERIC_USERINFO_ENDPOINT", "https://idp.example.com/userinfo") + assert is_sso_provider_fully_configured() is True + + def test_saml_metadata_url_is_ready_when_runtime_installed(self, monkeypatch): + from litellm.proxy.auth import auth_utils + + monkeypatch.setenv("SAML_IDP_METADATA_URL", "https://idp.example.com/metadata.xml") + monkeypatch.setattr(auth_utils.importlib.util, "find_spec", lambda name: object()) + assert auth_utils.is_sso_provider_fully_configured() is True + + def test_saml_metadata_url_is_not_ready_without_runtime(self, monkeypatch): + """Regression: python3-saml (``onelogin.saml2``) is an optional + dependency; SAMLAuthHandler fails closed on every request when it is + not installed, so IdP metadata alone must not read as ready.""" + from litellm.proxy.auth import auth_utils + + monkeypatch.setenv("SAML_IDP_METADATA_URL", "https://idp.example.com/metadata.xml") + monkeypatch.setattr(auth_utils.importlib.util, "find_spec", lambda name: None) + assert auth_utils.is_sso_provider_fully_configured() is False + + def test_saml_check_does_not_raise_when_package_entirely_absent(self, monkeypatch): + """Regression: `importlib.util.find_spec("onelogin.saml2.auth")` + raises ModuleNotFoundError (not merely returns None) when the + TOP-LEVEL `onelogin` package is not installed at all, which is + exactly the real-world "optional extra not installed" case. If the + gate does not catch this, every password login 500s instead of + falling back, on a deployment that configured SAML metadata but + skipped the extra.""" + from litellm.proxy.auth import auth_utils + + def _raise(name: str): + raise ModuleNotFoundError("No module named 'onelogin'") + + monkeypatch.setenv("SAML_IDP_METADATA_URL", "https://idp.example.com/metadata.xml") + monkeypatch.setattr(auth_utils.importlib.util, "find_spec", _raise) + assert auth_utils.is_sso_provider_fully_configured() is False + + def test_incomplete_earlier_provider_does_not_mask_a_ready_later_one(self, monkeypatch): + """Regression: a stray GOOGLE_CLIENT_ID with no secret (e.g. a + leftover from a migration) must not stop the check from reaching a + fully configured Microsoft provider set alongside it — every + provider is evaluated independently, not in a first-match order.""" + from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured + + monkeypatch.setenv("GOOGLE_CLIENT_ID", "google-client") + monkeypatch.setenv("MICROSOFT_CLIENT_ID", "ms-client") + monkeypatch.setenv("MICROSOFT_CLIENT_SECRET", "ms-secret") + monkeypatch.setenv("MICROSOFT_TENANT", "ms-tenant") + assert is_sso_provider_fully_configured() is True class TestIsRequestBodySafeBlocksAwsIdentitySelectors: diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index 1c66acf8678..8d93d801bfd 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -6,6 +6,7 @@ to login_utils.py for better reusability. """ import os +from contextlib import ExitStack from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -598,3 +599,203 @@ class TestEncodeUiSessionJwt: request.cookies = {"token": token} with patch("litellm.proxy.proxy_server.master_key", "sk-master-for-tests"): assert _user_id_from_session_cookie(request) == "cornell-user" + + +def _patch_sso_configured(stack: ExitStack, *, configured: bool) -> None: + stack.enter_context( + patch( # test-quality-ok: no HTTP boundary here; same internal the pre-existing tests above already mock + "litellm.proxy.auth.login_utils.is_sso_provider_fully_configured", return_value=configured + ) + ) + + +def _patch_successful_admin_login_deps(stack: ExitStack) -> None: + """The collaborators a real admin login exercises past the SSO gate: + generating the UI session key, syncing the admin role, and reading the + experimental-login flag. Shared so the two "still allowed" tests below + don't each repeat the same three-mock wiring.""" + stack.enter_context( + patch( # test-quality-ok: internal orchestration, no HTTP boundary; matches pre-existing tests + "litellm.proxy.auth.login_utils.generate_key_helper_fn", + new_callable=AsyncMock, + return_value={"token": "test-token", "user_id": LITELLM_PROXY_ADMIN_NAME}, + ) + ) + stack.enter_context( + patch( # test-quality-ok: internal orchestration, no HTTP boundary; matches pre-existing tests + "litellm.proxy.auth.login_utils.user_update", + new_callable=AsyncMock, + return_value=None, + ) + ) + stack.enter_context( + patch( # test-quality-ok: internal orchestration, no HTTP boundary; matches pre-existing tests + "litellm.proxy.auth.login_utils.get_secret_bool", + return_value=False, + ) + ) + + +class TestDisablePasswordLoginWhenSSOEnabled: + """`disable_password_login_when_sso_enabled` must reject every + username/password login attempt (including the UI_USERNAME/UI_PASSWORD + admin fallback) once SSO is configured, so SSO becomes the only way to + reach the Admin UI. It must not affect logins when SSO is unconfigured, + so admins can never lock themselves out with no SSO to fall back to.""" + + @pytest.mark.asyncio + async def test_rejects_correct_admin_credentials_when_sso_configured(self): + master_key = "sk-1234" + ui_username = "admin" + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + with patch.dict(os.environ, {"UI_USERNAME": ui_username, "UI_PASSWORD": master_key}): + with ExitStack() as stack: + _patch_sso_configured(stack, configured=True) + with pytest.raises(ProxyException) as exc_info: + await authenticate_user( + username=ui_username, + password=master_key, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={"disable_password_login_when_sso_enabled": True}, + ) + + assert exc_info.value.type == ProxyErrorTypes.auth_error + assert exc_info.value.code == "403" + # The credential comparison must never even run. + mock_prisma_client.db.litellm_usertable.find_first.assert_not_called() + + @pytest.mark.asyncio + async def test_rejects_correct_db_user_credentials_when_sso_configured(self): + master_key = "sk-1234" + user_email = "test@example.com" + password = "correct-password" + + mock_user = LiteLLM_UserTable( + user_id="test-user-123", + user_email=user_email, + password=hash_token(token=password), + user_role=LitellmUserRoles.INTERNAL_USER, + ) + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=mock_user) + + with patch.dict(os.environ, {"UI_USERNAME": "admin", "UI_PASSWORD": "unrelated"}): + with ExitStack() as stack: + _patch_sso_configured(stack, configured=True) + with pytest.raises(ProxyException) as exc_info: + await authenticate_user( + username=user_email, + password=password, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={"disable_password_login_when_sso_enabled": True}, + ) + + assert exc_info.value.code == "403" + mock_prisma_client.db.litellm_usertable.find_first.assert_not_called() + + @pytest.mark.asyncio + async def test_allows_password_login_when_setting_enabled_but_sso_not_configured(self): + """The setting alone must not lock out an admin who has not actually + configured SSO — there would be no fallback left.""" + master_key = "sk-1234" + ui_username = "admin" + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + with patch.dict( + os.environ, + { + "UI_USERNAME": ui_username, + "UI_PASSWORD": master_key, + "DATABASE_URL": "postgresql://test:test@localhost/test", + }, + clear=True, + ): + with ExitStack() as stack: + _patch_sso_configured(stack, configured=False) + _patch_successful_admin_login_deps(stack) + result = await authenticate_user( + username=ui_username, + password=master_key, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={"disable_password_login_when_sso_enabled": True}, + ) + + assert isinstance(result, LoginResult) + assert result.user_id == LITELLM_PROXY_ADMIN_NAME + + @pytest.mark.asyncio + async def test_allows_password_login_when_sso_env_is_incomplete(self): + """Regression: a lone MICROSOFT_CLIENT_ID with no client secret or + tenant makes has_user_setup_sso() True, but a real SSO sign-in would + fail. The gate must read the real env (no is_sso_provider_fully_configured + mock here) and still let password login through, or an admin who set + one env var by mistake is locked out with no way in.""" + master_key = "sk-1234" + ui_username = "admin" + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + with patch.dict( + os.environ, + { + "UI_USERNAME": ui_username, + "UI_PASSWORD": master_key, + "DATABASE_URL": "postgresql://test:test@localhost/test", + "MICROSOFT_CLIENT_ID": "ms-client-id-only", + }, + clear=True, + ): + with ExitStack() as stack: + _patch_successful_admin_login_deps(stack) + result = await authenticate_user( + username=ui_username, + password=master_key, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={"disable_password_login_when_sso_enabled": True}, + ) + + assert isinstance(result, LoginResult) + assert result.user_id == LITELLM_PROXY_ADMIN_NAME + + @pytest.mark.asyncio + async def test_allows_password_login_when_sso_configured_but_setting_not_enabled(self): + """SSO being configured must not, by itself, disable the password + fallback: the setting is opt-in.""" + master_key = "sk-1234" + ui_username = "admin" + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) + + with patch.dict( + os.environ, + { + "UI_USERNAME": ui_username, + "UI_PASSWORD": master_key, + "DATABASE_URL": "postgresql://test:test@localhost/test", + }, + clear=True, + ): + with ExitStack() as stack: + _patch_sso_configured(stack, configured=True) + _patch_successful_admin_login_deps(stack) + result = await authenticate_user( + username=ui_username, + password=master_key, + master_key=master_key, + prisma_client=mock_prisma_client, + general_settings={}, + ) + + assert isinstance(result, LoginResult) + assert result.user_id == LITELLM_PROXY_ADMIN_NAME diff --git a/tests/test_litellm/proxy/auth/test_onboarding.py b/tests/test_litellm/proxy/auth/test_onboarding.py index d55a5472af1..524b655b465 100644 --- a/tests/test_litellm/proxy/auth/test_onboarding.py +++ b/tests/test_litellm/proxy/auth/test_onboarding.py @@ -231,7 +231,7 @@ async def test_claim_token_rejects_already_used_link(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with patch("litellm.proxy.proxy_server.prisma_client", prisma): @@ -254,7 +254,7 @@ async def test_claim_token_rejects_expired_link(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with patch("litellm.proxy.proxy_server.prisma_client", prisma): @@ -275,7 +275,7 @@ async def test_claim_token_rejects_mismatched_user_id(): data = InvitationClaim( invitation_link="invite-abc", user_id="wrong-user", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with patch("litellm.proxy.proxy_server.prisma_client", prisma): @@ -296,7 +296,7 @@ async def test_claim_token_rejects_missing_onboarding_token(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with ( @@ -322,7 +322,7 @@ async def test_claim_token_rejects_wrong_onboarding_session(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) request = _make_claim_request( _make_onboarding_token(invitation_link="other-invite") @@ -351,7 +351,7 @@ async def test_claim_token_rejects_invalid_bearer_token(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) request = _make_claim_request("sk-regular-key") @@ -380,7 +380,7 @@ async def test_claim_token_rejects_concurrent_reuse_before_password_write(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with ( @@ -418,7 +418,7 @@ async def test_claim_token_sets_accepted_at_after_password_written(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) mock_token_response = {"token": "sk-generated-key", "user_id": "user-123"} @@ -477,7 +477,7 @@ async def test_claim_token_rolls_back_invite_when_session_key_mint_fails(): data = InvitationClaim( invitation_link="invite-abc", user_id="user-123", - password="NewP@ssw0rd", + password="NewP@ssw0rd123", ) with ( diff --git a/tests/test_litellm/proxy/auth/test_password_policy.py b/tests/test_litellm/proxy/auth/test_password_policy.py new file mode 100644 index 00000000000..f6e7d443907 --- /dev/null +++ b/tests/test_litellm/proxy/auth/test_password_policy.py @@ -0,0 +1,136 @@ +""" +Tests for the configurable password-strength policy in +`litellm.proxy.auth.password_policy`, enforced on every path that persists a +new or changed password for a locally-managed user. +""" + +import pytest + +from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.proxy.auth.password_policy import ( + DEFAULT_MIN_LENGTH, + MIN_ALLOWED_LENGTH, + PasswordPolicy, + get_password_policy, + validate_password_policy, +) + +STRONG_PASSWORD = "Str0ng!Passw0rd" + + +def test_get_password_policy_defaults_to_pif_baseline(): + policy = get_password_policy({}) + assert policy == PasswordPolicy( + min_length=DEFAULT_MIN_LENGTH, + require_uppercase=True, + require_lowercase=True, + require_numbers=True, + require_special_characters=True, + ) + + +def test_get_password_policy_reads_overrides_from_general_settings(): + policy = get_password_policy( + { + "password_policy_min_length": 20, + "password_policy_require_uppercase": False, + "password_policy_require_lowercase": False, + "password_policy_require_numbers": False, + "password_policy_require_special_characters": False, + } + ) + assert policy == PasswordPolicy( + min_length=20, + require_uppercase=False, + require_lowercase=False, + require_numbers=False, + require_special_characters=False, + ) + + +def test_validate_password_policy_accepts_strong_password(): + assert validate_password_policy(STRONG_PASSWORD, {}) is None + + +@pytest.mark.parametrize( + "password,expected_fragment", + [ + ("Sh0rt!Pw", "12 characters"), + ("weakpassword123!", "uppercase"), + ("WEAKPASSWORD123!", "lowercase"), + ("WeakPassword!!!!", "number"), + ("WeakPassword12345", "special character"), + ], +) +def test_validate_password_policy_rejects_each_missing_class(password, expected_fragment): + with pytest.raises(ProxyException) as exc_info: + validate_password_policy(password, {}) + assert exc_info.value.code == "400" + assert exc_info.value.type == ProxyErrorTypes.validation_error + assert exc_info.value.param == "password" + assert expected_fragment in exc_info.value.message + + +def test_validate_password_policy_reports_every_violation_at_once(): + with pytest.raises(ProxyException) as exc_info: + validate_password_policy("weak", {}) + assert "12 characters" in exc_info.value.message + assert "uppercase" in exc_info.value.message + assert "number" in exc_info.value.message + assert "special character" in exc_info.value.message + + +def test_validate_password_policy_honors_relaxed_config(): + general_settings = { + "password_policy_min_length": MIN_ALLOWED_LENGTH, + "password_policy_require_special_characters": False, + } + # 8 chars, has upper/lower/number, no special char: fails default policy, + # passes the relaxed one above. + validate_password_policy("Abcd1234", general_settings) + with pytest.raises(ProxyException): + validate_password_policy("Abcd1234", {}) + + +def test_validate_password_policy_honors_stricter_min_length(): + general_settings = {"password_policy_min_length": 20} + with pytest.raises(ProxyException) as exc_info: + validate_password_policy(STRONG_PASSWORD, general_settings) + assert "20 characters" in exc_info.value.message + + +@pytest.mark.parametrize("configured_min_length", [0, -1, -100, 1, 7]) +def test_get_password_policy_floors_nonpositive_or_too_low_min_length(configured_min_length): + """A misconfigured min_length must never disable the length check + entirely: it floors at MIN_ALLOWED_LENGTH instead of passing through.""" + policy = get_password_policy({"password_policy_min_length": configured_min_length}) + assert policy.min_length == MIN_ALLOWED_LENGTH + + +def test_validate_password_policy_rejects_short_password_even_with_zero_min_length_configured(): + general_settings = {"password_policy_min_length": 0} + with pytest.raises(ProxyException) as exc_info: + validate_password_policy("a", general_settings) + assert f"{MIN_ALLOWED_LENGTH} characters" in exc_info.value.message + + +def test_get_password_policy_ignores_boolean_min_length(): + """`bool` is a subclass of `int` in Python; a stray `true`/`false` value + must not silently coerce into a min_length of 1 or 0.""" + policy = get_password_policy({"password_policy_min_length": False}) + assert policy.min_length == DEFAULT_MIN_LENGTH + + +def test_validate_password_policy_rejects_unicode_letter_as_special_character(): + """Regression: an ASCII-only `[^A-Za-z0-9]` check would miscount an + accented letter as the required special character, so a letters-and- + digits-only password like this one (no real symbol) must still be + rejected.""" + with pytest.raises(ProxyException) as exc_info: + validate_password_policy("Passwörd1234", {}) + assert "special character" in exc_info.value.message + + +def test_validate_password_policy_accepts_real_special_character_with_unicode_letters(): + """Same base password as the rejection test above, plus an actual symbol.""" + assert validate_password_policy("Passwörd1234!", {}) is None diff --git a/tests/test_litellm/proxy/db/test_exception_handler.py b/tests/test_litellm/proxy/db/test_exception_handler.py index d80e3acb4b8..c685d778c0e 100644 --- a/tests/test_litellm/proxy/db/test_exception_handler.py +++ b/tests/test_litellm/proxy/db/test_exception_handler.py @@ -1,12 +1,14 @@ import asyncio import json import sys +from typing import Final from unittest.mock import MagicMock, patch import httpx import pytest from fastapi import HTTPException, Request from prisma import errors as prisma_errors +from prisma.engine.errors import BinaryNotFoundError, EngineConnectionError from prisma.errors import ( ClientNotConnectedError, DataError, @@ -317,6 +319,43 @@ def test_is_database_service_unavailable_error_in_chain_sees_through_wrapping(): assert PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(ValueError("nope")) is False +def test_find_database_service_unavailable_error_in_chain_returns_the_wrapped_outage_itself(): + """Wording a 503 by the kind of outage needs the wrapped database error, not the ValueError + get_user_object wrapped it in, so the finder must hand back the inner exception.""" + outage = _wrapped_like_get_user_object(ConnectionError("can't reach database server")) + found = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(outage) + assert isinstance(found, ConnectionError) + assert found is outage.__context__ + missing_user = _wrapped_like_get_user_object(Exception()) + assert PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(missing_user) is None + + +def _raised_while_handling(inner, outer): + try: + raise inner + except BaseException: + try: + raise outer + except BaseException as surfaced: + return surfaced + + +def test_permanent_fault_outranks_the_transient_error_that_surfaced_it(): + """A reconnect that dies on a missing engine binary raises the transport error last, with the + BinaryNotFoundError left as __context__. The binary is what keeps the database down, so both the + finder and the 503 wording must pick it over the outer transient error, whichever way they nest.""" + permanent = BinaryNotFoundError("query engine binary not found") + transient_over_permanent = _raised_while_handling(permanent, httpx.ConnectError("connection refused")) + permanent_over_transient = _raised_while_handling(httpx.ConnectError("connection refused"), permanent) + + for chain in (transient_over_permanent, permanent_over_transient): + assert PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(chain) is permanent + message = PrismaDBExceptionHandler.database_unavailable_message(chain) + assert "BinaryNotFoundError" in message + assert "will not clear by retrying" in message + assert "temporarily unreachable" not in message + + def test_is_database_service_unavailable_error_in_chain_terminates_on_a_cause_cycle(): """The walk must terminate on a pathological __cause__ cycle rather than hang. Neither link is an outage, so the bounded walk returns False instead of looping forever.""" @@ -508,6 +547,51 @@ def test_permanent_prisma_faults_are_still_reported_as_service_problems(prisma_e assert PrismaDBExceptionHandler.is_database_service_unavailable_error(prisma_error) is True +RECONNECTABLE_CLIENT_STATE_FAULTS = (prisma_errors.ClientNotConnectedError, prisma_errors.HTTPClientClosedError) + + +@pytest.mark.parametrize("prisma_error", PERMANENT_PRISMA_FAULTS) +def test_permanent_prisma_faults_are_worded_as_not_retryable(prisma_error): + """A 503 for a fault that never heals must not tell the operator to wait. + + The status stays 503 (the service is at fault), but the message has to say + the outage is not transient and name the engine fault, or an operator + watching a version-skewed engine keeps retrying a request that can never + succeed. The two client-state faults a reconnect can repair keep the retry + wording.""" + reconnectable = isinstance(prisma_error, RECONNECTABLE_CLIENT_STATE_FAULTS) + message = PrismaDBExceptionHandler.database_unavailable_message(prisma_error) + + assert PrismaDBExceptionHandler.is_permanent_database_fault(prisma_error) is (not reconnectable) + assert message.startswith("Service Unavailable") + assert ("temporarily unreachable" in message) is reconnectable + assert ("Please retry shortly" in message) is reconnectable + assert ("will not clear by retrying" in message) is (not reconnectable) + assert (type(prisma_error).__name__ in message) is (not reconnectable) + + +@pytest.mark.parametrize( + "transient_error", + [ + pytest.param(httpx.ConnectError("All connection attempts failed"), id="ConnectError"), + pytest.param(ConnectionError("connection refused"), id="ConnectionError"), + pytest.param(EngineConnectionError(), id="EngineConnectionError"), + pytest.param(prisma_errors.PrismaError("can't reach database server"), id="P1001_text"), + pytest.param( + ProxyException(message="no db", type=ProxyErrorTypes.no_db_connection, param=None, code=503), + id="ProxyException", + ), + ], +) +def test_transient_outages_keep_the_retry_wording(transient_error): + """A genuine outage is expected to come back, so the retry guidance is the + right message and must not be replaced by the permanent-fault text.""" + assert PrismaDBExceptionHandler.is_permanent_database_fault(transient_error) is False + assert PrismaDBExceptionHandler.database_unavailable_message(transient_error) == ( + "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." + ) + + @pytest.mark.parametrize( "transient_error", [ @@ -579,3 +663,40 @@ def test_is_deadlock_error_matches_postgres_deadlock(error): def test_is_deadlock_error_excludes_non_deadlocks(error): """Non-deadlock prisma errors, connectivity failures, and non-prisma exceptions are not treated as deadlocks.""" assert PrismaDBExceptionHandler.is_deadlock_error(error) is False + + +MOCKED_PRISMA_PREDICATES: Final = ( + PrismaDBExceptionHandler.is_database_infrastructure_error, + PrismaDBExceptionHandler.is_database_transport_error, + PrismaDBExceptionHandler.is_deadlock_error, + PrismaDBExceptionHandler.is_prisma_engine_internal_error, + PrismaDBExceptionHandler.is_database_service_unavailable_error, +) + + +@pytest.mark.parametrize("predicate", MOCKED_PRISMA_PREDICATES, ids=lambda p: p.__name__) +def test_predicates_answer_false_for_a_plain_exception_when_prisma_is_mocked(predicate): + """Suites that swap ``sys.modules["prisma"]`` for a ``MagicMock`` hand the + predicates mocks in place of prisma's error classes. ``isinstance`` against + a mock raises ``TypeError``; the predicate must instead answer for the + non-prisma checks it still has.""" + with patch.dict(sys.modules, {"prisma": MagicMock()}): + assert predicate(Exception("db connection dropped")) is False + + +def test_infrastructure_error_still_recognizes_transport_errors_when_prisma_is_mocked(): + """Skipping the prisma classes must not skip the checks that do not need them.""" + with patch.dict(sys.modules, {"prisma": MagicMock()}): + no_db: Final = ProxyException(message="no db", type=ProxyErrorTypes.no_db_connection, param=None, code=503) + assert PrismaDBExceptionHandler.is_database_infrastructure_error(httpx.ConnectError("refused")) is True + assert PrismaDBExceptionHandler.is_database_infrastructure_error(no_db) is True + + +def test_connection_error_answers_when_prisma_is_mocked_after_import(): + """``prisma.engine`` is already loaded in a real process, so a mock parent + still resolves ``prisma.engine.errors``; its classes are then mocks too.""" + import prisma.engine.errors # noqa: F401 + + with patch.dict(sys.modules, {"prisma": MagicMock()}): + assert PrismaDBExceptionHandler.is_database_connection_error(Exception("x")) is False + assert PrismaDBExceptionHandler.is_database_connection_error(httpx.ConnectError("refused")) is True diff --git a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py index f4da8c941a4..c3f7b0100d8 100644 --- a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py +++ b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py @@ -18,7 +18,7 @@ def test_ui_discovery_endpoints_with_defaults(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): @@ -41,7 +41,7 @@ def test_ui_discovery_endpoints_with_custom_server_root_path(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/litellm"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): @@ -66,7 +66,7 @@ def test_ui_discovery_endpoints_with_proxy_base_url_when_set(): "litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com", ), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): @@ -91,7 +91,7 @@ def test_ui_discovery_endpoints_with_sso_configured_and_auto_redirect_enabled(): "litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com", ), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch.dict( os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true", "DISABLE_ADMIN_UI": "false"}, @@ -121,7 +121,7 @@ def test_ui_discovery_endpoints_with_sso_configured_and_auto_redirect_not_set_de "litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com", ), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): # Ensure AUTO_REDIRECT_UI_LOGIN_TO_SSO is not set (simulate default) @@ -148,7 +148,7 @@ def test_ui_discovery_endpoints_with_sso_configured_but_auto_redirect_disabled() "litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com", ), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch.dict( os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "false", "DISABLE_ADMIN_UI": "false"}, @@ -174,7 +174,7 @@ def test_ui_discovery_endpoints_with_sso_not_configured_but_auto_redirect_enable with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict( os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true", "DISABLE_ADMIN_UI": "false"}, @@ -203,7 +203,7 @@ def test_ui_discovery_endpoints_both_routes_return_same_data(): "litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com", ), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch.dict( os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true", "DISABLE_ADMIN_UI": "false"}, @@ -228,7 +228,7 @@ def test_ui_discovery_endpoints_with_auto_redirect_via_general_settings(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch( "litellm.proxy.proxy_server.general_settings", {"auto_redirect_ui_login_to_sso": True}, @@ -254,7 +254,7 @@ def test_ui_discovery_endpoints_with_auto_redirect_env_var_overrides_general_set with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=True), patch( "litellm.proxy.proxy_server.general_settings", {"auto_redirect_ui_login_to_sso": False}, @@ -281,7 +281,7 @@ def test_ui_discovery_endpoints_with_admin_ui_disabled(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "true"}, clear=False), ): @@ -311,7 +311,7 @@ def test_ui_discovery_endpoints_is_control_plane_true_when_workers_configured(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch("litellm.proxy.proxy_server.proxy_config", mock_config), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): @@ -336,7 +336,7 @@ def test_ui_discovery_endpoints_hide_default_credentials_hint_default_false(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): os.environ.pop("LITELLM_HIDE_DEFAULT_CREDENTIALS_HINT", None) @@ -357,7 +357,7 @@ def test_ui_discovery_endpoints_hide_default_credentials_hint_via_env_var(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch.dict( os.environ, { @@ -384,7 +384,7 @@ def test_ui_discovery_endpoints_hide_default_credentials_hint_via_general_settin with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch( "litellm.proxy.proxy_server.general_settings", {"hide_default_credentials_hint": True}, @@ -411,7 +411,7 @@ def test_ui_discovery_endpoints_is_control_plane_false_when_no_workers(): with ( patch("litellm.proxy.utils.get_server_root_path", return_value="/"), patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), - patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), + patch("litellm.proxy.auth.auth_utils.has_user_setup_sso", return_value=False), patch("litellm.proxy.proxy_server.proxy_config", mock_config), patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False), ): diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 4ee6741ee02..84f7611c0c0 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -842,24 +842,37 @@ async def test_presidio_filter_scope_initializer(monkeypatch): params_input = LitellmParams(guardrail="presidio", mode="pre_call", presidio_filter_scope="input") guardrail_dict = {"guardrail_name": "g1"} - cb = initialize_presidio(params_input, guardrail_dict) - assert cb is created[0] + callbacks = initialize_presidio(params_input, guardrail_dict) + assert callbacks == (created[0],) assert created[0].apply_to_output is False # output-only created.clear() params_output = LitellmParams(guardrail="presidio", mode="pre_call", presidio_filter_scope="output") - cb = initialize_presidio(params_output, guardrail_dict) + callbacks = initialize_presidio(params_output, guardrail_dict) assert len(created) == 1 + assert callbacks == (created[0],) assert created[0].apply_to_output is True - # both -> expect two callbacks (input + output) + # both -> expect two callbacks (input + output), both returned, input first created.clear() params_both = LitellmParams(guardrail="presidio", mode="pre_call", presidio_filter_scope="both") - cb = initialize_presidio(params_both, guardrail_dict) + callbacks = initialize_presidio(params_both, guardrail_dict) assert len(created) == 2 - assert any(not c.apply_to_output for c in created) - assert any(c.apply_to_output for c in created) + assert callbacks == tuple(created) + assert callbacks[0].apply_to_output is False + assert callbacks[1].apply_to_output is True + + # both + output_parse_pii -> three callbacks, all returned, input first + created.clear() + params_all = LitellmParams( + guardrail="presidio", mode="pre_call", presidio_filter_scope="both", output_parse_pii=True + ) + callbacks = initialize_presidio(params_all, guardrail_dict) + assert len(created) == 3 + assert callbacks == tuple(created) + assert callbacks[0].apply_to_output is False + assert mgr.added[-3:] == list(created) @pytest.mark.asyncio @@ -3116,6 +3129,18 @@ def test_update_in_memory_applies_analyze_chunk_size(): assert guardrail.presidio_analyze_chunk_size_bytes == 99_000 +def test_update_in_memory_keeps_output_masker_from_unmasking(): + masker = _OPTIONAL_PresidioPIIMasking(mock_testing=True, apply_to_output=True, output_parse_pii=False) + unmasker = _OPTIONAL_PresidioPIIMasking(mock_testing=True, output_parse_pii=True) + params = LitellmParams(guardrail="presidio", mode="pre_call", output_parse_pii=True) + + masker.update_in_memory_litellm_params(params) + unmasker.update_in_memory_litellm_params(params) + + assert (masker.apply_to_output, masker.output_parse_pii) == (True, False) + assert (unmasker.apply_to_output, unmasker.output_parse_pii) == (False, True) + + def test_merge_drops_truncated_same_type_fragment_from_overlap(): """A boundary entity seen truncated by chunk 1 and whole by chunk 2 must merge to the single full span; keeping both overlapping spans corrupts the diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 2c0735970d3..24742e1bac2 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -1,3 +1,4 @@ +from collections.abc import Iterable from unittest.mock import AsyncMock, MagicMock import pytest @@ -491,6 +492,144 @@ def test_repeated_db_sync_does_not_accumulate_runner_instances(): cb_list[:] = snapshot +PRESIDIO_SIBLINGS_GID = "55555555-5555-5555-5555-555555555555" +PRESIDIO_SIBLINGS_NAME = "presidio-siblings" + + +def _presidio_db_guardrail(pii_entities_config: dict[str, str]) -> Guardrail: + return Guardrail( + guardrail_id=PRESIDIO_SIBLINGS_GID, + guardrail_name=PRESIDIO_SIBLINGS_NAME, + litellm_params={ + "guardrail": "presidio", + "mode": "pre_call", + "default_on": True, + "output_parse_pii": True, + "presidio_filter_scope": "both", + "presidio_analyzer_api_base": "https://fakelink.com/v1/presidio/analyze", + "presidio_anonymizer_api_base": "https://fakelink.com/v1/presidio/anonymize", + "pii_entities_config": pii_entities_config, + }, + ) + + +def _presidio_callbacks_in(cb_list: Iterable[object]) -> list[CustomGuardrail]: + return [ + callback + for callback in cb_list + if isinstance(callback, CustomGuardrail) and getattr(callback, "guardrail_name", None) == PRESIDIO_SIBLINGS_NAME + ] + + +def test_presidio_siblings_are_tracked_and_deleted_together(): + """ + A presidio guardrail scoped to both stages registers the pre_call primary plus + the post_call unmask and mask-output siblings. Deleting the guardrail must remove + all three from every callback list, not just the primary. + """ + import litellm + + handler = InMemoryGuardrailHandler() + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + handler.initialize_guardrail(_presidio_db_guardrail({"EMAIL_ADDRESS": "MASK"})) + + registered = _presidio_callbacks_in(litellm.callbacks) + assert len(registered) == 3 + primary = handler.guardrail_id_to_custom_guardrail[PRESIDIO_SIBLINGS_GID] + siblings = handler.guardrail_id_to_sibling_callbacks[PRESIDIO_SIBLINGS_GID] + assert primary is registered[0] + assert siblings == tuple(registered[1:]) + assert [sibling.event_hook for sibling in siblings] == [GuardrailEventHooks.post_call] * 2 + + for cb_list in lists[1:]: + cb_list.extend(registered) + + handler.delete_in_memory_guardrail(PRESIDIO_SIBLINGS_GID) + + for cb_list in lists: + assert _presidio_callbacks_in(cb_list) == [] + assert PRESIDIO_SIBLINGS_GID not in handler.guardrail_id_to_custom_guardrail + assert PRESIDIO_SIBLINGS_GID not in handler.guardrail_id_to_sibling_callbacks + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + +def test_update_in_memory_guardrail_reaches_presidio_siblings_and_keeps_their_stage(): + import litellm + + handler = InMemoryGuardrailHandler() + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + handler.initialize_guardrail(_presidio_db_guardrail({"EMAIL_ADDRESS": "MASK", "IP_ADDRESS": "MASK"})) + tracked = _presidio_callbacks_in(litellm.callbacks) + roles_before = [ + (callback.apply_to_output, callback.output_parse_pii, callback.event_hook) for callback in tracked + ] + assert roles_before == [ + (False, True, [GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call]), + (False, True, GuardrailEventHooks.post_call), + (True, False, GuardrailEventHooks.post_call), + ] + + updated = Guardrail( + guardrail_id=PRESIDIO_SIBLINGS_GID, + guardrail_name=PRESIDIO_SIBLINGS_NAME, + litellm_params=LitellmParams( + guardrail="presidio", + mode="pre_call", + default_on=True, + output_parse_pii=True, + presidio_filter_scope="both", + presidio_analyzer_api_base="https://fakelink.com/v1/presidio/analyze", + presidio_anonymizer_api_base="https://fakelink.com/v1/presidio/anonymize", + pii_entities_config={"EMAIL_ADDRESS": "MASK"}, + ), + ) + handler.update_in_memory_guardrail(guardrail_id=PRESIDIO_SIBLINGS_GID, guardrail=updated) + + assert [callback.pii_entities_config for callback in tracked] == [{"EMAIL_ADDRESS": "MASK"}] * 3 + assert [ + (callback.apply_to_output, callback.output_parse_pii, callback.event_hook) for callback in tracked + ] == roles_before + assert _presidio_callbacks_in(litellm.callbacks) == tracked + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + +def test_repeated_db_sync_replaces_presidio_siblings_instead_of_leaking_stale_ones(): + """ + The callback manager dedupes custom loggers by their scalar attributes, so a + leaked post_call sibling blocks the re-initialized sibling from registering and + keeps serving the previous entity config. After every DB re-sync, each callback + list must hold exactly the three current instances, all on the latest config. + """ + import litellm + + handler = InMemoryGuardrailHandler() + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + entity_configs = [{"EMAIL_ADDRESS": "MASK"}, {"EMAIL_ADDRESS": "MASK", "IP_ADDRESS": "MASK"}] + for cycle in range(4): + latest = entity_configs[cycle % 2] + handler.sync_guardrail_from_db(_presidio_db_guardrail(latest)) + for cb_list in lists[1:]: + cb_list.extend(_presidio_callbacks_in(litellm.callbacks)) + + for cb_list in lists: + current = _presidio_callbacks_in(cb_list) + assert len({id(callback) for callback in current}) == 3 + assert all(callback.pii_entities_config == latest for callback in current) + finally: + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + def _judge_guardrail(guardrail_id: str) -> Guardrail: return Guardrail( guardrail_id=guardrail_id, diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 8bce967b316..f231eb66a50 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4220,3 +4220,88 @@ async def test_user_new_persists_model_max_budget( ) assert captured["user_data"].get("model_max_budget") == expected_written + + +@pytest.fixture +def _admin_prisma(mocker): + """A mocked prisma_client wired in as proxy_server's module globals, for + the password-policy tests below (mirrors the pattern every other test in + this file repeats per-test; consolidated here since these three share it + verbatim).""" + mock_prisma_client = mocker.MagicMock() + mocker.patch( # test-quality-ok: same module-global mocking every test in this file already uses + "litellm.proxy.proxy_server.prisma_client", mock_prisma_client + ) + mocker.patch( # test-quality-ok: same module-global mocking every test in this file already uses + "litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin" + ) + return mock_prisma_client + + +@pytest.mark.asyncio +async def test_user_update_rejects_weak_password(_admin_prisma): + """/user/update must reject a password that fails the configured + policy before it ever reaches the DB write.""" + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + user_request = UpdateUserRequest(user_id="target-user", password="short1!") + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + with pytest.raises(ProxyException) as exc_info: + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) + + assert exc_info.value.code == "400" + _admin_prisma.db.litellm_usertable.find_first.assert_not_called() + + +@pytest.mark.asyncio +async def test_user_update_rejects_weak_password_against_configured_policy(_admin_prisma, mocker): + """A password that meets the default policy but not a stricter + admin-configured one must still be rejected.""" + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + mocker.patch( # test-quality-ok: same module-global mocking every test in this file already uses + "litellm.proxy.proxy_server.general_settings", + {"password_policy_min_length": 24}, + ) + + user_request = UpdateUserRequest(user_id="target-user", password="Str0ng!Passw0rd") + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + with pytest.raises(ProxyException) as exc_info: + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) + + assert "24 characters" in exc_info.value.message + + +@pytest.mark.asyncio +async def test_user_update_hashes_and_persists_strong_password(_admin_prisma, mocker): + """A password meeting the policy is hashed (never stored in plaintext) + and reaches the DB write.""" + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + mock_prisma_client = _admin_prisma + existing_user = mocker.MagicMock() + existing_user.model_dump.return_value = {"user_id": "target-user"} + existing_user.user_id = "target-user" + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user"}) + mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) + + strong_password = "Str0ng!Passw0rd" + user_request = UpdateUserRequest(user_id="target-user", password=strong_password) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) + + written_data = mock_prisma_client.update_data.call_args.kwargs["data"] + assert written_data.get("password") is not None + assert written_data["password"] != strong_password diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index 87e0319f6a1..bca97915347 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -4555,3 +4555,114 @@ def test_create_file_async_pre_call_hook_rejection_blocks_upload(monkeypatch, ll assert response.status_code == 400, response.text assert "file upload not allowed" in response.text assert provider_route.call_count == 0 + + +def test_create_file_non_batch_over_max_file_size_mb_rejected_before_forwarding(monkeypatch, llm_router: Router): + """max_file_size_mb applies to every purpose, unlike the batch-only max_batch_file_size_mb.""" + import litellm.proxy.proxy_server as ps + + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + monkeypatch.setitem(ps.general_settings, "max_file_size_mb", 1) + + oversized = b"x" * (2 * 1024 * 1024) + try: + response = client.post( + "/v1/files", + files={"file": ("labels.jsonl", oversized, "application/octet-stream")}, + data={"purpose": "user_data"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 413, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "file" + assert "max_file_size_mb" in error["message"] + assert "1 MB" in error["message"] + assert forwarded_calls == [] + + +def test_create_file_non_batch_under_max_file_size_mb_forwards(monkeypatch, llm_router: Router): + import litellm.proxy.proxy_server as ps + + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + monkeypatch.setitem(ps.general_settings, "max_file_size_mb", 1) + + try: + response = client.post( + "/v1/files", + files={"file": ("labels.jsonl", b"small content", "application/octet-stream")}, + data={"purpose": "user_data"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 200, response.text + assert len(forwarded_calls) == 1 + + +def test_create_file_blocked_extension_rejected_before_forwarding(monkeypatch, llm_router: Router): + import litellm.proxy.proxy_server as ps + + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + monkeypatch.setitem(ps.general_settings, "blocked_file_extensions", [".exe", ".sh"]) + + try: + response = client.post( + "/v1/files", + files={"file": ("payload.exe", b"MZ\x90\x00", "application/octet-stream")}, + data={"purpose": "user_data"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "file" + assert ".exe" in error["message"] + assert "blocked_file_extensions" in error["message"] + assert forwarded_calls == [] + + +def test_create_file_blocked_extension_unset_allows_everything(monkeypatch, llm_router: Router): + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + + try: + response = client.post( + "/v1/files", + files={"file": ("payload.exe", b"MZ\x90\x00", "application/octet-stream")}, + data={"purpose": "user_data"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 200, response.text + assert len(forwarded_calls) == 1 + + +def test_create_file_path_traversal_filename_rejected_before_forwarding(monkeypatch, llm_router: Router): + """A filename carrying a directory-traversal component must never reach storage or the provider.""" + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + + try: + response = client.post( + "/v1/files", + files={"file": ("../../etc/passwd", b"malicious content", "text/plain")}, + data={"purpose": "user_data"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "file" + assert "traversal" in error["message"].lower() + assert forwarded_calls == [] diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_general_upload_validation.py b/tests/test_litellm/proxy/openai_files_endpoint/test_general_upload_validation.py new file mode 100644 index 00000000000..9f7dd914e4a --- /dev/null +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_general_upload_validation.py @@ -0,0 +1,134 @@ +import io + +import pytest + +from litellm.proxy._types import ProxyException +from litellm.proxy.openai_files_endpoints.general_upload_validation import ( + MB, + UploadedFileBlockedExtension, + UploadedFileTooLarge, + UploadedFileUnsafeFilename, + check_blocked_extension, + check_unsafe_filename, + check_upload_file_size, + raise_upload_validation_failure, +) + + +def test_size_under_cap_allowed(): + assert check_upload_file_size(b"x" * 100, 1) is None + + +def test_size_over_cap_rejected_for_bytes(): + content = b"x" * (2 * MB) + assert check_upload_file_size(content, 1) == UploadedFileTooLarge(size_bytes=len(content), limit_mb=1) + + +def test_size_over_cap_rejected_for_binaryio_and_restores_caller_position(): + """The handle is caller-owned; inspecting its size must not discard where the caller had it.""" + content = b"x" * (2 * MB) + handle = io.BytesIO(content) + handle.seek(17) + assert check_upload_file_size(handle, 1) == UploadedFileTooLarge(size_bytes=len(content), limit_mb=1) + assert handle.tell() == 17 + + +def test_size_under_cap_allowed_for_binaryio_restores_caller_position(): + handle = io.BytesIO(b"x" * 100) + handle.seek(42) + assert check_upload_file_size(handle, 1) is None + assert handle.tell() == 42 + + +def test_size_exactly_at_cap_allowed(): + content = b"x" * MB + assert check_upload_file_size(content, 1) is None + + +def test_no_cap_skips_size_check(): + assert check_upload_file_size(b"x" * (10 * MB), None) is None + + +@pytest.mark.parametrize("cap", [0, -3]) +def test_nonpositive_cap_disables_size_check(cap): + assert check_upload_file_size(b"x" * (10 * MB), cap) is None + + +def test_blocked_extension_rejected(): + assert check_blocked_extension("payload.exe", (".exe", ".sh")) == UploadedFileBlockedExtension(extension=".exe") + + +def test_blocked_extension_match_is_case_insensitive(): + assert check_blocked_extension("payload.EXE", (".exe",)) == UploadedFileBlockedExtension(extension=".exe") + + +def test_blocked_extension_match_is_case_insensitive_for_configured_value(): + """A config entry like blocked_file_extensions: ['.EXE'] must still catch a lowercase upload.""" + assert check_blocked_extension("payload.exe", (".EXE",)) == UploadedFileBlockedExtension(extension=".exe") + + +def test_extension_not_in_blocklist_allowed(): + assert check_blocked_extension("report.pdf", (".exe", ".sh")) is None + + +def test_empty_blocklist_allows_everything(): + assert check_blocked_extension("payload.exe", ()) is None + + +def test_no_filename_skips_extension_check(): + assert check_blocked_extension(None, (".exe",)) is None + + +def test_path_traversal_filename_rejected(): + assert check_unsafe_filename("../../etc/passwd") == UploadedFileUnsafeFilename(filename="../../etc/passwd") + + +def test_windows_style_path_traversal_filename_rejected(): + assert check_unsafe_filename("..\\..\\windows\\system32\\config") == UploadedFileUnsafeFilename( + filename="..\\..\\windows\\system32\\config" + ) + + +def test_traversal_embedded_after_extension_rejected(): + assert check_unsafe_filename("report.jsonl/../../etc/cron.d/evil") == UploadedFileUnsafeFilename( + filename="report.jsonl/../../etc/cron.d/evil" + ) + + +def test_null_byte_filename_rejected(): + assert check_unsafe_filename("report.pdf\x00.exe") == UploadedFileUnsafeFilename(filename="report.pdf\x00.exe") + + +@pytest.mark.parametrize("filename", ["report.pdf", ".env", "a.b.c.jsonl", "my file (1).csv", None]) +def test_ordinary_filenames_allowed(filename): + assert check_unsafe_filename(filename) is None + + +@pytest.mark.parametrize( + "failure, expected_code, expected_fragments", + [ + ( + UploadedFileTooLarge(size_bytes=15728640, limit_mb=10), + "413", + ("15.0 MB", "max_file_size_mb", "10 MB", "not forwarded"), + ), + ( + UploadedFileBlockedExtension(extension=".exe"), + "400", + (".exe", "blocked_file_extensions", "not forwarded"), + ), + ( + UploadedFileUnsafeFilename(filename="../../etc/passwd"), + "400", + ("../../etc/passwd", "traversal", "not forwarded"), + ), + ], +) +def test_failures_map_to_openai_shaped_proxy_exceptions(failure, expected_code, expected_fragments): + with pytest.raises(ProxyException) as exc_info: + raise_upload_validation_failure(failure) + assert exc_info.value.code == expected_code + assert exc_info.value.type == "invalid_request_error" + assert exc_info.value.param == "file" + for fragment in expected_fragments: + assert fragment in exc_info.value.message diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py index af37dbe85fe..45460dcecf1 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py @@ -29,7 +29,7 @@ def _install_login_mocks(monkeypatch, raise_on_auth: bool = False) -> None: """ from litellm.proxy import proxy_server as ps - async def _fake_auth(username, password, master_key, prisma_client): + async def _fake_auth(username, password, master_key, prisma_client, general_settings=None): if raise_on_auth: raise Exception("boom-auth-failure") fake = MagicMock() diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py b/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py index 5cc22cca7a0..778acc1baab 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py @@ -234,7 +234,7 @@ def test_claim_onboarding_link_happy(client, monkeypatch, mock_prisma): json={ "invitation_link": "inv-123", "user_id": "user-abc", - "password": "hunter2", + "password": "Hunter2Strong!", }, headers={"Authorization": f"Bearer {onboarding_jwt}"}, ) @@ -260,7 +260,7 @@ def test_claim_onboarding_link_invalid_invite_401(client, monkeypatch, mock_pris json={ "invitation_link": "missing", "user_id": "user-abc", - "password": "hunter2", + "password": "Hunter2Strong!", }, headers={"Authorization": "Bearer irrelevant"}, ) @@ -287,7 +287,7 @@ def test_claim_onboarding_link_user_id_mismatch_401( json={ "invitation_link": "inv-123", "user_id": "user-attacker", - "password": "hunter2", + "password": "Hunter2Strong!", }, headers={"Authorization": "Bearer irrelevant"}, ) @@ -339,7 +339,7 @@ def test_claim_onboarding_link_bad_onboarding_jwt_401( json={ "invitation_link": "inv-123", "user_id": "user-abc", - "password": "hunter2", + "password": "Hunter2Strong!", }, headers={"Authorization": f"Bearer {bogus_jwt}"}, ) diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 31430da71e8..8006f64ba41 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -1,3 +1,4 @@ +import re from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch @@ -756,6 +757,45 @@ def test_public_agent_hub_returns_empty_when_no_public_groups(): assert response.json() == [] +# --------------------------------------------------------------------------- +# /public/agents/fields +# --------------------------------------------------------------------------- + + +def test_bedrock_agentcore_runtime_arn_validation_pattern_accepts_full_resource_path(): + """Regression for LIT-6737: the AgentCore agent_runtime_arn field's + validation_pattern must accept a complete runtime ARN whose resource part + is itself multi-segment (``runtime/``), and reject the exact + truncated shape a naive split("/")-by-position parse used to produce (the + ARN cut off right after the ``runtime`` resource type). + """ + app_instance = FastAPI() + app_instance.include_router(router) + test_client = TestClient(app_instance) + + response = test_client.get("/public/agents/fields") + assert response.status_code == 200 + agents = response.json() + + bedrock_agentcore = next((a for a in agents if a["agent_type"] == "bedrock_agentcore"), None) + assert bedrock_agentcore is not None, "bedrock_agentcore agent type not found" + assert bedrock_agentcore["model_template"] == "bedrock/agentcore/{agent_runtime_arn}" + + fields_by_key = {f["key"]: f for f in bedrock_agentcore["credential_fields"]} + arn_field = fields_by_key["agent_runtime_arn"] + assert arn_field["required"] is True + assert arn_field["include_in_litellm_params"] is False + + pattern = arn_field.get("validation_pattern") + assert pattern, "agent_runtime_arn must ship a validation_pattern so the UI can reject a truncated ARN" + + full_arn = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime/hosted_agent_4vm3i-BaTdfOELAs" + truncated_arn = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime" + + assert re.match(pattern, full_arn), "the validator must accept a complete runtime ARN" + assert not re.match(pattern, truncated_arn), "the validator must reject the truncated ARN" + + # --------------------------------------------------------------------------- # /public/endpoints # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 91fca8f1e27..b18fc373cae 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -130,6 +130,7 @@ def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch): password="secret", master_key="test-master-key", prisma_client=mock_prisma_client, + general_settings={}, ) mock_create_ui_token_object.assert_called_once_with( login_result=mock_login_result, diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index dc256ccf718..709447d23c0 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -3006,6 +3006,128 @@ def test_update_mcp_semantic_filter_settings_requires_proxy_admin(monkeypatch): app.dependency_overrides.pop(user_api_key_auth, None) +class TestMcpToolSearchSettingsEndpoints: + """`litellm_settings.mcp_tool_search` drives the native `mcp_tool_search` virtual tool, so the UI must round-trip it.""" + + @staticmethod + def _override_auth(role: LitellmUserRoles): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="u", api_key="hashed", user_role=role + ) + + def test_get_returns_stored_values_and_field_schema(self, mock_proxy_config, mock_auth, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", object()) + mock_proxy_config["config"]["litellm_settings"]["mcp_tool_search"] = { + "embedding_model": "text-embedding-3-small", + "core_tools": ["treasury-get_rates"], + } + + resp = client.get("/get/mcp_tool_search_settings") + + assert resp.status_code == 200, resp.text + assert resp.json()["values"] == { + "embedding_model": "text-embedding-3-small", + "top_k": 5, + "similarity_threshold": 0.0, + "core_tools": ["treasury-get_rates"], + } + assert resp.json()["field_schema"]["properties"]["core_tools"]["type"] == "array" + + def test_update_requires_proxy_admin(self, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + self._override_auth(LitellmUserRoles.INTERNAL_USER) + try: + resp = client.patch("/update/mcp_tool_search_settings", json={"top_k": 3}) + finally: + app.dependency_overrides.clear() + assert resp.status_code == 403 + + def test_update_persists_and_applies_in_memory(self, mock_proxy_config, monkeypatch): + import litellm + + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + monkeypatch.setattr(litellm, "mcp_tool_search", None) + self._override_auth(LitellmUserRoles.PROXY_ADMIN) + payload = { + "embedding_model": "text-embedding-3-small", + "top_k": 3, + "similarity_threshold": 0.25, + "core_tools": ["treasury-get_rates"], + } + try: + resp = client.patch("/update/mcp_tool_search_settings", json=payload) + finally: + app.dependency_overrides.clear() + + assert resp.status_code == 200, resp.text + assert mock_proxy_config["save_call_count"]() == 1 + assert litellm.mcp_tool_search == payload + assert mock_proxy_config["config"]["litellm_settings"]["mcp_tool_search"] == payload + + def test_update_rejects_out_of_range_top_k(self, mock_proxy_config, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + self._override_auth(LitellmUserRoles.PROXY_ADMIN) + try: + resp = client.patch("/update/mcp_tool_search_settings", json={"top_k": 0}) + finally: + app.dependency_overrides.clear() + assert resp.status_code == 422 + assert mock_proxy_config["save_call_count"]() == 0 + + +def test_upload_logo_requires_proxy_admin(monkeypatch): + """Any authenticated key could previously write a file to the server's disk here.""" + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + async def _internal_user_auth(): + return UserAPIKeyAuth( + user_id="internal-user-1", + api_key="hashed-internal-key", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + app.dependency_overrides[user_api_key_auth] = _internal_user_auth + try: + resp = client.post( + "/upload/logo", + files={"file": ("logo.png", b"\x89PNG\r\n\x1a\n" + b"x" * 32, "image/png")}, + ) + assert resp.status_code == 403 + assert "proxy admin" in resp.json()["detail"].lower() + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_upload_logo_allows_proxy_admin(monkeypatch, tmp_path): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + async def _admin_auth(): + return UserAPIKeyAuth( + user_id="admin-1", + api_key="hashed-admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + app.dependency_overrides[user_api_key_auth] = _admin_auth + try: + resp = client.post( + "/upload/logo", + files={"file": ("logo.png", b"\x89PNG\r\n\x1a\n" + b"x" * 32, "image/png")}, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["status"] == "success" + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + uploaded_path = resp.json().get("file_path") + if uploaded_path and os.path.exists(uploaded_path): + os.remove(uploaded_path) + + class TestPtuCostAttributionUISetting: """``enable_ptu_cost_attribution`` is derived from the environment on every GET. diff --git a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py new file mode 100644 index 00000000000..e449d4392d8 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import importlib.util +import subprocess +import sys +import zipfile +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from types import MappingProxyType, ModuleType +from typing import Final, Protocol, cast + +import pytest + + +class _CommandRunner(Protocol): + def __call__( + self, + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, + ) -> subprocess.CompletedProcess[str]: ... + + +class _VerifierModule(Protocol): + main: Callable[ + [ + Sequence[str] | None, + Mapping[str, str] | None, + Callable[[Path], ModuleType | None], + _CommandRunner, + ], + int, + ] + + +_REPO_ROOT: Final = Path(__file__).resolve().parents[3] +_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "verify_linux_native_wheel.py" +_SPEC: Final = importlib.util.spec_from_file_location("verify_linux_native_wheel", _MODULE_PATH) +assert _SPEC is not None and _SPEC.loader is not None +_LOADED_VERIFIER: Final = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _LOADED_VERIFIER +_SPEC.loader.exec_module(_LOADED_VERIFIER) +verifier: Final = cast(_VerifierModule, _LOADED_VERIFIER) + +_EXPECTED_TAG: Final = "cp310-abi3-linux_x86_64" +_NATIVE_MEMBER: Final = "litellm/rust_bridge/_native.abi3.so" +_DIST_INFO: Final = "litellm-1.100.0.dist-info" + + +def _write_wheel( + tmp_path: Path, + *, + filename_tag: str, + metadata_tags: tuple[str, ...] | None = (_EXPECTED_TAG,), + dist_info: str = _DIST_INFO, + duplicate_wheel: bool = False, +) -> Path: + wheel: Final = tmp_path / f"litellm-1.100.0-{filename_tag}.whl" + with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(_NATIVE_MEMBER, b"synthetic native extension") + archive.writestr( + f"{dist_info}/METADATA", + "Metadata-Version: 2.1\nName: litellm\nVersion: 1.100.0\n", + ) + archive.writestr( + f"{dist_info}/RECORD", + f"{_NATIVE_MEMBER},,\n{dist_info}/WHEEL,,\n", + ) + if metadata_tags is not None: + wheel_metadata: Final = ( + "Wheel-Version: 1.0\nGenerator: regression-test\nRoot-Is-Purelib: false\n" + + "".join(f"Tag: {tag}\n" for tag in metadata_tags) + ) + archive.writestr(f"{dist_info}/WHEEL", wheel_metadata) + if duplicate_wheel: + archive.writestr(f"{dist_info}/WHEEL", wheel_metadata) + return wheel + + +def _fake_subprocess_run( + command: tuple[str, ...], + *, + check: bool, + capture_output: bool, + text: bool, +) -> subprocess.CompletedProcess[str]: + assert check and capture_output and text + if command == ("rustc", "--version"): + return subprocess.CompletedProcess(command, 0, stdout="rustc 1.98.0 (regression-test)\n", stderr="") + if "--sections" in command: + return subprocess.CompletedProcess(command, 0, stdout="[ 1] .text PROGBITS\n", stderr="") + if "--dyn-syms" in command: + return subprocess.CompletedProcess(command, 0, stdout="PyInit__native\n", stderr="") + raise AssertionError(f"unexpected subprocess command: {command}") + + +class _NativeModuleWithPanicHook(ModuleType): + def _panic_for_test(self) -> None: + return None + + +def _run_verifier( + wheel: Path, + *, + exposes_panic: bool = False, +) -> int: + native_module: Final = ( + _NativeModuleWithPanicHook("litellm.rust_bridge._native") + if exposes_panic + else ModuleType("litellm.rust_bridge._native") + ) + + def _fake_load_native_module(_: Path) -> ModuleType: + return native_module + + environment: Final = MappingProxyType({"GITHUB_STEP_SUMMARY": str(wheel.parent / "summary.md")}) + return verifier.main( + (str(_MODULE_PATH), str(wheel)), + environment, + _fake_load_native_module, + _fake_subprocess_run, + ) + + +def test_accepts_expected_release_wheel_tags(tmp_path: Path) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + + assert _run_verifier(wheel) == 0 + + +def test_rejects_cp312_version_specific_wheel(tmp_path: Path) -> None: + tag: Final = "cp312-cp312-linux_x86_64" + wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) + + assert _run_verifier(wheel) == 1 + + +def test_rejects_non_linux_platform_tag(tmp_path: Path) -> None: + tag: Final = "cp310-abi3-win_amd64" + wheel: Final = _write_wheel(tmp_path, filename_tag=tag, metadata_tags=(tag,)) + + assert _run_verifier(wheel) == 1 + + +@pytest.mark.parametrize( + "metadata_tags", + (None, ("cp312-cp312-linux_x86_64",)), + ids=("missing", "mismatched"), +) +def test_rejects_missing_or_mismatched_wheel_metadata_tag( + tmp_path: Path, + metadata_tags: tuple[str, ...] | None, +) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG, metadata_tags=metadata_tags) + + assert _run_verifier(wheel) == 1 + + +def test_rejects_wheel_metadata_from_wrong_dist_info_directory( + tmp_path: Path, +) -> None: + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + dist_info="decoy-1.0.0.dist-info", + ) + + assert _run_verifier(wheel) == 1 + + +def test_rejects_duplicate_wheel_metadata_tags(tmp_path: Path) -> None: + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + metadata_tags=(_EXPECTED_TAG, _EXPECTED_TAG), + ) + + assert _run_verifier(wheel) == 1 + + +def test_rejects_duplicate_wheel_metadata_file(tmp_path: Path) -> None: + with pytest.warns(UserWarning, match="Duplicate name"): + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + duplicate_wheel=True, + ) + + assert _run_verifier(wheel) == 1 + + +def test_rejects_production_module_exposing_panic_hook(tmp_path: Path) -> None: + wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) + + assert _run_verifier(wheel, exposes_panic=True) == 1 diff --git a/tests/test_litellm/test_circleci_rust_toolchain.py b/tests/test_litellm/test_circleci_rust_toolchain.py index c35ced51e16..800ca21b95d 100644 --- a/tests/test_litellm/test_circleci_rust_toolchain.py +++ b/tests/test_litellm/test_circleci_rust_toolchain.py @@ -17,28 +17,27 @@ Two invariants are pinned here: Windows job, so the check accepts either. A new job that syncs without one falls back to the unpinned path, which is exactly the regression a static check catches at PR time and a green CI run does not. - 2. `install_rust` itself pins what it downloads: an explicit rustup version in - the URL, a verified SHA-256, and an exact toolchain version rather than a - channel name. - -The Windows job predates `install_rust` and provisions its toolchain inline, so -invariant 2 is scoped to `install_rust`; invariant 1 covers both. + 2. Both installers pin what they download: an explicit rustup version, a + verified SHA-256, and the exact toolchain in `rust-toolchain.toml`. """ from __future__ import annotations import re from pathlib import Path +from typing import Final import pytest import yaml REPO_ROOT = Path(__file__).resolve().parents[2] CONFIG = REPO_ROOT / ".circleci" / "config.yml" +TOOLCHAIN: Final = REPO_ROOT / "rust-toolchain.toml" BUILDS_WORKSPACE = re.compile(r"\buv\s+(?:sync|build)\b") RUSTUP_ARCHIVE_URL = re.compile(r"https://static\.rust-lang\.org/rustup/archive/\d+\.\d+\.\d+/") -EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?\d+\.\d+\.\d+\"?") +EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?(\d+\.\d+\.\d+)\"?") +TOOLCHAIN_CHANNEL: Final = re.compile(r'^channel = "(\d+\.\d+\.\d+)"$', re.MULTILINE) def _config() -> dict[str, object]: @@ -57,6 +56,12 @@ def _step_text(step: object) -> str: return "" +def _pinned_toolchain() -> str: + match: Final = TOOLCHAIN_CHANNEL.search(TOOLCHAIN.read_text()) + assert match is not None, "rust-toolchain.toml must pin an exact channel" + return match.group(1) + + def _without_comments(text: str) -> str: return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) @@ -142,7 +147,17 @@ def test_install_rust_verifies_the_installer_checksum(install_rust_command: str) def test_install_rust_pins_an_exact_toolchain_version(install_rust_command: str) -> None: - assert EXACT_TOOLCHAIN.search(install_rust_command), ( - "install_rust must pin an exact toolchain version (e.g. 1.97.1); a channel name like " + match: Final = EXACT_TOOLCHAIN.search(install_rust_command) + assert match is not None, ( + "install_rust must pin an exact toolchain version (e.g. 1.98.0); a channel name like " "stable/beta/nightly makes the compiler drift with whatever upstream published that day" ) + assert match.group(1) == _pinned_toolchain() + + +def test_windows_installer_matches_the_repo_toolchain() -> None: + windows_steps: Final = _step_lists()["job using_litellm_on_windows"] + windows_command: Final = "\n".join(_step_text(step) for step in windows_steps) + match: Final = EXACT_TOOLCHAIN.search(windows_command) + assert match is not None + assert match.group(1) == _pinned_toolchain() diff --git a/tests/test_litellm/test_logging.py b/tests/test_litellm/test_logging.py index 087a1c8b3ad..bf3757e6886 100644 --- a/tests/test_litellm/test_logging.py +++ b/tests/test_litellm/test_logging.py @@ -1,26 +1,30 @@ import ast import asyncio import json +import logging import re import sys +import time +from io import StringIO from pathlib import Path from typing import List import pytest -import logging - import litellm from litellm._logging import ( _COLOR_LOG_FORMAT, + _MAX_SCRUBBED_ACCESS_ARG, _PLAIN_LOG_FORMAT, ALL_LOGGERS, + AccessLogRedactionFilter, CorrelationContextFilter, CorrelationPlainFormatter, JsonFormatter, LevelRoutingStreamHandler, SecretRedactionFilter, StdoutLogTruncationFilter, + _get_uvicorn_json_log_config, _initialize_loggers_with_handler, _parse_json_logs_env, _plain_log_format, @@ -968,3 +972,209 @@ def test_plain_log_format_survives_none_streams(): """sys.stdout/sys.stderr can be None in embedded interpreters; import must not crash.""" assert _plain_log_format(None, None) == _PLAIN_LOG_FORMAT assert _plain_log_format(_FakeStream(True), None) == _PLAIN_LOG_FORMAT + + +# --------------------------------------------------------------------------- +# Access-log redaction (LIT-5909) +# --------------------------------------------------------------------------- + +_LEAKED_KEY = "sk-mx5ous1o9Iezz5fj3pkLuA" + + +def _access_record(full_path: str) -> logging.LogRecord: + """A record shaped exactly like the one uvicorn.access emits per request.""" + return logging.LogRecord( + name="uvicorn.access", + level=logging.INFO, + pathname="", + lineno=0, + msg='%s - "%s %s HTTP/%s" %d', + args=("127.0.0.1:1", "GET", full_path, "1.1", 200), + exc_info=None, + ) + + +@pytest.mark.parametrize( + "full_path", + [ + f"/key/info?key={_LEAKED_KEY}", + f"/global/spend/report?api_key={_LEAKED_KEY}&start_date=2026-08-01", + f"/key/spend/report?api_key={_LEAKED_KEY}", + f"/spend/logs?api_key={_LEAKED_KEY}", + f"/user/daily/activity?api_key={_LEAKED_KEY}", + f"/gemini/v1beta/models/gemini-2.0-flash:generateContent?key={_LEAKED_KEY}", + ], +) +def test_access_log_filter_redacts_a_credential_query_parameter(full_path): + record = _access_record(full_path) + assert AccessLogRedactionFilter().filter(record) is True + assert _LEAKED_KEY not in record.getMessage() + assert "REDACTED" in record.getMessage() + + +def test_access_log_filter_keeps_the_record_formattable_by_uvicorn(): + """uvicorn's AccessFormatter unpacks record.args, so the filter must scrub the + args in place rather than collapse them the way SecretRedactionFilter does.""" + from uvicorn.logging import AccessFormatter + + record = _access_record(f"/key/info?key={_LEAKED_KEY}") + AccessLogRedactionFilter().filter(record) + assert isinstance(record.args, tuple) + assert len(record.args) == 5 + + formatted = AccessFormatter('%(client_addr)s - "%(request_line)s" %(status_code)s', use_colors=False).format(record) + assert _LEAKED_KEY not in formatted + assert "GET" in formatted + assert "200 OK" in formatted + + +@pytest.mark.parametrize( + "full_path, want", + [ + # The delimiter must survive so the logged request line stays well formed. + (f"/key/info?key={_LEAKED_KEY}&page=2", "/key/info?REDACTED&page=2"), + ("/download?sig=AbCd1234%2Fxy&page=2", "/download?REDACTED&page=2"), + ( + f"/global/spend/report?api_key={_LEAKED_KEY}&start_date=2026-01-01", + "/global/spend/report?REDACTED&start_date=2026-01-01", + ), + ("/sso/callback?client_secret=abcdefgh12345&state=xyz", "/sso/callback?REDACTED&state=xyz"), + (f"/v1/models?token={_LEAKED_KEY}&page=2", "/v1/models?REDACTED&page=2"), + ], +) +def test_access_log_filter_keeps_the_query_delimiter(full_path, want): + record = _access_record(full_path) + AccessLogRedactionFilter().filter(record) + assert record.args[2] == want + + +@pytest.mark.parametrize( + "full_path, want", + [ + # Both the param name and the value are encoded, so neither is literal text + # the patterns can see, yet the request parser decodes it into a working key. + (f"/key/info?k%65y=sk%2D{_LEAKED_KEY[3:]}", "/key/info?REDACTED"), + (f"/key/info?k%65y=sk%2D{_LEAKED_KEY[3:]}&page=2", "/key/info?REDACTED"), + (f"/v1/models/sk%2D{_LEAKED_KEY[3:]}", "REDACTED"), + # A decoded credential must never be echoed back: it can carry a newline and + # forge a following log line. + (f"/v1/models?k%65y=sk%2D{_LEAKED_KEY[3:]}%0AINFO:%20forged", "/v1/models?REDACTED"), + ], +) +def test_access_log_filter_redacts_a_percent_encoded_credential(full_path, want): + record = _access_record(full_path) + AccessLogRedactionFilter().filter(record) + assert record.args[2] == want + + +@pytest.mark.parametrize( + "full_path", + [ + "/v1/models?filter=gpt%2D4o&page=2", + "/gemini/v1beta/models/gemini-2.0-flash%3AgenerateContent", + ], +) +def test_access_log_filter_leaves_harmless_percent_encoding_alone(full_path): + """Decoding is a detector, not a rewrite, so a request line with no credential + in it survives encoded exactly as the client sent it.""" + record = _access_record(full_path) + AccessLogRedactionFilter().filter(record) + assert record.args[2] == full_path + + +def test_access_log_filter_caps_how_much_of_a_request_target_it_scans(): + """The request target is the only input to the secret regex an unauthenticated + caller controls end to end, so it is bounded before it is scanned, and the + dropped tail must not reach the log either.""" + record = _access_record("/v1/models?u=" + "a://" * 8192 + f"&key={_LEAKED_KEY}") + + started = time.perf_counter() + AccessLogRedactionFilter().filter(record) + elapsed = time.perf_counter() - started + + scrubbed = record.args[2] + assert _LEAKED_KEY not in scrubbed + assert len(scrubbed) < 1024 + assert elapsed < 1.0, f"scrubbing one access line took {elapsed:.2f}s" + + +@pytest.mark.parametrize("chars_before_the_cut", range(1, 12)) +def test_access_log_filter_never_logs_a_half_scanned_credential(chars_before_the_cut): + """Cutting mid-value would leave a prefix too short for the key= pattern to match, + and that prefix would then be logged raw, so the cut lands on a param boundary.""" + prefix = "/v1/models?u=" + padding = _MAX_SCRUBBED_ACCESS_ARG - len(prefix) - len("&key=") - chars_before_the_cut + record = _access_record(f"{prefix}{'a' * padding}&key={_LEAKED_KEY}") + + AccessLogRedactionFilter().filter(record) + + assert f"key={_LEAKED_KEY[:chars_before_the_cut]}" not in record.args[2] + + +def test_access_log_filter_leaves_a_credential_free_request_line_intact(): + record = _access_record("/v1/chat/completions") + AccessLogRedactionFilter().filter(record) + assert record.getMessage() == '127.0.0.1:1 - "GET /v1/chat/completions HTTP/1.1" 200' + + +def test_access_log_filter_redacts_a_record_that_carries_no_positional_args(): + record = logging.LogRecord( + name="uvicorn.access", + level=logging.INFO, + pathname="", + lineno=0, + msg=f'127.0.0.1:1 - "GET /key/info?key={_LEAKED_KEY} HTTP/1.1" 200', + args=None, + exc_info=None, + ) + assert AccessLogRedactionFilter().filter(record) is True + assert _LEAKED_KEY not in record.getMessage() + + +def _emit_access_line(full_path: str) -> str: + """Hand one real record to uvicorn.access and return what a handler wrote out.""" + from uvicorn.logging import AccessFormatter + + logger = logging.getLogger("uvicorn.access") + stream = StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(AccessFormatter('%(client_addr)s - "%(request_line)s" %(status_code)s', use_colors=False)) + saved_level, saved_propagate = logger.level, logger.propagate + logger.addHandler(handler) + logger.setLevel(logging.INFO) + logger.propagate = False + try: + logger.handle(_access_record(full_path)) + finally: + logger.removeHandler(handler) + logger.setLevel(saved_level) + logger.propagate = saved_propagate + return stream.getvalue() + + +def test_uvicorn_access_logger_redacts_a_credential_it_is_handed(): + """Registration happens at litellm import; without it the filter never runs.""" + emitted = _emit_access_line(f"/key/info?key={_LEAKED_KEY}") + + assert _LEAKED_KEY not in emitted + assert "REDACTED" in emitted + + +def test_access_redaction_survives_the_uvicorn_json_log_config(): + """litellm hands uvicorn a dictConfig when json_logs is on. dictConfig clears a + logger's handlers but not its filters, so redaction has to still be attached.""" + import logging.config + + names = ("uvicorn", "uvicorn.error", "uvicorn.access") + saved = tuple((logging.getLogger(n), logging.getLogger(n).handlers[:], logging.getLogger(n).level) for n in names) + try: + logging.config.dictConfig(_get_uvicorn_json_log_config()) + emitted = _emit_access_line(f"/key/info?key={_LEAKED_KEY}") + + assert _LEAKED_KEY not in emitted + assert "REDACTED" in emitted + finally: + for lg, handlers, level in saved: + lg.handlers[:] = handlers + lg.setLevel(level) + lg.propagate = True diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index ff5db18a3ea..dcaa6cfd602 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -6,6 +6,7 @@ import logging import os import threading from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -1537,6 +1538,91 @@ async def test_ageneric_api_call_deployment_model_overrides_alias(): ), f"Expected deployment model 'vertex_ai/gemini-2.5-flash', got '{captured['model']}'" +@pytest.mark.asyncio +async def test_ageneric_api_call_resolves_realtime_session_model(): + """ + Regression for #36742: realtime client secret requests carry the model inside `session` too, and the proxy + fills it with the pre-routing model group name. The underlying litellm function reads session.model first, + so it must see the resolved deployment, while a caller's nested transcription model stays untouched. + """ + routed: Final = AsyncMock(return_value={"result": "ok"}) + + router = litellm.Router( + model_list=[ + { + "model_name": "my-realtime-group", + "litellm_params": { + "model": "openai/gpt-realtime-2.1-mini", + "api_key": "fake-key", + }, + "model_info": {"mode": "realtime"}, + } + ] + ) + + await router._ageneric_api_call_with_fallbacks( + model="my-realtime-group", + original_function=routed, + session={ + "type": "realtime", + "model": "my-realtime-group", + "audio": {"input": {"transcription": {"model": "gpt-4o-transcribe"}}}, + }, + ) + + sent: Final = routed.call_args.kwargs + assert sent["model"] == "openai/gpt-realtime-2.1-mini" + assert sent["session"]["model"] == "openai/gpt-realtime-2.1-mini" + assert sent["session"]["audio"]["input"]["transcription"]["model"] == "gpt-4o-transcribe" + + +@pytest.mark.asyncio +async def test_ageneric_api_call_does_not_add_session_model(): + """ + A session that never carried a model must not gain one from routing: the underlying function then falls back + to the resolved `model` kwarg itself, and the outgoing session body keeps the caller's shape. + """ + routed: Final = AsyncMock(return_value={"result": "ok"}) + + router = litellm.Router( + model_list=[ + { + "model_name": "my-realtime-group", + "litellm_params": { + "model": "openai/gpt-realtime-2.1-mini", + "api_key": "fake-key", + }, + "model_info": {"mode": "realtime"}, + } + ] + ) + + await router._ageneric_api_call_with_fallbacks( + model="my-realtime-group", + original_function=routed, + session={"type": "realtime"}, + ) + + sent: Final = routed.call_args.kwargs + assert sent["model"] == "openai/gpt-realtime-2.1-mini" + assert sent["session"] == {"type": "realtime"} + + +@pytest.mark.parametrize( + "session, expected", + [ + ({"type": "realtime", "model": "my-realtime-group"}, {"session": {"type": "realtime", "model": "resolved"}}), + ({"type": "realtime"}, {}), + (None, {}), + ("not-a-session", {}), + ], +) +def test_with_router_resolved_session_model(session, expected): + from litellm.router import _with_router_resolved_session_model + + assert dict(_with_router_resolved_session_model(session, "resolved")) == expected + + def test_router_get_model_access_groups_team_only_models(): """ Test that Router.get_model_access_groups returns the correct response for team-only models @@ -11907,3 +11993,55 @@ class TestPreRoutingTierDrivesFallbacks: response = await router.acompletion(model="smart-router", messages=[{"role": "user", "content": "hi"}]) assert response.choices[0].message.content == "from backup-b" + + +@pytest.mark.asyncio +async def test_prompt_management_factory_marks_injection_for_every_deployment(monkeypatch): + """The factory stamps a provisional deployment's model_info into kwargs before the + prompt pass runs, then routes on the returned model, so any deployment can end up + billed. An injection recorded there must carry the every-deployment sentinel, never + the provisional deployment's id, or a differently-billed deployment loses the credit.""" + import time + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + router = litellm.Router( + model_list=[ + { + "model_name": "cached-claude", + "litellm_params": { + "model": "anthropic_cache_control_hook/claude-sonnet-5", + "prompt_id": "cache-points", + }, + "model_info": {"id": "provisional-dep"}, + } + ] + ) + captured: dict = {} + + async def _capture_acompletion(**kwargs): + captured.update(kwargs) + return litellm.ModelResponse() + + monkeypatch.setattr(litellm, "acompletion", _capture_acompletion) + logging_obj = LiteLLMLogging( + model="cached-claude", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + start_time=time.time(), + litellm_call_id="lit-6445", + function_id="f", + ) + await router.acompletion( + model="cached-claude", + messages=[ + {"role": "system", "content": "a static system prompt"}, + {"role": "user", "content": "hi"}, + ], + cache_control_injection_points=[{"location": "message", "role": "system"}], + litellm_logging_obj=logging_obj, + ) + bucket = captured.get("litellm_metadata") or captured["metadata"] + assert captured["model_info"]["id"] == "provisional-dep" + assert bucket["litellm_gateway_injected_cache"] == "" diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index 7d50694c805..303efcab9c7 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -1,8 +1,10 @@ import logging import logging.config import sys +import time from collections.abc import Callable from io import StringIO +from typing import Final from unittest.mock import patch import pytest @@ -67,6 +69,50 @@ def test_redact_string_catches_secret_patterns(): assert redact_string(normal) == normal +@pytest.mark.parametrize( + "connection_string", + [ + "postgres://admin:pass3cret@db.example.com:5432/mydb", + "redis://:pass3cret@cache.example.com:6379", + "postgres://admin:pass/s3cret@db.example.com:5432/mydb", + "amqp://admin:pass:s3cret@rabbit:5672", + "https://ad@min:pass3cret@host", + # An unencoded "@" inside the password, with a ":" after it + "postgresql://admin:p@ss3cret:2026@db.example.com:5432/mydb", + "amqp://guest:gu@st3cret:1@rabbit:5672/", + # An AWS RDS IAM auth token is a presigned query string used as the + # password, so the userinfo runs to several hundred characters. + "postgresql://litellm:host%3A5432%2F%3FAction%3Dconnect%26X-Amz-Signature%3D" + + "f" * 540 + + "s3cret@db.host:5432/litellm", + ], +) +def test_redact_string_still_catches_connection_string_credentials(connection_string): + """The bounded userinfo pattern must keep matching real connection strings.""" + assert "s3cret" not in redact_string(connection_string) + + +def _redaction_cost(url_bytes: int) -> float: + url: Final = "/x?u=" + "a://" * (url_bytes // 4) + + def once() -> float: + started = time.perf_counter() + redact_string(url) + return time.perf_counter() - started + + return min(once() for _ in range(3)) + + +def test_redact_string_stays_sub_quadratic_on_a_long_adversarial_url(): + """Access-log redaction runs on attacker-controlled request lines, so quadrupling + a URL of scheme separators must not multiply the cost by sixteen. Comparing two + sizes rather than asserting a wall-clock ceiling keeps this honest on a slow box: + the unbounded pattern this replaced cost 5s at 4 KB and 314s at 16 KB.""" + growth: Final = _redaction_cost(16 * 1024) / _redaction_cost(4 * 1024) + + assert growth < 11.0, f"cost grew {growth:.1f}x for 4x the URL length" + + def test_redact_string_catches_minimum_length_virtual_key(): """Regression test for LIT-4355: keys at the enforced 16-char minimum (MINIMUM_CUSTOM_KEY_LENGTH) must be treated as key-shaped by the scrubber.""" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 0790b41c349..200cfd02197 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -41,7 +41,6 @@ from litellm.utils import ( _snapshot_exception_for_hook, async_post_call_failure_deployment_hook, client, - get_api_key, get_llm_provider, get_non_default_completion_params, get_optional_params_image_gen, @@ -4917,17 +4916,6 @@ def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkey _invalidate_model_cost_lowercase_map() -def test_ai21_api_key_is_resolved_from_the_documented_env_var(monkeypatch: pytest.MonkeyPatch) -> None: - """The ai21 branch resolved a misspelled env var, so the name every other ai21 code path - reads, and the only name documented, was ignored.""" - monkeypatch.setattr(litellm, "api_key", None) - monkeypatch.setattr(litellm, "ai21_key", None) - monkeypatch.delenv("AI211_API_KEY", raising=False) - monkeypatch.setenv("AI21_API_KEY", "sk-ai21-resolved-from-env") - - assert get_api_key(llm_provider="ai21", dynamic_api_key=None) == "sk-ai21-resolved-from-env" - - class _JsonCapture(logging.Handler): def __init__(self): super().__init__() diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 914b15222e8..fe21965e3c9 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 22362 + "limit": 22356 }, "LIT002": { - "limit": 26774 + "limit": 26771 }, "LIT003": { "limit": 269 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16499 + "limit": 16486 }, "LIT011": { "limit": 5535 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx index 5bc8c2a5b1c..357f924cbe7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx @@ -82,6 +82,40 @@ const langgraphInfo: AgentCreateInfo = { ], }; +const FULL_RUNTIME_ARN = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime/hosted_agent_4vm3i-BaTdfOELAs"; + +const BEDROCK_AGENTCORE_AGENT = { + agent_id: "agent-3", + agent_name: "bedrock-agent", + agent_card_params: { name: "bedrock-agent", description: "agentcore agent", url: "", version: "1.0.0", skills: [] }, + litellm_params: { + custom_llm_provider: "bedrock", + model: `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + }, +}; + +const bedrockAgentcoreInfo: AgentCreateInfo = { + agent_type: "bedrock_agentcore", + agent_type_display_name: "Bedrock AgentCore", + description: "Bedrock AgentCore runtimes", + logo_url: "/b.png", + use_a2a_form_fields: false, + litellm_params_template: { custom_llm_provider: "bedrock" }, + model_template: "bedrock/agentcore/{agent_runtime_arn}", + credential_fields: [ + { + key: "agent_runtime_arn", + label: "Agent Runtime ARN", + field_type: "text", + required: true, + include_in_litellm_params: false, + validation_pattern: "^arn:aws[a-zA-Z0-9-]*:bedrock-agentcore:[a-z0-9-]+:[0-9]{12}:runtime/.+$", + validation_message: + 'Enter the complete Bedrock AgentCore runtime ARN, including the runtime ID after "runtime/".', + }, + ], +}; + const setup = () => userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); const renderView = () => { @@ -284,6 +318,69 @@ describe("AgentInfoView update payload", () => { expect(patchedPayload().object_permission).toEqual(existingMcpGrants); }); + it("preserves the full AgentCore runtime ARN (including the resource id after runtime/) across an unedited save", async () => { + vi.mocked(networking.getAgentCreateMetadata).mockResolvedValue([bedrockAgentcoreInfo]); + vi.mocked(networking.getAgentInfo).mockResolvedValue(BEDROCK_AGENTCORE_AGENT as never); + const user = setup(); + renderView(); + await openEditor(user); + + expect(await screen.findByLabelText("Agent Runtime ARN")).toHaveValue(FULL_RUNTIME_ARN); + + await save(user); + + expect((patchedPayload().litellm_params as Record).model).toBe( + `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + ); + }); + + it("blocks the save and shows a validation error when the Agent Runtime ARN is truncated", async () => { + vi.mocked(networking.getAgentCreateMetadata).mockResolvedValue([bedrockAgentcoreInfo]); + vi.mocked(networking.getAgentInfo).mockResolvedValue(BEDROCK_AGENTCORE_AGENT as never); + const user = setup(); + renderView(); + await openEditor(user); + + const arnField = await screen.findByLabelText("Agent Runtime ARN"); + await user.clear(arnField); + fireEvent.change(arnField, { + target: { value: "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime" }, + }); + await user.click(screen.getByRole("button", { name: "Save Changes" })); + + expect( + await screen.findByText( + 'Enter the complete Bedrock AgentCore runtime ARN, including the runtime ID after "runtime/".', + ), + ).toBeInTheDocument(); + expect(networking.patchAgentCall).not.toHaveBeenCalled(); + }); + + it("renders and saves normally when a field's validation_pattern is not a valid regex", async () => { + const infoWithBadPattern: AgentCreateInfo = { + ...bedrockAgentcoreInfo, + credential_fields: [ + { + ...bedrockAgentcoreInfo.credential_fields[0], + validation_pattern: "(unterminated", + }, + ], + }; + vi.mocked(networking.getAgentCreateMetadata).mockResolvedValue([infoWithBadPattern]); + vi.mocked(networking.getAgentInfo).mockResolvedValue(BEDROCK_AGENTCORE_AGENT as never); + const user = setup(); + renderView(); + await openEditor(user); + + expect(await screen.findByLabelText("Agent Runtime ARN")).toHaveValue(FULL_RUNTIME_ARN); + + await save(user); + + expect((patchedPayload().litellm_params as Record).model).toBe( + `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + ); + }); + it("reloads the agent and leaves edit mode when the edit is cancelled", async () => { const user = setup(); renderView(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.test.ts new file mode 100644 index 00000000000..0c2d2500776 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.test.ts @@ -0,0 +1,73 @@ +import { describe, it, expect } from "vitest"; +import { detectAgentType, extractModelTemplateValues, parseDynamicAgentForForm } from "./agent_type_utils"; +import type { AgentCreateInfo } from "@/components/networking"; +import type { Agent } from "@/components/agents/types"; + +const FULL_RUNTIME_ARN = "arn:aws:bedrock-agentcore:eu-central-1:123456789012:runtime/hosted_agent_4vm3i-BaTdfOELAs"; + +const bedrockAgentcoreInfo: AgentCreateInfo = { + agent_type: "bedrock_agentcore", + agent_type_display_name: "Bedrock AgentCore", + model_template: "bedrock/agentcore/{agent_runtime_arn}", + credential_fields: [ + { + key: "agent_runtime_arn", + label: "Agent Runtime ARN", + required: true, + include_in_litellm_params: false, + }, + ], +}; + +describe("extractModelTemplateValues", () => { + it("recovers a placeholder value that itself contains '/' (an AWS ARN resource path)", () => { + const values = extractModelTemplateValues( + "bedrock/agentcore/{agent_runtime_arn}", + `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + ); + + expect(values.agent_runtime_arn).toBe(FULL_RUNTIME_ARN); + }); + + it("recovers a placeholder value with no '/' (single path segment)", () => { + const values = extractModelTemplateValues("langgraph/{assistant_id}", "langgraph/asst_1"); + + expect(values.assistant_id).toBe("asst_1"); + }); + + it("returns no match when the model does not fit the template", () => { + const values = extractModelTemplateValues("langgraph/{assistant_id}", "azure_ai/agents/asst_1"); + + expect(values).toEqual({}); + }); +}); + +describe("parseDynamicAgentForForm", () => { + it("preserves the full runtime ARN, including the resource id after 'runtime/', when populating the edit form", () => { + const agent = { + agent_id: "agent-1", + agent_name: "bedrock-agent", + agent_card_params: { description: "" }, + litellm_params: { + custom_llm_provider: "bedrock", + model: `bedrock/agentcore/${FULL_RUNTIME_ARN}`, + }, + } as unknown as Agent; + + const values = parseDynamicAgentForForm(agent, bedrockAgentcoreInfo); + + expect(values.agent_runtime_arn).toBe(FULL_RUNTIME_ARN); + }); +}); + +describe("detectAgentType", () => { + it("detects bedrock_agentcore agents from the model prefix", () => { + const agent = { + agent_id: "agent-1", + agent_name: "bedrock-agent", + litellm_params: { model: `bedrock/agentcore/${FULL_RUNTIME_ARN}` }, + } as unknown as Agent; + + expect(detectAgentType(agent)).toBe("bedrock_agentcore"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts index f91590c5732..506f355c519 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_type_utils.ts @@ -25,6 +25,29 @@ export const detectAgentType = (agent: Agent): string => { return "a2a"; }; +const escapeRegExp = (segment: string): string => segment.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +/** + * Reverses a `model_template` (e.g. "bedrock/agentcore/{agent_runtime_arn}") against a stored + * `model` string to recover the placeholder values that produced it. Builds a regex from the + * template's literal segments rather than matching by split("/") position, because a + * placeholder's value can itself contain "/" (an AWS ARN's "runtime/" resource path, + * a Vertex AI reasoning engine's "projects/.../reasoningEngines/..." resource id) and would + * otherwise be cut off at the first one. + */ +export const extractModelTemplateValues = (template: string, model: string): Record => { + // Splitting on a regex with a capturing group interleaves the captured placeholder + // names between the surrounding literal segments, e.g. "a/{x}/b" -> ["a/", "x", "/b"]. + const parts = template.split(/\{([a-zA-Z0-9_]+)\}/g); + const fieldNames = parts.filter((_part, index) => index % 2 === 1); + const pattern = parts.map((part, index) => (index % 2 === 1 ? "(.+)" : escapeRegExp(part))).join(""); + + const match = model.match(new RegExp(`^${pattern}$`)); + if (!match) return {}; + + return Object.fromEntries(fieldNames.map((name, index) => [name, match[index + 1]])); +}; + /** * Parses agent data for dynamic form fields (non-A2A agents). * Extracts values from litellm_params based on the agent type metadata. @@ -35,24 +58,18 @@ export const parseDynamicAgentForForm = (agent: Agent, agentTypeInfo: AgentCreat description: agent.agent_card_params?.description || "", }; + const templateValues = + agentTypeInfo.model_template && agent.litellm_params?.model + ? extractModelTemplateValues(agentTypeInfo.model_template, agent.litellm_params.model) + : {}; + // Extract credential field values from litellm_params for (const field of agentTypeInfo.credential_fields) { if (field.include_in_litellm_params !== false) { values[field.key] = agent.litellm_params?.[field.key] || field.default_value || ""; - } else { - // For fields not in litellm_params (like agent_id), try to extract from model string - if (agentTypeInfo.model_template && agent.litellm_params?.model) { - const model = agent.litellm_params.model; - const templateParts = agentTypeInfo.model_template.split("/"); - const modelParts = model.split("/"); - - // Find the placeholder position and extract the value - templateParts.forEach((part, index) => { - if (part === `{${field.key}}` && modelParts[index]) { - values[field.key] = modelParts[index]; - } - }); - } + } else if (templateValues[field.key] !== undefined) { + // For fields not in litellm_params (like agent_runtime_arn), recover from the model string + values[field.key] = templateValues[field.key]; } } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx index 04a8b0df9d9..0f3355fe073 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/dynamic_agent_form_fields.tsx @@ -24,58 +24,80 @@ interface DynamicAgentFormFieldsProps { export const unmountedDynamicFieldNames = (mountedPanels: readonly string[]): readonly string[] => mountedPanels.includes(AGENT_FORM_CONFIG.cost.key) ? [] : COST_FIELD_NAMES; -const CredentialField = ({ field }: { field: AgentCredentialFieldMetadata }) => ( - - {({ value, onChange, ref, ...control }) => { - const text = typeof value === "string" ? value : ""; - if (field.field_type === "password") { - return ( - - ); - } - if (field.field_type === "textarea") { - return ( -