diff --git a/.github/workflows/image-scan.yml b/.github/workflows/image-scan.yml index 3e2bca8779d..d769b6ab8af 100644 --- a/.github/workflows/image-scan.yml +++ b/.github/workflows/image-scan.yml @@ -53,7 +53,7 @@ jobs: if: >- github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 30 + timeout-minutes: 60 permissions: contents: read steps: diff --git a/.github/workflows/interactions-spec-canary.yml b/.github/workflows/interactions-spec-canary.yml index a2e4646dce5..89ebdd87a7c 100644 --- a/.github/workflows/interactions-spec-canary.yml +++ b/.github/workflows/interactions-spec-canary.yml @@ -49,9 +49,9 @@ jobs: id: compliance run: | uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \ - --interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \ + --interactions-spec-url=https://aistudio.google.com/api-reference/interactions.openapi.json \ --tb=short -vv -p no:rerunfailures -p no:pytest-retry \ - --allow-hosts=127.0.0.1,::1,localhost,ai.google.dev + --allow-hosts=127.0.0.1,::1,localhost,ai.google.dev,aistudio.google.com tracking-issue: name: tracking-issue diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index cbeea71ad0f..fd1d2973951 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -21,7 +21,7 @@ jobs: pull-requests: read actions: read runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 services: redis: image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f1456b1fde8..8ac157c94f5 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -45,7 +45,7 @@ jobs: id: rust-bridge-key if: steps.changes.outputs.decision != 'skip' env: - RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml') }} + RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'packaging/litellm-proxy/pyproject.toml') }} run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT" - name: Set up Python @@ -98,6 +98,18 @@ jobs: mkdir -p rust-bridge-dist/litellm/rust_bridge cp litellm/rust_bridge/_native.abi3.so rust-bridge-dist/litellm/rust_bridge/_native.abi3.so + - name: Build the gateway wheel once for test shards + if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' + timeout-minutes: 15 + working-directory: packaging/litellm-proxy + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_GHA_VERSION: litellm-ci-v1 + SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} + CARGO_INCREMENTAL: "0" + run: uv run --no-project --with maturin==1.15.0 maturin build --profile dev --strip --out ../../rust-bridge-dist + - name: Upload Rust bridge artifact if: steps.changes.outputs.decision != 'skip' timeout-minutes: 10 @@ -1024,7 +1036,7 @@ jobs: if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))" @@ -1486,7 +1498,7 @@ jobs: if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so fi diff --git a/Dockerfile b/Dockerfile index 4cef63c8d71..94557bcf68c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,6 +76,7 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ COPY pyproject.toml uv.lock ./ COPY enterprise/pyproject.toml enterprise/ COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ +COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/ # Install third-party dependencies (cached unless pyproject.toml/uv.lock change) RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ @@ -101,7 +102,8 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/ RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh # Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ +RUN uv build --package litellm-proxy --wheel --out-dir /wheels +RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \ --extra proxy \ --extra proxy-runtime \ --group admin-mcp \ @@ -111,6 +113,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra bedrock-realtime \ --python python3.13 +RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl + RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ prisma generate --schema=./schema.prisma diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 20f810686a3..2ebffc50b5e 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -74,6 +74,7 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ COPY pyproject.toml uv.lock ./ COPY enterprise/pyproject.toml enterprise/ COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ +COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/ # Install third-party dependencies (cached unless pyproject.toml/uv.lock change) RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ @@ -99,7 +100,8 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/ RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh # Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ +RUN uv build --package litellm-proxy --wheel --out-dir /wheels +RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \ --extra proxy \ --extra proxy-runtime \ --group admin-mcp \ @@ -109,6 +111,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra bedrock-realtime \ --python python3.13 +RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl + RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ prisma generate --schema=./schema.prisma diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index a31e926256a..4cfc6d4e081 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -8,7 +8,6 @@ import re from collections.abc import Callable, Mapping, Sequence from dataclasses import replace as dataclasses_replace from datetime import datetime, timedelta -from types import MappingProxyType from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast from pydantic import TypeAdapter, ValidationError @@ -766,7 +765,6 @@ class CheckBatchCost: credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {} _file_content = await afile_content( file_id=cast(str, raw_output_file_id), - _litellm_internal_model_credentials=MappingProxyType(dict(credentials)), **cast(FileContentCallOptions, credentials), ) @@ -867,10 +865,7 @@ class CheckBatchCost: error_file_failed_requests: Final = await count_error_file_failed_requests( response, custom_llm_provider=batch_file_provider, - litellm_params={ - **credentials, - "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)), - }, + litellm_params=credentials, ) batch_result: Final = ( output_file_result @@ -1029,11 +1024,7 @@ class CheckBatchCost: custom_llm_provider=deployment_info.litellm_params.custom_llm_provider, ) cost_provider: Final = cast(BatchFileProvider, llm_provider) - credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {} - file_access: Final = { - **credentials, - "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)), - } + file_access: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {} output_content: Final = ( await fetch_batch_managed_file_content( response.output_file_id, custom_llm_provider=cost_provider, litellm_params=file_access diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 91c5e2c5f39..aac4469be20 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -127,7 +127,6 @@ class _ManagedFileRetrieve(Protocol): self, *, file_id: str, - _litellm_internal_model_credentials: Mapping[str, object] | None = None, **kwargs: Unpack[FileRetrieveCallOptions], ) -> OpenAIFileObject: ... @@ -400,7 +399,7 @@ class _RouterFileCallKwargs(TypedDict, total=False): _MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str]) _FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"}) -_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"}) +_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id"}) @dataclass(frozen=True, slots=True) @@ -459,15 +458,7 @@ async def _delete_provider_file( provider_file_id: str, request_data: Mapping[str, object], ) -> None: - credentials: Final = llm_router.registry().credentials(model_id, form="with_provider") - delete_data: Final = { - **{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS}, - **( - {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))} - if credentials is not None - else {} - ), - } + delete_data: Final = {k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS} await llm_router.afile_delete( model=model_id, file_id=provider_file_id, @@ -1160,7 +1151,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): *, provider_file_id: str, call_options: Mapping[str, object], - internal_model_credentials: Mapping[str, object] | None = None, ) -> OpenAIFileObject: """Retrieve provider file details with transient retries and SDK retries disabled.""" retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve) @@ -1172,14 +1162,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if attempt > 0: await self._sleep(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS[attempt - 1]) try: - if internal_model_credentials is None: - return await retrieve_file( - file_id=provider_file_id, - **retrieve_options, - ) return await retrieve_file( file_id=provider_file_id, - _litellm_internal_model_credentials=MappingProxyType(dict(internal_model_credentials)), **retrieve_options, ) except Exception as error: @@ -1221,7 +1205,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): self._afile_retrieve_with_retries( provider_file_id=provider_file_id, call_options=router_credentials, - internal_model_credentials=router_credentials, ), timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS, ) @@ -2866,15 +2849,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): exception_dict = {} for model_id, provider_file_id in specific_model_file_id_mapping.items(): try: - # Cloud-storage providers (e.g. Bedrock S3) validate file ids - # against the deployment's configured bucket, which they only - # trust from this immutable server-side snapshot, never from - # request params. - credentials = llm_router.registry().credentials(model_id, form="with_provider") - if credentials is not None: - data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials))) - else: - data.pop("_litellm_internal_model_credentials", None) return cast( HttpxBinaryResponseContent, await llm_router.afile_content( diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 658bed831ba..1e1c7d73091 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -41,10 +41,14 @@ COPY pyproject.toml README.md rust-toolchain.toml ./ COPY .cargo ./.cargo COPY litellm-rust ./litellm-rust COPY model_prices_and_context_window.json ./ +COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/ RUN --mount=type=cache,target=/root/.cache/uv \ mkdir -p litellm/rust_bridge /wheels && \ touch litellm/__init__.py litellm/rust_bridge/__init__.py && \ + uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \ + python -c "import maturin; maturin.build_wheel('/wheels')" && \ + cd packaging/litellm-proxy && \ uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \ python -c "import maturin; maturin.build_wheel('/wheels')" @@ -85,6 +89,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=uv.lock,target=uv.lock \ --mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \ --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \ + --mount=type=bind,source=packaging/litellm-proxy/pyproject.toml,target=packaging/litellm-proxy/pyproject.toml \ uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ --extra proxy \ --extra proxy-runtime \ @@ -98,7 +103,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ COPY . . RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-install-project --no-default-groups --no-editable \ + uv sync --frozen --no-install-project --no-install-package litellm-proxy --no-default-groups --no-editable \ --extra proxy \ --extra proxy-runtime \ --group admin-mcp \ @@ -108,7 +113,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --python python3.13 RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \ - uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl + uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl /wheels/litellm_proxy-*.whl # PYTHONPATH=/app makes the source tree shadow the installed package, so the # compiled Rust extension must live next to the source or it is never imported. diff --git a/license_cache.json b/license_cache.json index cd8a54d72a0..c53cd8021e2 100644 --- a/license_cache.json +++ b/license_cache.json @@ -56,5 +56,42 @@ "vcrpy:8.1.1": "MIT", "langchain-openai:1.1.14": "MIT", "grpc-google-iam-v1:0.14.4": "Apache 2.0", - "tornado:6.5.5": "Apache-2.0" + "tornado:6.5.5": "Apache-2.0", + "packaging:24.0": "Apache Software License", + "huggingface-hub:0.34.0": "Apache", + "async-timeout:4.0.3": "Apache 2", + "pydantic-settings:2.14.1": "MIT", + "granian:2.7.4": "BSD-3-Clause", + "granian:2.8.0": "BSD-3-Clause", + "hiredis:3.0.0": "MIT", + "hiredis:3.5.0": "MIT", + "InquirerPy:0.3.4": "MIT", + "tomlkit:0.13.3": "MIT", + "semver:3.0.4": "Copyright (c) 2013, Konstantine Rybnikov All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of the python-semver org nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ", + "requests:2.32.0": "Apache-2.0", + "keyring:25.6.0": "MIT License", + "python3-saml:1.16.0": "MIT", + "mlflow:3.17.2": "Copyright 2018 Databricks, Inc. All rights reserved.\n \n \t\t\t\tApache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n \n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n \n 1. Definitions.\n \n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n \n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n \n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n \n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n \n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n \n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n \n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n \n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n \n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n \n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n \n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n \n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n \n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n \n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n \n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n \n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n \n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n \n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n \n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n \n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n \n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n \n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n \n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n \n END OF TERMS AND CONDITIONS\n APPENDIX: How to apply the Apache License to your work.\n \n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n \n Copyright [yyyy] [name of copyright owner]\n \n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n ", + "google-cloud-speech:2.40.0": "Apache-2.0", + "protobuf:5.26.1": "3-Clause BSD License", + "aws-sdk-bedrock-runtime:0.10.0": "Apache-2.0", + "opentelemetry-instrumentation-fastapi:0.66b1": "Apache-2.0", + "reportlab:5.0.1": "BSD license (see license.txt for details), Copyright (c) 2000-2025, ReportLab Inc.", + "basedpyright:1.39.7": "MIT License", + "mypy:1.20.1": "MIT", + "keyring:25.7.0": "MIT", + "pytest-socket:0.8.1": "MIT", + "tomli:2.4.1": "MIT", + "botocore-stubs:1.43.14": "MIT", + "types-boto3:1.43.30": "MIT", + "nvidia-riva-client:2.16.0": "MIT", + "playwright:1.61.0": "Apache-2.0", + "psutil:7.2.2": "BSD-3-Clause", + "httptools:0.7.1": "MIT", + "pyarrow:23.0.1": "Apache-2.0", + "langchain:1.3.9": "MIT", + "traceloop-sdk:0.34.0": "Apache-2.0", + "claude-agent-sdk:0.2.165": "MIT", + "tornado:6.5.10": "Apache-2.0", + "tornado:6.6": "Apache-2.0" } \ No newline at end of file diff --git a/litellm-rust/crates/router-core/src/call.rs b/litellm-rust/crates/router-core/src/call.rs index 3a30c779958..a118cac28c1 100644 --- a/litellm-rust/crates/router-core/src/call.rs +++ b/litellm-rust/crates/router-core/src/call.rs @@ -4,13 +4,7 @@ //! `should_retry_this_error` and `_time_to_sleep_before_retry` in `litellm/router.py`, and //! `run_async_fallback` in `router_utils/fallback_event_handlers.py`. -use std::{ - borrow::Cow, - future::Future, - pin::Pin, - sync::Arc, - time::{Duration, Instant}, -}; +use std::{borrow::Cow, future::Future, pin::Pin, sync::Arc, time::Duration}; use serde_json::{Map, Value}; @@ -1231,7 +1225,7 @@ impl Core { .unrouted_attempt(snapshot, op, request, target, (retry, max_retries), at) .await; } - let pick_started = Instant::now(); + let pick_started = self.health.now_time(); let (deployment, candidates) = match self .picked(snapshot, op, request, target, inputs, retry, at) .await @@ -1248,7 +1242,11 @@ impl Core { self.host.observe(&RouterEvent::Selected { model_group: candidates.model_group.clone(), deployment_id: deployment.id.clone(), - pick: pick_started.elapsed(), + pick: self + .health + .now_time() + .duration_since(pick_started) + .unwrap_or_default(), }); if !self.host.supports(op, deployment.provider()) { return Err(Error::Config(ConfigError::Unsupported(Unsupported::Op { @@ -1274,7 +1272,7 @@ impl Core { }, None, snapshot.settings.dynamic.default_ref(), - jiff::Timestamp::now(), + self.health.timestamp(), ), Ok(None) ); diff --git a/litellm-rust/crates/router-core/src/health.rs b/litellm-rust/crates/router-core/src/health.rs index c8055eafe31..eaa61dac7ba 100644 --- a/litellm-rust/crates/router-core/src/health.rs +++ b/litellm-rust/crates/router-core/src/health.rs @@ -247,6 +247,10 @@ impl Health { self.clock.now() } + pub(crate) fn timestamp(&self) -> jiff::Timestamp { + jiff::Timestamp::try_from(self.clock.now()).unwrap_or(jiff::Timestamp::UNIX_EPOCH) + } + fn now(&self) -> f64 { unix_seconds(self.clock.now()) } diff --git a/litellm-rust/crates/router-core/src/lookup.rs b/litellm-rust/crates/router-core/src/lookup.rs index 39c235d6f7b..8fef0ba222f 100644 --- a/litellm-rust/crates/router-core/src/lookup.rs +++ b/litellm-rust/crates/router-core/src/lookup.rs @@ -433,6 +433,10 @@ impl<'a> Lookup<'a> { .collect() } + pub fn has_team_wildcards(&self, team_id: &str) -> bool { + self.host.team_patterns.iter().any(|(t, _)| t == team_id) + } + /// The deployments `team_id`'s own wildcard pattern routes `name` to /// (`_core_team_pattern_routers`, keyed by `team_public_model_name`). pub fn team_wildcard(&self, name: &str, team_id: &str) -> Vec> { diff --git a/litellm-rust/crates/router-core/src/resolve.rs b/litellm-rust/crates/router-core/src/resolve.rs index 9075fc36c53..d897716b1e5 100644 --- a/litellm-rust/crates/router-core/src/resolve.rs +++ b/litellm-rust/crates/router-core/src/resolve.rs @@ -223,14 +223,20 @@ pub(crate) fn wildcard_name<'a>( registry: &'a Registry, name: &'a str, specific_deployment: bool, + team_id: Option<&str>, ) -> Option<&'a str> { let model = registry.alias_of(name).unwrap_or(name); - (registry.has_wildcards() + let lookup = crate::lookup::Lookup::new(registry); + let team_patterns = team_id.is_some_and(|team| lookup.has_team_wildcards(team)); + let team_misses = || team_id.is_none_or(|team| lookup.team_wildcard(model, team).is_empty()); + let global_misses = || registry.has_wildcards() && registry.wildcard_for(model).is_none(); + ((team_patterns || registry.has_wildcards()) && !specific_deployment && !registry.has_name(model) && (registry.has_name(name) || registry.deployment(name).is_none()) && registry.routing_group_named(model).is_none() - && registry.wildcard_for(model).is_none()) + && team_misses() + && (team_patterns || global_misses())) .then_some(model) } @@ -339,15 +345,21 @@ pub fn resolve_request( let kept = reserved(model, deployments, caller, now)?; return resolved(ResolveKind::Group, model, kept); } - // `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one. - let team_owned = team_id - .map(|team| crate::lookup::Lookup::new(registry).team_wildcard(model, team)) - .filter(|deployments| !deployments.is_empty()); - if let Some(deployments) = team_owned { + // `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one, + // each tried as spelled and then under the name's provider (`get_pattern`). + let team_owned = team_id.and_then(|team| { + let lookup = crate::lookup::Lookup::new(registry); + [Some(model), caller.wildcard_request] + .into_iter() + .flatten() + .map(|request| (request, lookup.team_wildcard(request, team))) + .find(|(_, deployments)| !deployments.is_empty()) + }); + if let Some((request, deployments)) = team_owned { let routed = deployments.iter().flat_map(|deployment| { let public = deployment.model_info.team_public_model_name.as_deref(); - let pattern = public.unwrap_or(model); - routed_wildcard(pattern, model, std::slice::from_ref(deployment)) + let pattern = public.unwrap_or(request); + routed_wildcard(pattern, request, std::slice::from_ref(deployment)) }); let kept = reserved(model, routed.collect(), caller, now)?; return resolved(ResolveKind::Wildcard, model, kept); @@ -428,6 +440,7 @@ pub fn resolve_request( mod tests { use std::collections::BTreeMap; + use rstest::rstest; use serde_json::{Value, json}; use super::*; @@ -544,8 +557,11 @@ mod tests { #[test] fn a_name_no_pattern_matches_is_tried_under_its_provider() { let registry = registry(wildcard_group()); - assert_eq!(wildcard_name(®istry, "gpt-4o", false), Some("gpt-4o")); - assert_eq!(wildcard_name(®istry, "openai/gpt-4o", false), None); + assert_eq!( + wildcard_name(®istry, "gpt-4o", false, None), + Some("gpt-4o") + ); + assert_eq!(wildcard_name(®istry, "openai/gpt-4o", false, None), None); let caller = Caller { wildcard_request: Some("openai/gpt-4o"), ..Caller::default() @@ -564,6 +580,48 @@ mod tests { assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o"); } + #[rstest] + fn a_bare_name_is_tried_under_its_provider_against_the_teams_own_wildcard() { + let registry = registry(json!([ + {"model_name": "model_name_tw_1", "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "tw", "team_id": "TW", "team_public_model_name": "openai/*"}}, + {"model_name": "g", "litellm_params": {"model": "openai/g"}, "model_info": {"id": "g"}}, + ])); + // No proxy-wide wildcard: only the team's patterns call for the provider spelling. + assert_eq!( + wildcard_name(®istry, "gpt-4o-mini", false, Some("TW")), + Some("gpt-4o-mini") + ); + assert_eq!(wildcard_name(®istry, "gpt-4o-mini", false, None), None); + assert_eq!( + wildcard_name(®istry, "openai/gpt-4o-mini", false, Some("TW")), + None + ); + let resolve = |team_id| { + let caller = Caller { + team_id, + wildcard_request: Some("openai/gpt-4o-mini"), + ..Caller::default() + }; + resolve_request( + ®istry, + "gpt-4o-mini", + false, + caller, + None, + None, + Timestamp::now(), + ) + .unwrap() + }; + let resolution = resolve(Some("TW")).unwrap(); + assert_eq!(resolution.kind, ResolveKind::Wildcard); + assert_eq!(resolution.group, "gpt-4o-mini"); + assert_eq!(ids(Some(resolution.clone())), ["tw"]); + assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o-mini"); + assert!(resolve(Some("OTHER")).is_none()); + } + #[test] fn a_deployment_named_by_id_refuses_the_endpoint_like_any_other() { let registry = registry(json!([ diff --git a/litellm-rust/crates/router-core/src/select.rs b/litellm-rust/crates/router-core/src/select.rs index 1972b2cbd90..c7f3b1f9c31 100644 --- a/litellm-rust/crates/router-core/src/select.rs +++ b/litellm-rust/crates/router-core/src/select.rs @@ -281,8 +281,12 @@ impl Core { // No deployment for the name: route to the first default fallback. let default_target = crate::resolve::default_target(&snapshot.settings.dynamic); // `get_pattern`: a name no pattern matches is tried again under its inferred provider. - let wildcard_name = - crate::resolve::wildcard_name(registry, name, request.specific_deployment); + let wildcard_name = crate::resolve::wildcard_name( + registry, + name, + request.specific_deployment, + request.team_id.as_deref(), + ); let wildcard_request = match wildcard_name { Some(model) => { let scope = StrategyScope::Default; @@ -308,7 +312,7 @@ impl Core { caller, default_target, snapshot.settings.dynamic.default_ref(), - jiff::Timestamp::now(), + self.health.timestamp(), )? .ok_or_else(|| Error::UnknownModel { // The name `_common_checks_available_deployment` looked for, past the alias. diff --git a/litellm-rust/crates/router-core/tests/call_loop.rs b/litellm-rust/crates/router-core/tests/call_loop.rs index 9f41371f255..a2a741d6135 100644 --- a/litellm-rust/crates/router-core/tests/call_loop.rs +++ b/litellm-rust/crates/router-core/tests/call_loop.rs @@ -1755,3 +1755,57 @@ async fn a_listed_pass_through_pick_names_the_group_an_alias_resolves_to(config: Error::NoPassThroughDeployments { model } if model == "primary" )); } + +struct StoppedClock(u64); + +impl litellm_router_core::health::Clock for StoppedClock { + fn now(&self) -> std::time::SystemTime { + std::time::UNIX_EPOCH + Duration::from_secs(self.0) + } +} + +fn utc_hour(hour: u64) -> u64 { + 1_767_225_600 + hour * 3_600 +} + +#[rstest] +#[case::inside_the_window(utc_hour(10), vec!["open"])] +#[case::outside_the_window(utc_hour(20), vec!["open", "reserved"])] +#[tokio::test] +async fn access_windows_follow_the_routers_clock(#[case] now: u64, #[case] expected: Vec<&str>) { + let config: RouterConfig = serde_json::from_value(json!({ + "model_list": [ + {"model_name": "chat", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "open"}}, + {"model_name": "chat", "litellm_params": {"model": "openai/b"}, "model_info": { + "id": "reserved", + "access_windows": [{"start": "09:00", "end": "17:00", "timezone": "UTC", "team_ids": ["batch"]}], + }}, + ] + })) + .unwrap(); + let router = Router::builder( + config, + RouterDynamicConfig::default(), + Arc::new(ScriptedHost::default()), + ) + .randomness(Arc::new(SeededRandomness::new(7))) + .clock(Arc::new(StoppedClock(now))) + .build() + .unwrap(); + let mut picked = std::collections::BTreeSet::new(); + for _ in 0..40 { + let deployment = router + .select(&Op::from("acompletion"), &request_for("chat")) + .await + .unwrap(); + picked.insert(deployment.id.clone()); + } + assert_eq!(picked.into_iter().collect::>(), expected); +} + +fn request_for(model: &str) -> RouterRequest { + RouterRequest { + model: model.to_owned(), + ..RouterRequest::default() + } +} diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index d272a235758..ca072368061 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -641,9 +641,11 @@ def extract_file_access_credentials( "bucket_name", "s3_endpoint_url", "s3_region_name", + "s3_bucket_name", + "s3_output_bucket_name", + "allow_legacy_cloud_file_ids", "timeout", "max_retries", - "_litellm_internal_model_credentials", *AWS_CREDENTIAL_KWARGS_KEYS, # A federated deployment holds no api_key, so without these the fetch that reads a # finished batch's output has nothing to authenticate with and its cost is never billed. diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 9b5630d81ad..d88f549b06b 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -22,7 +22,6 @@ from openai.types.batch import BatchRequestCounts import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.batches.handler import AnthropicBatchesHandler from litellm.llms.azure.batches.handler import AzureBatchesAPI @@ -585,7 +584,6 @@ def retrieve_batch( custom_llm_provider=custom_llm_provider, **kwargs, ) - add_trusted_model_credentials_to_litellm_params(litellm_params, kwargs) if litellm_logging_obj is not None: litellm_logging_obj.update_from_kwargs( kwargs=kwargs, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 0f07cdca29d..a2f7e10e252 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -41,12 +41,14 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import ( ) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, iter_items from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( prompt_cache_breakpoint_with_readable_mode, - responses_reasoning_items_from_thinking_blocks, whole_prompt_cache_breakpoint_or_none, with_prompt_cache_breakpoint, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + responses_reasoning_items_from_thinking_blocks, +) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( CompletionTransformationBridge, diff --git a/litellm/files/main.py b/litellm/files/main.py index c5172bc01ae..d1ddf6c4c4c 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -38,7 +38,6 @@ import litellm from litellm import get_secret_str from litellm.files.streaming import FileContentStreamingResponse from litellm.files.types import FileContentProvider, FileContentStreamingResult, FileRetrieveProvider -from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.azure.common_utils import get_azure_credentials @@ -422,10 +421,6 @@ def file_retrieve( ) if provider_config is not None: litellm_params_dict: Final = get_litellm_params(**kwargs) - add_trusted_model_credentials_to_litellm_params( - litellm_params_dict=litellm_params_dict, - kwargs=kwargs, - ) litellm_params_dict["api_key"] = optional_params.api_key litellm_params_dict["api_base"] = optional_params.api_base @@ -544,10 +539,6 @@ def file_delete( pass optional_params: Final = GenericLiteLLMParams(**kwargs) litellm_params_dict: Final = get_litellm_params(**kwargs) - add_trusted_model_credentials_to_litellm_params( - litellm_params_dict=litellm_params_dict, - kwargs=kwargs, - ) ### TIMEOUT LOGIC ### timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 # set timeout for 10 minutes by default @@ -737,10 +728,6 @@ def file_list( ) if provider_config is not None: litellm_params_dict: Final = get_litellm_params(**kwargs) - add_trusted_model_credentials_to_litellm_params( - litellm_params_dict=litellm_params_dict, - kwargs=kwargs, - ) litellm_params_dict["api_key"] = optional_params.api_key litellm_params_dict["api_base"] = optional_params.api_base @@ -890,10 +877,6 @@ def file_content( try: optional_params: Final = GenericLiteLLMParams(**kwargs) litellm_params_dict: Final = get_litellm_params(**kwargs) - add_trusted_model_credentials_to_litellm_params( - litellm_params_dict=litellm_params_dict, - kwargs=kwargs, - ) ### TIMEOUT LOGIC ### timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 client: Final = kwargs.get("client") diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 4e473c2f3f0..ea518d506fb 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -26,18 +26,18 @@ from litellm.exceptions import UnsupportedParamsError from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.factory import ( + tool_call_ids_answered_by_client, + tool_call_ids_answered_in, +) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT, cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_as_cache_control, with_prompt_cache_breakpoint, without_prompt_cache_breakpoints, ) -from litellm.litellm_core_utils.prompt_templates.factory import ( - tool_call_ids_answered_by_client, - tool_call_ids_answered_in, -) -from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field from litellm.llms.anthropic.common_utils import ( is_claude_code_one_shot_subagent_request, supports_anthropic_cache_control, diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 0579eb15baf..385522c2435 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -898,6 +898,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac def redacts_messages_itself(self) -> bool: return False + def exempt_from_message_redaction(self) -> bool: + """Router pre-call checks that key routing off the request and never log it return True, so + `turn_off_message_logging` leaves the success event they receive unredacted.""" + return False + def redact_standard_logging_payload_from_model_call_details(self, model_call_details: dict) -> dict: """ Redacts or excludes fields from StandardLoggingPayload before callbacks receive it. diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py index 0b8c128286a..d3cc04408b6 100644 --- a/litellm/interactions/background_cost_polling.py +++ b/litellm/interactions/background_cost_polling.py @@ -71,12 +71,12 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj _TERMINAL_STATUSES: Final = frozenset( - {"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"} + {"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action", "requires_continuation"} ) _POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"}) -_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"}) +_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action", "requires_continuation"}) SettlementOutcome: TypeAlias = Literal["billed", "released", "unsettled"] diff --git a/litellm/interactions/main.py b/litellm/interactions/main.py index acd43c9b830..cd144646f79 100644 --- a/litellm/interactions/main.py +++ b/litellm/interactions/main.py @@ -95,6 +95,7 @@ async def acreate( response_mime_type: str | None = None, # Continuation previous_interaction_id: str | None = None, + continuation_token: str | None = None, # Extra params extra_headers: dict[str, str] | None = None, extra_body: Mapping[str, object] | None = None, @@ -129,6 +130,7 @@ async def acreate( response_format: JSON schema for response format response_mime_type: MIME type of the response previous_interaction_id: ID of previous interaction for continuation + continuation_token: Token from a `requires_continuation` interaction to resume its long decode extra_headers: Additional headers extra_body: Additional body parameters timeout: Request timeout @@ -164,6 +166,7 @@ async def acreate( response_format=response_format, response_mime_type=response_mime_type, previous_interaction_id=previous_interaction_id, + continuation_token=continuation_token, extra_headers=extra_headers, extra_body=extra_body, timeout=timeout, @@ -224,6 +227,7 @@ def create( response_mime_type: str | None = None, # Continuation previous_interaction_id: str | None = None, + continuation_token: str | None = None, # Extra params extra_headers: dict[str, str] | None = None, extra_body: Mapping[str, object] | None = None, @@ -262,6 +266,7 @@ def create( response_format: JSON schema for response format response_mime_type: MIME type of the response previous_interaction_id: ID of previous interaction for continuation + continuation_token: Token from a `requires_continuation` interaction to resume its long decode extra_headers: Additional headers extra_body: Additional body parameters timeout: Request timeout diff --git a/litellm/interactions/utils.py b/litellm/interactions/utils.py index 3895a85061d..1d7465a99c1 100644 --- a/litellm/interactions/utils.py +++ b/litellm/interactions/utils.py @@ -20,6 +20,7 @@ INTERACTIONS_API_OPTIONAL_PARAMS: Final = { "response_format", "response_mime_type", "previous_interaction_id", + "continuation_token", "agent_config", } diff --git a/litellm/litellm_core_utils/cloud_storage_security.py b/litellm/litellm_core_utils/cloud_storage_security.py index 8e949694881..477775bb423 100644 --- a/litellm/litellm_core_utils/cloud_storage_security.py +++ b/litellm/litellm_core_utils/cloud_storage_security.py @@ -2,12 +2,10 @@ import posixpath import re from collections.abc import Mapping, Sequence from itertools import accumulate, repeat -from types import MappingProxyType from typing import Final from urllib.parse import quote, unquote from litellm._uuid import uuid -from litellm.litellm_core_utils.core_helpers import as_str_mapping VERTEX_AI_MANAGED_GCS_PREFIX: Final = "litellm-vertex-files/" BEDROCK_MANAGED_S3_BATCH_PREFIX: Final = "litellm-bedrock-files-" @@ -19,7 +17,6 @@ BEDROCK_MANAGED_S3_PREFIXES: Final = ( BEDROCK_MANAGED_S3_OUTPUT_PREFIX, ) MANAGED_CLOUD_STORAGE_SCHEMES: Final = ("s3://", "gs://") -_MAPPING_PROXY_TYPE: Final[type] = type(MappingProxyType({})) MAX_FILE_ID_DECODE_PASSES: Final = 8 @@ -138,13 +135,7 @@ def encode_s3_object_key_for_url(object_key: str) -> str: def should_allow_legacy_cloud_file_ids( litellm_params: Mapping[str, object] | None = None, ) -> bool: - value = None - if isinstance(litellm_params, Mapping): - trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials") - if isinstance(trusted_model_credentials, _MAPPING_PROXY_TYPE): - trusted_mapping: Final = as_str_mapping(trusted_model_credentials) - value = trusted_mapping.get("allow_legacy_cloud_file_ids") if trusted_mapping is not None else None - + value: Final = litellm_params.get("allow_legacy_cloud_file_ids") if litellm_params is not None else None if isinstance(value, bool): return value if isinstance(value, str): diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index d0ac5c00513..41c1baffab8 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -1,7 +1,6 @@ import reprlib -from collections.abc import Mapping, MutableMapping +from collections.abc import Mapping from dataclasses import dataclass, fields -from types import MappingProxyType from typing import Final, cast from pydantic import TypeAdapter, ValidationError @@ -61,6 +60,9 @@ OPTIONAL_KWARGS_KEYS: Final = ( "client_side_timeout", "gcs_bucket_name", "bucket_name", + "s3_bucket_name", + "s3_output_bucket_name", + "allow_legacy_cloud_file_ids", "s3_endpoint_url", "s3_region_name", "s3_access_key_id", @@ -292,19 +294,3 @@ def get_litellm_params( litellm_params[key] = kwargs[key] return litellm_params - - -def add_trusted_model_credentials_to_litellm_params( - litellm_params_dict: MutableMapping[str, object], kwargs: Mapping[str, object] -) -> None: - """ - Carry the immutable server-side credential snapshot into litellm_params. - - get_litellm_params has a fixed signature, so callers that need the snapshot to - survive into the logging object and the downstream file read have to re-add it. Only - a MappingProxyType is accepted, since providers resolve trusted configuration such - as a Bedrock file bucket from it and must not read a request-supplied mapping. - """ - trusted_model_credentials: Final = kwargs.get("_litellm_internal_model_credentials") - if isinstance(trusted_model_credentials, MappingProxyType): - litellm_params_dict["_litellm_internal_model_credentials"] = trusted_model_credentials diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 84fce5b9c90..fcc521cf6cf 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4440,7 +4440,11 @@ class Logging(LiteLLMLoggingBaseClass): if isinstance(callback, CustomLogger): # custom logger class from litellm.integrations.shadow_eval_logger import ShadowEvalLogger - model_call_details: dict = self.model_call_details + model_call_details: dict = ( + self._model_call_details_with_request_messages() + if callback.exempt_from_message_redaction() + else self.model_call_details + ) ################################## # call redaction hook for custom logger model_call_details = callback.redact_standard_logging_payload_from_model_call_details( @@ -5004,6 +5008,21 @@ class Logging(LiteLLMLoggingBaseClass): ) -> None: executor.submit(self.success_handler, result, start_time, end_time, cache_hit) + def _model_call_details_with_request_messages(self) -> dict[str, object]: # mutable-ok: loggers take a dict + """The success-event payload with the request messages put back where `turn_off_message_logging` + redacted them in place, for a callback that routes on the request and never logs it.""" + request_messages: Final = cast(object, self.messages) # cast-ok: the request messages are only put back as-is + standard_logging_object: Final = self.model_call_details.get("standard_logging_object") + return { + **self.model_call_details, + "messages": request_messages, + **( + {"standard_logging_object": {**standard_logging_object, "messages": request_messages}} + if isinstance(standard_logging_object, dict) + else {} + ), + } + def _get_completed_success_callbacks( self, *, asynchronous: bool ) -> tuple[str | Callable[..., object] | CustomLogger, ...]: diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 08ab2d1a2e5..0fed2cb2fce 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -19,7 +19,7 @@ from litellm.constants import ( VLLM_CHOICE_FIELDS, ) from litellm.litellm_core_utils.hidden_params import get_hidden_params -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( extract_reasoning_content, ) from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer @@ -1072,9 +1072,10 @@ def convert_to_model_response_object( if isinstance(e, APIError): raise + hide_reply: Final = response_type == "audio_transcription" received_args: Final = dict( - response_object=response_object, - model_response_object=model_response_object, + response_object="" if hide_reply else response_object, + model_response_object="" if hide_reply else model_response_object, response_type=response_type, stream=stream, start_time=start_time, diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 75f3a85ab43..fd16351e516 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2,52 +2,212 @@ Common utility functions used for translating messages across providers """ -import base64 -import binascii -import io import json -import mimetypes -import re -from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence +from collections.abc import Hashable, Iterable, Mapping, Sequence from dataclasses import dataclass -from graphlib import CycleError, TopologicalSorter -from itertools import chain, groupby, islice -from os import PathLike -from pathlib import Path -from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast +from itertools import chain, groupby +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast -from pydantic import TypeAdapter, ValidationError +from pydantic import TypeAdapter import litellm -from litellm import verbose_logger -from litellm.exceptions import UnsupportedParamsError -from litellm.litellm_core_utils.batch_utils import InMemoryFile -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data as extract_file_data, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_metadata as extract_file_metadata, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_file_ids_from_messages as get_file_ids_from_messages, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_format_from_file_id as get_format_from_file_id, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_image_mime_type_from_url as get_image_mime_type_from_url, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + infer_content_type_from_url_and_content as infer_content_type_from_url_and_content, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + update_messages_with_model_file_ids as update_messages_with_model_file_ids, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + update_responses_input_with_model_file_ids as update_responses_input_with_model_file_ids, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + update_responses_tools_with_model_file_ids as update_responses_tools_with_model_file_ids, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns as drop_lookaround_regex_patterns, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns as drop_non_python_regex_patterns, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_combinators_and_drop_non_python_regex_patterns as flatten_combinators_and_drop_non_python_regex_patterns, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators as flatten_top_level_schema_combinators, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + is_object_schema as is_object_schema, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_fragment_ref as schema_fragment_ref, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_grammar_branch_count as schema_grammar_branch_count, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_has_lookaround_pattern as schema_has_lookaround_pattern, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_has_open_object as schema_has_open_object, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_has_unresolvable_ref as schema_has_unresolvable_ref, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_optional_property_count as schema_optional_property_count, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_property_count as schema_property_count, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_string_formats as schema_string_formats, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_union_property_count as schema_union_property_count, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + tool_with_sanitized_parameters as tool_with_sanitized_parameters, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + LITELLM_INTERNAL_MESSAGE_FIELDS as LITELLM_INTERNAL_MESSAGE_FIELDS, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS as LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT as PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + cache_control_as_prompt_cache_breakpoint as cache_control_as_prompt_cache_breakpoint, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + chat_completion_prompt_cache_breakpoint_for_wire as chat_completion_prompt_cache_breakpoint_for_wire, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + chat_completion_prompt_cache_breakpoints_for_wire as chat_completion_prompt_cache_breakpoints_for_wire, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + prompt_cache_breakpoint_as_cache_control as prompt_cache_breakpoint_as_cache_control, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + prompt_cache_breakpoint_for_wire as prompt_cache_breakpoint_for_wire, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + prompt_cache_breakpoint_with_readable_mode as prompt_cache_breakpoint_with_readable_mode, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + prompt_cache_breakpoint_within as prompt_cache_breakpoint_within, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + whole_prompt_cache_breakpoint_or_none as whole_prompt_cache_breakpoint_or_none, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + with_prompt_cache_breakpoint as with_prompt_cache_breakpoint, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + without_prompt_cache_breakpoints as without_prompt_cache_breakpoints, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + ENCRYPTED_REASONING_SIGNATURE_PREFIX as ENCRYPTED_REASONING_SIGNATURE_PREFIX, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + anthropic_content_lists as anthropic_content_lists, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + converse_redacted_content as converse_redacted_content, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_content_from_signature as encrypted_content_from_signature, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_content_of_block as encrypted_content_of_block, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_reasoning_signature as encrypted_reasoning_signature, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + extract_reasoning_content as extract_reasoning_content, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + has_blank_redacted_data as has_blank_redacted_data, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + is_encrypted_reasoning_block as is_encrypted_reasoning_block, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + is_unsignable_thinking_block as is_unsignable_thinking_block, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + native_redacted_thinking_data as native_redacted_thinking_data, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + parse_content_for_reasoning as parse_content_for_reasoning, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + reasoning_content_from_thinking_blocks as reasoning_content_from_thinking_blocks, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + responses_reasoning_items_from_thinking_blocks as responses_reasoning_items_from_thinking_blocks, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + strip_encrypted_reasoning_from_messages as strip_encrypted_reasoning_from_messages, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS as MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + parse_tool_call_arguments as parse_tool_call_arguments, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + salvage_concatenated_tool_arguments as salvage_concatenated_tool_arguments, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + split_concatenated_json_objects as split_concatenated_json_objects, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + _image_part_url, # pyright: ignore[reportPrivateUsage] # Shared implementation for as_openai_image_part +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + anthropic_image_source_to_openai_url as anthropic_image_source_to_openai_url, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + embedding_input_is_all_media as embedding_input_is_all_media, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + split_embedding_media_for_tpm as split_embedding_media_for_tpm, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + split_media_for_tpm as split_media_for_tpm, # noqa: PLC0414 # public compatibility re-export +) from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionAudioObject, - ChatCompletionCachedContent, ChatCompletionFileObject, ChatCompletionImageObject, - ChatCompletionPromptCacheBreakpoint, - ChatCompletionReasoningItem, - ChatCompletionReasoningSummaryTextBlock, - ChatCompletionRedactedThinkingBlock, ChatCompletionResponseMessage, ChatCompletionTextObject, - ChatCompletionThinkingBlock, ChatCompletionToolParam, ChatCompletionUserMessage, - PromptCacheBreakpoint, ) from litellm.types.utils import ( Choices, - ExtractedFileData, - FileTypes, ModelResponse, - SpecialEnums, StreamingChoices, ) @@ -58,6 +218,10 @@ DEFAULT_USER_CONTINUE_MESSAGE: Final = ChatCompletionUserMessage(content="Please DEFAULT_ASSISTANT_CONTINUE_MESSAGE: Final = ChatCompletionAssistantMessage(content="Please continue.", role="assistant") +_extract_reasoning_content: Final = extract_reasoning_content +_get_image_mime_type_from_url: Final = get_image_mime_type_from_url +_parse_content_for_reasoning: Final = parse_content_for_reasoning + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LoggingClass @@ -245,228 +409,6 @@ _IMAGE_CONTENT_PART_TYPES: Final = ("image_url", "input_image", "image", "comput _IMAGE_SCAN_MAX_DEPTH: Final = 4 -@dataclass(frozen=True, slots=True) -class _TPMMediaProjection: - content: object - media: tuple[Mapping[str, object], ...] = () - - -def _tpm_image_part(url: str, detail: object = "auto") -> Mapping[str, object]: - return {"type": "image_url", "image_url": {"url": url, "detail": detail}} - - -def _tpm_blank_fields(fields: Mapping[str, object], names: tuple[str, ...]) -> Mapping[str, object]: - return {key: "" if key in names and isinstance(value, str) else value for key, value in fields.items()} - - -def _tpm_file_media(part: Mapping[str, object]) -> _TPMMediaProjection: - from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type - - nested: Final = part.get("type") == "file" - fields: Final = as_str_mapping(part.get("file")) if nested else part - if fields is None: - return _TPMMediaProjection(part) - raw_data: Final = fields.get("file_data") - data: Final = raw_data if isinstance(raw_data, str) else "" - declared: Final = next( - ( - value - for key in ("format", "mime_type", "content_type") - if isinstance(value := fields.get(key), str) and value - ), - "", - ) - mime: Final = (declared or parse_mime_type(data[:256]) or "").split(";", 1)[0].strip().lower() - if not mime.startswith(("image/", "audio/")): - return _TPMMediaProjection(part) - raw_url: Final = fields.get("file_url") or fields.get("file_id") - url: Final = data or (raw_url if isinstance(raw_url, str) else "") - header_end: Final = data.find(",", 0, 256) - encoded: Final = ( - data[header_end + 1 :] - if header_end >= 0 and data[:header_end].endswith(";base64") - else "" - if data.startswith(("data:", "https://", "http://", "gs://")) - else data - ) - media: Final = ( - _tpm_image_part( - url - if not data or url.startswith(("data:", "https://", "http://", "gs://")) - else f"data:{mime};base64,{url}", - fields.get("detail", "auto"), - ) - if mime.startswith("image/") - else {"type": "input_audio", "input_audio": {"data": encoded, "format": mime.removeprefix("audio/")}} - ) - projected: Final = _tpm_blank_fields(fields, ("file_data", "file_url")) - return _TPMMediaProjection({**part, "file": projected} if nested else projected, (media,)) - - -def _tpm_media_leaf(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - if part is None: - return _TPMMediaProjection(value) - kind: Final = part.get("type") - if kind in ("file", "input_file"): - return _tpm_file_media(part) - if kind == "input_audio": - audio: Final = as_str_mapping(part.get("input_audio")) - return _TPMMediaProjection( - {**part, "input_audio": _tpm_blank_fields(audio, ("data",))} if audio is not None else value, - ({"type": "input_audio", "input_audio": part.get("input_audio")},), - ) - if kind in ("image_url", "input_image", "image", "computer_screenshot"): - image: Final = as_str_mapping(part.get("image_url")) - source: Final = as_str_mapping(part.get("source")) - canonical: Final = _tpm_image_part( - _image_part_url({**part, "type": "input_image"} if kind == "computer_screenshot" else part) or "", - (image or part).get("detail", part.get("detail", "auto")), - ) - projected: Final = ( - {**part, "image_url": _tpm_blank_fields(image, ("url",))} - if image is not None and kind in ("image_url", "input_image", "computer_screenshot") - else {**part, "source": _tpm_blank_fields(source, ("data", "url"))} - if source is not None and kind == "image" and source.get("type") in ("base64", "url") - else _tpm_blank_fields(part, ("image_url", "url")) - ) - return _TPMMediaProjection(projected, (canonical,)) - return _TPMMediaProjection(value) - - -def _tpm_media_list(value: object, split: Callable[[object], _TPMMediaProjection]) -> _TPMMediaProjection: - if not isinstance(value, (list, tuple)): - return _TPMMediaProjection(value) - parts: Final = cast(Sequence[object], value) # cast-ok: finite JSON content arrays - projected: Final = tuple(map(split, parts)) - media: Final = tuple(chain.from_iterable(part.media for part in projected)) - content: Final = tuple(part.content for part in projected) - return _TPMMediaProjection(parts if not media else content if isinstance(value, tuple) else list(content), media) - - -def _tpm_replace_field(part: Mapping[str, object], key: str, projection: _TPMMediaProjection) -> _TPMMediaProjection: - return _TPMMediaProjection({**part, key: projection.content} if projection.media else part, projection.media) - - -def _tpm_document_media(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - source: Final = as_str_mapping(part.get("source")) if part is not None else None - if part is None or part.get("type") != "document" or source is None or source.get("type") != "content": - return _tpm_media_leaf(value) - content: Final = _tpm_media_list(source.get("content"), _tpm_media_leaf) - return _tpm_replace_field(part, "source", _tpm_replace_field(source, "content", content)) - - -def _tpm_tool_result_media(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - if part is None or part.get("type") != "tool_result": - return _tpm_document_media(value) - return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_document_media)) - - -def _tpm_code_output_media(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - url: Final = part.get("url") if part is not None else None - if part is None or part.get("type") != "image" or not isinstance(url, str) or not url: - return _TPMMediaProjection(value) - return _TPMMediaProjection(_tpm_blank_fields(part, ("url",)), (_tpm_image_part(url),)) - - -def _tpm_input_media(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - if part is None: - return _TPMMediaProjection(value) - kind: Final = part.get("type") - if kind in ("function_call_output", "custom_tool_call_output"): - return _tpm_replace_field(part, "output", _tpm_media_list(part.get("output"), _tpm_tool_result_media)) - if kind == "computer_call_output": - return _tpm_replace_field(part, "output", _tpm_media_leaf(part.get("output"))) - if kind == "code_interpreter_call": - return _tpm_replace_field(part, "outputs", _tpm_media_list(part.get("outputs"), _tpm_code_output_media)) - result: Final = part.get("result") - if kind == "image_generation_call" and isinstance(result, str) and result: - return _TPMMediaProjection( - _tpm_blank_fields(part, ("result",)), - (_tpm_image_part("data:application/octet-stream;base64," + result),), - ) - if kind == "message" or (kind is None and isinstance(part.get("role"), str)): - return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_tool_result_media)) - return _tpm_tool_result_media(value) - - -def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]: - projection: Final = _tpm_media_list(value, _tpm_input_media) - return projection.content, projection.media - - -_TPM_EMBEDDING_FILE_MIME_PREFIXES: Final = ("application/pdf", "video/") - - -def _tpm_embedding_file(data_uri: str) -> tuple[Mapping[str, object], ...]: - from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type - - mime: Final = (parse_mime_type(data_uri[:256]) or "").strip().lower() - if not mime.startswith(_TPM_EMBEDDING_FILE_MIME_PREFIXES): - return () - return ({"type": "file", "file": {"file_data": data_uri, "format": mime}},) - - -_BASE64_PAYLOAD: Final = re.compile(r"[A-Za-z0-9+/]*={0,2}") - - -def _is_base64_data_uri(value: str) -> bool: - header_end: Final = value.find(",", 0, 256) - if header_end < 0 or not value[:header_end].endswith(";base64"): - return False - payload_start: Final = header_end + 1 - if (len(value) - payload_start) % 4 or _BASE64_PAYLOAD.fullmatch(value, payload_start) is None: - return False - last_quantum: Final = value[max(payload_start, len(value) - 4) :] - return base64.b64encode(base64.b64decode(last_quantum)).decode("ascii") == last_quantum - - -def _tpm_embedding_file_block(part: Mapping[str, object]) -> _TPMMediaProjection: - projection: Final = _tpm_file_media(part) - fields: Final = as_str_mapping(part.get("file")) - data: Final = fields.get("file_data") if fields is not None else None - if projection.media or fields is None or not isinstance(data, str) or not _is_base64_data_uri(data): - return projection - media: Final = _tpm_embedding_file(data) - return _TPMMediaProjection({**part, "file": _tpm_blank_fields(fields, ("file_data",))} if media else part, media) - - -def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - if part is not None and part.get("type") == "file": - return _tpm_embedding_file_block(part) - if not isinstance(value, str) or not value.startswith("data:"): - return _tpm_media_leaf(value) - if not _is_base64_data_uri(value): - return _TPMMediaProjection(value) - media: Final = _tpm_file_media({"file_data": value}).media or _tpm_embedding_file(value) - return _TPMMediaProjection("" if media else value, media) - - -def _is_json_array(value: object) -> bool: - return isinstance(value, (list, tuple)) - - -def _tpm_embedding_item(value: object) -> _TPMMediaProjection: - return _tpm_media_list(value, _tpm_embedding_leaf) if _is_json_array(value) else _tpm_embedding_leaf(value) - - -def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]: - projection: Final = ( - _tpm_media_list(value, _tpm_embedding_item) if _is_json_array(value) else _tpm_embedding_leaf(value) - ) - return projection.content, projection.media - - -def embedding_input_is_all_media(value: object) -> bool: - items: Final = cast(Sequence[object], value) if _is_json_array(value) else (value,) # cast-ok: JSON input array - return all(_tpm_embedding_leaf(item).media for item in items) - - def _content_parts_contain_image(parts: Sequence[object]) -> bool: """Depth-bounded frontier walk over nested content lists, iterative because the repo bans recursion; an Anthropic tool_result nests its image parts exactly one level down.""" @@ -487,19 +429,6 @@ def _content_parts_contain_image(parts: Sequence[object]) -> bool: return False -def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) -> str | None: - """Data or remote URL for an Anthropic ``source`` block, in the form chat completions expects.""" - source_type: Final = image_source.get("type") - if source_type == "base64": - media_type: Final = image_source.get("media_type") or "image/jpeg" - image_data: Final = image_source.get("data") or "" - return f"data:{media_type};base64,{image_data}" if image_data else None - if source_type == "url": - url: Final = image_source.get("url") - return url if isinstance(url, str) else "" - return None - - @dataclass(frozen=True, slots=True) class InlineDocument: filename: str @@ -603,22 +532,6 @@ def messages_with_file_parts_as_image_urls(messages: Sequence[AllMessageValues]) return tuple(_typed_message_with_file_parts_as_image_urls(message) for message in messages) -def _image_part_url(part: Mapping[str, object]) -> str | None: - """The image URL carried by one content part, whichever of the three dialects wrote it.""" - part_type: Final = part.get("type") - if part_type in ("image_url", "input_image"): - image_url: Final = part.get("image_url") or part.get("url") - if isinstance(image_url, str): - return image_url - image_fields: Final = as_str_mapping(image_url) - url: Final = image_fields.get("url") if image_fields is not None else None - return url if isinstance(url, str) else None - if part_type == "image": - source: Final = part.get("source") - return anthropic_image_source_to_openai_url(source) if isinstance(source, Mapping) else None - return None - - def as_openai_image_part(part: Mapping[str, object]) -> ChatCompletionImageObject | None: """One image content part rewritten into chat-completions dialect, or None when it is not one. @@ -867,433 +780,6 @@ def get_completion_messages( return messages -def get_format_from_file_id(file_id: str | None) -> str | None: - """ - Gets format from file id - - unified_file_id = litellm_proxy:{};unified_id,{} - If not a unified file id, returns 'file' as default format - """ - from litellm.proxy.openai_files_endpoints.common_utils import ( - convert_b64_uid_to_unified_uid, - ) - - if not file_id: - return None - try: - transformed_file_id: Final = convert_b64_uid_to_unified_uid(file_id) - if transformed_file_id.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value): - match: Final = re.match( - f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}:(.*?);unified_id", - transformed_file_id, - ) - if match: - return match.group(1) - - return None - except Exception: - return None - - -def update_messages_with_model_file_ids( - messages: list[AllMessageValues], - model_id: str | None, - model_file_id_mapping: dict[str, dict[str, str]], -) -> list[AllMessageValues]: - """ - Updates messages with model file ids. - - For managed files (unified file IDs), uses model_file_id_mapping if it - resolves the id, otherwise decodes the base64-encoded unified file ID - and extracts the llm_output_file_id directly. Mirrors the Responses-API - sibling `update_responses_input_with_model_file_ids`. - - model_file_id_mapping: Dict[str, Dict[str, str]] = { - "litellm_proxy/file_id": { - "model_id": "provider_file_id" - } - } - """ - from litellm.proxy.openai_files_endpoints.common_utils import ( - convert_b64_uid_to_unified_uid, - get_original_file_id, - is_base64_encoded_unified_file_id, - is_model_embedded_id, - ) - - for message in messages: - if message.get("role") == "user": - content = message.get("content") - if content: - if isinstance(content, str): - continue - for c in content: - if not isinstance(c, dict): - # Content list items aren't always dicts. e.g. - # text_completion forwards a token-ids list/list-of- - # lists through this path. Skip non-dict items - # instead of indexing into them. - continue - if c.get("type") == "file": - file_object = cast(ChatCompletionFileObject, c) - file_object_file_field = file_object.get("file") - if not isinstance(file_object_file_field, dict): - # Content block has `type: "file"` but not the - # OpenAI Chat Completions shape (e.g. a LangChain - # v1 standardized file block, or a provider-native - # shape that also uses `type: "file"`). Nothing to - # remap here, so skip instead of crashing. - continue - file_id = file_object_file_field.get("file_id") - format = file_object_file_field.get("format", get_format_from_file_id(file_id)) - - if file_id: - provider_file_id = ( - model_file_id_mapping.get(file_id, {}).get(model_id) - if model_file_id_mapping and model_id is not None - else None - ) - if not provider_file_id and is_base64_encoded_unified_file_id(file_id): - unified_file_id = convert_b64_uid_to_unified_uid(file_id) - if "llm_output_file_id," in unified_file_id: - provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0] - if not provider_file_id and is_model_embedded_id(file_id): - provider_file_id = get_original_file_id(file_id) - file_object_file_field["file_id"] = provider_file_id or file_id - if format: - file_object_file_field["format"] = format - return messages - - -def update_responses_input_with_model_file_ids( - input: object, - model_id: str | None = None, - model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None, -) -> object: - """ - Updates responses API input with provider-specific file IDs. - File IDs are always inside the content array, not as direct input_file items. - - For managed files (unified file IDs), uses model_file_id_mapping if provided, - otherwise decodes the base64-encoded unified file ID and extracts the llm_output_file_id directly. - - Args: - input: The responses API input parameter - model_id: The model ID to use for looking up provider-specific file IDs - model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs - Format: {"litellm_file_id": {"model_id": "provider_file_id"}} - """ - from litellm.proxy.openai_files_endpoints.common_utils import ( - convert_b64_uid_to_unified_uid, - get_original_file_id, - is_base64_encoded_unified_file_id, - is_model_embedded_id, - ) - - if isinstance(input, str): - return input - - if not isinstance(input, list): - return input - - updated_input: Final = [] - for item in input: - if not isinstance(item, dict): - updated_input.append(item) - continue - - updated_item = item.copy() - content = item.get("content") - if isinstance(content, list): - updated_content = [] - for content_item in content: - if isinstance(content_item, dict) and content_item.get("type") == "input_file": - file_id = content_item.get("file_id") - if isinstance(file_id, str) and file_id: - provider_file_id = file_id # Default to original - - # Check if we have a mapping for this file ID - if model_file_id_mapping and model_id and file_id in model_file_id_mapping: - # Use the model-specific file ID from mapping - provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id - updated_content_item = content_item.copy() - updated_content_item["file_id"] = provider_file_id - updated_content.append(updated_content_item) - else: - # Check if this is a base64-encoded unified file ID without mapping - is_unified_file_id = is_base64_encoded_unified_file_id(file_id) - if is_unified_file_id: - # Fallback: decode unified file ID - unified_file_id = convert_b64_uid_to_unified_uid(file_id) - if "llm_output_file_id," in unified_file_id: - provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0] - - updated_content_item = content_item.copy() - updated_content_item["file_id"] = provider_file_id - updated_content.append(updated_content_item) - elif is_model_embedded_id(file_id): - updated_content_item = content_item.copy() - updated_content_item["file_id"] = get_original_file_id(file_id) - updated_content.append(updated_content_item) - else: - # Not a managed file, keep as-is - updated_content.append(content_item) - else: - updated_content.append(content_item) - else: - updated_content.append(content_item) - updated_item["content"] = updated_content - - updated_input.append(updated_item) - - return updated_input - - -def _decode_vector_store_ids_in_tools( - tools: list[dict[str, object]] | None, -) -> list[dict[str, object]] | None: - """ - Decodes unified (LiteLLM-managed) vector_store_ids in file_search tools to - provider-native IDs. Non-unified IDs are passed through unchanged. - - This runs unconditionally — no file-ID mapping is required. - """ - if not tools or not isinstance(tools, list): - return tools - - from litellm.llms.base_llm.managed_resources.utils import ( - is_base64_encoded_unified_id, - parse_unified_id, - ) - - updated_tools: Final = [] - for tool in tools: - if not isinstance(tool, dict) or tool.get("type") != "file_search": - updated_tools.append(tool) - continue - - vector_store_ids = tool.get("vector_store_ids") - if not isinstance(vector_store_ids, list): - updated_tools.append(tool) - continue - - decoded_ids = [] - for vs_id in vector_store_ids: - if not isinstance(vs_id, str) or not is_base64_encoded_unified_id(vs_id): - decoded_ids.append(vs_id) - continue - - parsed = parse_unified_id(vs_id) - provider_resource_id = parsed.get("provider_resource_id") if parsed else None - - if not provider_resource_id: - verbose_logger.warning( - "file_search tool contains unified vector_store_id '%s' that could " - "not be decoded to a provider resource ID — passing original ID. " - "Ensure the vector store was created via LiteLLM.", - vs_id, - ) - decoded_ids.append(vs_id) - else: - decoded_ids.append(provider_resource_id) - - updated_tools.append({**tool, "vector_store_ids": decoded_ids}) - - return updated_tools - - -def update_responses_tools_with_model_file_ids( - tools: list[dict[str, object]] | None, - model_id: str | None = None, - model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None, -) -> list[dict[str, object]] | None: - """ - Updates responses API tools with provider-specific file IDs. - - Pass 1 (always): decode unified vector_store_ids in file_search tools. - Pass 2 (needs mapping): map code_interpreter container file_ids to provider IDs. - - Args: - tools: The responses API tools parameter - model_id: The model ID to use for looking up provider-specific file IDs - model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs - Format: {"litellm_file_id": {"model_id": "provider_file_id"}} - """ - if not tools or not isinstance(tools, list): - return tools - - # Pass 1: decode unified vector_store_ids (no mapping needed) - tools = _decode_vector_store_ids_in_tools(tools) or tools - - # Pass 2: map code_interpreter file IDs (requires mapping) - if not model_file_id_mapping or not model_id: - return tools - - updated_tools: Final = [] - for tool in tools: - if not isinstance(tool, dict): - updated_tools.append(tool) - continue - - updated_tool = tool.copy() - - # Handle code_interpreter with container file_ids - if tool.get("type") == "code_interpreter": - container = tool.get("container") - if isinstance(container, dict): - container_file_ids = container.get("file_ids") - if isinstance(container_file_ids, list): - updated_file_ids = [] - for file_id in container_file_ids: - if isinstance(file_id, str): - # Check if we have a mapping for this file ID - if file_id in model_file_id_mapping: - # Map to provider-specific file ID - provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id - updated_file_ids.append(provider_file_id) - else: - updated_file_ids.append(file_id) - else: - updated_file_ids.append(file_id) - - # Update the tool with new file IDs - updated_container = container.copy() - updated_container["file_ids"] = updated_file_ids - updated_tool["container"] = updated_container - - updated_tools.append(updated_tool) - - return updated_tools - - -def extract_file_metadata(file_data: FileTypes) -> tuple[str | None, str | None]: - """ - Resolve (filename, content_type) without reading the file body. - - Mirrors extract_file_data's metadata resolution but never calls .read(), so - it stays O(1) on large uploads. Use this when only metadata is needed (batch - detection, GCS object naming) and the body must remain a streamable Path/handle. - """ - filename: str | None = None - content_type: str | None = None - file_content: object = None - - if isinstance(file_data, tuple): - if len(file_data) == 2: - filename, file_content = file_data - elif len(file_data) == 3: - filename, file_content, content_type = file_data - elif len(file_data) == 4: - filename, file_content, content_type, _ = file_data - elif isinstance(file_data, InMemoryFile): - filename = file_data.name - content_type = file_data.content_type - else: - file_content = file_data - - if filename is None: - if isinstance(file_content, PathLike): - filename = Path(file_content).name - elif isinstance(file_content, io.IOBase): - name_attr: Final = getattr(file_content, "name", None) - if isinstance(name_attr, str): - filename = Path(name_attr).name - - if not content_type: - guessed: Final = mimetypes.guess_type(filename)[0] if filename else None - content_type = guessed or "application/octet-stream" - - return filename, content_type - - -def extract_file_data(file_data: FileTypes) -> ExtractedFileData: - """ - Extracts and processes file data from various input formats. - - Args: - file_data: Can be a tuple of (filename, content, [content_type], [headers]) or direct file content - - Returns: - ExtractedFileData containing: - - filename: Name of the file if provided - - content: The file content in bytes - - content_type: MIME type of the file - - headers: Any additional headers - """ - # Parse the file_data based on its type - filename = None - file_content = None - content_type = None - file_headers: Mapping[str, str] = {} - - if isinstance(file_data, tuple): - if len(file_data) == 2: - filename, file_content = file_data - elif len(file_data) == 3: - filename, file_content, content_type = file_data - elif len(file_data) == 4: - filename, file_content, content_type, file_headers = file_data - elif isinstance(file_data, InMemoryFile): - filename = file_data.name - file_content = file_data - content_type = file_data.content_type - else: - file_content = file_data - # Convert content to bytes - if isinstance(file_content, str): - # Bare string inputs are rejected: when this helper runs in a proxy - # request handler the string came from an attacker-controlled form - # field, and opening it as a path is an arbitrary file read on the - # proxy host. SDK callers who want to upload from a path should - # either pass a pathlib.Path (a PathLike instance — see the branch - # below) or open the file themselves and pass the handle / bytes. - raise ValueError( - "extract_file_data does not accept bare str inputs. Pass bytes, " - "an open file handle, a (filename, content) tuple, or a " - "pathlib.Path. To upload a local file from a path, call " - "open(path, 'rb') yourself." - ) - if isinstance(file_content, PathLike): - # PathLike (pathlib.Path) is a Python-level type that HTTP form - # values can't fabricate. Treat as a local file path for SDK - # convenience. - if filename is None: - filename = Path(file_content).name - with open(file_content, "rb") as f: - content = f.read() - elif isinstance(file_content, io.IOBase): - # If it's a file-like object - # Try to get filename from file handle if not already set - if not filename and hasattr(file_content, "name"): - filename = Path(file_content.name).name - - content = file_content.read() - - if isinstance(content, str): - content = content.encode("utf-8") - # Reset file pointer to beginning - file_content.seek(0) - elif isinstance(file_content, bytes): - content = file_content - else: - raise ValueError(f"Unsupported file content type: {type(file_content)}") - - # Use provided content type or guess based on filename - if not content_type: - if filename: - guessed_type: Final = mimetypes.guess_type(filename)[0] - content_type = guessed_type if guessed_type else "application/octet-stream" - else: - content_type = "application/octet-stream" - - return ExtractedFileData( - filename=filename, - content=content, - content_type=content_type, - headers=file_headers, - ) - - # --------------------------------------------------------------------------- # Generic, dependency-free implementation of `unpack_defs` # --------------------------------------------------------------------------- @@ -1536,633 +1022,6 @@ def sanitize_input_schema_for_anthropic(input_schema: dict) -> "AnthropicInputSc return AnthropicInputSchema(**filtered) -_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("allOf", "anyOf", "oneOf") -_OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS: Final = ("enum", "const", "not") -_LOCAL_SCHEMA_REF_PREFIXES: Final = (("#/$defs/", "$defs"), ("#/definitions/", "definitions")) -_MAX_SCHEMA_FLATTEN_DEPTH: Final = 32 -_EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({}) - - -def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]: - properties: Final = schema.get("properties") - return properties if isinstance(properties, dict) else _EMPTY_SCHEMA - - -def _schema_branches(schema: Mapping[str, object], combinator: str) -> tuple[object, ...]: - branches: Final = schema.get(combinator) - return tuple(branches) if isinstance(branches, list) else () - - -def _schema_required_names(schema: Mapping[str, object]) -> frozenset[str]: - required: Final = schema.get("required") - if not isinstance(required, list): - return frozenset() - return frozenset(name for name in required if isinstance(name, str)) - - -def _combinator_required_names(combinator: str, branches: tuple[Mapping[str, object], ...]) -> frozenset[str]: - branch_names: Final = tuple(_schema_required_names(branch) for branch in branches) - if not branch_names: - return frozenset() - if combinator == "allOf": - return branch_names[0].union(*branch_names[1:]) - return branch_names[0].intersection(*branch_names[1:]) - - -def _resolve_local_schema_ref(root: Mapping[str, object], ref: str) -> Mapping[str, object] | None: - matched: Final = next( - ((prefix, container) for prefix, container in _LOCAL_SCHEMA_REF_PREFIXES if ref.startswith(prefix)), - None, - ) - if matched is None: - return None - prefix, container = matched - definitions: Final = root.get(container) - if not isinstance(definitions, dict): - return None - target: Final = definitions.get(ref[len(prefix) :]) - return target if isinstance(target, dict) else None - - -def _mergeable_branch( - root: Mapping[str, object], - branch: object, - seen_refs: frozenset[str], - depth: int, - expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work -) -> Mapping[str, object] | None: - if not isinstance(branch, dict) or depth > _MAX_SCHEMA_FLATTEN_DEPTH: - return None - ref: Final = branch.get("$ref") - if not isinstance(ref, str): - flattened: Final = _flatten_schema_against_root(branch, root, seen_refs, depth, expanded_refs) - if any(combinator in flattened for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS): - return None - return flattened - if ref in expanded_refs: - return expanded_refs[ref] - if ref in seen_refs: - return None - target: Final = _resolve_local_schema_ref(root, ref) - expanded: Final = ( - None - if target is None - else _mergeable_branch(root, target, seen_refs | frozenset((ref,)), depth + 1, expanded_refs) - ) - expanded_refs[ref] = expanded - return expanded - - -def _is_object_schema(schema: Mapping[str, object]) -> bool: - return schema.get("type") == "object" or ("type" not in schema and "properties" in schema) - - -def _flatten_schema_against_root( - schema: Mapping[str, object], - root: Mapping[str, object], - seen_refs: frozenset[str], - depth: int, - expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work -) -> Mapping[str, object]: - raw_branch_groups: Final = tuple( - ( - combinator, - tuple( - _mergeable_branch(root, branch, seen_refs, depth + 1, expanded_refs) - for branch in _schema_branches(schema, combinator) - ), - ) - for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS - if isinstance(schema.get(combinator), list) - ) - dropped: Final = ( - *(combinator for combinator, _ in raw_branch_groups), - *(key for key in _OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS if key in schema), - ) - if not dropped: - return schema - - if any(branch is None for _, group in raw_branch_groups for branch in group): - return schema - branch_groups: Final = tuple( - (combinator, tuple(branch for branch in group if branch is not None)) for combinator, group in raw_branch_groups - ) - branches: Final = tuple(branch for _, group in branch_groups for branch in group) - is_object_schema: Final = _is_object_schema(schema) or ( - "type" not in schema and branches != () and all(_is_object_schema(branch) for branch in branches) - ) - if not is_object_schema: - return schema - - merged_properties: Final = { - name: value for source in (*reversed(branches), schema) for name, value in _schema_properties(source).items() - } - required_names: Final = _schema_required_names(schema).union( - *(_combinator_required_names(combinator, group) for combinator, group in branch_groups) - ) - kept: Final = MappingProxyType({key: value for key, value in schema.items() if key not in dropped}) - required_update: Final = MappingProxyType({"required": sorted(required_names)}) if required_names else _EMPTY_SCHEMA - return { - **kept, - "type": "object", - "properties": merged_properties, - **required_update, - } - - -def flatten_top_level_schema_combinators(schema: Mapping[str, object]) -> Mapping[str, object]: - """Merge top-level ``allOf``/``anyOf``/``oneOf`` branches into an object tool schema. - - OpenAI's function-calling validator rejects tool ``parameters`` carrying - 'oneOf'/'anyOf'/'allOf'/'enum'/'const'/'not' at the top level (nested uses - are accepted), while lenient backends such as the ChatGPT backend Codex - talks to natively accept them, so an MCP tool declaring a top-level union - 400s through LiteLLM. Branch properties merge without clobbering (the - top-level schema wins, then earlier branches); ``required`` becomes the - top-level list plus the intersection of the branch lists for anyOf/oneOf - or their union for allOf. Branches that are local ``$ref``s - (``#/$defs/...`` or ``#/definitions/...``) are resolved first, each ref - at most once per call, and branches that are themselves combinators are - flattened recursively up to a fixed depth; a branch that cannot be fully - merged (a boolean schema, an external or cyclic ``$ref``, a non-object - union, or nesting past the depth cap) leaves the whole schema untouched so - OpenAI's own validation still applies. Non-object schemas pass through - unchanged and the input is never mutated. - """ - return _flatten_schema_against_root(schema, schema, frozenset(), 0, {}) - - -_SUBSCHEMA_KEYWORDS: Final = frozenset( - { - "additionalItems", - "additionalProperties", - "contains", - "else", - "if", - "items", - "not", - "propertyNames", - "then", - "unevaluatedItems", - "unevaluatedProperties", - } -) -_SUBSCHEMA_LIST_KEYWORDS: Final = frozenset({"allOf", "anyOf", "items", "oneOf", "prefixItems"}) -_SUBSCHEMA_MAP_KEYWORDS: Final = frozenset( - {"$defs", "definitions", "dependentSchemas", "patternProperties", "properties"} -) - -_MAX_SCHEMA_NESTING: Final = 1024 - - -def drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: - """Drop every regex in a schema position that Python's ``re`` cannot compile. - - OpenAI validates tool ``parameters`` against the 2020-12 metaschema with - ``jsonschema``'s format checker, which hands each ``pattern`` value and each - ``patternProperties`` key to ``re.compile``, so a regex written for an - ECMA-262 engine (Unicode property escapes such as ``\\p{Cc}``, as in Claude - Code's ``Artifact`` tool) is refused with "'...' is not a 'regex'" by every - model family on both the chat and Responses wires. Only schema positions are - walked (properties, items, combinators, ``$defs`` and the other applicators), - so a ``pattern`` key inside ``default``, ``examples``, ``const`` or vendor - extensions is data and stays. Outside strict mode the keyword is only a - hint, so dropping it costs the model a constraint and the caller nothing. - Compilable regexes and everything else pass through, the input is never - mutated, and the same object comes back when nothing was dropped. The walk - is level-order rather than recursive, rebuilt deepest level first, and stops - at more schema levels than a JSON parser admits, so a cyclic schema built in - code cannot spin it. - """ - return _schema_without_rejected_regex(schema, _is_not_python_regex) - - -def drop_lookaround_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: - """Drop every regex in a schema position that uses a lookaround assertion. - - Some Bedrock Converse families compile tool schema regexes with an engine that - has no lookahead or lookbehind and refuse the whole request over one. The ``(?=``, - ``(?!``, ``(?<=`` and ``(? Mapping[str, object]: - rebuilt: dict[int, Mapping[str, object]] = {} # mutable-ok: per-call memo of rewritten nodes, deepest level first - for level in reversed(tuple(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))): - rebuilt.update( - (id(node), rewritten) - for node in level - if (rewritten := _node_without_rejected_regex(node, rebuilt, rejected)) is not node - ) - return rebuilt.get(id(schema), schema) - - -def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str, object], ...]]: - frontier: tuple[Mapping[str, object], ...] = (schema,) # rebind-ok: level-order cursor, one level a round - while frontier: - yield frontier - frontier = tuple(child for node in frontier for child in _subschemas(node)) - - -def schema_fragment_ref(ref: str) -> str: - return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref - - -def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool: - definitions: Final = MappingProxyType(dict(_root_definitions(schema))) - if not _refs_in(schema) <= definitions.keys(): - return True - try: - tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order()) - except CycleError: - return True - return False - - -def schema_union_property_count(schema: Mapping[str, object]) -> int: - return sum(1 for _, node in _property_schemas(schema) if _is_union(node)) - - -def schema_optional_property_count(schema: Mapping[str, object]) -> int: - return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema)) - - -def schema_property_count(schema: Mapping[str, object]) -> int: - return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties"))) - - -def schema_grammar_branch_count(schema: Mapping[str, object]) -> int: - return sum(_grammar_branches(node) for node in _schema_nodes(schema)) - - -def schema_has_open_object(schema: Mapping[str, object]) -> bool: - return any(_is_open_object(node) for node in _schema_nodes(schema)) - - -def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool: - return any( - isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern) - for node in _schema_nodes(schema) - ) - - -def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]: - return frozenset( - value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str) - ) - - -def _may_be_string(node: Mapping[str, object]) -> bool: - node_type: Final = node.get("type") - return node_type is None or node_type == "string" or "string" in _list_at(node, "type") - - -def is_object_schema(node: Mapping[str, object]) -> bool: - node_type: Final = node.get("type") - return node_type == "object" or (isinstance(node_type, list) and "object" in node_type) - - -def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]: - return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING)) - - -_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...]) - - -def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]: - value: Final = container.get(key) - return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else () - - -def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]: - if container is None: - return iter(()) - return ((key, value) for key in container if (value := _schema_at(container, key)) is not None) - - -def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]: - for defs_key in ("$defs", "definitions"): - yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key))) - - -def _refs_in(schema: Mapping[str, object]) -> frozenset[str]: - return frozenset( - schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str) - ) - - -def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]: - return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema)) - - -def _is_union(node: Mapping[str, object]) -> bool: - return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1 - - -def _grammar_branches(node: Mapping[str, object]) -> int: - properties: Final = _schema_at(node, "properties") - if properties is None: - return 0 - optional: Final = len(_optional_property_names(node)) - return (1 << optional) * (len(properties) - optional + 1) - - -def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]: - properties: Final = _schema_at(node, "properties") - if properties is None: - return frozenset() - return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str)) - - -def _is_open_object(node: Mapping[str, object]) -> bool: - additional_properties: Final = node.get("additionalProperties") - if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node: - return True - return is_object_schema(node) and "properties" not in node and additional_properties is not False - - -def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]: - for key, value in node.items(): - if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict): - yield from (sub for sub in value.values() if isinstance(sub, dict)) - elif key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list): - yield from (sub for sub in value if isinstance(sub, dict)) - elif key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict): - yield value - - -def _node_without_rejected_regex( - node: Mapping[str, object], - rebuilt: Mapping[int, Mapping[str, object]], - rejected: Callable[[str], bool], -) -> Mapping[str, object]: - kept: Final = { - key: _keyword_value_rebuilt(key, value, rebuilt, rejected) - for key, value in node.items() - if key != "pattern" or not isinstance(value, str) or not rejected(value) - } - if len(kept) == len(node) and all(kept[key] is node[key] for key in kept): - return node - dropped_pattern_properties: Final = _dropped_pattern_properties(node, kept, rebuilt) - if not dropped_pattern_properties or kept.get("additionalProperties") is not False: - return kept - return {**kept, "additionalProperties": _any_of(dropped_pattern_properties)} - - -def _dropped_pattern_properties( - node: Mapping[str, object], - kept: Mapping[str, object], - rebuilt: Mapping[int, Mapping[str, object]], -) -> tuple[object, ...]: - before: Final = _schema_at(node, "patternProperties") - after: Final = _schema_at(kept, "patternProperties") - if before is None or after is None: - return () - return tuple(rebuilt.get(id(sub), sub) for name, sub in before.items() if name not in after) - - -def _schema_at(container: Mapping[str, object], key: str) -> Mapping[str, object] | None: - value: Final = container.get(key) - return value if isinstance(value, dict) else None - - -def _any_of(schemas: tuple[object, ...]) -> object: - return schemas[0] if len(schemas) == 1 else {"anyOf": list(schemas)} - - -def _keyword_value_rebuilt( - key: str, - value: object, - rebuilt: Mapping[int, Mapping[str, object]], - rejected: Callable[[str], bool], -) -> object: - if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict): - kept: Final = { - name: rebuilt.get(id(sub), sub) - for name, sub in value.items() - if key != "patternProperties" or not isinstance(name, str) or not rejected(name) - } - return value if len(kept) == len(value) and all(kept[name] is value[name] for name in kept) else kept - if key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list): - items: Final = [rebuilt.get(id(sub), sub) for sub in value] - return value if all(new is old for new, old in zip(items, value, strict=True)) else items - if key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict): - return rebuilt.get(id(value), value) - return value - - -def _is_not_python_regex(pattern: str) -> bool: - try: - re.compile(pattern) - except (re.error, RecursionError): - return True - return False - - -_REGEX_LOOKAROUND_RE: Final = re.compile(r"\(\? bool: - return _REGEX_LOOKAROUND_RE.search(pattern) is not None - - -def flatten_combinators_and_drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: - return flatten_top_level_schema_combinators(drop_non_python_regex_patterns(schema)) - - -def tool_with_sanitized_parameters( - tool: Mapping[str, object], - sanitize: Callable[[Mapping[str, object]], Mapping[str, object]], -) -> Mapping[str, object]: - """Run the tool's JSON schema through ``sanitize``: ``function.parameters`` on an - OpenAI tool, ``input_schema`` on an Anthropic one. The same object comes back when - nothing changed.""" - function: Final = _schema_at(tool, "function") - if function is not None: - parameters: Final = _schema_at(function, "parameters") - if parameters is None: - return tool - sanitized_parameters: Final = sanitize(parameters) - if sanitized_parameters is parameters: - return tool - return {**tool, "function": {**function, "parameters": sanitized_parameters}} - input_schema: Final = _schema_at(tool, "input_schema") - if input_schema is None: - return tool - sanitized_schema: Final = sanitize(input_schema) - return tool if sanitized_schema is input_schema else {**tool, "input_schema": sanitized_schema} - - -def get_image_mime_type_from_url(url: str) -> str | None: - """ - Get mime type for common image URLs - See gemini mime types: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/image-understanding#image-requirements - - Supported by Gemini: - application/pdf - audio/mpeg - audio/mp3 - audio/wav - audio/ogg - image/png - image/jpeg - image/webp - text/plain - video/mov - video/mpeg - video/mp4 - video/mpg - video/avi - video/wmv - video/mpegps - video/flv - """ - from urllib.parse import urlparse - - url = url.lower() - - # Parse URL to extract path without query parameters - # This handles URLs like: https://example.com/image.jpg?signature=... - parsed: Final = urlparse(url) - path: Final = parsed.path - - # Map file extensions to mime types - mime_types: Final = { - # Images - (".jpg", ".jpeg"): "image/jpeg", - (".png",): "image/png", - (".webp",): "image/webp", - # Videos - (".mp4",): "video/mp4", - (".mov",): "video/mov", - (".mpeg", ".mpg"): "video/mpeg", - (".avi",): "video/avi", - (".wmv",): "video/wmv", - (".mpegps",): "video/mpegps", - (".flv",): "video/flv", - # Audio - (".mp3",): "audio/mp3", - (".wav",): "audio/wav", - (".mpeg",): "audio/mpeg", - (".ogg",): "audio/ogg", - # Documents - (".pdf",): "application/pdf", - (".txt",): "text/plain", - } - - # Check each extension group against the URL - for extensions, mime_type in mime_types.items(): - if any(path.endswith(ext) for ext in extensions): - return mime_type - - return None - - -_get_image_mime_type_from_url = get_image_mime_type_from_url - - -def infer_content_type_from_url_and_content( - url: str, - content: bytes, - current_content_type: str | None = None, -) -> str: - """ - Infer content type from URL extension and binary content when content-type header is missing or generic. - - This helper implements a fallback strategy for determining MIME types when HTTP headers - are missing or provide generic values (like binary/octet-stream). It's commonly used - when processing images and documents from various sources (S3, URLs, etc.). - - Fallback Strategy: - 1. If current_content_type is valid (not None and not generic octet-stream), return it - 2. Try to infer from URL extension (handles query parameters) - 3. Try to detect from binary content signature (magic bytes) - 4. Raise ValueError if all methods fail - - Args: - url: The URL of the content (used to extract file extension) - content: The binary content (first ~100 bytes are sufficient for detection) - current_content_type: The current content-type from headers (may be None or generic) - - Returns: - str: The inferred MIME type (e.g., "image/png", "application/pdf") - - Raises: - ValueError: If content type cannot be determined by any method - - Example: - >>> content_type = infer_content_type_from_url_and_content( - ... url="https://s3.amazonaws.com/bucket/image.png?AWSAccessKeyId=123", - ... content=png_binary_data, - ... current_content_type="binary/octet-stream" - ... ) - >>> print(content_type) - "image/png" - """ - from litellm.litellm_core_utils.token_counter import get_image_type - - media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None - - # If we have a valid content type that's not generic, use it - if media_type and media_type not in [ - "binary/octet-stream", - "application/octet-stream", - ]: - return media_type - - # Extension to MIME type mapping - # Supports images, documents, and other common file types - extension_to_mime: Final = { - # Image formats - "jpg": "image/jpeg", - "jpeg": "image/jpeg", - "png": "image/png", - "gif": "image/gif", - "webp": "image/webp", - # Document formats - "pdf": "application/pdf", - "csv": "text/csv", - "doc": "application/msword", - "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "xls": "application/vnd.ms-excel", - "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "html": "text/html", - "txt": "text/plain", - "md": "text/markdown", - } - - # Try to infer from URL extension - if url: - extension: Final = url.split(".")[-1].lower().split("?")[0] # Remove query params - inferred_type: Final = extension_to_mime.get(extension) - if inferred_type: - return inferred_type - - # Try to detect from binary content signature (magic bytes) - if content: - detected_type: Final = get_image_type(content[:100]) - if detected_type: - type_to_mime: Final = { - "png": "image/png", - "jpeg": "image/jpeg", - "gif": "image/gif", - "webp": "image/webp", - "heic": "image/heic", - } - if detected_type in type_to_mime: - return type_to_mime[detected_type] - - # If all fallbacks failed, raise error - raise ValueError(f"Unable to determine content type from URL: {url}. Response content-type: {current_content_type}") - - def get_tool_call_names(tools: list[ChatCompletionToolParam]) -> list[str]: """ Get tool call names from tools @@ -2214,32 +1073,6 @@ def convert_custom_tool_format_to_responses_shape(format_obj: Mapping[str, objec } -def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]: - """ - Gets file ids from messages - """ - file_ids: Final = [] - for message in messages: - if message.get("role") == "user": - content = message.get("content") - if content: - if isinstance(content, str): - continue - for c in content: - if isinstance(c, dict) and c["type"] == "file": - file_object = cast(ChatCompletionFileObject, c) - file_object_file_field = file_object.get("file") - if not isinstance(file_object_file_field, dict): - # Content block has `type: "file"` but not the - # OpenAI Chat Completions shape. No file_id to - # extract, so skip instead of raising KeyError. - continue - file_id = file_object_file_field.get("file_id") - if file_id: - file_ids.append(file_id) - return file_ids - - def check_is_function_call(logging_obj: "LoggingClass") -> bool: from litellm.litellm_core_utils.prompt_templates.common_utils import ( is_function_call, @@ -2252,195 +1085,6 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool: return False -_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object]) -PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = ( - "set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)" -) - - -def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: - """A breakpoint the client already wrote on the block wins, TTL included.""" - if marker is None or target.get("prompt_cache_breakpoint") is not None: - return target - marked: Final = {**target, "prompt_cache_breakpoint": marker} - return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key - - -_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) -_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(ChatCompletionPromptCacheBreakpoint) -_AcceptedBreakpointT: Final = TypeVar("_AcceptedBreakpointT", bound=Mapping[str, object]) - - -def _accepted_prompt_cache_breakpoint( - accepted: TypeAdapter[_AcceptedBreakpointT], marker: object -) -> _AcceptedBreakpointT | None: - try: - return accepted.validate_python(marker) - except ValidationError: - return None - - -def prompt_cache_breakpoint_within( - accepted: TypeAdapter[_AcceptedBreakpointT], marker: object -) -> _AcceptedBreakpointT | None: - """The client's marker cut down to the ``accepted`` shape: an unsupported ttl is dropped and the mode kept, - and a marker whose mode the target cannot read is dropped whole.""" - whole: Final = _accepted_prompt_cache_breakpoint(accepted, marker) - if whole is not None: - return whole - fields: Final = _breakpoint_fields_or_none(marker) - without_ttl: Final = ( - None - if fields is None - else _accepted_prompt_cache_breakpoint(accepted, {key: item for key, item in fields.items() if key != "ttl"}) - ) - verbose_logger.debug( - "Dropping malformed prompt_cache_breakpoint %r under drop_params" - if without_ttl is None - else "Dropping the unsupported ttl of prompt_cache_breakpoint %r under drop_params", - marker, - ) - return without_ttl - - -def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: - if marker is None or not drop_params: - return marker - return prompt_cache_breakpoint_within(_PROMPT_CACHE_BREAKPOINT, marker) - - -def prompt_cache_breakpoint_with_readable_mode(marker: object, drop_params: bool) -> object: - """The client's marker as written, or None under drop_params when no target could read its mode, so the - target's own config decides which ttl and keys it keeps.""" - return None if prompt_cache_breakpoint_for_wire(marker, drop_params) is None else marker - - -def whole_prompt_cache_breakpoint_or_none(marker: object) -> PromptCacheBreakpoint | None: - return _accepted_prompt_cache_breakpoint(_PROMPT_CACHE_BREAKPOINT, marker) - - -def chat_completion_prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: - if marker is None or not drop_params: - return marker - return prompt_cache_breakpoint_within(_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT, marker) - - -def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: - """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block.""" - if block.get("cache_control") is None: - return block - unmarked: Final = {key: value for key, value in block.items() if key != "cache_control"} - return cast( # cast-ok: same block shape as the input with the marker key renamed - _MarkedT, with_prompt_cache_breakpoint(unmarked, PromptCacheBreakpoint(mode="explicit")) - ) - - -def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT: - """OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block. - A ``cache_control`` the client already wrote on the block wins, TTL included.""" - marker: Final = block.get("prompt_cache_breakpoint") - if marker is None: - return block - unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} - control: Final = ( - None - if block.get("cache_control") is not None - else _cache_control_for_prompt_cache_breakpoint(marker, drop_params) - ) - marked: Final = unmarked if control is None else {**unmarked, "cache_control": control} - return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed - - -def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None: - try: - breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker) - except ValidationError: - if not drop_params: - raise UnsupportedParamsError( - message=( - f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got " - f"{marker!r}. Fix the marker, or " - f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it." - ), - status_code=400, - ) - verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) - return None - ttl: Final = breakpoint_mark.get("ttl") - if ttl is not None and not drop_params: - raise UnsupportedParamsError( - message=( - f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint " - f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send " - "a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block " - "where the model offers a 1h cache." - ), - status_code=400, - ) - return ChatCompletionCachedContent(type="ephemeral") - - -_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"}) -_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object]) -_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object]) - - -def _breakpoint_fields_or_none(value: object) -> Mapping[str, object] | None: - try: - return _BREAKPOINT_FIELDS.validate_python(value) - except ValidationError: - return None - - -def _without_prompt_cache_breakpoint(value: object) -> object: - if not isinstance(value, Mapping): - return value - fields: Final = _BREAKPOINT_FIELDS.validate_python(value) - return {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"} - - -def _breakpoints_stripped_from(key: str, value: object) -> object: - if key in _BREAKPOINT_BLOCK_LIST_KEYS and isinstance(value, list): - return [_without_prompt_cache_breakpoint(block) for block in _BREAKPOINT_BLOCKS.validate_python(value)] - return _without_prompt_cache_breakpoint(value) if key == "function" else value - - -def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT: - """A message or tool definition with every ``prompt_cache_breakpoint`` removed: its own, its content parts', - its tool calls' and its ``function``'s.""" - stripped: Final = { - key: _breakpoints_stripped_from(key, value) for key, value in block.items() if key != "prompt_cache_breakpoint" - } - return cast(_MarkedT, stripped) # cast-ok: same block shape as the input minus the marker keys - - -def _part_with_chat_completion_prompt_cache_breakpoint(part: object) -> object: - if not isinstance(part, Mapping): - return part - fields: Final = _BREAKPOINT_FIELDS.validate_python(part) - marker: Final = fields.get("prompt_cache_breakpoint") - if marker is None: - return fields - unmarked: Final = {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"} - return with_prompt_cache_breakpoint(unmarked, chat_completion_prompt_cache_breakpoint_for_wire(marker, True)) - - -def chat_completion_prompt_cache_breakpoints_for_wire(message: _MarkedT, drop_params: bool) -> _MarkedT: - content: Final = message.get("content") - if not drop_params or not isinstance(content, list): - return message - parts: Final = [ - _part_with_chat_completion_prompt_cache_breakpoint(part) for part in _BREAKPOINT_BLOCKS.validate_python(content) - ] - return cast(_MarkedT, {**message, "content": parts}) # cast-ok: same message shape with its parts' markers trimmed - - -LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS: Final = frozenset( - {"thinking_blocks", "reasoning_items", "provider_specific_fields"} -) -LITELLM_INTERNAL_MESSAGE_FIELDS: Final = LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS | {"reasoning_content"} - - def _tool_call_without_provider_specific_fields(tool_call: object) -> object: if not isinstance(tool_call, dict): return tool_call @@ -2692,267 +1336,6 @@ def convert_prefix_message_to_non_prefix_messages( return new_messages -def _provider_text_or_none(value: object) -> str | None: - return cast(str | None, value) # cast-ok: reasoning fields arrive in untyped provider messages - - -def extract_reasoning_content(message: Mapping[str, object]) -> tuple[str | None, str | None]: - """ - Extract reasoning content and main content from a message. - - Args: - message (dict): The message dictionary that may contain reasoning_content - - Returns: - tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content) - """ - message_content: Final = message.get("content") - if "reasoning_content" in message: - return _provider_text_or_none(message["reasoning_content"]), _provider_text_or_none(message_content) - elif "reasoning" in message: - return _provider_text_or_none(message["reasoning"]), _provider_text_or_none(message_content) - elif isinstance(message_content, str): - return parse_content_for_reasoning(message_content) - return None, _provider_text_or_none(message_content) - - -_extract_reasoning_content = extract_reasoning_content - - -def _readable_thinking_text(block: Mapping[str, object]) -> str: - """The text a chat model can read back, empty for redacted blocks and malformed ones.""" - if block.get("type") != "thinking": - return "" - return str(block.get("thinking") or "") - - -def reasoning_content_from_thinking_blocks( - thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock], -) -> str: - """Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect. - - Redacted blocks carry no readable text, so they contribute nothing. - """ - return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block))) - - -ENCRYPTED_REASONING_SIGNATURE_PREFIX: Final = "litellm_encrypted_reasoning:" - - -def encrypted_reasoning_signature(encrypted_content: str) -> str: - """The opaque value a Responses API reasoning item's `encrypted_content` travels in. - - Anthropic clients echo a thinking block's `signature` and a redacted block's `data` - back verbatim, so either field can carry the encrypted reasoning across turns; the - prefix tells the two apart from a signature Anthropic minted. - """ - return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}" - - -def _carries_encrypted_reasoning(signature: object) -> bool: - return isinstance(signature, str) and signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX) - - -def encrypted_content_from_signature(signature: object) -> str | None: - if not isinstance(signature, str) or not _carries_encrypted_reasoning(signature): - return None - return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None - - -def _encrypted_reasoning_field(block: Mapping[str, object]) -> object: - match block.get("type"): - case "thinking": - return block.get("signature") - case "redacted_thinking": - return block.get("data") - case _: - return None - - -def encrypted_content_of_block(block: Mapping[str, object]) -> str | None: - return encrypted_content_from_signature(_encrypted_reasoning_field(block)) - - -def is_encrypted_reasoning_block(block: object) -> bool: - """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. - - Only the Responses API that minted the content can read it back, so an Anthropic - backend has to drop such a block rather than fail signature verification on it. - """ - if not isinstance(block, Mapping): - return False - mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance - return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping)) - - -def is_unsignable_thinking_block(block: object) -> bool: - """A thinking block Anthropic cannot accept on input. - - Anthropic verifies the signature cryptographically, so a block with a null, - empty, or missing signature (e.g. from an open-source reasoning model) is - rejected with a 400, and so is a block whose signature or data carries - another provider's encrypted reasoning. A signed block is kept whatever its - text, since ``display: omitted`` returns signed blocks with empty ``thinking``. - `redacted_thinking` blocks carry no signature and are always kept. - """ - if is_encrypted_reasoning_block(block): - return True - if not isinstance(block, Mapping): - return False - mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance - if mapping.get("type") != "thinking": - return False - signature: Final = mapping.get("signature") - return not (isinstance(signature, str) and len(signature) > 0) - - -def has_blank_redacted_data(block: object) -> bool: - match block: - case {"type": "redacted_thinking", "data": str() as data}: - return not data.strip() - case {"type": "redacted_thinking"}: - return True - case _: - return False - - -def native_redacted_thinking_data(data: str) -> str: - """Claude's own `data` string for a redacted_thinking block. - - Bedrock Converse carries it base64-wrapped as `redactedContent`, and LiteLLM passed that - wrapper through as `data` until it started decoding Converse replies, so histories saved - before then still hold it. Claude's own value never decodes twice, since its first byte - decodes to a protobuf tag outside the base64 alphabet. - """ - return _converse_wrapped_payload(data) or data - - -def converse_redacted_content(data: str) -> str: - """The `redactedContent` Bedrock Converse expects for a redacted_thinking block's `data`.""" - return base64.b64encode(native_redacted_thinking_data(data).encode()).decode() - - -def _converse_wrapped_payload(data: str) -> str | None: - try: - payload: Final = base64.b64decode(data, validate=True).decode("ascii") - _ = base64.b64decode(payload, validate=True) - except (binascii.Error, ValueError): - return None - else: - return payload - - -def strip_encrypted_reasoning_from_messages( - messages: object, - *, - should_strip: Callable[[Mapping[str, object]], bool] | None = None, -) -> None: - """Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from - Anthropic-shaped history. - - The whole block goes, the way #40280 drops undecryptable Responses ``input`` items: a - provider that did not mint the block rejects it signed (a foreign signature) and unsigned - (a missing signature) alike, so keeping its text as an unsigned thinking block only moves - the 400 from the router to the provider. - - Mutates the content lists in place: the router's fallback snapshot shares these - message objects, so a rebound list would replay the stripped blocks on the fallback hop. - """ - if not isinstance(messages, list): - return - for content in anthropic_content_lists(cast(list[object], messages)): # cast-ok: untyped client json - _strip_encrypted_reasoning_from_blocks(content, should_strip=should_strip) - - -def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: - return ( - cast(list[object], content) # cast-ok: narrowed by isinstance - for message in messages - if isinstance(message, Mapping) - for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance - if isinstance(content, list) - ) - - -def _strip_encrypted_reasoning_from_blocks( - content: object, - *, - should_strip: Callable[[Mapping[str, object]], bool] | None = None, -) -> None: - blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance - kept: Final = tuple( - block - for block in blocks - if not is_encrypted_reasoning_block(block) - or (should_strip is not None and not should_strip(cast(Mapping[str, object], block))) - ) - blocks[:] = kept - - -def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: - index, block = indexed_block - return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary" - - -def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> ChatCompletionReasoningItem | None: - summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload - ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text) - for block in group - if (text := _readable_thinking_text(block)) - ] - encrypted_content: Final = encrypted_content_of_block(group[0]) - if encrypted_content is not None: - return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content) - if not summary: - return None - return ChatCompletionReasoningItem(type="reasoning", summary=summary) - - -def responses_reasoning_items_from_thinking_blocks( - thinking_blocks: Iterable[Mapping[str, object]], -) -> tuple[ChatCompletionReasoningItem, ...]: - """Build Responses API `reasoning` input items from Anthropic thinking blocks. - - A block carrying encrypted reasoning replays the item it came from byte for byte; - a run of plain thinking blocks collapses into one summary-only item. No item carries - an `id`: the Responses API 404s on any id it did not mint itself and rejects an empty - one, while an item without an id is always accepted. - """ - return tuple( - item - for _, group in groupby(enumerate(thinking_blocks), key=_reasoning_replay_group_key) - if (item := _reasoning_item_from_block_group(tuple(block for _, block in group))) is not None - ) - - -def parse_content_for_reasoning( - message_text: str | None, -) -> tuple[str | None, str | None]: - """ - Parse the content for reasoning - - Returns: - - reasoning_content: The content of the reasoning - - content: The content of the message - """ - if not message_text: - return None, message_text - - reasoning_match: Final = re.match( - r"<(?:think|thinking|budget:thinking)>(.*?)(.*)", - message_text, - re.DOTALL, - ) - - if reasoning_match: - return reasoning_match.group(1), reasoning_match.group(2) - - return None, message_text - - -_parse_content_for_reasoning = parse_content_for_reasoning - - def _extract_base64_data(image_url: str) -> str: """ Extract pure base64 data from an image URL. @@ -3163,252 +1546,6 @@ def merge_consecutive_system_messages( ] -_DECODED_JSON: Final = TypeAdapter(object) - - -def _attempt_json_repair(s: str) -> object | None: - """ - Attempt to repair truncated JSON produced by LLM tool calls. - - Handles the most common truncation patterns where the model generates - valid JSON that is cut short (missing closing brackets/braces). - - Returns the parsed value on success, or None if repair fails. - """ - import json - - stripped: Final = s.rstrip() - if not stripped: - return None - - # Track the stack of unmatched openers to respect nesting order - opener_stack: Final[list] = [] - in_string = False - escape_next = False - - for ch in stripped: - if escape_next: - escape_next = False - continue - if ch == "\\": - if in_string: - escape_next = True - continue - if ch == '"': - in_string = not in_string - continue - if in_string: - continue - if ch == "{": - opener_stack.append("}") - elif ch == "[": - opener_stack.append("]") - elif ch in ("}", "]"): - if opener_stack and opener_stack[-1] == ch: - opener_stack.pop() - - if not opener_stack: - return None - - # Remove trailing comma before we close brackets - candidate = stripped.rstrip(",") - - # Close in reverse order of opening (respects nesting) - candidate += "".join(reversed(opener_stack)) - - try: - return _DECODED_JSON.validate_python(json.loads(candidate)) - except json.JSONDecodeError: - pass - - return None - - -def parse_tool_call_arguments( - arguments: str | None, - tool_name: str | None = None, - context: str | None = None, -) -> Any: - """ - Parse tool call arguments from a JSON string. - - When the JSON is malformed (e.g. truncated by the model), this function - attempts a lightweight repair (closing unmatched brackets/braces) before - raising an error. A warning is logged whenever repair succeeds so that - callers are aware the arguments were not perfectly formed. - - Args: - arguments: The JSON string containing tool arguments, or None. - tool_name: Optional name of the tool (for error messages). - context: Optional context string (e.g., "Anthropic Messages API"). - - Returns: - Parsed arguments (usually a dict, but may be any JSON-deserializable - type such as list, str, int, float, or None). Returns empty dict if - arguments is None or empty. - - Raises: - ValueError: If the arguments string is not valid JSON and cannot be repaired. - """ - import json - - if not arguments or not arguments.strip(): - return {} - - try: - return _DECODED_JSON.validate_python(json.loads(arguments)) - except json.JSONDecodeError as original_error: - repaired: Final = _attempt_json_repair(arguments) - if repaired is not None: - verbose_logger.warning( - "Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %.200s%s", - tool_name or "", - context or "unknown context", - len(arguments), - arguments, - "..." if len(arguments) > 200 else "", - ) - return repaired - - error_parts: Final = ["Failed to parse tool call arguments"] - - if tool_name: - error_parts.append(f"for tool '{tool_name}'") - if context: - error_parts.append(f"({context})") - - error_message: Final = " ".join(error_parts) + f". Error: {original_error}. Arguments: {arguments}" - - raise ValueError(error_message) from original_error - - -def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]: - """ - Split a string that contains one or more concatenated JSON objects into - a list of parsed dicts. - - LLM providers (notably Bedrock Claude Sonnet 4.5) sometimes return - multiple tool-call argument objects concatenated in a single - ``arguments`` string, e.g.:: - - '{"command":["curl",...]}{"command":["curl",...]}{"command":["curl",...]}' - - ``json.loads()`` fails on this with ``JSONDecodeError: Extra data``. - This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string - and extract each JSON object individually. - - The walk degrades gracefully: if the string is malformed or truncated - (e.g. a stream that ended mid-tool-call), whatever complete objects were - parsed before the bad tail are returned and the remainder is discarded - with a warning, rather than raising. The sole caller - (``_convert_to_bedrock_tool_call_invoke``) treats an empty result as - ``input={}`` so the conversation can continue instead of hard-failing. - - Returns - ------- - list[dict] - A list of parsed dicts – one per JSON object found. If *raw* is - empty, whitespace-only, or wholly unparseable, an empty list is - returned. - """ - import json - - raw = raw.strip() - if not raw: - return [] - - decoder: Final = json.JSONDecoder() - results: Final[list[dict[str, object]]] = [] - idx = 0 - length: Final = len(raw) - - while idx < length: - # Skip whitespace between objects - while idx < length and raw[idx] in " \t\n\r": - idx += 1 - if idx >= length: - break - - try: - decoded = decoder.raw_decode(raw, idx) - except json.JSONDecodeError as e: - verbose_logger.warning( - "split_concatenated_json_objects: discarding unparseable tool-call " - "arguments tail after %d complete object(s); decode_start=%d error=%s", - len(results), - idx, - e, - ) - break - obj = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) - if obj is not None: - results.append(dict(obj)) - else: - # Non-dict JSON value – wrap in empty dict (Bedrock requires - # toolUse.input to be an object). - results.append({}) - idx = decoded[1] - - return results - - -MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: Final = 8 - - -def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], ...]: - """Return complete concatenated JSON objects that are safe to expand. - - Identical objects collapse to the first one and are not capped. More than - ``MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS`` objects that are not all identical - returns an empty tuple. Anything that is not a full concatenation of JSON - objects returns an empty tuple. Repeated copies of the first object are not - retained, and once the cap is passed the rest of the string is only checked. - """ - stripped: Final = raw.strip() - if not stripped: - return () - decoder: Final = json.JSONDecoder() - length: Final = len(stripped) - idx = 0 # rebind-ok: cursor walks the concatenated JSON string - count = 0 # rebind-ok: counts complete objects without retaining duplicates - kept = () # rebind-ok: holds at most one object past the salvage cap - exceeded = False # rebind-ok: cap already passed, the tail is only validated - while idx < length: - while idx < length and stripped[idx] in " \t\n\r": - idx += 1 - if idx >= length: - break - try: - decoded = decoder.raw_decode(stripped, idx) - except json.JSONDecodeError: - return () - decoded_object = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) - if decoded_object is None: - return () - idx = decoded[1] - if exceeded: - continue - obj = dict(decoded_object) - count += 1 - if not kept: - kept = (obj,) - continue - if obj == kept[0] and len(kept) == 1: - continue - if len(kept) == 1 and count > 2 and count - 1 > MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: - exceeded = True - continue - if len(kept) == 1 and count > 2: - kept = (kept[0],) * (count - 1) - if len(kept) >= MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: - exceeded = True - continue - kept = (*kept, obj) - if exceeded: - return () - return kept - - def text_completion_prompt_to_messages(prompt: object) -> tuple[AllMessageValues, ...]: """ Wrap an OpenAI ``/v1/completions`` ``prompt`` into Chat Completion messages. diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index db56f390198..7520d69d4bb 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4291,7 +4291,7 @@ def _convert_to_bedrock_tool_call_invoke( - extract name - extract id """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( split_concatenated_json_objects, ) diff --git a/litellm/litellm_core_utils/prompt_templates/file_id_utils.py b/litellm/litellm_core_utils/prompt_templates/file_id_utils.py new file mode 100644 index 00000000000..db74ccf1c6d --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/file_id_utils.py @@ -0,0 +1,635 @@ +import io +import mimetypes +import re +from collections.abc import Mapping +from os import PathLike +from pathlib import Path +from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move + +from litellm import verbose_logger +from litellm.litellm_core_utils.batch_utils import InMemoryFile +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionFileObject, +) +from litellm.types.utils import ( + ExtractedFileData, + FileTypes, + SpecialEnums, +) + + +def get_format_from_file_id(file_id: str | None) -> str | None: + """ + Gets format from file id + + unified_file_id = litellm_proxy:{};unified_id,{} + If not a unified file id, returns 'file' as default format + """ + from litellm.proxy.openai_files_endpoints.common_utils import ( + convert_b64_uid_to_unified_uid, + ) + + if not file_id: + return None + try: + transformed_file_id: Final = convert_b64_uid_to_unified_uid(file_id) + if transformed_file_id.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value): + match: Final = re.match( + f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}:(.*?);unified_id", + transformed_file_id, + ) + if match: + return match.group(1) + + return None + except Exception: + return None + + +def update_messages_with_model_file_ids( + messages: list[AllMessageValues], + model_id: str | None, + model_file_id_mapping: dict[str, dict[str, str]], +) -> list[AllMessageValues]: + """ + Updates messages with model file ids. + + For managed files (unified file IDs), uses model_file_id_mapping if it + resolves the id, otherwise decodes the base64-encoded unified file ID + and extracts the llm_output_file_id directly. Mirrors the Responses-API + sibling `update_responses_input_with_model_file_ids`. + + model_file_id_mapping: Dict[str, Dict[str, str]] = { + "litellm_proxy/file_id": { + "model_id": "provider_file_id" + } + } + """ + from litellm.proxy.openai_files_endpoints.common_utils import ( + convert_b64_uid_to_unified_uid, + get_original_file_id, + is_base64_encoded_unified_file_id, + is_model_embedded_id, + ) + + for message in messages: + if message.get("role") == "user": + content = message.get("content") + if content: + if isinstance(content, str): + continue + for c in content: + if not isinstance(c, dict): + # Content list items aren't always dicts. e.g. + # text_completion forwards a token-ids list/list-of- + # lists through this path. Skip non-dict items + # instead of indexing into them. + continue + if c.get("type") == "file": + file_object = cast(ChatCompletionFileObject, c) + file_object_file_field = file_object.get("file") + if not isinstance(file_object_file_field, dict): + # Content block has `type: "file"` but not the + # OpenAI Chat Completions shape (e.g. a LangChain + # v1 standardized file block, or a provider-native + # shape that also uses `type: "file"`). Nothing to + # remap here, so skip instead of crashing. + continue + file_id = file_object_file_field.get("file_id") + format = file_object_file_field.get("format", get_format_from_file_id(file_id)) + + if file_id: + provider_file_id = ( + model_file_id_mapping.get(file_id, {}).get(model_id) + if model_file_id_mapping and model_id is not None + else None + ) + if not provider_file_id and is_base64_encoded_unified_file_id(file_id): + unified_file_id = convert_b64_uid_to_unified_uid(file_id) + if "llm_output_file_id," in unified_file_id: + provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0] + if not provider_file_id and is_model_embedded_id(file_id): + provider_file_id = get_original_file_id(file_id) + file_object_file_field["file_id"] = provider_file_id or file_id + if format: + file_object_file_field["format"] = format + return messages + + +def update_responses_input_with_model_file_ids( + input: object, + model_id: str | None = None, + model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None, +) -> object: + """ + Updates responses API input with provider-specific file IDs. + File IDs are always inside the content array, not as direct input_file items. + + For managed files (unified file IDs), uses model_file_id_mapping if provided, + otherwise decodes the base64-encoded unified file ID and extracts the llm_output_file_id directly. + + Args: + input: The responses API input parameter + model_id: The model ID to use for looking up provider-specific file IDs + model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs + Format: {"litellm_file_id": {"model_id": "provider_file_id"}} + """ + from litellm.proxy.openai_files_endpoints.common_utils import ( + convert_b64_uid_to_unified_uid, + get_original_file_id, + is_base64_encoded_unified_file_id, + is_model_embedded_id, + ) + + if isinstance(input, str): + return input + + if not isinstance(input, list): + return input + + updated_input: Final = [] + for item in input: + if not isinstance(item, dict): + updated_input.append(item) + continue + + updated_item = item.copy() + content = item.get("content") + if isinstance(content, list): + updated_content = [] + for content_item in content: + if isinstance(content_item, dict) and content_item.get("type") == "input_file": + file_id = content_item.get("file_id") + if isinstance(file_id, str) and file_id: + provider_file_id = file_id # Default to original + + # Check if we have a mapping for this file ID + if model_file_id_mapping and model_id and file_id in model_file_id_mapping: + # Use the model-specific file ID from mapping + provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id + updated_content_item = content_item.copy() + updated_content_item["file_id"] = provider_file_id + updated_content.append(updated_content_item) + else: + # Check if this is a base64-encoded unified file ID without mapping + is_unified_file_id = is_base64_encoded_unified_file_id(file_id) + if is_unified_file_id: + # Fallback: decode unified file ID + unified_file_id = convert_b64_uid_to_unified_uid(file_id) + if "llm_output_file_id," in unified_file_id: + provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0] + + updated_content_item = content_item.copy() + updated_content_item["file_id"] = provider_file_id + updated_content.append(updated_content_item) + elif is_model_embedded_id(file_id): + updated_content_item = content_item.copy() + updated_content_item["file_id"] = get_original_file_id(file_id) + updated_content.append(updated_content_item) + else: + # Not a managed file, keep as-is + updated_content.append(content_item) + else: + updated_content.append(content_item) + else: + updated_content.append(content_item) + updated_item["content"] = updated_content + + updated_input.append(updated_item) + + return updated_input + + +def _decode_vector_store_ids_in_tools( + tools: list[dict[str, object]] | None, +) -> list[dict[str, object]] | None: + """ + Decodes unified (LiteLLM-managed) vector_store_ids in file_search tools to + provider-native IDs. Non-unified IDs are passed through unchanged. + + This runs unconditionally — no file-ID mapping is required. + """ + if not tools or not isinstance(tools, list): + return tools + + from litellm.llms.base_llm.managed_resources.utils import ( + is_base64_encoded_unified_id, + parse_unified_id, + ) + + updated_tools: Final = [] + for tool in tools: + if not isinstance(tool, dict) or tool.get("type") != "file_search": + updated_tools.append(tool) + continue + + vector_store_ids = tool.get("vector_store_ids") + if not isinstance(vector_store_ids, list): + updated_tools.append(tool) + continue + + decoded_ids = [] + for vs_id in vector_store_ids: + if not isinstance(vs_id, str) or not is_base64_encoded_unified_id(vs_id): + decoded_ids.append(vs_id) + continue + + parsed = parse_unified_id(vs_id) + provider_resource_id = parsed.get("provider_resource_id") if parsed else None + + if not provider_resource_id: + verbose_logger.warning( + "file_search tool contains unified vector_store_id '%s' that could " + "not be decoded to a provider resource ID — passing original ID. " + "Ensure the vector store was created via LiteLLM.", + vs_id, + ) + decoded_ids.append(vs_id) + else: + decoded_ids.append(provider_resource_id) + + updated_tools.append({**tool, "vector_store_ids": decoded_ids}) + + return updated_tools + + +def update_responses_tools_with_model_file_ids( + tools: list[dict[str, object]] | None, + model_id: str | None = None, + model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None, +) -> list[dict[str, object]] | None: + """ + Updates responses API tools with provider-specific file IDs. + + Pass 1 (always): decode unified vector_store_ids in file_search tools. + Pass 2 (needs mapping): map code_interpreter container file_ids to provider IDs. + + Args: + tools: The responses API tools parameter + model_id: The model ID to use for looking up provider-specific file IDs + model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs + Format: {"litellm_file_id": {"model_id": "provider_file_id"}} + """ + if not tools or not isinstance(tools, list): + return tools + + # Pass 1: decode unified vector_store_ids (no mapping needed) + tools = _decode_vector_store_ids_in_tools(tools) or tools + + # Pass 2: map code_interpreter file IDs (requires mapping) + if not model_file_id_mapping or not model_id: + return tools + + updated_tools: Final = [] + for tool in tools: + if not isinstance(tool, dict): + updated_tools.append(tool) + continue + + updated_tool = tool.copy() + + # Handle code_interpreter with container file_ids + if tool.get("type") == "code_interpreter": + container = tool.get("container") + if isinstance(container, dict): + container_file_ids = container.get("file_ids") + if isinstance(container_file_ids, list): + updated_file_ids = [] + for file_id in container_file_ids: + if isinstance(file_id, str): + # Check if we have a mapping for this file ID + if file_id in model_file_id_mapping: + # Map to provider-specific file ID + provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id + updated_file_ids.append(provider_file_id) + else: + updated_file_ids.append(file_id) + else: + updated_file_ids.append(file_id) + + # Update the tool with new file IDs + updated_container = container.copy() + updated_container["file_ids"] = updated_file_ids + updated_tool["container"] = updated_container + + updated_tools.append(updated_tool) + + return updated_tools + + +def extract_file_metadata(file_data: FileTypes) -> tuple[str | None, str | None]: + """ + Resolve (filename, content_type) without reading the file body. + + Mirrors extract_file_data's metadata resolution but never calls .read(), so + it stays O(1) on large uploads. Use this when only metadata is needed (batch + detection, GCS object naming) and the body must remain a streamable Path/handle. + """ + filename: str | None = None + content_type: str | None = None + file_content: object = None + + if isinstance(file_data, tuple): + if len(file_data) == 2: + filename, file_content = file_data + elif len(file_data) == 3: + filename, file_content, content_type = file_data + elif len(file_data) == 4: + filename, file_content, content_type, _ = file_data + elif isinstance(file_data, InMemoryFile): + filename = file_data.name + content_type = file_data.content_type + else: + file_content = file_data + + if filename is None: + if isinstance(file_content, PathLike): + filename = Path(file_content).name + elif isinstance(file_content, io.IOBase): + name_attr: Final = getattr(file_content, "name", None) + if isinstance(name_attr, str): + filename = Path(name_attr).name + + if not content_type: + guessed: Final = mimetypes.guess_type(filename)[0] if filename else None + content_type = guessed or "application/octet-stream" + + return filename, content_type + + +def extract_file_data(file_data: FileTypes) -> ExtractedFileData: + """ + Extracts and processes file data from various input formats. + + Args: + file_data: Can be a tuple of (filename, content, [content_type], [headers]) or direct file content + + Returns: + ExtractedFileData containing: + - filename: Name of the file if provided + - content: The file content in bytes + - content_type: MIME type of the file + - headers: Any additional headers + """ + # Parse the file_data based on its type + filename = None + file_content = None + content_type = None + file_headers: Mapping[str, str] = {} + + if isinstance(file_data, tuple): + if len(file_data) == 2: + filename, file_content = file_data + elif len(file_data) == 3: + filename, file_content, content_type = file_data + elif len(file_data) == 4: + filename, file_content, content_type, file_headers = file_data + elif isinstance(file_data, InMemoryFile): + filename = file_data.name + file_content = file_data + content_type = file_data.content_type + else: + file_content = file_data + # Convert content to bytes + if isinstance(file_content, str): + # Bare string inputs are rejected: when this helper runs in a proxy + # request handler the string came from an attacker-controlled form + # field, and opening it as a path is an arbitrary file read on the + # proxy host. SDK callers who want to upload from a path should + # either pass a pathlib.Path (a PathLike instance — see the branch + # below) or open the file themselves and pass the handle / bytes. + raise ValueError( + "extract_file_data does not accept bare str inputs. Pass bytes, " + "an open file handle, a (filename, content) tuple, or a " + "pathlib.Path. To upload a local file from a path, call " + "open(path, 'rb') yourself." + ) + if isinstance(file_content, PathLike): + # PathLike (pathlib.Path) is a Python-level type that HTTP form + # values can't fabricate. Treat as a local file path for SDK + # convenience. + if filename is None: + filename = Path(file_content).name + with open(file_content, "rb") as f: + content = f.read() + elif isinstance(file_content, io.IOBase): + # If it's a file-like object + # Try to get filename from file handle if not already set + if not filename and hasattr(file_content, "name"): + filename = Path(file_content.name).name + + content = file_content.read() + + if isinstance(content, str): + content = content.encode("utf-8") + # Reset file pointer to beginning + file_content.seek(0) + elif isinstance(file_content, bytes): + content = file_content + else: + raise ValueError(f"Unsupported file content type: {type(file_content)}") + + # Use provided content type or guess based on filename + if not content_type: + if filename: + guessed_type: Final = mimetypes.guess_type(filename)[0] + content_type = guessed_type if guessed_type else "application/octet-stream" + else: + content_type = "application/octet-stream" + + return ExtractedFileData( + filename=filename, + content=content, + content_type=content_type, + headers=file_headers, + ) + + +def get_image_mime_type_from_url(url: str) -> str | None: + """ + Get mime type for common image URLs + See gemini mime types: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/image-understanding#image-requirements + + Supported by Gemini: + application/pdf + audio/mpeg + audio/mp3 + audio/wav + audio/ogg + image/png + image/jpeg + image/webp + text/plain + video/mov + video/mpeg + video/mp4 + video/mpg + video/avi + video/wmv + video/mpegps + video/flv + """ + from urllib.parse import urlparse + + url = url.lower() + + # Parse URL to extract path without query parameters + # This handles URLs like: https://example.com/image.jpg?signature=... + parsed: Final = urlparse(url) + path: Final = parsed.path + + # Map file extensions to mime types + mime_types: Final = { + # Images + (".jpg", ".jpeg"): "image/jpeg", + (".png",): "image/png", + (".webp",): "image/webp", + # Videos + (".mp4",): "video/mp4", + (".mov",): "video/mov", + (".mpeg", ".mpg"): "video/mpeg", + (".avi",): "video/avi", + (".wmv",): "video/wmv", + (".mpegps",): "video/mpegps", + (".flv",): "video/flv", + # Audio + (".mp3",): "audio/mp3", + (".wav",): "audio/wav", + (".mpeg",): "audio/mpeg", + (".ogg",): "audio/ogg", + # Documents + (".pdf",): "application/pdf", + (".txt",): "text/plain", + } + + # Check each extension group against the URL + for extensions, mime_type in mime_types.items(): + if any(path.endswith(ext) for ext in extensions): + return mime_type + + return None + + +_get_image_mime_type_from_url = get_image_mime_type_from_url + + +def infer_content_type_from_url_and_content( + url: str, + content: bytes, + current_content_type: str | None = None, +) -> str: + """ + Infer content type from URL extension and binary content when content-type header is missing or generic. + + This helper implements a fallback strategy for determining MIME types when HTTP headers + are missing or provide generic values (like binary/octet-stream). It's commonly used + when processing images and documents from various sources (S3, URLs, etc.). + + Fallback Strategy: + 1. If current_content_type is valid (not None and not generic octet-stream), return it + 2. Try to infer from URL extension (handles query parameters) + 3. Try to detect from binary content signature (magic bytes) + 4. Raise ValueError if all methods fail + + Args: + url: The URL of the content (used to extract file extension) + content: The binary content (first ~100 bytes are sufficient for detection) + current_content_type: The current content-type from headers (may be None or generic) + + Returns: + str: The inferred MIME type (e.g., "image/png", "application/pdf") + + Raises: + ValueError: If content type cannot be determined by any method + + Example: + >>> content_type = infer_content_type_from_url_and_content( + ... url="https://s3.amazonaws.com/bucket/image.png?AWSAccessKeyId=123", + ... content=png_binary_data, + ... current_content_type="binary/octet-stream" + ... ) + >>> print(content_type) + "image/png" + """ + from litellm.litellm_core_utils.token_counter import get_image_type + + media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None + + # If we have a valid content type that's not generic, use it + if media_type and media_type not in [ + "binary/octet-stream", + "application/octet-stream", + ]: + return media_type + + # Extension to MIME type mapping + # Supports images, documents, and other common file types + extension_to_mime: Final = { + # Image formats + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "png": "image/png", + "gif": "image/gif", + "webp": "image/webp", + # Document formats + "pdf": "application/pdf", + "csv": "text/csv", + "doc": "application/msword", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xls": "application/vnd.ms-excel", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "html": "text/html", + "txt": "text/plain", + "md": "text/markdown", + } + + # Try to infer from URL extension + if url: + extension: Final = url.split(".")[-1].lower().split("?")[0] # Remove query params + inferred_type: Final = extension_to_mime.get(extension) + if inferred_type: + return inferred_type + + # Try to detect from binary content signature (magic bytes) + if content: + detected_type: Final = get_image_type(content[:100]) + if detected_type: + type_to_mime: Final = { + "png": "image/png", + "jpeg": "image/jpeg", + "gif": "image/gif", + "webp": "image/webp", + "heic": "image/heic", + } + if detected_type in type_to_mime: + return type_to_mime[detected_type] + + # If all fallbacks failed, raise error + raise ValueError(f"Unable to determine content type from URL: {url}. Response content-type: {current_content_type}") + + +def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]: + """ + Gets file ids from messages + """ + file_ids: Final = [] + for message in messages: + if message.get("role") == "user": + content = message.get("content") + if content: + if isinstance(content, str): + continue + for c in content: + if isinstance(c, dict) and c["type"] == "file": + file_object = cast(ChatCompletionFileObject, c) + file_object_file_field = file_object.get("file") + if not isinstance(file_object_file_field, dict): + # Content block has `type: "file"` but not the + # OpenAI Chat Completions shape. No file_id to + # extract, so skip instead of raising KeyError. + continue + file_id = file_object_file_field.get("file_id") + if file_id: + file_ids.append(file_id) + return file_ids diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index ffc0e82e410..fa6f1c1ed0a 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -21,7 +21,9 @@ import litellm from litellm import verbose_logger from litellm.caching.caching import InMemoryCache from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, NON_LLM_CONNECTION_TIMEOUT -from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + infer_content_type_from_url_and_content, +) from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get from litellm.types.llms.openai import AllMessageValues diff --git a/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py new file mode 100644 index 00000000000..1d6d28c2680 --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py @@ -0,0 +1,471 @@ +import re +from collections.abc import Callable, Iterator, Mapping +from graphlib import CycleError, TopologicalSorter +from itertools import chain, islice +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter + +_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("allOf", "anyOf", "oneOf") +_OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS: Final = ("enum", "const", "not") +_LOCAL_SCHEMA_REF_PREFIXES: Final = (("#/$defs/", "$defs"), ("#/definitions/", "definitions")) +_MAX_SCHEMA_FLATTEN_DEPTH: Final = 32 +_EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]: + properties: Final = schema.get("properties") + return properties if isinstance(properties, dict) else _EMPTY_SCHEMA + + +def _schema_branches(schema: Mapping[str, object], combinator: str) -> tuple[object, ...]: + branches: Final = schema.get(combinator) + return tuple(branches) if isinstance(branches, list) else () + + +def _schema_required_names(schema: Mapping[str, object]) -> frozenset[str]: + required: Final = schema.get("required") + if not isinstance(required, list): + return frozenset() + return frozenset(name for name in required if isinstance(name, str)) + + +def _combinator_required_names(combinator: str, branches: tuple[Mapping[str, object], ...]) -> frozenset[str]: + branch_names: Final = tuple(_schema_required_names(branch) for branch in branches) + if not branch_names: + return frozenset() + if combinator == "allOf": + return branch_names[0].union(*branch_names[1:]) + return branch_names[0].intersection(*branch_names[1:]) + + +def _resolve_local_schema_ref(root: Mapping[str, object], ref: str) -> Mapping[str, object] | None: + matched: Final = next( + ((prefix, container) for prefix, container in _LOCAL_SCHEMA_REF_PREFIXES if ref.startswith(prefix)), + None, + ) + if matched is None: + return None + prefix, container = matched + definitions: Final = root.get(container) + if not isinstance(definitions, dict): + return None + target: Final = definitions.get(ref[len(prefix) :]) + return target if isinstance(target, dict) else None + + +def _mergeable_branch( + root: Mapping[str, object], + branch: object, + seen_refs: frozenset[str], + depth: int, + expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work +) -> Mapping[str, object] | None: + if not isinstance(branch, dict) or depth > _MAX_SCHEMA_FLATTEN_DEPTH: + return None + ref: Final = branch.get("$ref") + if not isinstance(ref, str): + flattened: Final = _flatten_schema_against_root(branch, root, seen_refs, depth, expanded_refs) + if any(combinator in flattened for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS): + return None + return flattened + if ref in expanded_refs: + return expanded_refs[ref] + if ref in seen_refs: + return None + target: Final = _resolve_local_schema_ref(root, ref) + expanded: Final = ( + None + if target is None + else _mergeable_branch(root, target, seen_refs | frozenset((ref,)), depth + 1, expanded_refs) + ) + expanded_refs[ref] = expanded + return expanded + + +def _is_object_schema(schema: Mapping[str, object]) -> bool: + return schema.get("type") == "object" or ("type" not in schema and "properties" in schema) + + +def _flatten_schema_against_root( + schema: Mapping[str, object], + root: Mapping[str, object], + seen_refs: frozenset[str], + depth: int, + expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work +) -> Mapping[str, object]: + raw_branch_groups: Final = tuple( + ( + combinator, + tuple( + _mergeable_branch(root, branch, seen_refs, depth + 1, expanded_refs) + for branch in _schema_branches(schema, combinator) + ), + ) + for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS + if isinstance(schema.get(combinator), list) + ) + dropped: Final = ( + *(combinator for combinator, _ in raw_branch_groups), + *(key for key in _OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS if key in schema), + ) + if not dropped: + return schema + + if any(branch is None for _, group in raw_branch_groups for branch in group): + return schema + branch_groups: Final = tuple( + (combinator, tuple(branch for branch in group if branch is not None)) for combinator, group in raw_branch_groups + ) + branches: Final = tuple(branch for _, group in branch_groups for branch in group) + is_object_schema: Final = _is_object_schema(schema) or ( + "type" not in schema and branches != () and all(_is_object_schema(branch) for branch in branches) + ) + if not is_object_schema: + return schema + + merged_properties: Final = { + name: value for source in (*reversed(branches), schema) for name, value in _schema_properties(source).items() + } + required_names: Final = _schema_required_names(schema).union( + *(_combinator_required_names(combinator, group) for combinator, group in branch_groups) + ) + kept: Final = MappingProxyType({key: value for key, value in schema.items() if key not in dropped}) + required_update: Final = MappingProxyType({"required": sorted(required_names)}) if required_names else _EMPTY_SCHEMA + return { + **kept, + "type": "object", + "properties": merged_properties, + **required_update, + } + + +def flatten_top_level_schema_combinators(schema: Mapping[str, object]) -> Mapping[str, object]: + """Merge top-level ``allOf``/``anyOf``/``oneOf`` branches into an object tool schema. + + OpenAI's function-calling validator rejects tool ``parameters`` carrying + 'oneOf'/'anyOf'/'allOf'/'enum'/'const'/'not' at the top level (nested uses + are accepted), while lenient backends such as the ChatGPT backend Codex + talks to natively accept them, so an MCP tool declaring a top-level union + 400s through LiteLLM. Branch properties merge without clobbering (the + top-level schema wins, then earlier branches); ``required`` becomes the + top-level list plus the intersection of the branch lists for anyOf/oneOf + or their union for allOf. Branches that are local ``$ref``s + (``#/$defs/...`` or ``#/definitions/...``) are resolved first, each ref + at most once per call, and branches that are themselves combinators are + flattened recursively up to a fixed depth; a branch that cannot be fully + merged (a boolean schema, an external or cyclic ``$ref``, a non-object + union, or nesting past the depth cap) leaves the whole schema untouched so + OpenAI's own validation still applies. Non-object schemas pass through + unchanged and the input is never mutated. + """ + return _flatten_schema_against_root(schema, schema, frozenset(), 0, {}) + + +_SUBSCHEMA_KEYWORDS: Final = frozenset( + { + "additionalItems", + "additionalProperties", + "contains", + "else", + "if", + "items", + "not", + "propertyNames", + "then", + "unevaluatedItems", + "unevaluatedProperties", + } +) +_SUBSCHEMA_LIST_KEYWORDS: Final = frozenset({"allOf", "anyOf", "items", "oneOf", "prefixItems"}) +_SUBSCHEMA_MAP_KEYWORDS: Final = frozenset( + {"$defs", "definitions", "dependentSchemas", "patternProperties", "properties"} +) + +_MAX_SCHEMA_NESTING: Final = 1024 + + +def drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: + """Drop every regex in a schema position that Python's ``re`` cannot compile. + + OpenAI validates tool ``parameters`` against the 2020-12 metaschema with + ``jsonschema``'s format checker, which hands each ``pattern`` value and each + ``patternProperties`` key to ``re.compile``, so a regex written for an + ECMA-262 engine (Unicode property escapes such as ``\\p{Cc}``, as in Claude + Code's ``Artifact`` tool) is refused with "'...' is not a 'regex'" by every + model family on both the chat and Responses wires. Only schema positions are + walked (properties, items, combinators, ``$defs`` and the other applicators), + so a ``pattern`` key inside ``default``, ``examples``, ``const`` or vendor + extensions is data and stays. Outside strict mode the keyword is only a + hint, so dropping it costs the model a constraint and the caller nothing. + Compilable regexes and everything else pass through, the input is never + mutated, and the same object comes back when nothing was dropped. The walk + is level-order rather than recursive, rebuilt deepest level first, and stops + at more schema levels than a JSON parser admits, so a cyclic schema built in + code cannot spin it. + """ + return _schema_without_rejected_regex(schema, _is_not_python_regex) + + +def drop_lookaround_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: + """Drop every regex in a schema position that uses a lookaround assertion. + + Some Bedrock Converse families compile tool schema regexes with an engine that + has no lookahead or lookbehind and refuse the whole request over one. The ``(?=``, + ``(?!``, ``(?<=`` and ``(? Mapping[str, object]: + rebuilt: dict[int, Mapping[str, object]] = {} # mutable-ok: per-call memo of rewritten nodes, deepest level first + for level in reversed(tuple(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))): + rebuilt.update( + (id(node), rewritten) + for node in level + if (rewritten := _node_without_rejected_regex(node, rebuilt, rejected)) is not node + ) + return rebuilt.get(id(schema), schema) + + +def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str, object], ...]]: + frontier: tuple[Mapping[str, object], ...] = (schema,) # rebind-ok: level-order cursor, one level a round + while frontier: + yield frontier + frontier = tuple(child for node in frontier for child in _subschemas(node)) + + +def schema_fragment_ref(ref: str) -> str: + return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref + + +def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool: + definitions: Final = MappingProxyType(dict(_root_definitions(schema))) + if not _refs_in(schema) <= definitions.keys(): + return True + try: + tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order()) + except CycleError: + return True + return False + + +def schema_union_property_count(schema: Mapping[str, object]) -> int: + return sum(1 for _, node in _property_schemas(schema) if _is_union(node)) + + +def schema_optional_property_count(schema: Mapping[str, object]) -> int: + return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema)) + + +def schema_property_count(schema: Mapping[str, object]) -> int: + return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties"))) + + +def schema_grammar_branch_count(schema: Mapping[str, object]) -> int: + return sum(_grammar_branches(node) for node in _schema_nodes(schema)) + + +def schema_has_open_object(schema: Mapping[str, object]) -> bool: + return any(_is_open_object(node) for node in _schema_nodes(schema)) + + +def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool: + return any( + isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern) + for node in _schema_nodes(schema) + ) + + +def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]: + return frozenset( + value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str) + ) + + +def _may_be_string(node: Mapping[str, object]) -> bool: + node_type: Final = node.get("type") + return node_type is None or node_type == "string" or "string" in _list_at(node, "type") + + +def is_object_schema(node: Mapping[str, object]) -> bool: + node_type: Final = node.get("type") + return node_type == "object" or (isinstance(node_type, list) and "object" in node_type) + + +def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]: + return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING)) + + +_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...]) + + +def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]: + value: Final = container.get(key) + return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else () + + +def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]: + if container is None: + return iter(()) + return ((key, value) for key in container if (value := _schema_at(container, key)) is not None) + + +def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]: + for defs_key in ("$defs", "definitions"): + yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key))) + + +def _refs_in(schema: Mapping[str, object]) -> frozenset[str]: + return frozenset( + schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str) + ) + + +def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]: + return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema)) + + +def _is_union(node: Mapping[str, object]) -> bool: + return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1 + + +def _grammar_branches(node: Mapping[str, object]) -> int: + properties: Final = _schema_at(node, "properties") + if properties is None: + return 0 + optional: Final = len(_optional_property_names(node)) + return (1 << optional) * (len(properties) - optional + 1) + + +def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]: + properties: Final = _schema_at(node, "properties") + if properties is None: + return frozenset() + return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str)) + + +def _is_open_object(node: Mapping[str, object]) -> bool: + additional_properties: Final = node.get("additionalProperties") + if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node: + return True + return is_object_schema(node) and "properties" not in node and additional_properties is not False + + +def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]: + for key, value in node.items(): + if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict): + yield from (sub for sub in value.values() if isinstance(sub, dict)) + elif key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list): + yield from (sub for sub in value if isinstance(sub, dict)) + elif key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict): + yield value + + +def _node_without_rejected_regex( + node: Mapping[str, object], + rebuilt: Mapping[int, Mapping[str, object]], + rejected: Callable[[str], bool], +) -> Mapping[str, object]: + kept: Final = { + key: _keyword_value_rebuilt(key, value, rebuilt, rejected) + for key, value in node.items() + if key != "pattern" or not isinstance(value, str) or not rejected(value) + } + if len(kept) == len(node) and all(kept[key] is node[key] for key in kept): + return node + dropped_pattern_properties: Final = _dropped_pattern_properties(node, kept, rebuilt) + if not dropped_pattern_properties or kept.get("additionalProperties") is not False: + return kept + return {**kept, "additionalProperties": _any_of(dropped_pattern_properties)} + + +def _dropped_pattern_properties( + node: Mapping[str, object], + kept: Mapping[str, object], + rebuilt: Mapping[int, Mapping[str, object]], +) -> tuple[object, ...]: + before: Final = _schema_at(node, "patternProperties") + after: Final = _schema_at(kept, "patternProperties") + if before is None or after is None: + return () + return tuple(rebuilt.get(id(sub), sub) for name, sub in before.items() if name not in after) + + +def _schema_at(container: Mapping[str, object], key: str) -> Mapping[str, object] | None: + value: Final = container.get(key) + return value if isinstance(value, dict) else None + + +def _any_of(schemas: tuple[object, ...]) -> object: + return schemas[0] if len(schemas) == 1 else {"anyOf": list(schemas)} + + +def _keyword_value_rebuilt( + key: str, + value: object, + rebuilt: Mapping[int, Mapping[str, object]], + rejected: Callable[[str], bool], +) -> object: + if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict): + kept: Final = { + name: rebuilt.get(id(sub), sub) + for name, sub in value.items() + if key != "patternProperties" or not isinstance(name, str) or not rejected(name) + } + return value if len(kept) == len(value) and all(kept[name] is value[name] for name in kept) else kept + if key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list): + items: Final = [rebuilt.get(id(sub), sub) for sub in value] + return value if all(new is old for new, old in zip(items, value, strict=True)) else items + if key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict): + return rebuilt.get(id(value), value) + return value + + +def _is_not_python_regex(pattern: str) -> bool: + try: + re.compile(pattern) + except (re.error, RecursionError): + return True + return False + + +_REGEX_LOOKAROUND_RE: Final = re.compile(r"\(\? bool: + return _REGEX_LOOKAROUND_RE.search(pattern) is not None + + +def flatten_combinators_and_drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: + return flatten_top_level_schema_combinators(drop_non_python_regex_patterns(schema)) + + +def tool_with_sanitized_parameters( + tool: Mapping[str, object], + sanitize: Callable[[Mapping[str, object]], Mapping[str, object]], +) -> Mapping[str, object]: + """Run the tool's JSON schema through ``sanitize``: ``function.parameters`` on an + OpenAI tool, ``input_schema`` on an Anthropic one. The same object comes back when + nothing changed.""" + function: Final = _schema_at(tool, "function") + if function is not None: + parameters: Final = _schema_at(function, "parameters") + if parameters is None: + return tool + sanitized_parameters: Final = sanitize(parameters) + if sanitized_parameters is parameters: + return tool + return {**tool, "function": {**function, "parameters": sanitized_parameters}} + input_schema: Final = _schema_at(tool, "input_schema") + if input_schema is None: + return tool + sanitized_schema: Final = sanitize(input_schema) + return tool if sanitized_schema is input_schema else {**tool, "input_schema": sanitized_schema} diff --git a/litellm/litellm_core_utils/prompt_templates/prompt_cache_breakpoint_utils.py b/litellm/litellm_core_utils/prompt_templates/prompt_cache_breakpoint_utils.py new file mode 100644 index 00000000000..27cf83210b7 --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/prompt_cache_breakpoint_utils.py @@ -0,0 +1,212 @@ +from collections.abc import Mapping +from typing import Final, TypeVar, cast # noqa: TID251 # Existing casts retained unchanged in this pure move + +from pydantic import TypeAdapter, ValidationError + +from litellm import verbose_logger +from litellm.exceptions import UnsupportedParamsError +from litellm.types.llms.openai import ( + ChatCompletionCachedContent, + ChatCompletionPromptCacheBreakpoint, + PromptCacheBreakpoint, +) + +_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object]) + + +PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = ( + "set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)" +) + + +def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: + """A breakpoint the client already wrote on the block wins, TTL included.""" + if marker is None or target.get("prompt_cache_breakpoint") is not None: + return target + marked: Final = {**target, "prompt_cache_breakpoint": marker} + return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key + + +_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) + + +_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(ChatCompletionPromptCacheBreakpoint) + + +_AcceptedBreakpointT: Final = TypeVar("_AcceptedBreakpointT", bound=Mapping[str, object]) + + +def _accepted_prompt_cache_breakpoint( + accepted: TypeAdapter[_AcceptedBreakpointT], marker: object +) -> _AcceptedBreakpointT | None: + try: + return accepted.validate_python(marker) + except ValidationError: + return None + + +def prompt_cache_breakpoint_within( + accepted: TypeAdapter[_AcceptedBreakpointT], marker: object +) -> _AcceptedBreakpointT | None: + """The client's marker cut down to the ``accepted`` shape: an unsupported ttl is dropped and the mode kept, + and a marker whose mode the target cannot read is dropped whole.""" + whole: Final = _accepted_prompt_cache_breakpoint(accepted, marker) + if whole is not None: + return whole + fields: Final = _breakpoint_fields_or_none(marker) + without_ttl: Final = ( + None + if fields is None + else _accepted_prompt_cache_breakpoint(accepted, {key: item for key, item in fields.items() if key != "ttl"}) + ) + verbose_logger.debug( + "Dropping malformed prompt_cache_breakpoint %r under drop_params" + if without_ttl is None + else "Dropping the unsupported ttl of prompt_cache_breakpoint %r under drop_params", + marker, + ) + return without_ttl + + +def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: + if marker is None or not drop_params: + return marker + return prompt_cache_breakpoint_within(_PROMPT_CACHE_BREAKPOINT, marker) + + +def prompt_cache_breakpoint_with_readable_mode(marker: object, drop_params: bool) -> object: + """The client's marker as written, or None under drop_params when no target could read its mode, so the + target's own config decides which ttl and keys it keeps.""" + return None if prompt_cache_breakpoint_for_wire(marker, drop_params) is None else marker + + +def whole_prompt_cache_breakpoint_or_none(marker: object) -> PromptCacheBreakpoint | None: + return _accepted_prompt_cache_breakpoint(_PROMPT_CACHE_BREAKPOINT, marker) + + +def chat_completion_prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: + if marker is None or not drop_params: + return marker + return prompt_cache_breakpoint_within(_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT, marker) + + +def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: + """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block.""" + if block.get("cache_control") is None: + return block + unmarked: Final = {key: value for key, value in block.items() if key != "cache_control"} + return cast( # cast-ok: same block shape as the input with the marker key renamed + _MarkedT, with_prompt_cache_breakpoint(unmarked, PromptCacheBreakpoint(mode="explicit")) + ) + + +def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT: + """OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block. + A ``cache_control`` the client already wrote on the block wins, TTL included.""" + marker: Final = block.get("prompt_cache_breakpoint") + if marker is None: + return block + unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} + control: Final = ( + None + if block.get("cache_control") is not None + else _cache_control_for_prompt_cache_breakpoint(marker, drop_params) + ) + marked: Final = unmarked if control is None else {**unmarked, "cache_control": control} + return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed + + +def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None: + try: + breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + if not drop_params: + raise UnsupportedParamsError( + message=( + f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got " + f"{marker!r}. Fix the marker, or " + f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it." + ), + status_code=400, + ) + verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + ttl: Final = breakpoint_mark.get("ttl") + if ttl is not None and not drop_params: + raise UnsupportedParamsError( + message=( + f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint " + f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send " + "a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block " + "where the model offers a 1h cache." + ), + status_code=400, + ) + return ChatCompletionCachedContent(type="ephemeral") + + +_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"}) + + +_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object]) + + +def _breakpoint_fields_or_none(value: object) -> Mapping[str, object] | None: + try: + return _BREAKPOINT_FIELDS.validate_python(value) + except ValidationError: + return None + + +def _without_prompt_cache_breakpoint(value: object) -> object: + if not isinstance(value, Mapping): + return value + fields: Final = _BREAKPOINT_FIELDS.validate_python(value) + return {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"} + + +def _breakpoints_stripped_from(key: str, value: object) -> object: + if key in _BREAKPOINT_BLOCK_LIST_KEYS and isinstance(value, list): + return [_without_prompt_cache_breakpoint(block) for block in _BREAKPOINT_BLOCKS.validate_python(value)] + return _without_prompt_cache_breakpoint(value) if key == "function" else value + + +def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT: + """A message or tool definition with every ``prompt_cache_breakpoint`` removed: its own, its content parts', + its tool calls' and its ``function``'s.""" + stripped: Final = { + key: _breakpoints_stripped_from(key, value) for key, value in block.items() if key != "prompt_cache_breakpoint" + } + return cast(_MarkedT, stripped) # cast-ok: same block shape as the input minus the marker keys + + +def _part_with_chat_completion_prompt_cache_breakpoint(part: object) -> object: + if not isinstance(part, Mapping): + return part + fields: Final = _BREAKPOINT_FIELDS.validate_python(part) + marker: Final = fields.get("prompt_cache_breakpoint") + if marker is None: + return fields + unmarked: Final = {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"} + return with_prompt_cache_breakpoint(unmarked, chat_completion_prompt_cache_breakpoint_for_wire(marker, True)) + + +def chat_completion_prompt_cache_breakpoints_for_wire(message: _MarkedT, drop_params: bool) -> _MarkedT: + content: Final = message.get("content") + if not drop_params or not isinstance(content, list): + return message + parts: Final = [ + _part_with_chat_completion_prompt_cache_breakpoint(part) for part in _BREAKPOINT_BLOCKS.validate_python(content) + ] + return cast(_MarkedT, {**message, "content": parts}) # cast-ok: same message shape with its parts' markers trimmed + + +LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS: Final = frozenset( + {"thinking_blocks", "reasoning_items", "provider_specific_fields"} +) + + +LITELLM_INTERNAL_MESSAGE_FIELDS: Final = LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS | {"reasoning_content"} diff --git a/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py b/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py new file mode 100644 index 00000000000..817b1fa5915 --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py @@ -0,0 +1,274 @@ +import base64 +import binascii +import re +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from itertools import groupby +from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move + +from litellm.types.llms.openai import ( + ChatCompletionReasoningItem, + ChatCompletionReasoningSummaryTextBlock, + ChatCompletionRedactedThinkingBlock, + ChatCompletionThinkingBlock, +) + + +def _provider_text_or_none(value: object) -> str | None: + return cast(str | None, value) # cast-ok: reasoning fields arrive in untyped provider messages + + +def extract_reasoning_content(message: Mapping[str, object]) -> tuple[str | None, str | None]: + """ + Extract reasoning content and main content from a message. + + Args: + message (dict): The message dictionary that may contain reasoning_content + + Returns: + tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content) + """ + message_content: Final = message.get("content") + if "reasoning_content" in message: + return _provider_text_or_none(message["reasoning_content"]), _provider_text_or_none(message_content) + elif "reasoning" in message: + return _provider_text_or_none(message["reasoning"]), _provider_text_or_none(message_content) + elif isinstance(message_content, str): + return parse_content_for_reasoning(message_content) + return None, _provider_text_or_none(message_content) + + +_extract_reasoning_content = extract_reasoning_content + + +def _readable_thinking_text(block: Mapping[str, object]) -> str: + """The text a chat model can read back, empty for redacted blocks and malformed ones.""" + if block.get("type") != "thinking": + return "" + return str(block.get("thinking") or "") + + +def reasoning_content_from_thinking_blocks( + thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock], +) -> str: + """Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect. + + Redacted blocks carry no readable text, so they contribute nothing. + """ + return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block))) + + +ENCRYPTED_REASONING_SIGNATURE_PREFIX: Final = "litellm_encrypted_reasoning:" + + +def encrypted_reasoning_signature(encrypted_content: str) -> str: + """The opaque value a Responses API reasoning item's `encrypted_content` travels in. + + Anthropic clients echo a thinking block's `signature` and a redacted block's `data` + back verbatim, so either field can carry the encrypted reasoning across turns; the + prefix tells the two apart from a signature Anthropic minted. + """ + return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}" + + +def _carries_encrypted_reasoning(signature: object) -> bool: + return isinstance(signature, str) and signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX) + + +def encrypted_content_from_signature(signature: object) -> str | None: + if not isinstance(signature, str) or not _carries_encrypted_reasoning(signature): + return None + return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None + + +def _encrypted_reasoning_field(block: Mapping[str, object]) -> object: + match block.get("type"): + case "thinking": + return block.get("signature") + case "redacted_thinking": + return block.get("data") + case _: + return None + + +def encrypted_content_of_block(block: Mapping[str, object]) -> str | None: + return encrypted_content_from_signature(_encrypted_reasoning_field(block)) + + +def is_encrypted_reasoning_block(block: object) -> bool: + """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. + + Only the Responses API that minted the content can read it back, so an Anthropic + backend has to drop such a block rather than fail signature verification on it. + """ + if not isinstance(block, Mapping): + return False + mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance + return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping)) + + +def is_unsignable_thinking_block(block: object) -> bool: + """A thinking block Anthropic cannot accept on input. + + Anthropic verifies the signature cryptographically, so a block with a null, + empty, or missing signature (e.g. from an open-source reasoning model) is + rejected with a 400, and so is a block whose signature or data carries + another provider's encrypted reasoning. A signed block is kept whatever its + text, since ``display: omitted`` returns signed blocks with empty ``thinking``. + `redacted_thinking` blocks carry no signature and are always kept. + """ + if is_encrypted_reasoning_block(block): + return True + if not isinstance(block, Mapping): + return False + mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance + if mapping.get("type") != "thinking": + return False + signature: Final = mapping.get("signature") + return not (isinstance(signature, str) and len(signature) > 0) + + +def has_blank_redacted_data(block: object) -> bool: + match block: + case {"type": "redacted_thinking", "data": str() as data}: + return not data.strip() + case {"type": "redacted_thinking"}: + return True + case _: + return False + + +def native_redacted_thinking_data(data: str) -> str: + """Claude's own `data` string for a redacted_thinking block. + + Bedrock Converse carries it base64-wrapped as `redactedContent`, and LiteLLM passed that + wrapper through as `data` until it started decoding Converse replies, so histories saved + before then still hold it. Claude's own value never decodes twice, since its first byte + decodes to a protobuf tag outside the base64 alphabet. + """ + return _converse_wrapped_payload(data) or data + + +def converse_redacted_content(data: str) -> str: + """The `redactedContent` Bedrock Converse expects for a redacted_thinking block's `data`.""" + return base64.b64encode(native_redacted_thinking_data(data).encode()).decode() + + +def _converse_wrapped_payload(data: str) -> str | None: + try: + payload: Final = base64.b64decode(data, validate=True).decode("ascii") + _ = base64.b64decode(payload, validate=True) + except (binascii.Error, ValueError): + return None + else: + return payload + + +def strip_encrypted_reasoning_from_messages( + messages: object, + *, + should_strip: Callable[[Mapping[str, object]], bool] | None = None, +) -> None: + """Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from + Anthropic-shaped history. + + The whole block goes, the way #40280 drops undecryptable Responses ``input`` items: a + provider that did not mint the block rejects it signed (a foreign signature) and unsigned + (a missing signature) alike, so keeping its text as an unsigned thinking block only moves + the 400 from the router to the provider. + + Mutates the content lists in place: the router's fallback snapshot shares these + message objects, so a rebound list would replay the stripped blocks on the fallback hop. + """ + if not isinstance(messages, list): + return + for content in anthropic_content_lists(cast(list[object], messages)): # cast-ok: untyped client json + _strip_encrypted_reasoning_from_blocks(content, should_strip=should_strip) + + +def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: + return ( + cast(list[object], content) # cast-ok: narrowed by isinstance + for message in messages + if isinstance(message, Mapping) + for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance + if isinstance(content, list) + ) + + +def _strip_encrypted_reasoning_from_blocks( + content: object, + *, + should_strip: Callable[[Mapping[str, object]], bool] | None = None, +) -> None: + blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance + kept: Final = tuple( + block + for block in blocks + if not is_encrypted_reasoning_block(block) + or (should_strip is not None and not should_strip(cast(Mapping[str, object], block))) + ) + blocks[:] = kept + + +def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: + index, block = indexed_block + return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary" + + +def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> ChatCompletionReasoningItem | None: + summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload + ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text) + for block in group + if (text := _readable_thinking_text(block)) + ] + encrypted_content: Final = encrypted_content_of_block(group[0]) + if encrypted_content is not None: + return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content) + if not summary: + return None + return ChatCompletionReasoningItem(type="reasoning", summary=summary) + + +def responses_reasoning_items_from_thinking_blocks( + thinking_blocks: Iterable[Mapping[str, object]], +) -> tuple[ChatCompletionReasoningItem, ...]: + """Build Responses API `reasoning` input items from Anthropic thinking blocks. + + A block carrying encrypted reasoning replays the item it came from byte for byte; + a run of plain thinking blocks collapses into one summary-only item. No item carries + an `id`: the Responses API 404s on any id it did not mint itself and rejects an empty + one, while an item without an id is always accepted. + """ + return tuple( + item + for _, group in groupby(enumerate(thinking_blocks), key=_reasoning_replay_group_key) + if (item := _reasoning_item_from_block_group(tuple(block for _, block in group))) is not None + ) + + +def parse_content_for_reasoning( + message_text: str | None, +) -> tuple[str | None, str | None]: + """ + Parse the content for reasoning + + Returns: + - reasoning_content: The content of the reasoning + - content: The content of the message + """ + if not message_text: + return None, message_text + + reasoning_match: Final = re.match( + r"<(?:think|thinking|budget:thinking)>(.*?)(.*)", + message_text, + re.DOTALL, + ) + + if reasoning_match: + return reasoning_match.group(1), reasoning_match.group(2) + + return None, message_text + + +_parse_content_for_reasoning = parse_content_for_reasoning diff --git a/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py b/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py new file mode 100644 index 00000000000..317ffa7b3b1 --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py @@ -0,0 +1,252 @@ +import json +from typing import Any, Final # noqa: TID251 # Existing annotations retained unchanged in this pure move + +from pydantic import TypeAdapter + +from litellm import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping + +_DECODED_JSON: Final = TypeAdapter(object) + + +def _attempt_json_repair(s: str) -> object | None: + """ + Attempt to repair truncated JSON produced by LLM tool calls. + + Handles the most common truncation patterns where the model generates + valid JSON that is cut short (missing closing brackets/braces). + + Returns the parsed value on success, or None if repair fails. + """ + import json + + stripped: Final = s.rstrip() + if not stripped: + return None + + # Track the stack of unmatched openers to respect nesting order + opener_stack: Final[list] = [] + in_string = False + escape_next = False + + for ch in stripped: + if escape_next: + escape_next = False + continue + if ch == "\\": + if in_string: + escape_next = True + continue + if ch == '"': + in_string = not in_string + continue + if in_string: + continue + if ch == "{": + opener_stack.append("}") + elif ch == "[": + opener_stack.append("]") + elif ch in ("}", "]"): + if opener_stack and opener_stack[-1] == ch: + opener_stack.pop() + + if not opener_stack: + return None + + # Remove trailing comma before we close brackets + candidate = stripped.rstrip(",") + + # Close in reverse order of opening (respects nesting) + candidate += "".join(reversed(opener_stack)) + + try: + return _DECODED_JSON.validate_python(json.loads(candidate)) + except json.JSONDecodeError: + pass + + return None + + +def parse_tool_call_arguments( + arguments: str | None, + tool_name: str | None = None, + context: str | None = None, +) -> Any: + """ + Parse tool call arguments from a JSON string. + + When the JSON is malformed (e.g. truncated by the model), this function + attempts a lightweight repair (closing unmatched brackets/braces) before + raising an error. A warning is logged whenever repair succeeds so that + callers are aware the arguments were not perfectly formed. + + Args: + arguments: The JSON string containing tool arguments, or None. + tool_name: Optional name of the tool (for error messages). + context: Optional context string (e.g., "Anthropic Messages API"). + + Returns: + Parsed arguments (usually a dict, but may be any JSON-deserializable + type such as list, str, int, float, or None). Returns empty dict if + arguments is None or empty. + + Raises: + ValueError: If the arguments string is not valid JSON and cannot be repaired. + """ + import json + + if not arguments or not arguments.strip(): + return {} + + try: + return _DECODED_JSON.validate_python(json.loads(arguments)) + except json.JSONDecodeError as original_error: + repaired: Final = _attempt_json_repair(arguments) + if repaired is not None: + verbose_logger.warning( + "Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %.200s%s", + tool_name or "", + context or "unknown context", + len(arguments), + arguments, + "..." if len(arguments) > 200 else "", + ) + return repaired + + error_parts: Final = ["Failed to parse tool call arguments"] + + if tool_name: + error_parts.append(f"for tool '{tool_name}'") + if context: + error_parts.append(f"({context})") + + error_message: Final = " ".join(error_parts) + f". Error: {original_error}. Arguments: {arguments}" + + raise ValueError(error_message) from original_error + + +def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]: + """ + Split a string that contains one or more concatenated JSON objects into + a list of parsed dicts. + + LLM providers (notably Bedrock Claude Sonnet 4.5) sometimes return + multiple tool-call argument objects concatenated in a single + ``arguments`` string, e.g.:: + + '{"command":["curl",...]}{"command":["curl",...]}{"command":["curl",...]}' + + ``json.loads()`` fails on this with ``JSONDecodeError: Extra data``. + This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string + and extract each JSON object individually. + + The walk degrades gracefully: if the string is malformed or truncated + (e.g. a stream that ended mid-tool-call), whatever complete objects were + parsed before the bad tail are returned and the remainder is discarded + with a warning, rather than raising. The sole caller + (``_convert_to_bedrock_tool_call_invoke``) treats an empty result as + ``input={}`` so the conversation can continue instead of hard-failing. + + Returns + ------- + list[dict] + A list of parsed dicts – one per JSON object found. If *raw* is + empty, whitespace-only, or wholly unparseable, an empty list is + returned. + """ + import json + + raw = raw.strip() + if not raw: + return [] + + decoder: Final = json.JSONDecoder() + results: Final[list[dict[str, object]]] = [] + idx = 0 + length: Final = len(raw) + + while idx < length: + # Skip whitespace between objects + while idx < length and raw[idx] in " \t\n\r": + idx += 1 + if idx >= length: + break + + try: + decoded = decoder.raw_decode(raw, idx) + except json.JSONDecodeError as e: + verbose_logger.warning( + "split_concatenated_json_objects: discarding unparseable tool-call " + "arguments tail after %d complete object(s); decode_start=%d error=%s", + len(results), + idx, + e, + ) + break + obj = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) + if obj is not None: + results.append(dict(obj)) + else: + # Non-dict JSON value – wrap in empty dict (Bedrock requires + # toolUse.input to be an object). + results.append({}) + idx = decoded[1] + + return results + + +MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: Final = 8 + + +def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], ...]: + """Return complete concatenated JSON objects that are safe to expand. + + Identical objects collapse to the first one and are not capped. More than + ``MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS`` objects that are not all identical + returns an empty tuple. Anything that is not a full concatenation of JSON + objects returns an empty tuple. Repeated copies of the first object are not + retained, and once the cap is passed the rest of the string is only checked. + """ + stripped: Final = raw.strip() + if not stripped: + return () + decoder: Final = json.JSONDecoder() + length: Final = len(stripped) + idx = 0 # rebind-ok: cursor walks the concatenated JSON string + count = 0 # rebind-ok: counts complete objects without retaining duplicates + kept = () # rebind-ok: holds at most one object past the salvage cap + exceeded = False # rebind-ok: cap already passed, the tail is only validated + while idx < length: + while idx < length and stripped[idx] in " \t\n\r": + idx += 1 + if idx >= length: + break + try: + decoded = decoder.raw_decode(stripped, idx) + except json.JSONDecodeError: + return () + decoded_object = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) + if decoded_object is None: + return () + idx = decoded[1] + if exceeded: + continue + obj = dict(decoded_object) + count += 1 + if not kept: + kept = (obj,) + continue + if obj == kept[0] and len(kept) == 1: + continue + if len(kept) == 1 and count > 2 and count - 1 > MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: + exceeded = True + continue + if len(kept) == 1 and count > 2: + kept = (kept[0],) * (count - 1) + if len(kept) >= MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: + exceeded = True + continue + kept = (*kept, obj) + if exceeded: + return () + return kept diff --git a/litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py b/litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py new file mode 100644 index 00000000000..a4cb52d689c --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py @@ -0,0 +1,259 @@ +import base64 +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move + +from litellm.litellm_core_utils.core_helpers import as_str_mapping + + +@dataclass(frozen=True, slots=True) +class _TPMMediaProjection: + content: object + media: tuple[Mapping[str, object], ...] = () + + +def _tpm_image_part(url: str, detail: object = "auto") -> Mapping[str, object]: + return {"type": "image_url", "image_url": {"url": url, "detail": detail}} + + +def _tpm_blank_fields(fields: Mapping[str, object], names: tuple[str, ...]) -> Mapping[str, object]: + return {key: "" if key in names and isinstance(value, str) else value for key, value in fields.items()} + + +def _tpm_file_media(part: Mapping[str, object]) -> _TPMMediaProjection: + from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type + + nested: Final = part.get("type") == "file" + fields: Final = as_str_mapping(part.get("file")) if nested else part + if fields is None: + return _TPMMediaProjection(part) + raw_data: Final = fields.get("file_data") + data: Final = raw_data if isinstance(raw_data, str) else "" + declared: Final = next( + ( + value + for key in ("format", "mime_type", "content_type") + if isinstance(value := fields.get(key), str) and value + ), + "", + ) + mime: Final = (declared or parse_mime_type(data[:256]) or "").split(";", 1)[0].strip().lower() + if not mime.startswith(("image/", "audio/")): + return _TPMMediaProjection(part) + raw_url: Final = fields.get("file_url") or fields.get("file_id") + url: Final = data or (raw_url if isinstance(raw_url, str) else "") + header_end: Final = data.find(",", 0, 256) + encoded: Final = ( + data[header_end + 1 :] + if header_end >= 0 and data[:header_end].endswith(";base64") + else "" + if data.startswith(("data:", "https://", "http://", "gs://")) + else data + ) + media: Final = ( + _tpm_image_part( + url + if not data or url.startswith(("data:", "https://", "http://", "gs://")) + else f"data:{mime};base64,{url}", + fields.get("detail", "auto"), + ) + if mime.startswith("image/") + else {"type": "input_audio", "input_audio": {"data": encoded, "format": mime.removeprefix("audio/")}} + ) + projected: Final = _tpm_blank_fields(fields, ("file_data", "file_url")) + return _TPMMediaProjection({**part, "file": projected} if nested else projected, (media,)) + + +def _tpm_media_leaf(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + if part is None: + return _TPMMediaProjection(value) + kind: Final = part.get("type") + if kind in ("file", "input_file"): + return _tpm_file_media(part) + if kind == "input_audio": + audio: Final = as_str_mapping(part.get("input_audio")) + return _TPMMediaProjection( + {**part, "input_audio": _tpm_blank_fields(audio, ("data",))} if audio is not None else value, + ({"type": "input_audio", "input_audio": part.get("input_audio")},), + ) + if kind in ("image_url", "input_image", "image", "computer_screenshot"): + image: Final = as_str_mapping(part.get("image_url")) + source: Final = as_str_mapping(part.get("source")) + canonical: Final = _tpm_image_part( + _image_part_url({**part, "type": "input_image"} if kind == "computer_screenshot" else part) or "", + (image or part).get("detail", part.get("detail", "auto")), + ) + projected: Final = ( + {**part, "image_url": _tpm_blank_fields(image, ("url",))} + if image is not None and kind in ("image_url", "input_image", "computer_screenshot") + else {**part, "source": _tpm_blank_fields(source, ("data", "url"))} + if source is not None and kind == "image" and source.get("type") in ("base64", "url") + else _tpm_blank_fields(part, ("image_url", "url")) + ) + return _TPMMediaProjection(projected, (canonical,)) + return _TPMMediaProjection(value) + + +def _tpm_media_list(value: object, split: Callable[[object], _TPMMediaProjection]) -> _TPMMediaProjection: + if not isinstance(value, (list, tuple)): + return _TPMMediaProjection(value) + parts: Final = cast(Sequence[object], value) # cast-ok: finite JSON content arrays + projected: Final = tuple(map(split, parts)) + media: Final = tuple(chain.from_iterable(part.media for part in projected)) + content: Final = tuple(part.content for part in projected) + return _TPMMediaProjection(parts if not media else content if isinstance(value, tuple) else list(content), media) + + +def _tpm_replace_field(part: Mapping[str, object], key: str, projection: _TPMMediaProjection) -> _TPMMediaProjection: + return _TPMMediaProjection({**part, key: projection.content} if projection.media else part, projection.media) + + +def _tpm_document_media(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + source: Final = as_str_mapping(part.get("source")) if part is not None else None + if part is None or part.get("type") != "document" or source is None or source.get("type") != "content": + return _tpm_media_leaf(value) + content: Final = _tpm_media_list(source.get("content"), _tpm_media_leaf) + return _tpm_replace_field(part, "source", _tpm_replace_field(source, "content", content)) + + +def _tpm_tool_result_media(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + if part is None or part.get("type") != "tool_result": + return _tpm_document_media(value) + return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_document_media)) + + +def _tpm_code_output_media(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + url: Final = part.get("url") if part is not None else None + if part is None or part.get("type") != "image" or not isinstance(url, str) or not url: + return _TPMMediaProjection(value) + return _TPMMediaProjection(_tpm_blank_fields(part, ("url",)), (_tpm_image_part(url),)) + + +def _tpm_input_media(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + if part is None: + return _TPMMediaProjection(value) + kind: Final = part.get("type") + if kind in ("function_call_output", "custom_tool_call_output"): + return _tpm_replace_field(part, "output", _tpm_media_list(part.get("output"), _tpm_tool_result_media)) + if kind == "computer_call_output": + return _tpm_replace_field(part, "output", _tpm_media_leaf(part.get("output"))) + if kind == "code_interpreter_call": + return _tpm_replace_field(part, "outputs", _tpm_media_list(part.get("outputs"), _tpm_code_output_media)) + result: Final = part.get("result") + if kind == "image_generation_call" and isinstance(result, str) and result: + return _TPMMediaProjection( + _tpm_blank_fields(part, ("result",)), + (_tpm_image_part("data:application/octet-stream;base64," + result),), + ) + if kind == "message" or (kind is None and isinstance(part.get("role"), str)): + return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_tool_result_media)) + return _tpm_tool_result_media(value) + + +def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]: + projection: Final = _tpm_media_list(value, _tpm_input_media) + return projection.content, projection.media + + +_TPM_EMBEDDING_FILE_MIME_PREFIXES: Final = ("application/pdf", "video/") + + +def _tpm_embedding_file(data_uri: str) -> tuple[Mapping[str, object], ...]: + from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type + + mime: Final = (parse_mime_type(data_uri[:256]) or "").strip().lower() + if not mime.startswith(_TPM_EMBEDDING_FILE_MIME_PREFIXES): + return () + return ({"type": "file", "file": {"file_data": data_uri, "format": mime}},) + + +_BASE64_PAYLOAD: Final = re.compile(r"[A-Za-z0-9+/]*={0,2}") + + +def _is_base64_data_uri(value: str) -> bool: + header_end: Final = value.find(",", 0, 256) + if header_end < 0 or not value[:header_end].endswith(";base64"): + return False + payload_start: Final = header_end + 1 + if (len(value) - payload_start) % 4 or _BASE64_PAYLOAD.fullmatch(value, payload_start) is None: + return False + last_quantum: Final = value[max(payload_start, len(value) - 4) :] + return base64.b64encode(base64.b64decode(last_quantum)).decode("ascii") == last_quantum + + +def _tpm_embedding_file_block(part: Mapping[str, object]) -> _TPMMediaProjection: + projection: Final = _tpm_file_media(part) + fields: Final = as_str_mapping(part.get("file")) + data: Final = fields.get("file_data") if fields is not None else None + if projection.media or fields is None or not isinstance(data, str) or not _is_base64_data_uri(data): + return projection + media: Final = _tpm_embedding_file(data) + return _TPMMediaProjection({**part, "file": _tpm_blank_fields(fields, ("file_data",))} if media else part, media) + + +def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + if part is not None and part.get("type") == "file": + return _tpm_embedding_file_block(part) + if not isinstance(value, str) or not value.startswith("data:"): + return _tpm_media_leaf(value) + if not _is_base64_data_uri(value): + return _TPMMediaProjection(value) + media: Final = _tpm_file_media({"file_data": value}).media or _tpm_embedding_file(value) + return _TPMMediaProjection("" if media else value, media) + + +def _is_json_array(value: object) -> bool: + return isinstance(value, (list, tuple)) + + +def _tpm_embedding_item(value: object) -> _TPMMediaProjection: + return _tpm_media_list(value, _tpm_embedding_leaf) if _is_json_array(value) else _tpm_embedding_leaf(value) + + +def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]: + projection: Final = ( + _tpm_media_list(value, _tpm_embedding_item) if _is_json_array(value) else _tpm_embedding_leaf(value) + ) + return projection.content, projection.media + + +def embedding_input_is_all_media(value: object) -> bool: + items: Final = cast(Sequence[object], value) if _is_json_array(value) else (value,) # cast-ok: JSON input array + return all(_tpm_embedding_leaf(item).media for item in items) + + +def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) -> str | None: + """Data or remote URL for an Anthropic ``source`` block, in the form chat completions expects.""" + source_type: Final = image_source.get("type") + if source_type == "base64": + media_type: Final = image_source.get("media_type") or "image/jpeg" + image_data: Final = image_source.get("data") or "" + return f"data:{media_type};base64,{image_data}" if image_data else None + if source_type == "url": + url: Final = image_source.get("url") + return url if isinstance(url, str) else "" + return None + + +def _image_part_url(part: Mapping[str, object]) -> str | None: + """The image URL carried by one content part, whichever of the three dialects wrote it.""" + part_type: Final = part.get("type") + if part_type in ("image_url", "input_image"): + image_url: Final = part.get("image_url") or part.get("url") + if isinstance(image_url, str): + return image_url + image_fields: Final = as_str_mapping(image_url) + url: Final = image_fields.get("url") if image_fields is not None else None + return url if isinstance(url, str) else None + if part_type == "image": + source: Final = part.get("source") + return anthropic_image_source_to_openai_url(source) if isinstance(source, Mapping) else None + return None diff --git a/litellm/litellm_core_utils/routing_common_utils.py b/litellm/litellm_core_utils/routing_common_utils.py index 62bdc1c10a7..d13ff4059aa 100644 --- a/litellm/litellm_core_utils/routing_common_utils.py +++ b/litellm/litellm_core_utils/routing_common_utils.py @@ -409,7 +409,7 @@ def deployments_the_caller_may_use( return () -_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "web_search_preview"}) +_WEB_SEARCH_TOOL_KEYS: Final = frozenset({"googleSearch", "google_search"}) def _deployment_supports_web_search(deployment: dict) -> bool: @@ -433,11 +433,18 @@ def _deployment_supports_web_search(deployment: dict) -> bool: def is_web_search_request(request_kwargs: Mapping[str, object]) -> bool: + """OpenAI ``web_search`` / ``web_search_preview``, Anthropic ``web_search_``, Gemini + ``googleSearch`` / ``google_search``.""" tools: Final[object] = request_kwargs.get("tools") return isinstance(tools, (list, tuple)) and any( isinstance(tool, dict) - and isinstance(tool_type := tool.get("type"), str) - and tool_type in _WEB_SEARCH_TOOL_TYPES + and ( + any(key in tool for key in _WEB_SEARCH_TOOL_KEYS) + or ( + isinstance(tool_type := tool.get("type"), str) + and (tool_type == "web_search" or tool_type.startswith("web_search_")) + ) + ) for tool in tools ) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 52a5026c44f..e56dd83a277 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -706,7 +706,7 @@ def _data_uri_media_scope(model: str, media: Sequence[Mapping[str, object]]) -> def split_embedding_input_tokens( value: _EmbeddingInputT, media_models: Callable[[], Sequence[str]], fallback_model: str | None = None ) -> tuple[_EmbeddingInputT, int]: - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( embedding_input_is_all_media, split_embedding_media_for_tpm, ) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index ac4addb0847..a6f6ec3d598 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -37,18 +37,9 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( DEFS_MAX_INLINED_BYTES, invalid_type_message, - is_object_schema, is_present_non_string, is_unhashable_value_behind, sanitize_input_schema_for_anthropic, - schema_fragment_ref, - schema_grammar_branch_count, - schema_has_lookaround_pattern, - schema_has_open_object, - schema_has_unresolvable_ref, - schema_optional_property_count, - schema_string_formats, - schema_union_property_count, unpack_defs, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( @@ -59,6 +50,17 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( inline_remote_image_urls, is_unformatted_pdf_link_image, ) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + is_object_schema, + schema_fragment_ref, + schema_grammar_branch_count, + schema_has_lookaround_pattern, + schema_has_open_object, + schema_has_unresolvable_ref, + schema_optional_property_count, + schema_string_formats, + schema_union_property_count, +) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( message_field, parts_of, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index fd36ce9e43f..8211f404041 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -29,17 +29,19 @@ from litellm.constants import ( ) from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import as_str_mapping, lax_bool -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_file_ids_from_messages, - is_encrypted_reasoning_block, - native_redacted_thinking_data, -) from litellm.litellm_core_utils.prompt_templates.factory import THOUGHT_SIGNATURE_SEPARATOR +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_file_ids_from_messages, +) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( find_anthropic_server_tool_result, message_field, parts_of, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + is_encrypted_reasoning_block, + native_redacted_thinking_data, +) from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.wif import ( diff --git a/litellm/llms/anthropic/files/transformation.py b/litellm/llms/anthropic/files/transformation.py index db14d3366ff..98548a58481 100644 --- a/litellm/llms/anthropic/files/transformation.py +++ b/litellm/llms/anthropic/files/transformation.py @@ -21,7 +21,9 @@ import httpx from openai.types.file_deleted import FileDeleted from pydantic import ConfigDict, TypeAdapter -from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import ( diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index afcd94c5b2c..5379dcd6d68 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -123,15 +123,23 @@ def create_tool_name_mapping( from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingChoice from litellm.litellm_core_utils.prompt_templates.common_utils import ( - anthropic_image_source_to_openai_url, anthropic_inline_document, - parse_tool_call_arguments, - reasoning_content_from_thinking_blocks, - with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, ) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + with_prompt_cache_breakpoint, +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + reasoning_content_from_thinking_blocks, +) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + parse_tool_call_arguments, +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + anthropic_image_source_to_openai_url, +) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, reasoning_effort_from_thinking_silently, diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index ec09987837c..85f042f621c 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -20,7 +20,7 @@ from litellm.litellm_core_utils.asyncify import ( run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_with_readable_mode, with_prompt_cache_breakpoint, diff --git a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index 5d82b4f3d96..20dd13a329e 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -13,7 +13,7 @@ from litellm._logging import redact_internal_details_from_client_message from litellm._uuid import uuid from litellm.exceptions import APIError, MidStreamFallbackError from litellm.litellm_core_utils.core_helpers import has_attributes, iter_items -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.messages.native.utils import ( diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 501b74802b5..2210d904440 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -16,9 +16,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, anthropic_inline_document, +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + with_prompt_cache_breakpoint, +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, responses_reasoning_items_from_thinking_blocks, - with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 0fe7caa1e16..ca036fa4bdf 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -14,17 +14,21 @@ from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_ from litellm.litellm_core_utils.allowed_tools_tool_choice import is_allowed_tools_tool_choice from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( - chat_completion_prompt_cache_breakpoints_for_wire, drop_tool_reference_parts_from_tool_messages, - flatten_combinators_and_drop_non_python_regex_patterns, hoist_images_from_tool_messages, system_messages_first, - tool_with_sanitized_parameters, - without_prompt_cache_breakpoints, ) from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, ) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + chat_completion_prompt_cache_breakpoints_for_wire, + without_prompt_cache_breakpoints, +) from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index f87beb056fd..6a0740b2f15 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -11,7 +11,9 @@ from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION -from litellm.litellm_core_utils.prompt_templates.common_utils import prompt_cache_breakpoint_within +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + prompt_cache_breakpoint_within, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.common_utils import BaseAzureLLM diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 487958d6d55..93ff6d8170a 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -21,6 +21,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_tool_reference_parts_from_tool_messages, filter_value_from_dict, is_pdf_file_part, +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 03fbc35a97f..b93d40738ea 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -38,25 +38,27 @@ from litellm.litellm_core_utils.core_helpers import ( safe_deep_copy, ) from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - is_object_schema, - native_redacted_thinking_data, - parse_content_for_reasoning, - tool_with_sanitized_parameters, -) from litellm.litellm_core_utils.prompt_templates.factory import ( BedrockConverseMessagesProcessor, _bedrock_converse_messages_pt, _bedrock_tools_pt, make_valid_bedrock_tool_name, ) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + is_object_schema, + tool_with_sanitized_parameters, +) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( CONVERTED_SYSTEM_NOTE, is_system_message, message_field, parts_of, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + native_redacted_thinking_data, + parse_content_for_reasoning, +) from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.anthropic.chat.transformation import ( @@ -77,6 +79,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_model_supports_regex_lookaround, bedrock_native_output_format, bedrock_native_output_is_opt_in, + bedrock_native_output_retry_requested, bedrock_supports_tool_search, deployment_model_info, requests_output_config_format, @@ -1425,6 +1428,16 @@ class AmazonConverseConfig(BaseConfig): deployment_model_info(litellm_params), ) + def uses_native_output( + self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] + ) -> bool: + return bedrock_native_output_retry_requested( + bedrock_capability_model(model, litellm_params.get("base_model")), + optional_params, + self.custom_llm_provider or "bedrock_converse", + deployment_model_info(litellm_params), + ) + def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict): """ Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget. diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index c6b3e7dd677..57cc39a03f4 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -16,7 +16,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers from litellm.litellm_core_utils.optional_imports import ensure_optional_import -from litellm.litellm_core_utils.prompt_templates.common_utils import native_redacted_thinking_data +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + native_redacted_thinking_data, +) from litellm.llms.anthropic.chat.handler import ( ModelResponseIterator as AnthropicModelResponseIterator, ) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py index 16dde81ca47..eac9843331c 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Final, cast from httpx import Response from litellm import verbose_logger -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( parse_content_for_reasoning, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index 62e1bee5217..1c38642e062 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -34,6 +34,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_capability_model, bedrock_chat_uses_native_structured_output, bedrock_cleaned_output_schema, + bedrock_native_output_retry_requested, bedrock_supports_tool_search, deployment_model_info, get_anthropic_beta_from_headers, @@ -161,6 +162,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): ) -> bool: return requests_opt_in_native_output(model, optional_params, deployment_model_info(litellm_params)) + def uses_native_output( + self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] + ) -> bool: + return bedrock_native_output_retry_requested( + model, optional_params, model_info=deployment_model_info(litellm_params) + ) + def map_openai_params( self, non_default_params: dict, diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py index 8069c3cf7e7..79742d0810c 100644 --- a/litellm/llms/bedrock/chat/native_output_fallback.py +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -1,4 +1,5 @@ from collections.abc import Callable, Coroutine +from dataclasses import dataclass from functools import partial from typing import Final, TypeAlias @@ -6,6 +7,7 @@ from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.common_utils import ( NativeOutputFallback, + bedrock_strict_tools_fallback, is_bedrock_native_output_rejection, warn_bedrock_native_output_rejected, warn_bedrock_strict_tools_dropped, @@ -17,7 +19,15 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse] NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult] NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None] -_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None] + + +@dataclass(frozen=True, slots=True) +class _SelectedFallback: + fallback: NativeOutputFallback + native_output_retry: bool + + +_RetryFor: TypeAlias = Callable[[BaseLLMException], _SelectedFallback | None] def _native_output_retry( @@ -47,17 +57,42 @@ def _fallback_for( fallback: NativeOutputFallbackFactory | None, strict_tools_fallback: NativeOutputFallbackFactory | None, model: str, -) -> NativeOutputFallback | None: +) -> _SelectedFallback | None: native_output_retry: Final = _native_output_retry(error, fallback, model) if native_output_retry is not None: - return native_output_retry - return _strict_tools_retry(error, strict_tools_fallback, model) + return _SelectedFallback(native_output_retry, native_output_retry=True) + strict_tools_retry: Final = _strict_tools_retry(error, strict_tools_fallback, model) + if strict_tools_retry is None: + return None + return _SelectedFallback(strict_tools_retry, native_output_retry=False) + + +def _strict_tools_retry_after_native_output_retry( + error: BaseLLMException, + retry: _SelectedFallback, + strict_tools_fallback: NativeOutputFallbackFactory | None, + model: str, +) -> NativeOutputFallback | None: + if ( + not retry.native_output_retry + or strict_tools_fallback is None + or not is_claude_strict_tools_rejection(error.status_code, error.message) + ): + return None + composed_retry: Final = bedrock_strict_tools_fallback( + model, retry.fallback.optional_params, retry.fallback.litellm_params + ) + if composed_retry is not None: + warn_bedrock_strict_tools_dropped(model) + return composed_retry async def _await_with_native_output_fallback( first_attempt: Coroutine[object, object, NativeOutputResponse], attempt: NativeOutputAttempt, retry_for: _RetryFor, + strict_tools_fallback: NativeOutputFallbackFactory | None, + model: str, ) -> NativeOutputResponse: try: return await first_attempt @@ -65,8 +100,17 @@ async def _await_with_native_output_fallback( retry: Final = retry_for(error) if retry is None: raise - second_attempt: Final = attempt(retry) - return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt + try: + second_attempt: Final = attempt(retry.fallback) + return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt + except BaseLLMException as error: + composed_retry: Final = _strict_tools_retry_after_native_output_retry( + error, retry, strict_tools_fallback, model + ) + if composed_retry is None: + raise + third_attempt: Final = attempt(composed_retry) + return await third_attempt if isinstance(third_attempt, Coroutine) else third_attempt def call_with_native_output_fallback( @@ -86,7 +130,15 @@ def call_with_native_output_fallback( retry: Final = retry_for(error) if retry is None: raise - return attempt(retry) + try: + return attempt(retry.fallback) + except BaseLLMException as retry_error: + composed_retry: Final = _strict_tools_retry_after_native_output_retry( + retry_error, retry, strict_tools_fallback, model + ) + if composed_retry is None: + raise + return attempt(composed_retry) if isinstance(first_attempt, Coroutine): - return _await_with_native_output_fallback(first_attempt, attempt, retry_for) + return _await_with_native_output_fallback(first_attempt, attempt, retry_for, strict_tools_fallback, model) return first_attempt diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 8e80c61413e..af7db118409 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -36,7 +36,9 @@ from litellm import verbose_logger from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.core_helpers import decoded_json_object, lax_bool, require_attributes from litellm.litellm_core_utils.optional_imports import ensure_optional_import -from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_property_count, +) from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.base_llm.anthropic_messages.transformation import ( @@ -293,6 +295,15 @@ def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "be return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key=key) +def _bedrock_native_output_enabled( + model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None +) -> bool: + deployment_flag: Final = (model_info or {}).get("supports_native_structured_output") + if isinstance(deployment_flag, bool): + return deployment_flag + return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) + + _DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object]) @@ -313,9 +324,9 @@ def _bedrock_native_output_opt_in( def bedrock_supports_native_structured_output( model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None ) -> bool: - return _bedrock_model_supports( - model, "supports_native_structured_output", custom_llm_provider - ) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) + return _bedrock_native_output_enabled(model, custom_llm_provider, model_info) or _bedrock_native_output_opt_in( + model, custom_llm_provider, model_info + ) def _bedrock_map_native_output_flag(model: str) -> object: @@ -350,7 +361,7 @@ def bedrock_chat_opts_into_native_structured_output( custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None, ) -> bool: - return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) or ( + return _bedrock_native_output_enabled(model, custom_llm_provider, model_info) or ( thinking_requested and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) ) @@ -371,15 +382,15 @@ def bedrock_chat_uses_native_structured_output( def bedrock_native_output_is_deployment_opt_in( model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None ) -> bool: - return not _bedrock_model_supports( - model, "supports_native_structured_output", custom_llm_provider - ) and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) + return not _bedrock_native_output_enabled(model, custom_llm_provider, model_info) and _bedrock_native_output_opt_in( + model, custom_llm_provider, model_info + ) def bedrock_native_output_is_opt_in( model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None ) -> bool: - return not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) and ( + return not _bedrock_native_output_enabled(model, custom_llm_provider, model_info) and ( _bedrock_native_output_opt_in(model, custom_llm_provider, model_info) or _bedrock_native_output_replaces_unforceable_tool(model, model_info) ) @@ -388,7 +399,21 @@ def bedrock_native_output_is_opt_in( def without_bedrock_native_output_opt_in(litellm_params: Mapping[str, object]) -> Mapping[str, object]: return { **litellm_params, - "model_info": {**deployment_model_info(litellm_params), "supports_bedrock_native_structured_output": False}, + "model_info": { + **deployment_model_info(litellm_params), + "supports_bedrock_native_structured_output": False, + }, + } + + +def for_bedrock_json_tool_retry(litellm_params: Mapping[str, object]) -> Mapping[str, object]: + opt_in_disabled: Final = without_bedrock_native_output_opt_in(litellm_params) + return { + **opt_in_disabled, + "model_info": { + **deployment_model_info(opt_in_disabled), + "supports_native_structured_output": False, + }, } @@ -396,6 +421,7 @@ _BEDROCK_NATIVE_OUTPUT_UNSUPPORTED: Final = "output_config.format: Extra inputs _BEDROCK_NATIVE_OUTPUT_REJECTIONS: Final = ( _BEDROCK_NATIVE_OUTPUT_UNSUPPORTED, "compilation exceeded limits: Schema is too complex", + "The compiled grammar is too large", "output_config.format.schema:", ) @@ -476,8 +502,27 @@ def requests_output_config_format(optional_params: Mapping[str, object]) -> bool def requests_native_output(anthropic_optional_params: Mapping[str, object]) -> bool: - return isinstance(anthropic_optional_params.get("output_format"), Mapping) or requests_output_config_format( - anthropic_optional_params + output_config: Final = json_object_or_none(anthropic_optional_params.get("outputConfig")) + return ( + isinstance(anthropic_optional_params.get("output_format"), Mapping) + or requests_output_config_format(anthropic_optional_params) + or ( + anthropic_optional_params.get("json_mode") is True + and output_config is not None + and isinstance(output_config.get("textFormat"), Mapping) + ) + ) + + +def bedrock_native_output_retry_requested( + model: str, + optional_params: Mapping[str, object], + custom_llm_provider: str = "bedrock", + model_info: Mapping[str, object] | None = None, +) -> bool: + return requests_native_output(optional_params) and ( + bedrock_native_output_is_opt_in(model, custom_llm_provider, model_info) + or _bedrock_native_output_enabled(model, custom_llm_provider, model_info) ) @@ -519,7 +564,7 @@ def bedrock_native_output_schema( model_info: Mapping[str, object] | None = None, ) -> Mapping[str, object] | None: native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider) - if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider): + if _bedrock_native_output_enabled(model, custom_llm_provider, model_info): return schema if native_schema is None else native_schema if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info): return native_schema diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 91035ec8f44..4d8eae72144 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -41,9 +41,11 @@ from litellm.litellm_core_utils.cloud_storage_security import ( ) from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, text_completion_prompt_to_messages, ) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -237,15 +239,6 @@ class _S3RequestTarget: request_params: _BedrockS3RequestParams -class _TrustedS3ModelCredentials(LiteLLMBaseModel): - """The S3 buckets the server trusts file ids against, from the deployment snapshot.""" - - model_config = ConfigDict(extra="ignore") - - s3_bucket_name: str | None = None - s3_output_bucket_name: str | None = None - - def extract_s3_uri_from_file_id(file_id: str) -> str: """ Resolve a Bedrock file id to its S3 URI. @@ -269,27 +262,31 @@ def extract_s3_uri_from_file_id(file_id: str) -> str: raise ValueError("file_id must be a managed LiteLLM S3 file id") -_S3_BUCKET_REQUIRED_ERROR: Final = "S3 bucket_name is required. Set 's3_bucket_name' in proxy config or AWS_S3_BUCKET_NAME for Bedrock file content retrieval." +_S3_BUCKET_REQUIRED_ERROR: Final = ( + "S3 bucket_name is required. Pass 's3_bucket_name', set it in proxy config, " + "or set AWS_S3_BUCKET_NAME for Bedrock file content retrieval." +) -def _trusted_s3_model_credentials(litellm_params: Mapping[str, object]) -> _TrustedS3ModelCredentials: - trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials") - if not isinstance(trusted_model_credentials, MappingProxyType): - return _TrustedS3ModelCredentials() - snapshot: Final[dict[str, object]] = {} - snapshot.update(trusted_model_credentials) # any-ok: untyped snapshot - return _TrustedS3ModelCredentials.model_validate(snapshot) +def _s3_param(litellm_params: Mapping[str, object], key: str, env_var: str) -> str | None: + value: Final = litellm_params.get(key) + return value if isinstance(value, str) and value else os.getenv(env_var) + + +def _input_bucket_name(litellm_params: Mapping[str, object]) -> str | None: + return _s3_param(litellm_params, "s3_bucket_name", "AWS_S3_BUCKET_NAME") + + +def _output_bucket_name(litellm_params: Mapping[str, object]) -> str | None: + return _s3_param(litellm_params, "s3_output_bucket_name", "AWS_S3_OUTPUT_BUCKET_NAME") def get_configured_s3_bucket_name(litellm_params: Mapping[str, object]) -> str: """ - Resolve the server-configured S3 bucket for Bedrock file operations. - - Only trusts the immutable server-side credential snapshot or the - environment; never a request-supplied param, since the bucket is what - `validate_managed_cloud_file_id` checks file ids against. + Resolve the S3 bucket for Bedrock file operations: the call's ``s3_bucket_name``, + else ``AWS_S3_BUCKET_NAME``. File ids are validated against this bucket. """ - bucket_name: Final = _trusted_s3_model_credentials(litellm_params).s3_bucket_name or os.getenv("AWS_S3_BUCKET_NAME") + bucket_name: Final = _input_bucket_name(litellm_params) if not bucket_name: raise ValueError(_S3_BUCKET_REQUIRED_ERROR) return bucket_name @@ -297,16 +294,13 @@ def get_configured_s3_bucket_name(litellm_params: Mapping[str, object]) -> str: def get_configured_s3_bucket_names(litellm_params: Mapping[str, object]) -> tuple[str, ...]: """ - Resolve the server-configured S3 buckets a Bedrock file id may live in. + Resolve the S3 buckets a Bedrock file id may live in. Bedrock batch outputs land in ``s3_output_bucket_name`` when it differs from - the input bucket, so retrieval validates against both. Same trust rules as - ``get_configured_s3_bucket_name``: only the immutable credential snapshot or - the environment, never a request param. + the input bucket, so retrieval validates against both. """ - trusted: Final = _trusted_s3_model_credentials(litellm_params) - input_bucket: Final = trusted.s3_bucket_name or os.getenv("AWS_S3_BUCKET_NAME") - output_bucket: Final = trusted.s3_output_bucket_name or os.getenv("AWS_S3_OUTPUT_BUCKET_NAME") + input_bucket: Final = _input_bucket_name(litellm_params) + output_bucket: Final = _output_bucket_name(litellm_params) buckets: Final = tuple(dict.fromkeys(bucket for bucket in (input_bucket, output_bucket) if bucket)) if not buckets: raise ValueError(_S3_BUCKET_REQUIRED_ERROR) @@ -522,12 +516,7 @@ def _output_location_still_unlisted(litellm_params: Mapping[str, object]) -> boo def _listing_bucket_name(litellm_params: Mapping[str, object], purpose: str | None) -> str: if purpose != "batch_output": return get_configured_s3_bucket_name(litellm_params) - trusted: Final = _trusted_s3_model_credentials(litellm_params) - return ( - trusted.s3_output_bucket_name - or os.getenv("AWS_S3_OUTPUT_BUCKET_NAME") - or get_configured_s3_bucket_name(litellm_params) - ) + return _output_bucket_name(litellm_params) or get_configured_s3_bucket_name(litellm_params) def _listed_object_created_at(entry: ET.Element) -> int: diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index 8b4e2e5b5c5..51f37364245 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -40,15 +40,17 @@ from litellm.llms.bedrock.common_utils import ( BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS, BedrockError, apply_bedrock_invoke_structured_output, - bedrock_native_output_is_deployment_opt_in, + bedrock_native_output_format, + bedrock_native_output_retry_requested, bedrock_supports_tool_search, ensure_bedrock_anthropic_messages_tool_names, + for_bedrock_json_tool_retry, get_anthropic_beta_from_headers, is_bedrock_native_output_rejection, + json_object_or_none, normalize_bedrock_opus_output_config_effort, normalize_custom_field_on_tools, normalize_tool_input_schema_types_for_bedrock_invoke, - requests_native_output, requests_output_config_format, strip_unsupported_bedrock_invoke_output_config_keys, supports_bedrock_prompt_cache_ttl, @@ -133,13 +135,24 @@ class AmazonAnthropicClaudeMessagesConfig( ) -> GenericLiteLLMParams | None: model_info: Final = _MODEL_INFO.validate_python(litellm_params.model_info) if not ( - requests_native_output(anthropic_messages_optional_request_params) - and bedrock_native_output_is_deployment_opt_in(model, model_info=model_info) + bedrock_native_output_retry_requested( + model, anthropic_messages_optional_request_params, model_info=model_info + ) ): return None - return litellm_params.model_copy( - update={"model_info": {**(model_info or {}), "supports_bedrock_native_structured_output": False}} + legacy_output_format: Final = json_object_or_none( + anthropic_messages_optional_request_params.get("output_format") ) + output_config: Final = json_object_or_none(anthropic_messages_optional_request_params.get("output_config")) + output_config_format: Final = ( + json_object_or_none(output_config.get("format")) if output_config is not None else None + ) + output_format: Final = legacy_output_format if legacy_output_format is not None else output_config_format + if output_format is None or (bedrock_native_output_format(model, output_format, model_info=model_info) is None): + return None + retry_params: Final = for_bedrock_json_tool_retry({"model_info": model_info}) + retry_model_info: Final = _MODEL_INFO.validate_python(retry_params.get("model_info")) + return litellm_params.model_copy(update={"model_info": retry_model_info}) def should_retry_anthropic_messages_without_native_output( self, e: httpx.HTTPStatusError, request_body: Mapping[str, object], model: str diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index 1f7b1712d68..7190a4f28c6 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -24,7 +24,9 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.litellm_core_utils.prompt_templates.common_utils import chat_completion_prompt_cache_breakpoint_for_wire +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + chat_completion_prompt_cache_breakpoint_for_wire, +) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items diff --git a/litellm/llms/custom_httpx/container_handler.py b/litellm/llms/custom_httpx/container_handler.py index a9b96217379..e89bceb043a 100644 --- a/litellm/llms/custom_httpx/container_handler.py +++ b/litellm/llms/custom_httpx/container_handler.py @@ -210,7 +210,7 @@ def _prepare_multipart_file_upload( Returns: Tuple of (files_dict, headers_without_content_type) """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 859211bd40d..a173cdb50f2 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -19,12 +19,14 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo should_convert_tool_call_to_json_mode, ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_reasoning_content, merge_consecutive_system_messages, strip_litellm_internal_message_fields, strip_name_from_message, strip_prefill_prefix_field, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + extract_reasoning_content, +) from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index 1c8a056116f..2c8b22459bd 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -158,7 +158,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming Deepgram response: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming Deepgram response: {e}") from None def _reconstruct_diarized_transcript(self, words: list) -> str: """ diff --git a/litellm/llms/elevenlabs/audio_transcription/transformation.py b/litellm/llms/elevenlabs/audio_transcription/transformation.py index 3dc42f35d3c..b09aa97f6fe 100644 --- a/litellm/llms/elevenlabs/audio_transcription/transformation.py +++ b/litellm/llms/elevenlabs/audio_transcription/transformation.py @@ -160,7 +160,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming ElevenLabs response: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming ElevenLabs response: {e}") from None def get_complete_url( self, diff --git a/litellm/llms/gemini/audio_transcription/transformation.py b/litellm/llms/gemini/audio_transcription/transformation.py index 3457b22faa4..ca4953445ee 100644 --- a/litellm/llms/gemini/audio_transcription/transformation.py +++ b/litellm/llms/gemini/audio_transcription/transformation.py @@ -138,13 +138,13 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): except ValueError: raise GeminiError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Gemini Interactions API: {raw_response.text}", + message="Received non-JSON response from Gemini Interactions API", ) parsed: Final = GeminiTranscriptionInteractionResponse.model_validate(response_json) if parsed.status != "completed": raise GeminiError( status_code=raw_response.status_code, - message=f"Gemini transcription interaction did not complete (status={parsed.status}): {raw_response.text}", + message=f"Gemini transcription interaction did not complete (status={parsed.status})", ) text_contents: Final = tuple( content diff --git a/litellm/llms/gemini/files/transformation.py b/litellm/llms/gemini/files/transformation.py index f9a7e230ea9..17460c0295a 100644 --- a/litellm/llms/gemini/files/transformation.py +++ b/litellm/llms/gemini/files/transformation.py @@ -14,7 +14,9 @@ from openai.types.file_deleted import FileDeleted from typing_extensions import ReadOnly, Required from litellm._logging import verbose_logger -from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.files.transformation import ( BaseFilesConfig, diff --git a/litellm/llms/gemini/interactions/transformation.py b/litellm/llms/gemini/interactions/transformation.py index 344ab56277c..a02afa3a13f 100644 --- a/litellm/llms/gemini/interactions/transformation.py +++ b/litellm/llms/gemini/interactions/transformation.py @@ -139,6 +139,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): "response_format", "response_mime_type", "previous_interaction_id", + "continuation_token", ] def validate_environment( @@ -219,6 +220,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): "environment", "response_modalities", "previous_interaction_id", + "continuation_token", ] dropped_keys: Final = ( frozenset({"agent_config"}) @@ -314,7 +316,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): ) -> InteractionsAPIStreamingResponse: """Parse streaming chunk.""" verbose_logger.debug("Google AI Interactions streaming chunk: %s", parsed_chunk) - return InteractionsAPIStreamingResponse(**parsed_chunk) + return InteractionsAPIStreamingResponse.model_validate(parsed_chunk) # GET / DELETE / CANCEL - just build URLs, responses match spec directly diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 3386cd452da..c5a0040eb80 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -7,10 +7,10 @@ import json from collections.abc import Coroutine from typing import Final, Literal, cast, overload -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( get_image_mime_type_from_url, ) -from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, ) diff --git a/litellm/llms/manus/files/transformation.py b/litellm/llms/manus/files/transformation.py index 991aa4cbc32..2db9f827a64 100644 --- a/litellm/llms/manus/files/transformation.py +++ b/litellm/llms/manus/files/transformation.py @@ -20,7 +20,9 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import ( diff --git a/litellm/llms/mistral/audio_transcription/transformation.py b/litellm/llms/mistral/audio_transcription/transformation.py index 3eb9e122265..34200effb18 100644 --- a/litellm/llms/mistral/audio_transcription/transformation.py +++ b/litellm/llms/mistral/audio_transcription/transformation.py @@ -133,7 +133,7 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise MistralAudioTranscriptionException( - message=raw_response.text, + message="Received non-JSON response from Mistral audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/mistral/files/transformation.py b/litellm/llms/mistral/files/transformation.py index 908567e88fb..1924e80677f 100644 --- a/litellm/llms/mistral/files/transformation.py +++ b/litellm/llms/mistral/files/transformation.py @@ -17,7 +17,9 @@ from openai.types.file_deleted import FileDeleted from pydantic import ConfigDict from typing_extensions import ReadOnly, TypedDict -from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 59020b058a2..c6f3a7a8280 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -12,6 +12,8 @@ from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_images_from_message, +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( extract_reasoning_content, ) from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE @@ -353,7 +355,7 @@ class OllamaChatConfig(BaseConfig): del response_json_message["thinking"] elif response_json_message.get("content") is not None: # parse reasoning content from content - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( parse_content_for_reasoning, ) diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 6dbcb050e11..565444f3f48 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -75,7 +75,7 @@ class _OllamaGenerateReasoning(LiteLLMBaseModel): def split(self) -> tuple[str | None, str | None]: """Reasoning reaches `/api/generate` either in the top-level `thinking` field or inline in `` tags, never both. The field wins, matching `ollama_chat`.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( parse_content_for_reasoning, ) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 0479ac5c36c..044c0131699 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -21,20 +21,24 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo should_convert_tool_call_to_json_mode, ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( - chat_completion_prompt_cache_breakpoints_for_wire, - drop_non_python_regex_patterns, drop_tool_reference_parts_from_tool_messages, - flatten_combinators_and_drop_non_python_regex_patterns, get_tool_call_names, hoist_images_from_tool_messages, system_messages_first, - tool_with_sanitized_parameters, - without_prompt_cache_breakpoints, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, convert_url_to_base64, ) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + chat_completion_prompt_cache_breakpoints_for_wire, + without_prompt_cache_breakpoints, +) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.base_utils import ( diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 3cb798b7318..3d0abb1a375 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -21,10 +21,12 @@ from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( safe_convert_created_field, ) -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - cache_control_as_prompt_cache_breakpoint, +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( drop_non_python_regex_patterns, flatten_combinators_and_drop_non_python_regex_patterns, +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_for_wire, whole_prompt_cache_breakpoint_or_none, with_prompt_cache_breakpoint, diff --git a/litellm/llms/openai/transcriptions/whisper_transformation.py b/litellm/llms/openai/transcriptions/whisper_transformation.py index eb4f1b6c9ce..947d6175122 100644 --- a/litellm/llms/openai/transcriptions/whisper_transformation.py +++ b/litellm/llms/openai/transcriptions/whisper_transformation.py @@ -141,7 +141,4 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): if any(key in raw_response_json for key in TranscriptionResponse.model_fields): return TranscriptionResponse(**raw_response_json) else: - raise ValueError( - "Invalid response format. Received response does not match the expected format. Got: ", - raw_response_json, - ) + raise ValueError("Invalid response format. Received response does not match the expected format.") diff --git a/litellm/llms/ovhcloud/audio_transcription/transformation.py b/litellm/llms/ovhcloud/audio_transcription/transformation.py index dd61bfb0c13..50bed1fef78 100644 --- a/litellm/llms/ovhcloud/audio_transcription/transformation.py +++ b/litellm/llms/ovhcloud/audio_transcription/transformation.py @@ -144,7 +144,7 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise OVHCloudException( - message=raw_response.text, + message="Received non-JSON response from OVHCloud audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/scaleway/audio_transcription/transformation.py b/litellm/llms/scaleway/audio_transcription/transformation.py index 8913cc362b3..2cc3ac17073 100644 --- a/litellm/llms/scaleway/audio_transcription/transformation.py +++ b/litellm/llms/scaleway/audio_transcription/transformation.py @@ -134,7 +134,7 @@ class ScalewayAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise ScalewayAudioTranscriptionException( - message=raw_response.text, + message="Received non-JSON response from Scaleway audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py index 26de755d5ca..09a09ef96dc 100644 --- a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py @@ -183,7 +183,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB except ValueError: raise VertexAIError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Vertex AI Gemini transcription: {raw_response.text}", + message="Received non-JSON response from Vertex AI Gemini transcription", ) parsed: Final = VertexGeminiTranscriptionResponse.model_validate(response_json) texts: Final = tuple( diff --git a/litellm/llms/vertex_ai/audio_transcription/transformation.py b/litellm/llms/vertex_ai/audio_transcription/transformation.py index 4fdf93a6307..017d7ff7b2f 100644 --- a/litellm/llms/vertex_ai/audio_transcription/transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/transformation.py @@ -172,7 +172,7 @@ class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase) except ValueError: raise VertexAIError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Google Speech-to-Text: {raw_response.text}", + message="Received non-JSON response from Google Speech-to-Text", ) parsed: Final = VertexSpeechToTextRecognizeResponse.model_validate(response_json) transcripts: Final = tuple( diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index fb952d805f0..1f03de4625c 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -35,7 +35,7 @@ from litellm.litellm_core_utils.cloud_storage_security import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, extract_file_metadata, ) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 3c32e6d2355..6e801ed5756 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -20,9 +20,6 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, has_attributes from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_image_mime_type_from_url, -) from litellm.litellm_core_utils.prompt_templates.factory import ( GeminiMediaSource, convert_to_anthropic_image_obj, @@ -33,6 +30,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( get_thought_signature_from_tool, response_schema_prompt, ) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_image_mime_type_from_url, +) from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, async_inline_remote_media from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels diff --git a/litellm/llms/watsonx/audio_transcription/transformation.py b/litellm/llms/watsonx/audio_transcription/transformation.py index 35fda05c028..b160ee65802 100644 --- a/litellm/llms/watsonx/audio_transcription/transformation.py +++ b/litellm/llms/watsonx/audio_transcription/transformation.py @@ -175,7 +175,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran try: raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: - raise ValueError(f"Error transforming response to json: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming response to json: {e}") # Extract only valid fields for TranscriptionResponse.__init__() # TranscriptionResponse only accepts 'text' and 'usage' in __init__() @@ -191,10 +191,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran response_kwargs["usage"] = usage if not response_kwargs: - raise ValueError( - "Invalid response format. Received response does not match the expected format. Got: ", - response_object, - ) + raise ValueError("Invalid response format. Received response does not match the expected format.") response: Final = TranscriptionResponse(**response_kwargs) diff --git a/litellm/llms/xai/audio_transcription/transformation.py b/litellm/llms/xai/audio_transcription/transformation.py index e29c3d9334a..c212dce26fe 100644 --- a/litellm/llms/xai/audio_transcription/transformation.py +++ b/litellm/llms/xai/audio_transcription/transformation.py @@ -39,7 +39,7 @@ class _XAISttWord(LiteLLMBaseModel): class _XAISttResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", hide_input_in_errors=True) text: str = "" language: str = "unknown" duration: float | None = None diff --git a/litellm/llms/xai/files/transformation.py b/litellm/llms/xai/files/transformation.py index 88f2fc51983..a4260ec7ff2 100644 --- a/litellm/llms/xai/files/transformation.py +++ b/litellm/llms/xai/files/transformation.py @@ -13,7 +13,9 @@ from openai.types.file_deleted import FileDeleted from pydantic import ConfigDict from typing_extensions import ReadOnly, TypedDict -from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj diff --git a/litellm/main.py b/litellm/main.py index 32155b989c2..71f6113aaf2 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -144,7 +144,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_route_for_request, bedrock_strict_tools_fallback, deployment_model_info, - without_bedrock_native_output_opt_in, + for_bedrock_json_tool_retry, without_bedrock_route_prefix, ) from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config @@ -4374,7 +4374,7 @@ def _native_output_fallback( first_attempt_params: Mapping[str, object], first_attempt_litellm_params: Mapping[str, object], ) -> NativeOutputFallback: - litellm_params: Final = without_bedrock_native_output_opt_in(first_attempt_litellm_params) + litellm_params: Final = for_bedrock_json_tool_retry(first_attempt_litellm_params) remapped_params: Final = ctx.optional_params_for_model_info(deployment_model_info(litellm_params)) return NativeOutputFallback( optional_params={ @@ -4505,7 +4505,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu converse_attempt, ( partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params)) - if AmazonConverseConfig().uses_opt_in_native_output(converse_model, converse_params, litellm_params) + if AmazonConverseConfig().uses_native_output(converse_model, converse_params, litellm_params) else None ), converse_model, @@ -4561,7 +4561,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu ( partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) if isinstance(provider_config, AmazonAnthropicClaudeConfig) - and provider_config.uses_opt_in_native_output(model, invoke_params, litellm_params) + and provider_config.uses_native_output(model, invoke_params, litellm_params) else None ), model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1bda9d24c8e..ad57685da04 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2242,7 +2242,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2331,7 +2331,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2420,7 +2420,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2504,7 +2504,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2593,7 +2593,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2715,7 +2715,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2759,7 +2759,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2803,7 +2803,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2847,7 +2847,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2891,7 +2891,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -4046,6 +4046,8 @@ "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, + "deprecation_date": "2027-09-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, @@ -4068,7 +4070,7 @@ "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512, - "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models" + "source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" }, "azure_ai/claude-opus-5": { "deprecation_date": "2027-07-08", @@ -13458,13 +13460,16 @@ "supports_reasoning": true }, "azure_ai/mistral-document-ai-2512": { + "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.003, "mode": "ocr", "supported_endpoints": [ "/v1/ocr" ], - "source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512" + "source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/mistral-ocr-4-0" }, "azure_ai/mistral-ocr-4-0": { "litellm_provider": "azure_ai", @@ -75499,7 +75504,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75539,7 +75544,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -76502,6 +76507,33 @@ "supports_tool_choice": true, "supports_vision": true }, + "us-gov.xai.grok-4.7": { + "supports_regex_lookaround": false, + "supported_endpoints": ["/v1/chat/completions"], + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supports_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-Fast": { "cache_read_input_token_cost": 1.4e-08, "input_cost_per_token": 6e-07, @@ -78470,7 +78502,7 @@ "cache_creation_input_token_cost": 1e-05, "cache_creation_input_token_cost_above_1hr": 1.6e-05, "cache_read_input_token_cost": 4e-07, - "deprecation_date": "2027-10-01", + "deprecation_date": "2027-01-15", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 8e-06, "litellm_provider": "azure_ai", @@ -78480,7 +78512,7 @@ "output_cost_per_token": 4e-05, "prompt_cache_min_tokens": 512.0, "search_context_cost_per_query": {"search_context_size_high":0.02,"search_context_size_low":0.02,"search_context_size_medium":0.02}, - "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models", + "source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -86471,5 +86503,32 @@ "supports_system_messages": false, "supports_video_input": true, "supports_vision": true + }, + "databricks/databricks-kimi-k3-1-pro": { + "cache_creation_input_token_cost": 1.99997e-06, + "cache_read_input_token_cost": 1.9999e-07, + "input_cost_per_token": 1.99997e-06, + "input_dbu_cost_per_token": 2.8571e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "metadata": {"notes":"Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference."}, + "mode": "chat", + "output_cost_per_token": 9.99999e-06, + "output_dbu_cost_per_token": 0.000142857, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 100.0 } } diff --git a/litellm/models/plugin_upload.py b/litellm/models/plugin_upload.py index bd5e85c79cc..95db82dec79 100644 --- a/litellm/models/plugin_upload.py +++ b/litellm/models/plugin_upload.py @@ -41,6 +41,14 @@ class PluginArchiveMetadata: description: str | None = None has_agent_skill_metadata: bool = False updated_at: datetime | None = None + created_at: datetime | None = None + created_by: str | None = None + + +@dataclass(frozen=True, slots=True) +class PluginReview: + approved_by: str + security_scan_passed: bool @dataclass(frozen=True, slots=True) @@ -51,6 +59,7 @@ class PluginPublication: manifest: Mapping[str, object] created_by: str | None content: bytes | None = None + review: PluginReview | None = None @dataclass(frozen=True, slots=True) @@ -66,6 +75,10 @@ class PublishedPlugin: version: str enabled: bool updated_at: datetime + created_at: datetime | None = None + created_by: str | None = None + review: PluginReview | None = None + published_by: str | None = None @dataclass(frozen=True, slots=True) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 7d9e230f8ef..5cb63c98d56 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -7683,6 +7683,17 @@ "PluginListItem": { "description": "Plugin item in list responses.", "properties": { + "approved_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Approved By" + }, "author": { "anyOf": [ { @@ -7715,6 +7726,17 @@ ], "title": "Created At" }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, "description": { "anyOf": [ { @@ -7737,6 +7759,17 @@ ], "title": "Domain" }, + "download_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Download Count" + }, "enabled": { "title": "Enabled", "type": "boolean" @@ -7801,6 +7834,22 @@ ], "title": "Namespace" }, + "published_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published By" + }, + "security_scan_passed": { + "default": false, + "title": "Security Scan Passed", + "type": "boolean" + }, "source": { "additionalProperties": { "type": "string" @@ -59415,6 +59464,14 @@ }, "title": "Skills", "type": "array" + }, + "tags": { + "default": [], + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -59690,6 +59747,17 @@ }, "SkillHubSkill": { "properties": { + "approved_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Approved By" + }, "content_url": { "anyOf": [ { @@ -59701,6 +59769,29 @@ ], "title": "Content Url" }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, "description": { "anyOf": [ { @@ -59732,7 +59823,7 @@ "type": "null" } ], - "description": "Gateway ZIP downloads, included in proxy-admin catalog listings", + "description": "Gateway ZIP downloads, including repeat downloads", "title": "Download Count" }, "download_url": { @@ -59779,6 +59870,22 @@ ], "title": "Namespace" }, + "published_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published By" + }, + "security_scan_passed": { + "default": false, + "title": "Security Scan Passed", + "type": "boolean" + }, "source": { "additionalProperties": { "type": "string" @@ -60434,6 +60541,66 @@ "title": "Limit", "type": "integer" } + }, + { + "in": "query", + "name": "tag", + "required": false, + "schema": { + "default": "", + "title": "Tag", + "type": "string" + } + }, + { + "in": "query", + "name": "security_scan_passed", + "required": false, + "schema": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Security Scan Passed" + } + }, + { + "in": "query", + "name": "sort_by", + "required": false, + "schema": { + "default": "name", + "enum": [ + "name", + "download_count", + "updated_at", + "created_at", + "created_by", + "published_by", + "approved_by", + "security_scan_passed" + ], + "title": "Sort By", + "type": "string" + } + }, + { + "in": "query", + "name": "sort_order", + "required": false, + "schema": { + "default": "asc", + "enum": [ + "asc", + "desc" + ], + "title": "Sort Order", + "type": "string" + } } ], "responses": { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d3ac264518e..b82edab85ab 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2813,6 +2813,14 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): description="How long a blocked source address, or source address and username, stays blocked. Every attempt from a blocked key, right or wrong, is refused with 429 before the password is checked; the block is not extended by refused attempts. Set under `general_settings` in config.yaml. Defaults to 300", ) allowed_routes: list | None = Field(None, description="Proxy API Endpoints you want users to be able to access") + allowed_client_side_s3_buckets: tuple[str, ...] | None = Field( + None, + description="S3 bucket names a client may send as `s3_bucket_name` or `s3_output_bucket_name` in a request body, e.g. on /v1/batches. Any other bucket is rejected unless client-side credentials are allowed", + ) + allowed_client_side_s3_endpoint_urls: tuple[str, ...] | None = Field( + None, + description="Exact `s3_endpoint_url` values a client may send in a request body. Any other value is rejected unless client-side credentials are allowed", + ) reject_clientside_metadata_tags: bool | None = Field( None, description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.", diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index 26cb0a2f3e6..3534dd167de 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -60,6 +60,7 @@ from litellm.proxy.common_utils.claude_code_marketplace_auth import ( ) from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.proxy.deps.repositories import PluginUploadRepositoryDep +from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review from litellm.repositories.table_repositories import ClaudeCodePluginRepository from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.claude_code_endpoints import ( @@ -666,6 +667,7 @@ async def register_plugin( response_model=ListPluginsResponse, ) async def list_plugins( + uploads: PluginUploadRepositoryDep, enabled_only: bool = False, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ) -> ListPluginsResponse: @@ -678,6 +680,17 @@ async def list_plugins( Returns: List of plugins with their metadata. """ + catalog: Final = await list_visible_plugins(enabled_only, user_api_key_dict) + if uploads is None: + raise _error_response(500, CommonProxyErrors.db_not_connected_error.value) + counts: Final = await uploads.download_counts(tuple(plugin.id for plugin in catalog.plugins)) + return ListPluginsResponse( + plugins=[plugin.model_copy(update={"download_count": counts.get(plugin.id, 0)}) for plugin in catalog.plugins], + count=catalog.count, + ) + + +async def list_visible_plugins(enabled_only: bool, user_api_key_dict: UserAPIKeyAuth) -> ListPluginsResponse: try: prisma_client: Final = await get_prisma_client() @@ -693,9 +706,14 @@ async def list_plugins( # Parse manifest to get additional fields manifest = json.loads(p.manifest_json) if p.manifest_json else {} + review = read_plugin_review(MANIFEST_FIELDS.validate_python(manifest)) plugin_list.append( PluginListItem( id=p.id, + created_by=p.created_by, + approved_by=review.approved_by if review else None, + published_by=read_plugin_publisher(MANIFEST_FIELDS.validate_python(manifest)), + security_scan_passed=review.security_scan_passed if review else False, name=p.name, version=p.version, description=p.description, diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 161ff2fdfcd..2b5fc6d877e 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -7,10 +7,10 @@ from functools import lru_cache from itertools import accumulate, chain, repeat from logging import Logger from types import MappingProxyType -from typing import Any, Final, Protocol +from typing import Any, Final, Literal, Protocol, TypeAlias from fastapi import HTTPException, Request, status -from pydantic import PositiveInt, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, PositiveInt, TypeAdapter, ValidationError import litellm from litellm import Router, constants, provider_list @@ -442,6 +442,9 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( # S3; ``sagemaker_base_url`` redirects all SageMaker traffic; # ``deployment_url`` redirects SAP deployments. "s3_endpoint_url", + "s3_bucket_name", + "s3_output_bucket_name", + "allow_legacy_cloud_file_ids", "sagemaker_base_url", "deployment_url", # NVIDIA Riva fields consumed by the audio-transcription handler @@ -479,6 +482,48 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( ) +_ClientSideAllowlistSetting: TypeAlias = Literal[ + "allowed_client_side_s3_buckets", "allowed_client_side_s3_endpoint_urls" +] + +_PROXY_ALLOWLIST_SETTING_BY_PARAM: Final[Mapping[str, _ClientSideAllowlistSetting]] = MappingProxyType( + { + "s3_bucket_name": "allowed_client_side_s3_buckets", + "s3_output_bucket_name": "allowed_client_side_s3_buckets", + "s3_endpoint_url": "allowed_client_side_s3_endpoint_urls", + } +) + + +class _ClientSideStorageAllowlist(BaseModel): + model_config = ConfigDict(frozen=True) + + allowed_client_side_s3_buckets: tuple[str, ...] | None = None + allowed_client_side_s3_endpoint_urls: tuple[str, ...] | None = None + + def admits(self, setting: _ClientSideAllowlistSetting, value: object) -> bool: + match setting: + case "allowed_client_side_s3_buckets": + return value in (self.allowed_client_side_s3_buckets or ()) + case "allowed_client_side_s3_endpoint_urls": + return value in (self.allowed_client_side_s3_endpoint_urls or ()) + + +def _banned_param_error(param: str) -> ValueError: + setting: Final = _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param) + allowlist_hint: Final = ( + "" if setting is None else f" To allow specific values only, list them in `general_settings.{setting}`." + ) + return ValueError( + f"Rejected Request: {param} is not allowed in request body. " + "Clientside passthrough requires explicit admin opt-in via " + "either `general_settings.allow_client_side_credentials = true` " + "(proxy-wide) or `configurable_clientside_auth_params` on the " + "deployment in your proxy config.yaml. " + f"Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997{allowlist_hint}", + ) + + def _check_banned_params( body: Mapping[str, object], general_settings: dict, @@ -508,6 +553,10 @@ def _check_banned_params( # Proxy-wide opt-in: every banned param is permitted, exit # entirely so the rest of the loop doesn't waste work. return + if ( + setting := _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param) + ) is not None and _ClientSideStorageAllowlist.model_validate(general_settings).admits(setting, body[param]): + continue if ( _allow_model_level_clientside_configurable_parameters( model=model, @@ -523,14 +572,7 @@ def _check_banned_params( # ``api_base`` with an unallowed ``langfuse_host`` is still # rejected for the second field. continue - raise ValueError( - f"Rejected Request: {param} is not allowed in request body. " - "Clientside passthrough requires explicit admin opt-in via " - "either `general_settings.allow_client_side_credentials = true` " - "(proxy-wide) or `configurable_clientside_auth_params` on the " - "deployment in your proxy config.yaml. " - "Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997", - ) + raise _banned_param_error(param) _FALLBACK_FIELDS: Final[tuple[str, ...]] = ( diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index bbb78e30a4b..512a9166e31 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -49,7 +49,6 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( # noqa: F401 # BATCH_CREATE_HIDDEN_PARAM, _is_base64_encoded_unified_file_id, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export add_deployment_model_info, - add_internal_model_credentials, apply_team_provider_credentials, authorize_model_for_key, batch_cost_poller_is_active, @@ -763,13 +762,6 @@ async def retrieve_batch( detail={"error": "LLM Router not initialized. Ensure models added to proxy."}, ) - if unified_batch_id: - add_internal_model_credentials( - data=data, - llm_router=llm_router, - model_id=unified_model_id, - ) - response = await llm_router.aretrieve_batch(**data) set_hidden_param(response, "unified_batch_id", unified_batch_id) if unified_batch_id: diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 1f8404fc37c..1a4d7ddeadb 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -23,7 +23,7 @@ def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None: def openssl_enforces_fips() -> bool: - """Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5.""" + """Python's OpenSSL runs with FIPS-only defaults and itself refuses scrypt.""" try: import _hashlib @@ -32,7 +32,7 @@ def openssl_enforces_fips() -> bool: except (ImportError, AttributeError, ValueError): return False try: - _hashlib.new("md5", b"", usedforsecurity=True) + _hashlib.scrypt(b"", salt=b"\0" * 16, n=2, r=1, p=1, dklen=16) except ValueError: return True return False diff --git a/litellm/proxy/discovery_endpoints/skill_hub_endpoints.py b/litellm/proxy/discovery_endpoints/skill_hub_endpoints.py index 93d1ace435d..ffddbd31048 100644 --- a/litellm/proxy/discovery_endpoints/skill_hub_endpoints.py +++ b/litellm/proxy/discovery_endpoints/skill_hub_endpoints.py @@ -3,7 +3,8 @@ import hashlib from collections.abc import Mapping from dataclasses import dataclass, replace from datetime import datetime -from typing import Annotated, Final +from itertools import chain +from typing import Annotated, Final, Literal from urllib.parse import quote import httpx @@ -33,7 +34,7 @@ from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketp enable_plugin, get_plugin_downloads, header_key_caller, - list_plugins, + list_visible_plugins, plugin_archive_source_matches, plugin_archive_url, plugin_downloader, @@ -65,6 +66,7 @@ from litellm.proxy.discovery_endpoints.skill_import import ( parse_skill_source, select_imported_skill, ) +from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review from litellm.types.proxy.claude_code_endpoints import PluginListItem from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import AgentSkillsIndexEntry from litellm.types.proxy.skill_hub import ( @@ -78,6 +80,7 @@ from litellm.types.proxy.skill_hub import ( SkillHubSkill, SkillHubVersion, SkillHubVersions, + SkillSortField, ) router: Final = APIRouter(prefix="/skill-hub/skills", tags=["Skills Hub"]) @@ -121,6 +124,11 @@ def _skill_response(request: Request, plugin: PluginListItem, download_count: in namespace=plugin.namespace, keywords=None if plugin.keywords is None else tuple(plugin.keywords), updated_at=None if plugin.updated_at is None else datetime.fromisoformat(plugin.updated_at), + created_at=None if plugin.created_at is None else datetime.fromisoformat(plugin.created_at), + created_by=plugin.created_by, + approved_by=plugin.approved_by, + published_by=plugin.published_by, + security_scan_passed=plugin.security_scan_passed, visibility="public" if plugin.enabled else "private", source={"source": "archive", "url": _download_url(request, plugin.name)} if local_archive else plugin.source, download_url=_download_url(request, plugin.name) if local_archive else None, @@ -137,28 +145,74 @@ async def list_skills( query: str = "", offset: Annotated[int, Query(ge=0)] = 0, limit: Annotated[int, Query(ge=1, le=100)] = 50, + tag: str = "", + security_scan_passed: bool | None = None, + sort_by: SkillSortField = "name", + sort_order: Literal["asc", "desc"] = "asc", ) -> SkillHubList: """Browse the caller's public and granted private skills.""" - catalog: Final = await list_plugins(enabled_only=False, user_api_key_dict=caller) - needle: Final = query.casefold() + catalog: Final = await list_visible_plugins(enabled_only=False, user_api_key_dict=caller) + needle: Final = query.strip().casefold() + selected_tag: Final = tag.strip().casefold() matched: Final = tuple( plugin for plugin in sorted(catalog.plugins, key=lambda item: item.name) - if needle in " ".join((plugin.name, plugin.description or "", *(plugin.keywords or ()))).casefold() + if needle + in " ".join( + ( + plugin.name, + plugin.description or "", + plugin.domain or "", + plugin.namespace or "", + plugin.created_by or "", + plugin.approved_by or "", + plugin.published_by or "", + *(plugin.keywords or ()), + ) + ).casefold() + and (not selected_tag or selected_tag in {word.casefold() for word in plugin.keywords or ()}) + and (security_scan_passed is None or plugin.security_scan_passed == security_scan_passed) ) - page: Final = matched[offset : offset + limit] counts: Final[Mapping[str, int]] = ( - await _require_store(uploads).download_counts(tuple(plugin.id for plugin in page)) - if is_proxy_admin(caller) + await _require_store(uploads).download_counts(tuple(plugin.id for plugin in matched)) + if sort_by == "download_count" else {} ) + ordered: Final = sorted( + matched, key=lambda plugin: _skill_sort_key(plugin, sort_by, counts), reverse=sort_order == "desc" + ) + page: Final = ordered[offset : offset + limit] + page_counts: Final = ( + await _require_store(uploads).download_counts(tuple(plugin.id for plugin in page)) + if sort_by != "download_count" + else counts + ) return SkillHubList( - skills=tuple(_skill_response(request, plugin, counts.get(plugin.id)) for plugin in page), + skills=tuple(_skill_response(request, plugin, page_counts.get(plugin.id)) for plugin in page), count=len(matched), next_offset=offset + limit if offset + limit < len(matched) else None, + tags=tuple( + sorted(set(chain.from_iterable(plugin.keywords or () for plugin in catalog.plugins)), key=str.casefold) + ), ) +def _skill_sort_key(plugin: PluginListItem, field: SkillSortField, counts: Mapping[str, int]) -> str | float: + if field == "download_count": + return counts.get(plugin.id, 0) + if field == "security_scan_passed": + return int(plugin.security_scan_passed) + if field in {"created_at", "updated_at"}: + value: Final = plugin.created_at if field == "created_at" else plugin.updated_at + return datetime.fromisoformat(value).timestamp() if value else 0 + return { + "name": plugin.name, + "created_by": plugin.created_by or "", + "published_by": plugin.published_by or "", + "approved_by": plugin.approved_by or "", + }[field].casefold() + + @router.get("/{name}", response_model=SkillHubSkill) async def get_skill( request: Request, @@ -176,6 +230,7 @@ async def get_skill( def skill_metadata_response(request: Request, metadata: PluginArchiveMetadata) -> SkillHubSkill: manifest: Final = MANIFEST_FIELDS.validate_json(metadata.manifest_json or "{}") + review: Final = read_plugin_review(manifest) return _skill_response( request, PluginListItem.model_validate( @@ -187,7 +242,11 @@ def skill_metadata_response(request: Request, metadata: PluginArchiveMetadata) - "source": dict(metadata.source), "has_agent_skill_metadata": metadata.has_agent_skill_metadata, "enabled": metadata.enabled, - "created_at": None, + "created_at": metadata.created_at.isoformat() if metadata.created_at is not None else None, + "created_by": metadata.created_by, + "approved_by": review.approved_by if review else None, + "published_by": read_plugin_publisher(manifest), + "security_scan_passed": review.security_scan_passed if review else False, "updated_at": metadata.updated_at.isoformat() if metadata.updated_at is not None else None, "domain": manifest.get("domain"), "namespace": manifest.get("namespace"), @@ -416,7 +475,11 @@ def skill_publication_response( "version": published.version, "enabled": published.enabled, "has_agent_skill_metadata": has_agent_skill_metadata(publication.manifest), - "created_at": None, + "created_at": published.created_at.isoformat() if published.created_at is not None else None, + "created_by": published.created_by, + "approved_by": published.review.approved_by if published.review else None, + "published_by": published.published_by, + "security_scan_passed": published.review.security_scan_passed if published.review else False, "updated_at": published.updated_at.isoformat(), "domain": publication.manifest.get("domain"), "namespace": publication.manifest.get("namespace"), diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 01a8d48e402..ffbaa20c69c 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -58,6 +58,8 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( split_media_for_tpm, ) from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c2fb9deb920..ae37b58360d 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -5483,7 +5483,9 @@ async def generate_key_helper_fn( if query_type == "insert_data": user_row = cast( # cast-ok: table_name="user" is the insert_data branch returning the user row "prisma_models.LiteLLM_UserTable | None", - await prisma_client.insert_data(data=user_data, table_name="user"), + await prisma_client.insert_data( + data=user_data, table_name="user", allow_existing_user=request_type != "user" + ), ) if user_row is None: diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index e48f68a2d0a..66c2e54f5c6 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -1294,7 +1294,7 @@ async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_grou if litellm.default_internal_user_params: default_role = litellm.default_internal_user_params.get("user_role") - new_user_request: Final = NewUserRequest( + new_user_request: Final = SCIMNewUserRequest( user_id=user_id, user_email=user_id, # We don't have email from group membership user_alias=None, @@ -1873,7 +1873,7 @@ async def create_user( admin_group: Final = await _get_scim_admin_group() resolved_role: Final = _resolve_scim_user_role(user.groups or [], admin_group, default_role) - new_user_request: Final = NewUserRequest( + new_user_request: Final = SCIMNewUserRequest( user_id=user_id, user_email=user_data["user_email"], user_alias=user_data["user_alias"], diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index f498dc1b5e1..80df6a95cc2 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -78,7 +78,6 @@ from litellm.proxy._types import ( LitellmUserRoles, Member, NewTeamRequest, - NewUserRequest, NewUserRequestTeam, NewUserResponse, ProxyErrorTypes, @@ -164,6 +163,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import ( MicrosoftGraphAPIUserGroupResponse, MicrosoftServicePrincipalTeam, RoleMappings, + SSONewUserRequest, TeamMappings, ) from litellm.types.proxy.ui_sso import ParsedOpenIDResult @@ -2915,7 +2915,7 @@ async def insert_sso_user( if user_defined_values["user_role"] is None: user_defined_values["user_role"] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY - new_user_request: Final = NewUserRequest( + new_user_request: Final = SSONewUserRequest( user_id=user_defined_values["user_id"], user_email=normalize_email(user_defined_values["user_email"]), user_alias=_get_sso_user_alias(result_openid), @@ -2924,11 +2924,9 @@ async def insert_sso_user( budget_duration=user_defined_values["budget_duration"], sso_user_id=user_defined_values["user_id"], auto_create_key=False, + metadata={"auth_provider": getattr(result_openid, "provider")} if hasattr(result_openid, "provider") else {}, ) - if result_openid and hasattr(result_openid, "provider"): - new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")} - from litellm.proxy.proxy_server import premium_user with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True): diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 437c439b8d2..1ae0d475d8c 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -742,31 +742,6 @@ def apply_team_provider_credentials( prepare_data_with_credentials(data=data, credentials={**credentials, **explicit_request_credentials(data)}) -def add_internal_model_credentials( - data: dict, - llm_router: "Router", - model_id: str | None, -) -> None: - """ - Attach the deployment's immutable server-side credential snapshot to a router-routed - batch call (in-place). - - Cost accounting for a completed batch reads the batch's output file, and the Bedrock - file config resolves its bucket only from this snapshot, never from a request param, - because the bucket is what managed file ids are validated against. Without it that - read fails and the batch's cost is never recorded. - """ - if model_id is None: - return - try: - credentials: Final = llm_router.registry().credentials(model_id, form="with_provider") - except Exception: # noqa: BLE001 # the snapshot only enables cost accounting; a batch whose deployment no longer resolves must still be retrievable - return - if credentials is None: - return - data["_litellm_internal_model_credentials"] = MappingProxyType(dict(credentials)) - - def add_deployment_model_info( data: dict, llm_router: Optional["Router"], @@ -792,7 +767,6 @@ def prepare_data_with_credentials( data: dict, credentials: dict, file_id: str | None = None, - include_internal_credentials: bool = False, ) -> None: """ Update data dictionary with model credentials (in-place). @@ -801,12 +775,8 @@ def prepare_data_with_credentials( data: Data dictionary to update credentials: Credentials from router file_id: Optional original file_id to set (for decoded file IDs) - include_internal_credentials: Preserve an immutable server-side snapshot - for code paths that must distinguish proxy config from request params. """ data.update(credentials) - if include_internal_credentials: - data["_litellm_internal_model_credentials"] = MappingProxyType(dict(credentials)) data.pop("custom_llm_provider", None) if file_id is not None: diff --git a/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py b/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py index 457af931f56..3b809688d6c 100644 --- a/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py +++ b/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py @@ -43,7 +43,6 @@ class FileContentStreamingHandler: data=resolved_streaming_data, credentials=credentials, file_id=original_file_id, - include_internal_credentials=True, ) resolved_streaming_data.pop("model", None) resolved_streaming_provider: Final = cast(str, credentials["custom_llm_provider"]) diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index e613d744fb0..5ce0ed8b493 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -77,7 +77,6 @@ from litellm.proxy.openai_files_endpoints.batch_guardrails import ( ) from litellm.proxy.openai_files_endpoints.common_utils import ( # noqa: F401 # legacy module exports _is_base64_encoded_unified_file_id, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - add_internal_model_credentials, apply_team_provider_credentials, authorize_model_for_key, deployment_drop_params, @@ -434,7 +433,7 @@ async def route_create_file( ) storage: Final = explicit_storage or (LITELLM_DB_STORAGE_BACKEND_NAME if executed_model is not None else None) if storage is not None: - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) from litellm.proxy.openai_files_endpoints.storage_backend_service import ( @@ -1091,7 +1090,6 @@ async def get_file_content( model: Final = cast(str | None, data.get("model")) if model: - add_internal_model_credentials(data=data, llm_router=llm_router, model_id=model) response = await llm_router.afile_content( **{ "model": model, @@ -1180,7 +1178,6 @@ async def get_file_content( data=data, credentials=credentials, file_id=original_file_id, # Use decoded file ID if from encoded ID - include_internal_credentials=True, ) response = await litellm.afile_content( custom_llm_provider=credentials["custom_llm_provider"], @@ -1354,7 +1351,6 @@ async def get_file( data=data, credentials=credentials, file_id=original_file_id, - include_internal_credentials=True, ) response = await litellm.afile_retrieve( @@ -1568,7 +1564,6 @@ async def delete_file( data=data, credentials=credentials, file_id=original_file_id, - include_internal_credentials=True, ) response = await litellm.afile_delete( @@ -1973,7 +1968,7 @@ async def list_files( ) elif should_route and credentials is not None: # Use model-based routing with credentials from config - prepare_data_with_credentials(data=data, credentials=credentials, include_internal_credentials=True) + prepare_data_with_credentials(data=data, credentials=credentials) response = await litellm.afile_list( custom_llm_provider=credentials["custom_llm_provider"], purpose=purpose, @@ -1997,7 +1992,7 @@ async def list_files( user_api_key_dict=user_api_key_dict, operation_context="file list", ) - prepare_data_with_credentials(data=data, credentials=credentials, include_internal_credentials=True) + prepare_data_with_credentials(data=data, credentials=credentials) response = await litellm.afile_list( custom_llm_provider=credentials["custom_llm_provider"], purpose=purpose, diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index b8cdea0dea4..cb1e51d0deb 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -24,7 +24,9 @@ from litellm.proxy._types import ( CommonProxyErrors, ) from litellm.proxy.db.health_check_latest import latest_by_model_name +from litellm.proxy.deps.repositories import PluginUploadRepositoryDep from litellm.proxy.utils import get_custom_url +from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review from litellm.repositories.table_repositories import ClaudeCodePluginRepository from litellm.router_strategy.complexity_router.fuse_presets import FusePresetCatalog, get_fuse_presets from litellm.types.agents import AgentCard @@ -71,6 +73,9 @@ class _EndpointEntry(TypedDict): class _PluginRow(Protocol): + @property + def created_by(self) -> str | None: ... + @property def id(self) -> str: ... @@ -366,7 +371,7 @@ async def get_mcp_servers(): "/public/skill_hub", tags=["public", "Claude Code Marketplace"], ) -async def public_skill_hub(): +async def public_skill_hub(uploads: PluginUploadRepositoryDep): """Return enabled (public) Claude Code skills, or none when the marketplace requires a key.""" from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( MANIFEST_FIELDS, @@ -385,13 +390,22 @@ async def public_skill_hub(): raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) try: plugins: Final = await _plugin_table(prisma_client).find_many(where={"enabled": True}) + counts: Final[Mapping[str, int]] = ( + await uploads.download_counts(tuple(plugin.id for plugin in plugins)) if uploads else {} + ) items: Final = [] for plugin in plugins: raw = plugin.manifest_json or {} manifest = json.loads(raw) if isinstance(raw, str) else raw + review = read_plugin_review(MANIFEST_FIELDS.validate_python(manifest)) items.append( PluginListItem( id=plugin.id, + download_count=counts.get(plugin.id), + created_by=plugin.created_by, + approved_by=review.approved_by if review else None, + published_by=read_plugin_publisher(MANIFEST_FIELDS.validate_python(manifest)), + security_scan_passed=review.security_scan_passed if review else False, name=plugin.name, enabled=plugin.enabled, created_at=str(plugin.created_at) if plugin.created_at else None, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 148d465099f..08942851af9 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -271,6 +271,10 @@ REQUIRED_ONE_OF_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, str]]] = Map {"acreate_interaction": ("model", "agent"), "acreate_agent": ("id", "name")} ) +BODY_PARAM_ALTERNATIVES_BY_ROUTE: Final[Mapping[str, Mapping[str, tuple[str, ...]]]] = MappingProxyType( + {"acreate_interaction": MappingProxyType({"input": ("continuation_token",)})} +) + JSON_OBJECT_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_JSON_OBJECT_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter( Mapping[str, object] | None @@ -320,8 +324,11 @@ def _find_missing_required_body_param( ) if missing_merge_base_param is not None: return MissingBodyParam(name=missing_merge_base_param, model_deployments_loaded=True) + alternatives: Final = BODY_PARAM_ALTERNATIVES_BY_ROUTE.get(route_type, {}) missing_present_params: Final = tuple( - param for param in REQUIRED_PRESENT_BODY_PARAMS_BY_ROUTE.get(route_type, ()) if param not in data + param + for param in REQUIRED_PRESENT_BODY_PARAMS_BY_ROUTE.get(route_type, ()) + if param not in data and not any(alternative in data for alternative in alternatives.get(param, ())) ) if not missing_present_params: return None diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 3489943af20..0c32b329465 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -314,6 +314,7 @@ if TYPE_CHECKING: from opentelemetry.trace import Span as _Span from prisma import models as prisma_models from prisma.client import TransactionManager + from prisma.errors import UniqueViolationError from prisma.models import LiteLLM_DeprecatedVerificationToken from prisma.types import HttpConfig, LiteLLM_VerificationTokenInclude @@ -5588,6 +5589,25 @@ _LOCK_TEAMS_BY_ID_SQL: Final = ( ) +def _user_unique_conflict_message(metadata: object, user_id: object) -> str: + match metadata: + case {"target": ["user_id"]}: + return f"User with id {user_id} already exists" + case {"target": ["sso_user_id"]}: + return "User with this sso_user_id already exists" + case _: + return "User conflicts with an existing unique value" + + +def _raise_user_unique_conflict(error: "UniqueViolationError", user_id: object, *, allow_existing: bool) -> NoReturn: + if allow_existing: + raise error + raise HTTPException( + status_code=409, + detail={"error": _user_unique_conflict_message(error.meta, user_id)}, + ) from error + + class PrismaClient: spend_log_transactions: list = [] _spend_log_transactions_lock = asyncio.Lock() @@ -6475,12 +6495,15 @@ class PrismaClient: max_tries=3, # maximum number of retries max_time=10, # maximum total time to retry for on_backoff=on_backoff, # specifying the function to call on backoff + giveup=lambda exc: isinstance(exc, HTTPException) and exc.status_code == 409, ) @log_db_metrics async def insert_data( self, data: Mapping[str, object], table_name: Literal["user", "key", "config", "spend", "team", "user_notification"], + *, + allow_existing_user: bool = True, ): """ Add a key to the database. If it already exists, do nothing. @@ -6516,14 +6539,20 @@ class PrismaClient: verbose_proxy_logger.info("Data Inserted into Keys Table") return new_verification_token elif table_name == "user": + from prisma.errors import UniqueViolationError + db_data = self.jsonify_object(data=data) try: - new_user_row: Final = await UserRepository(self).upsert_user(db_data) + new_user_row: Final = await UserRepository(self).upsert_user( + db_data, allow_existing=allow_existing_user + ) if new_user_row is None: raise HTTPException( status_code=403, detail="SSO for more than 5 users requires a LiteLLM Enterprise license. Existing users can still sign in.", ) + except UniqueViolationError as e: + _raise_user_unique_conflict(e, db_data["user_id"], allow_existing=allow_existing_user) except Exception as e: if ( "Foreign key constraint failed on the field: `LiteLLM_UserTable_organization_id_fkey (index)`" diff --git a/litellm/repositories/plugin_publication.py b/litellm/repositories/plugin_publication.py index f5fab458c6b..6f94076c3e0 100644 --- a/litellm/repositories/plugin_publication.py +++ b/litellm/repositories/plugin_publication.py @@ -1,15 +1,18 @@ import hashlib import json +from collections.abc import Mapping +from dataclasses import asdict from datetime import datetime, timezone from typing import TYPE_CHECKING, Final, Protocol -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from semver import Version from litellm.models.plugin_upload import ( PluginPublication, PluginPublicationBase, PluginPublicationError, + PluginReview, PublishedPlugin, ) from litellm.repositories.prisma_protocols import TableActions @@ -23,6 +26,22 @@ if TYPE_CHECKING: _MANIFEST: Final = TypeAdapter(dict[str, object]) _SOURCE: Final = TypeAdapter(dict[str, str]) +_REVIEW: Final = TypeAdapter(PluginReview) +_PUBLISHER: Final = TypeAdapter[str | None](str | None) + + +def read_plugin_review(manifest: Mapping[str, object]) -> PluginReview | None: + try: + return _REVIEW.validate_python(manifest.get("_litellm_review")) + except ValidationError: + return None + + +def read_plugin_publisher(manifest: Mapping[str, object]) -> str | None: + try: + return _PUBLISHER.validate_python(manifest.get("_litellm_published_by")) + except ValidationError: + return None class PluginPublicationTransaction(Protocol): @@ -122,7 +141,22 @@ async def publish_plugin( ) if isinstance(version, PluginPublicationError): return version - manifest_json: Final = json.dumps({**publication.manifest, "version": version}) + current_manifest: Final = _MANIFEST.validate_json(current.manifest_json or "{}") if current is not None else {} + same_source: Final = current is not None and current_manifest.get("source") == publication.manifest.get("source") + review: Final = publication.review or (read_plugin_review(current_manifest) if same_source else None) + publisher: Final = publication.review.approved_by if publication.review is not None else publication.created_by + manifest_json: Final = json.dumps( + { + **{ + key: value + for key, value in publication.manifest.items() + if key not in {"_litellm_review", "_litellm_published_by"} + }, + "version": version, + **({"_litellm_published_by": publisher} if publisher is not None else {}), + **({"_litellm_review": asdict(review)} if review is not None else {}), + } + ) if publication.content is not None and verified_archive(publication.content, manifest_json) is None: return PluginPublicationError("conflict", "Uploaded archive does not match its published source digest") timestamp: Final = datetime.now(timezone.utc) @@ -141,8 +175,18 @@ async def publish_plugin( await tx.litellm_claudecodepluginupload.create( data={"plugin_name": publication.name, "content": Base64.encode(publication.content)} ) - await _snapshot(tx, created, publication.content, publication.created_by) - return PublishedPlugin(created.id, created.name, version, created.enabled, created.updated_at or timestamp) + await _snapshot(tx, created, publication.content, publisher) + return PublishedPlugin( + created.id, + created.name, + version, + created.enabled, + created.updated_at or timestamp, + created.created_at, + created.created_by, + review, + publisher, + ) previous_archive: Final = await tx.litellm_claudecodepluginupload.find_unique(where={"plugin_name": current.name}) previous_content: Final = ( verified_archive(previous_archive.content.decode(), current.manifest_json) @@ -152,11 +196,7 @@ async def publish_plugin( history_count: Final = await tx.litellm_claudecodepluginversion.count(where={"plugin_id": current.id}, take=1) if history_count == 0: await _snapshot(tx, current, previous_content, None) - unchanged_content: Final = ( - previous_content - if _MANIFEST.validate_json(current.manifest_json or "{}").get("source") == publication.manifest.get("source") - else None - ) + unchanged_content: Final = previous_content if same_source else None content: Final = publication.content if publication.content is not None else unchanged_content updated: Final = await tx.litellm_claudecodeplugintable.update(where={"id": current.id}, data=data) if updated is None: @@ -171,5 +211,15 @@ async def publish_plugin( "update": {"content": Base64.encode(content)}, }, ) - await _snapshot(tx, updated, content, publication.created_by) - return PublishedPlugin(updated.id, updated.name, version, updated.enabled, updated.updated_at or timestamp) + await _snapshot(tx, updated, content, publisher) + return PublishedPlugin( + updated.id, + updated.name, + version, + updated.enabled, + updated.updated_at or timestamp, + updated.created_at, + updated.created_by, + review, + publisher, + ) diff --git a/litellm/repositories/plugin_upload_repository.py b/litellm/repositories/plugin_upload_repository.py index 99e226e442b..8045abf9af9 100644 --- a/litellm/repositories/plugin_upload_repository.py +++ b/litellm/repositories/plugin_upload_repository.py @@ -287,4 +287,6 @@ def _archive_metadata(plugin: "prisma_models.LiteLLM_ClaudeCodePluginTable") -> description=plugin.description, has_agent_skill_metadata=has_agent_skill_metadata(manifest), updated_at=plugin.updated_at, + created_at=plugin.created_at, + created_by=plugin.created_by, ) diff --git a/litellm/repositories/skill_submission_repository.py b/litellm/repositories/skill_submission_repository.py index 829aa0aadbf..03821da042a 100644 --- a/litellm/repositories/skill_submission_repository.py +++ b/litellm/repositories/skill_submission_repository.py @@ -7,12 +7,13 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # Prisma's generated delegates are dynamic from urllib.parse import urlsplit -from pydantic import BaseModel, ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.models.plugin_upload import ( PluginPublication, PluginPublicationBase, PluginPublicationError, + PluginReview, PublishedPlugin, uploaded_plugin_source, ) @@ -312,7 +313,9 @@ class SkillSubmissionRepository: if approve and content is not None and publication_base_url else _SOURCE.validate_python(submission.proposal.manifest.get("source")) ) - publication: Final = await _publish_submission(transaction, row, source, content) if approve else None + publication: Final = ( + await _publish_submission(transaction, row, source, content, reviewer) if approve else None + ) if isinstance(publication, PluginPublicationError): return publication reviewed_at: Final = datetime.now(timezone.utc) @@ -346,6 +349,7 @@ async def _publish_submission( row: "prisma_models.LiteLLM_SkillSubmission", source: Mapping[str, str], content: bytes | None, + reviewer: str, ) -> PublishedPlugin | PluginPublicationError: stored: Final = _StoredProposal.model_validate_json(row.proposal_json) proposal: Final = stored.proposal(row.plugin_name) @@ -363,6 +367,24 @@ async def _publish_submission( manifest={**stored.manifest, "source": dict(source)}, content=content, created_by=row.submitted_by, + review=PluginReview(reviewer, scan_passed_for_archive(row.scan_json, source.get("sha256"))), ), base, ) + + +def scan_passed_for_archive(scan_json: str | None, digest: str | None) -> bool: + if not scan_json or not digest: + return False + try: + scan: Final = SkillScan.model_validate_json(scan_json) + except ValidationError: + return False + return ( + scan.status == "completed" + and scan.archive_sha256 == digest + and scan.report is not None + and scan.report.complete + and scan.report.risk_score == 0 + and not scan.report.findings + ) diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py index 2622b6d5a2a..f39d28a3039 100644 --- a/litellm/repositories/user_repository.py +++ b/litellm/repositories/user_repository.py @@ -166,7 +166,9 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): """ return await _count_billable_users(self.table) - async def upsert_user(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_UserTable | None": + async def upsert_user( + self, data: Mapping[str, object], *, allow_existing: bool = True + ) -> "prisma_models.LiteLLM_UserTable | None": """Return the user row, or None when a new free SSO user would exceed capacity.""" scope: Final = _FREE_SSO_CREATION.get() if ( @@ -174,6 +176,8 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): or (scope[0] is not None and scope[0] != data["user_id"]) or scope[1] is not asyncio.current_task() ): + if not allow_existing: + return await self.table.create(data=dict(data)) return await self.table.upsert( where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}} ) diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index 0b769326012..3deceac07aa 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, update_responses_tools_with_model_file_ids, ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index d8277ec9d33..e085c4d7a2e 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -38,7 +38,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import ( ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, update_responses_tools_with_model_file_ids, ) diff --git a/litellm/responses/mcp/stream_round_billing.py b/litellm/responses/mcp/stream_round_billing.py index 5cb256a8ac0..be331a91475 100644 --- a/litellm/responses/mcp/stream_round_billing.py +++ b/litellm/responses/mcp/stream_round_billing.py @@ -1,5 +1,5 @@ import asyncio -from collections.abc import Callable, Coroutine, Sequence +from collections.abc import Callable, Sequence from dataclasses import dataclass from datetime import datetime from typing import Final, TypeAlias @@ -297,12 +297,8 @@ class StreamedRoundBilling: self._close(parked) self._bank((self._dispatch(row, billed, self._logging_obj.attempt_scoped_copy()),), row) - def _dispatch( - self, row: _RoundRow, billed: _BilledRounds, logger: LiteLLMLoggingObj - ) -> BilledRoundsDispatch | Coroutine[object, object, None]: + def _dispatch(self, row: _RoundRow, billed: _BilledRounds, logger: LiteLLMLoggingObj) -> BilledRoundsDispatch: dispatch: Final = self._log_success(_SuccessLog(logger, row, billed.chat_usage)) - if billed.chat_usage is None: - return dispatch return BilledRoundsDispatch(dispatch, billed.spend) @staticmethod diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 249c6370b86..288e4a7119d 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -53,7 +53,7 @@ from litellm.litellm_core_utils.internal_call_metadata import ( requests_background, without_budget_reservation, ) -from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend +from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, BilledRoundsSpend, UncachedPrice from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers @@ -638,6 +638,24 @@ class BaseResponsesAPIStreamingIterator: self._handle_failure(e) raise + def _deferred_logging_dispatch( + self, logging_coroutine: Coroutine[object, object, None], logging_response: object + ) -> BilledRoundsDispatch | Coroutine[object, object, None]: + response: Final[object] = getattr(logging_response, "response", None) + if not isinstance(response, ResponsesAPIResponse): + return logging_coroutine + price: Final = ( + UncachedPrice(0.0, None) + if self._completed_response_cache_hit + else self.logging_obj.uncached_price(response) + ) + spend: Final = BilledRoundsSpend( + ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response.usage), + price.cost or 0.0, + price.cost_breakdown, + ) + return BilledRoundsDispatch(logging_coroutine, spend) + def _log_completed_response(self, *, is_async: bool) -> None: if self._completed_response_logged: return @@ -662,7 +680,9 @@ class BaseResponsesAPIStreamingIterator: if deferred_dispatch_armed: # End-of-stream guardrail scans write guardrail_information after # the terminal event; dispatching now would snapshot metadata early. - self.logging_obj._deferred_stream_complete_args = (logging_coroutine,) + self.logging_obj._deferred_stream_complete_args = ( + self._deferred_logging_dispatch(logging_coroutine, logging_response), + ) else: spawn_logging_task( asyncio.get_running_loop(), logging_coroutine, task_name="Responses stream success logging" diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index 688b52f44a3..e2c621ccb7d 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -1907,10 +1907,9 @@ class RustEndpoints: if self._scheduler is None: from litellm.scheduler import Scheduler - self._scheduler = ( - Scheduler(polling_interval=self._polling_interval) # pyright: ignore[reportArgumentType] # as configured - if self._polling_interval is not None - else Scheduler() + self._scheduler = Scheduler( + polling_interval=self._polling_interval, # pyright: ignore[reportArgumentType] # as configured + redis_cache=self._core.routing_cache.redis_cache, ) return self._scheduler diff --git a/litellm/router_rust/invoke/walk.py b/litellm/router_rust/invoke/walk.py index 2c1481f068a..17e20119f43 100644 --- a/litellm/router_rust/invoke/walk.py +++ b/litellm/router_rust/invoke/walk.py @@ -264,10 +264,18 @@ class WalkMetadata: def hop_kwargs(self, event: Mapping[str, object]) -> dict[str, object]: # mutable-ok: a fresh dict for the loggers """The kwargs the level held after a hop (``kwargs["model"] = target``), as the - fallback loggers see them.""" + fallback loggers see them: the request's own metadata (the proxy's ``user_api_key_*`` + labels) with the level's walk stamps laid over it, as ``run_async_fallback``'s + ``_hop_metadata`` copies it.""" scope: Final = self._scope(event.get("scope")) model: Final = scope.metadata.get("model_group", event.get("to")) - return {**self._request, **scope.kwargs, "model": model, self._field: dict(scope.metadata)} + request_bucket: Final = as_str_mapping(self._request.get(self._field)) or {} + return { + **self._request, + **scope.kwargs, + "model": model, + self._field: {**request_bucket, **scope.metadata}, + } def log_retry(self, event: Mapping[str, object], error: BaseException | None) -> None: """A ``retry_logged`` event: the failure joins the scope's ``previous_models``.""" diff --git a/litellm/router_rust/surface/health.py b/litellm/router_rust/surface/health.py index b16adc38af9..0653518d0d7 100644 --- a/litellm/router_rust/surface/health.py +++ b/litellm/router_rust/surface/health.py @@ -279,6 +279,7 @@ class RustHealthCallbacks: is_caller_scoped_auth_failure, is_caller_timeout_408, is_failure_outside_selected_deployment, + is_response_lookup_not_found, is_video_lookup_not_found, ) @@ -288,6 +289,7 @@ class RustHealthCallbacks: is_failure_outside_selected_deployment(exception if isinstance(exception, BaseException) else None) or (isinstance(exception, Exception) and is_background_response_poll_not_found(exception, litellm_params)) or is_video_lookup_not_found(exception, logged) + or is_response_lookup_not_found(exception, logged) or is_caller_timeout_408(logged, status) # pyright: ignore[reportArgumentType] # the status as raised or is_caller_scoped_auth_failure(self._deployment_of(deployment_id), status) # pyright: ignore[reportArgumentType] # see above ) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 03485ebd5f8..4c918776223 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -108,6 +108,26 @@ def is_video_lookup_not_found(exception: object, failure_kwargs: Mapping[str, ob return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in VIDEO_ID_CALL_TYPES +# The calls on an existing response id. ``compact_responses`` is left out: it sends a model and input, +# so its 404 can mean a missing model rather than a missing response. +RESPONSE_ID_CALL_TYPES: Final = frozenset( + { + "aget_responses", + "get_responses", + "adelete_responses", + "delete_responses", + "acancel_responses", + "cancel_responses", + "alist_input_items", + } +) + + +def is_response_lookup_not_found(exception: object, failure_kwargs: Mapping[str, object]) -> bool: + """Whether a call on an existing response id failed with a provider 404: the response is not on that account.""" + return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in RESPONSE_ID_CALL_TYPES + + _EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = ( # ContentPolicyViolationError subclasses BadRequestError, so it must be checked first. (litellm.ContentPolicyViolationError, "ContentPolicyViolationErrorAllowedFails"), diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index f3425596afa..062886b44fd 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -51,7 +51,7 @@ from litellm.exceptions import AuthenticationError, BadRequestError from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider, get_llm_provider from litellm.litellm_core_utils.litellm_logging import has_sensitive_keyword -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( anthropic_content_lists, encrypted_content_of_block, strip_encrypted_reasoning_from_messages, diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index 81012d38b65..e5d944b3aef 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -163,6 +163,9 @@ class PromptCachingDeploymentCheck(CustomLogger): return healthy_deployments + def exempt_from_message_redaction(self) -> bool: + return True + async def async_log_success_event( self, kwargs: _PromptCachingSuccessEventKwargs, diff --git a/litellm/rust_bridge/responses/detectors.py b/litellm/rust_bridge/responses/detectors.py index 763b63d6742..2fdb2c44b54 100644 --- a/litellm/rust_bridge/responses/detectors.py +++ b/litellm/rust_bridge/responses/detectors.py @@ -67,7 +67,7 @@ def _tools(request: NativeCall) -> Sequence[object]: def managed_resource_ids(request: NativeCall) -> bool: """Python replaces managed file IDs in Responses input and tools with provider IDs.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, update_responses_tools_with_model_file_ids, ) diff --git a/litellm/types/interactions/generated.py b/litellm/types/interactions/generated.py index c4f7a2d8a7b..4d808742434 100644 --- a/litellm/types/interactions/generated.py +++ b/litellm/types/interactions/generated.py @@ -176,11 +176,16 @@ class Status1(Enum): incomplete = "incomplete" budget_exceeded = "budget_exceeded" queued = "queued" + requires_continuation = "requires_continuation" class InteractionStatusUpdate(LiteLLMBaseModel): interaction_id: str | None = None status: Status1 | None = None + continuation_token: str | None = Field( + None, + description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + ) event_type: Literal["interaction.status_update"] = "interaction.status_update" event_id: str | None = Field( None, @@ -343,6 +348,7 @@ class Status3(Enum): INCOMPLETE = "incomplete" BUDGET_EXCEEDED = "budget_exceeded" QUEUED = "queued" + REQUIRES_CONTINUATION = "requires_continuation" class ModelOption(RootModel[str]): @@ -604,6 +610,9 @@ class ToolChoice(RootModel[ToolChoiceType | ToolChoiceConfig]): class Usage(LiteLLMBaseModel): total_input_tokens: int | None = Field(None, description="Number of tokens in the prompt (context).") + initial_input_tokens: int | None = Field( + None, description="Number of tokens in the prompt of the first hop of a long decode." + ) input_tokens_by_modality: list[ModalityTokens] | None = Field( None, description="A breakdown of input token usage by modality." ) @@ -625,6 +634,7 @@ class Usage(LiteLLMBaseModel): None, description="A breakdown of tool-use token usage by modality." ) total_reasoning_tokens: int | None = Field(None, description="Number of tokens of thoughts for thinking models.") + total_thought_tokens: int | None = Field(None, description="Number of tokens of thoughts for thinking models.") total_tokens: int | None = Field( None, description="Total token count for the interaction request (prompt + responses + other\ninternal tokens).", @@ -787,6 +797,10 @@ class Interaction(LiteLLMBaseModel): description="The mime type of the response. This is required if response_format is set.", ) previous_interaction_id: str | None = Field(None, description="The ID of the previous interaction, if any.") + continuation_token: str | None = Field( + None, + description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + ) input: str | list[Content] | list[Turn] | Content | None = Field( None, description="The inputs for the interaction." ) @@ -843,6 +857,10 @@ class CreateModelInteractionParams(LiteLLMBaseModel): description="The mime type of the response. This is required if response_format is set.", ) previous_interaction_id: str | None = Field(None, description="The ID of the previous interaction, if any.") + continuation_token: str | None = Field( + None, + description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + ) input: str | list[Content] | list[Turn] | Content = Field(..., description="The inputs for the interaction.") generation_config: GenerationConfig | None = Field( None, @@ -894,6 +912,10 @@ class CreateAgentInteractionParams(LiteLLMBaseModel): description="The mime type of the response. This is required if response_format is set.", ) previous_interaction_id: str | None = Field(None, description="The ID of the previous interaction, if any.") + continuation_token: str | None = Field( + None, + description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + ) input: str | list[Content] | list[Turn] | Content = Field(..., description="The inputs for the interaction.") agent_config: DynamicAgentConfig | DeepResearchAgentConfig | None = Field( None, description="Configuration for the agent.", discriminator="type" @@ -1080,6 +1102,7 @@ class InteractionsAPIResponse(BaseLiteLLMOpenAIResponseObject): # New schema field (Api-Revision: 2026-05-20). steps: list[dict[str, Any]] | None = None usage: dict[str, Any] | None = None + continuation_token: str | None = None _hidden_params: dict = PrivateAttr(default_factory=dict) @@ -1126,6 +1149,7 @@ class InteractionsAPIStreamingResponse(BaseLiteLLMOpenAIResponseObject): step: dict[str, Any] | None = None interaction_id: str | None = None interaction: dict[str, Any] | None = None + continuation_token: str | None = None _hidden_params: dict = PrivateAttr(default_factory=dict) diff --git a/litellm/types/llms/gemini_audio_transcription.py b/litellm/types/llms/gemini_audio_transcription.py index c1cf3452945..59dd15aa13e 100644 --- a/litellm/types/llms/gemini_audio_transcription.py +++ b/litellm/types/llms/gemini_audio_transcription.py @@ -75,7 +75,7 @@ class GeminiTranscriptionUsage(LiteLLMBaseModel): class GeminiTranscriptionInteractionResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", hide_input_in_errors=True) id: str | None = None status: str | None = None diff --git a/litellm/types/llms/vertex_ai_gemini_transcription.py b/litellm/types/llms/vertex_ai_gemini_transcription.py index e78aafaca9f..d7bc136bcd1 100644 --- a/litellm/types/llms/vertex_ai_gemini_transcription.py +++ b/litellm/types/llms/vertex_ai_gemini_transcription.py @@ -68,7 +68,7 @@ class VertexGeminiTranscriptionUsageMetadata(LiteLLMBaseModel): class VertexGeminiTranscriptionResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", hide_input_in_errors=True) candidates: tuple[VertexGeminiTranscriptionCandidate, ...] = () usageMetadata: VertexGeminiTranscriptionUsageMetadata | None = None diff --git a/litellm/types/llms/vertex_ai_speech_to_text.py b/litellm/types/llms/vertex_ai_speech_to_text.py index f3d343d13ce..3dea7610150 100644 --- a/litellm/types/llms/vertex_ai_speech_to_text.py +++ b/litellm/types/llms/vertex_ai_speech_to_text.py @@ -40,6 +40,8 @@ class VertexSpeechToTextResponseMetadata(LiteLLMBaseModel): class VertexSpeechToTextRecognizeResponse(LiteLLMBaseModel): + model_config = ConfigDict(hide_input_in_errors=True) + results: list[VertexSpeechToTextResult] = Field(default=[]) metadata: VertexSpeechToTextResponseMetadata | None = None diff --git a/litellm/types/proxy/claude_code_endpoints.py b/litellm/types/proxy/claude_code_endpoints.py index 5d8479b8ce1..dfbf1de17af 100644 --- a/litellm/types/proxy/claude_code_endpoints.py +++ b/litellm/types/proxy/claude_code_endpoints.py @@ -154,6 +154,11 @@ class PluginListItem(LiteLLMBaseModel): enabled: bool created_at: str | None updated_at: str | None + created_by: str | None = None + approved_by: str | None = None + published_by: str | None = None + security_scan_passed: bool = False + download_count: int | None = None has_agent_skill_metadata: bool = Field(default=False, exclude=True) diff --git a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py index fc0b73f77b6..e79b509da1b 100644 --- a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py @@ -120,6 +120,7 @@ class BulkNewUserItem(NewUserRequest): model_config = ConfigDict(extra="forbid", protected_namespaces=()) + user_id: str | None = None auto_create_key: bool = False @field_validator("send_invite_email") diff --git a/litellm/types/proxy/management_endpoints/scim_v2.py b/litellm/types/proxy/management_endpoints/scim_v2.py index fadb309e0cd..72a39c61967 100644 --- a/litellm/types/proxy/management_endpoints/scim_v2.py +++ b/litellm/types/proxy/management_endpoints/scim_v2.py @@ -13,6 +13,7 @@ from pydantic import ( from pydantic_core.core_schema import SerializerFunctionWrapHandler from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.proxy.management_endpoints.user_requests import NewUserRequest SCIM_ENTERPRISE_USER_SCHEMA: Final = "urn:ietf:params:scim:schemas:extension:enterprise:2.0:User" SCIM_ENTERPRISE_METADATA_KEY: Final = "scim_enterprise" @@ -26,6 +27,14 @@ _SCIM_SERIALIZED_FIELDS_ADAPTER: Final = TypeAdapter( ) +class SCIMNewUserRequest(NewUserRequest): + """Preserve opaque directory IDs when provisioning SCIM users and group members.""" + + model_config = ConfigDict(frozen=True) + + user_id: str | None = None + + class LiteLLM_UserScimMetadata(LiteLLMBaseModel): """ Scim metadata stored in LiteLLM_UserTable.metadata diff --git a/litellm/types/proxy/management_endpoints/ui_sso.py b/litellm/types/proxy/management_endpoints/ui_sso.py index affb6d2e9ce..38ed799f38b 100644 --- a/litellm/types/proxy/management_endpoints/ui_sso.py +++ b/litellm/types/proxy/management_endpoints/ui_sso.py @@ -1,12 +1,19 @@ from typing import Literal -from pydantic import Field +from pydantic import ConfigDict, Field from typing_extensions import TypedDict from litellm.types.proxy.auth.user_roles import KeyManagementRoutes, LitellmUserRoles +from litellm.types.proxy.management_endpoints.user_requests import NewUserRequest from litellm.types.utils import LiteLLMPydanticObjectBase +class SSONewUserRequest(NewUserRequest): + model_config = ConfigDict(frozen=True) + + user_id: str | None = None + + class LiteLLM_UpperboundKeyGenerateParams(LiteLLMPydanticObjectBase): """ Set default upperbound to max budget a key called via `/key/generate` can be. diff --git a/litellm/types/proxy/management_endpoints/user_requests.py b/litellm/types/proxy/management_endpoints/user_requests.py index e53dc4152c3..edda1f78430 100644 --- a/litellm/types/proxy/management_endpoints/user_requests.py +++ b/litellm/types/proxy/management_endpoints/user_requests.py @@ -1,7 +1,7 @@ from collections.abc import Mapping -from typing import Literal +from typing import Annotated, Final, Literal -from pydantic import Field, field_validator, model_validator +from pydantic import AfterValidator, Field, field_validator, model_validator from litellm.models.user import LiteLLM_UserTable from litellm.types.proxy.auth.user_roles import LitellmUserRoles @@ -15,7 +15,17 @@ class NewUserRequestTeam(LiteLLMPydanticObjectBase): user_role: Literal["user", "admin"] = "user" +def _validate_custom_user_id(value: str) -> str: + normalized: Final = value.strip() + if not normalized: + raise ValueError("user_id must not be empty or contain only whitespace") + if "\x00" in normalized: + raise ValueError("user_id must not contain NUL characters") + return normalized + + class NewUserRequest(GenerateRequestBase): + user_id: Annotated[str, AfterValidator(_validate_custom_user_id)] | None = None tpm_limit: int | None = Field(default=None, ge=0) rpm_limit: int | None = Field(default=None, ge=0) max_budget: float | None = None diff --git a/litellm/types/proxy/skill_hub.py b/litellm/types/proxy/skill_hub.py index d575defcda7..050c3cb7eb8 100644 --- a/litellm/types/proxy/skill_hub.py +++ b/litellm/types/proxy/skill_hub.py @@ -7,6 +7,17 @@ from pydantic import ConfigDict, Field from litellm.models.skill_submission import SubmissionOperation, SubmissionStatus from litellm.types.llms.base import LiteLLMBaseModel +SkillSortField: TypeAlias = Literal[ + "name", + "download_count", + "updated_at", + "created_at", + "created_by", + "published_by", + "approved_by", + "security_scan_passed", +] + class SkillHubSkill(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) @@ -16,6 +27,11 @@ class SkillHubSkill(LiteLLMBaseModel): description: str | None = None version: str | None = None updated_at: datetime | None = None + created_at: datetime | None = None + created_by: str | None = None + approved_by: str | None = None + published_by: str | None = None + security_scan_passed: bool = False visibility: Literal["public", "private"] source: Mapping[str, str] download_url: str | None = None @@ -23,12 +39,11 @@ class SkillHubSkill(LiteLLMBaseModel): domain: str | None = None namespace: str | None = None keywords: tuple[str, ...] | None = None - download_count: int | None = Field( - None, description="Gateway ZIP downloads, included in proxy-admin catalog listings" - ) + download_count: int | None = Field(None, description="Gateway ZIP downloads, including repeat downloads") class SkillHubList(LiteLLMBaseModel): + tags: tuple[str, ...] = () model_config = ConfigDict(frozen=True) skills: tuple[SkillHubSkill, ...] diff --git a/litellm/types/router.py b/litellm/types/router.py index 3363d113fc7..a9f28fbc673 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -427,6 +427,7 @@ class CredentialLiteLLMParams(LiteLLMBaseModel): ## OBJECT STORAGE (files / batches) ## gcs_bucket_name: str | None = None bucket_name: str | None = None + allow_legacy_cloud_file_ids: bool | None = None ## AWS BEDROCK / SAGEMAKER ## aws_access_key_id: str | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ba29a67a36c..7c137b72c3e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -703,6 +703,8 @@ class CallTypes(str, Enum): adelete_responses = "adelete_responses" cancel_responses = "cancel_responses" acancel_responses = "acancel_responses" + compact_responses = "compact_responses" + acompact_responses = "acompact_responses" alist_input_items = "alist_input_items" llm_passthrough_route = "llm_passthrough_route" allm_passthrough_route = "allm_passthrough_route" @@ -806,6 +808,8 @@ CallTypesLiteral = Literal[ "send_message", "aresponses", "responses", + "compact_responses", + "acompact_responses", "acreate_skill", "acreate_realtime_client_secret", "arealtime_calls", @@ -2985,6 +2989,8 @@ _DETECTED_LANGUAGES: Final = TypeAdapter(tuple[TranscriptionDetectedLanguage, .. class TranscriptionResponse(OpenAIObject): + model_config = ConfigDict(hide_input_in_errors=True) + text: str | None = None usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None languages: Sequence[TranscriptionDetectedLanguage] | None = None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1bda9d24c8e..ad57685da04 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2242,7 +2242,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2331,7 +2331,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2420,7 +2420,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2504,7 +2504,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2593,7 +2593,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2715,7 +2715,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2759,7 +2759,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2803,7 +2803,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2847,7 +2847,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2891,7 +2891,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -4046,6 +4046,8 @@ "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, + "deprecation_date": "2027-09-01", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, @@ -4068,7 +4070,7 @@ "supports_xhigh_reasoning_effort": true, "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512, - "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models" + "source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" }, "azure_ai/claude-opus-5": { "deprecation_date": "2027-07-08", @@ -13458,13 +13460,16 @@ "supports_reasoning": true }, "azure_ai/mistral-document-ai-2512": { + "deprecation_date": "2027-02-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.003, "mode": "ocr", "supported_endpoints": [ "/v1/ocr" ], - "source": "https://ai.azure.com/catalog/models/mistral-document-ai-2512" + "source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", + "successor_model": "azure_ai/mistral-ocr-4-0" }, "azure_ai/mistral-ocr-4-0": { "litellm_provider": "azure_ai", @@ -75499,7 +75504,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75539,7 +75544,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": false, + "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -76502,6 +76507,33 @@ "supports_tool_choice": true, "supports_vision": true }, + "us-gov.xai.grok-4.7": { + "supports_regex_lookaround": false, + "supported_endpoints": ["/v1/chat/completions"], + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", + "supports_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-Fast": { "cache_read_input_token_cost": 1.4e-08, "input_cost_per_token": 6e-07, @@ -78470,7 +78502,7 @@ "cache_creation_input_token_cost": 1e-05, "cache_creation_input_token_cost_above_1hr": 1.6e-05, "cache_read_input_token_cost": 4e-07, - "deprecation_date": "2027-10-01", + "deprecation_date": "2027-01-15", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "input_cost_per_token": 8e-06, "litellm_provider": "azure_ai", @@ -78480,7 +78512,7 @@ "output_cost_per_token": 4e-05, "prompt_cache_min_tokens": 512.0, "search_context_cost_per_query": {"search_context_size_high":0.02,"search_context_size_low":0.02,"search_context_size_medium":0.02}, - "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models", + "source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -86471,5 +86503,32 @@ "supports_system_messages": false, "supports_video_input": true, "supports_vision": true + }, + "databricks/databricks-kimi-k3-1-pro": { + "cache_creation_input_token_cost": 1.99997e-06, + "cache_read_input_token_cost": 1.9999e-07, + "input_cost_per_token": 1.99997e-06, + "input_dbu_cost_per_token": 2.8571e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "metadata": {"notes":"Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference."}, + "mode": "chat", + "output_cost_per_token": 9.99999e-06, + "output_dbu_cost_per_token": 0.000142857, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "token_count_framing": 100.0 } } diff --git a/osv-scanner.toml b/osv-scanner.toml index 45fb7635088..11c2d4fbd87 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -83,11 +83,6 @@ id = "GHSA-45c3-73f7-pv4m" ignoreUntil = 2026-11-27 reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.0; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships" -[[IgnoredVulns]] -id = "GHSA-xcx6-jx8p-x3h5" -ignoreUntil = 2026-11-24 -reason = "@azure/msal-node is fixed only in 7.1.0 (published 2026-10-13), but it is dev-only in vscode-extension via @vscode/vsce > @azure/identity, and the latest @azure/identity (4.14.0) still requires msal-node ^6.0.0, so the lock cannot move without forcing a major override; the advisory is in the auth_code flow, which extension packaging does not run; bump and remove this entry once @azure/identity accepts msal-node >= 7.1.0" - [[PackageOverrides]] name = "pillow" version = "12.3.0" diff --git a/packaging/litellm-proxy/pyproject.toml b/packaging/litellm-proxy/pyproject.toml new file mode 100644 index 00000000000..172f40a7228 --- /dev/null +++ b/packaging/litellm-proxy/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "litellm-proxy" +version = "1.109.0" +description = "Opt-in Rust HTTP forwarding server for the LiteLLM Python proxy" +requires-python = ">=3.10, <3.16" +license = "MIT" + +[build-system] +requires = ["maturin==1.15.0"] +build-backend = "maturin" + +[tool.maturin] +manifest-path = "../../litellm-rust/crates/gateway/Cargo.toml" +bindings = "bin" +targets = [{ name = "litellm-gateway", kind = "bin" }] +profile = "release" +editable-profile = "dev" +include = [ + { path = "rust-toolchain.toml", format = "sdist" }, + { path = ".cargo/config.toml", format = "sdist" }, + { path = "model_prices_and_context_window.json", format = "sdist" }, +] diff --git a/pyproject.toml b/pyproject.toml index ee6cb6c1dcc..0b81618355d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -359,6 +359,7 @@ exclude = [ ] [tool.uv] +concurrent-builds = 1 cache-keys = [ { file = "pyproject.toml" }, { file = "rust-toolchain.toml" }, @@ -395,11 +396,12 @@ brotli = "2026-10-22T14:01:00Z" pypdf = "2026-10-28T17:01:00Z" [tool.uv.sources] +litellm-proxy = { workspace = true } litellm-proxy-extras = { workspace = true } litellm-enterprise = { workspace = true } [tool.uv.workspace] -members = ["enterprise", "litellm-proxy-extras"] +members = ["enterprise", "litellm-proxy-extras", "packaging/litellm-proxy"] [tool.commitizen] version = "1.109.0" diff --git a/scripts/build_core_distribution.py b/scripts/build_core_distribution.py index 41d7208d2c3..c458f6cb44f 100644 --- a/scripts/build_core_distribution.py +++ b/scripts/build_core_distribution.py @@ -1,4 +1,4 @@ -"""Build the independent core distribution from shared repository sources.""" +"""Build standalone core or gateway distributions from shared repository sources.""" import argparse import shutil @@ -19,8 +19,16 @@ SOURCES: Final = ( "LICENSE", ) +IN_TREE_RUST_PATH: Final = "../../litellm-rust/" -def stage_core_distribution(source: Path, destination: Path) -> None: + +class BuildOptions(argparse.Namespace): + package: str + out_dir: Path + sdist_only: bool + + +def stage_core_distribution(source: Path, destination: Path, *, package: str = "litellm-core") -> None: version: Final = subprocess.run( ["uv", "version", "--short"], cwd=source, check=True, capture_output=True, text=True ).stdout.strip() @@ -53,8 +61,7 @@ def stage_core_distribution(source: Path, destination: Path) -> None: )(directory, names) destination.mkdir(parents=True, exist_ok=True) - for name in SOURCES: - path: Final = source / name + for name, path in ((name, source / name) for name in SOURCES if package == "litellm-core" or name != "litellm"): if path.is_dir(): shutil.copytree( path, @@ -63,15 +70,19 @@ def stage_core_distribution(source: Path, destination: Path) -> None: ) else: shutil.copy2(path, destination / name) - shutil.copy2(source / "packaging/litellm-core/pyproject.toml", destination / "pyproject.toml") + template: Final = (source / "packaging" / package / "pyproject.toml").read_text() + if package == "litellm-proxy" and IN_TREE_RUST_PATH not in template: + raise SystemExit(f"packaging/{package}/pyproject.toml no longer points at {IN_TREE_RUST_PATH}") + metadata: Final = template.replace(IN_TREE_RUST_PATH, "litellm-rust/") + (destination / "pyproject.toml").write_text(metadata) subprocess.run(["uv", "version", version, "--frozen"], cwd=destination, check=True) -def build_core_distribution(output: Path, *, sdist_only: bool = False) -> None: +def build_core_distribution(output: Path, *, sdist_only: bool = False, package: str = "litellm-core") -> None: output_path: Final = output.resolve() with tempfile.TemporaryDirectory(prefix="litellm-core-") as temporary: stage: Final = Path(temporary) - stage_core_distribution(ROOT, stage) + stage_core_distribution(ROOT, stage, package=package) subprocess.run( ["uv", "build", "--python", sys.executable, "--out-dir", str(output_path)] + (["--sdist"] if sdist_only else []), @@ -82,12 +93,13 @@ def build_core_distribution(output: Path, *, sdist_only: bool = False) -> None: def main() -> None: parser: Final = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--package", choices=("litellm-core", "litellm-proxy"), default="litellm-core") parser.add_argument("--out-dir", type=Path, default=ROOT / "dist/core") parser.add_argument( "--sdist-only", action="store_true", help="Build only the source archive, without compiling a wheel" ) - args: Final = parser.parse_args() - build_core_distribution(Path(args.out_dir), sdist_only=args.sdist_only) + args: Final = parser.parse_args(namespace=BuildOptions()) + build_core_distribution(Path(args.out_dir), sdist_only=args.sdist_only, package=args.package) if __name__ == "__main__": diff --git a/tests/base_sdk_tests/test_core_distribution.py b/tests/base_sdk_tests/test_core_distribution.py index c22fa4e54fc..04536630200 100644 --- a/tests/base_sdk_tests/test_core_distribution.py +++ b/tests/base_sdk_tests/test_core_distribution.py @@ -20,12 +20,21 @@ else: ROOT: Final = Path(__file__).resolve().parents[2] +@pytest.mark.parametrize( + ("package_cli_args", "package"), + [ + (("--package", "litellm-proxy"), "litellm-proxy"), + ((), "litellm-core"), + ], +) @pytest.mark.parametrize("sdist_only", [False, True]) -def test_build_command_selects_requested_distributions(tmp_path: Path, sdist_only: bool) -> None: +def test_build_command_selects_requested_distributions( + tmp_path: Path, sdist_only: bool, package_cli_args: tuple[str, ...], package: str +) -> None: from scripts.build_core_distribution import main output: Final = tmp_path / "dist" - arguments: Final = ["build_core_distribution.py", "--out-dir", str(output)] + ( + arguments: Final = ["build_core_distribution.py", *package_cli_args, "--out-dir", str(output)] + ( ["--sdist-only"] if sdist_only else [] ) with ( @@ -42,6 +51,7 @@ def test_build_command_selects_requested_distributions(tmp_path: Path, sdist_onl assert command[command.index("--out-dir") + 1] == str(output) assert run.call_args.kwargs["check"] is True assert run.call_args.kwargs["cwd"] == stage.call_args.args[1] + assert stage.call_args.kwargs["package"] == package assert not stage.call_args.args[1].exists() @@ -53,7 +63,10 @@ def source_repository(tmp_path: Path) -> Path: return source -def test_staging_stamps_release_version_without_modifying_sources(tmp_path: Path, source_repository: Path) -> None: +@pytest.mark.parametrize("package", ["litellm-core", "litellm-proxy"]) +def test_staging_stamps_release_version_without_modifying_sources( + tmp_path: Path, source_repository: Path, package: str +) -> None: from scripts.build_core_distribution import SOURCES, stage_core_distribution source: Final = source_repository @@ -61,16 +74,44 @@ def test_staging_stamps_release_version_without_modifying_sources(tmp_path: Path path: Final = source / name path.write_text(f"shared {name}") (source / "pyproject.toml").write_text('[project]\nname = "litellm"\nversion = "9.8.7rc1"\n') - manifest: Final = source / "packaging/litellm-core/pyproject.toml" + manifest: Final = source / "packaging" / package / "pyproject.toml" manifest.parent.mkdir(parents=True) - manifest.write_bytes((ROOT / "packaging/litellm-core/pyproject.toml").read_bytes()) + manifest.write_bytes((ROOT / "packaging" / package / "pyproject.toml").read_bytes()) original: Final = manifest.read_bytes() stage: Final = tmp_path / "stage" - stage_core_distribution(source, stage) - assert tomllib.loads((stage / "pyproject.toml").read_text())["project"]["version"] == "9.8.7rc1" + stage_core_distribution(source, stage, package=package) + staged_project: Final = tomllib.loads((stage / "pyproject.toml").read_text()) + assert staged_project["project"]["name"] == package + assert staged_project["project"]["version"] == "9.8.7rc1" + if package == "litellm-proxy": + assert staged_project["tool"]["maturin"]["targets"] == [{"name": "litellm-gateway", "kind": "bin"}] + assert not (stage / "litellm").exists() + assert all( + (stage / name).exists() + for name in ("litellm-rust", ".cargo", "rust-toolchain.toml", "model_prices_and_context_window.json") + ) assert manifest.read_bytes() == original assert (source / "pyproject.toml").read_text() == '[project]\nname = "litellm"\nversion = "9.8.7rc1"\n' - assert all((stage / name).read_bytes() == (source / name).read_bytes() for name in SOURCES) + expected: Final = tuple(name for name in SOURCES if package == "litellm-core" or name != "litellm") + assert all((stage / name).read_bytes() == (source / name).read_bytes() for name in expected) + + +def test_gateway_staging_refuses_manifest_without_in_tree_rust_path(tmp_path: Path, source_repository: Path) -> None: + from scripts.build_core_distribution import SOURCES, stage_core_distribution + + source: Final = source_repository + for name in SOURCES: + (source / name).write_text(f"shared {name}") + (source / "pyproject.toml").write_text('[project]\nname = "litellm"\nversion = "9.8.7"\n') + manifest: Final = source / "packaging" / "litellm-proxy" / "pyproject.toml" + manifest.parent.mkdir(parents=True) + manifest.write_text( + (ROOT / "packaging" / "litellm-proxy" / "pyproject.toml") + .read_text() + .replace("../../litellm-rust/", "../../rust/") + ) + with pytest.raises(SystemExit, match="no longer points at"): + stage_core_distribution(source, tmp_path / "stage", package="litellm-proxy") @pytest.fixture(scope="module") @@ -99,7 +140,9 @@ def test_core_wheel_metadata_and_resources(distributions: tuple[Path, Path]) -> assert metadata["Version"] == tomllib.loads((ROOT / "pyproject.toml").read_text())["project"]["version"] assert not metadata.get_all("Provides-Extra") assert not any(n.endswith(".dist-info/entry_points.txt") for n in names) - requirements: Final = tomllib.loads((ROOT / "packaging/litellm-core/pyproject.toml").read_text())["project"]["dependencies"] + requirements: Final = tomllib.loads((ROOT / "packaging/litellm-core/pyproject.toml").read_text())["project"][ + "dependencies" + ] for python_version in ("3.10", "3.11", "3.12", "3.13", "3.14"): environment: Final = {"python_version": python_version, "python_full_version": python_version + ".0"} assert { diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index cd951c6cd28..d89badfb33e 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -89,6 +89,7 @@ importlib-metadata: >=6.8.0 # Apache 2.0 License tokenizers: >=0.20.2 # Apache 2.0 License jinja2: >=3.1.4 # BSD 3-Clause License litellm-proxy-extras: >=0.1.1 # MIT License +litellm-proxy: >=1.109.0 # MIT, verified in repository LICENSE for litellm-rust and packaging/litellm-proxy litellm-enterprise: >=0.1.1 # LiteLLM Enterprise License a2a-sdk: >=0.3.22 # Apache 2.0 license anyio: >=4.5.0 # Unknown license diff --git a/tests/integration/_support/device_login.py b/tests/integration/_support/device_login.py index 55519678f40..13a7169a01a 100644 --- a/tests/integration/_support/device_login.py +++ b/tests/integration/_support/device_login.py @@ -25,6 +25,7 @@ from pydantic import JsonValue CHATGPT_AUTH_HOST: Final = "auth.openai.com" GITHUB_HOST: Final = "github.com" GITHUB_API_HOST: Final = "api.github.com" +COPILOT_API_HOST: Final = "api.githubcopilot.com" AUTH_HOSTS: Final = (CHATGPT_AUTH_HOST, GITHUB_HOST, GITHUB_API_HOST) CHATGPT_ACCOUNT: Final = "acct-device-login" _FAR_FUTURE: Final = 4102444800 @@ -79,7 +80,17 @@ COPILOT_MINTED_KEY: Final = "tid=device-login;minted" COPILOT_REJECTED_KEY: Final = "tid=device-login;rejected" CONTROL_KEY: Final = "device-login-control-key" ACCEPTED_BEARERS: Final = frozenset( - {CHATGPT_STORED, CHATGPT_REFRESHED, CHATGPT_FIRST_LOGIN, COPILOT_STORED_KEY, COPILOT_MINTED_KEY, CONTROL_KEY} + { + CHATGPT_STORED, + CHATGPT_REFRESHED, + CHATGPT_FIRST_LOGIN, + COPILOT_ACCESS, + COPILOT_FIRST_LOGIN_ACCESS, + COPILOT_UNSAVED_KEY_ACCESS, + COPILOT_STORED_KEY, + COPILOT_MINTED_KEY, + CONTROL_KEY, + } ) REJECTED_DETAIL: Final = "the scripted provider rejected this bearer token" _DEVICE_AUTH_ID: Final = "device-auth-synthetic" @@ -180,6 +191,10 @@ def _auth_hosts(api_url: str, grant: threading.Event) -> Callable[[Request], Rep return _json(200, {"access_token": COPILOT_FIRST_LOGIN_ACCESS, "token_type": "bearer"}) case ("api.github.com", "GET", "/copilot_internal/v2/token"): return _copilot_api_key(request, api_url) + case ("api.github.com", "GET", "/copilot_internal/user"): + if request.headers.get("authorization") == f"token {COPILOT_REJECTED_ACCESS}": + return _json(401, {"message": "Bad credentials"}) + return _json(200, {"endpoints": {"api": f"https://{COPILOT_API_HOST}"}}) case _: return _switched_off() @@ -204,9 +219,22 @@ class Switches: switch.clear() +@dataclass(frozen=True, slots=True) +class _ProviderAPI: + plain: Wire + secure: Wire + + @property + def url(self) -> str: + return self.plain.url + + def drain(self) -> tuple[Request, ...]: + return (*self.plain.drain(), *self.secure.drain()) + + @dataclass(frozen=True, slots=True) class Peers: - api: Wire + api: _ProviderAPI auth: Wire tunnel: str cert: Path @@ -228,6 +256,14 @@ class Peers: def auth_connections(self) -> tuple[str, ...]: return tuple(self.authorities.get_nowait() for _ in range(self.authorities.qsize())) + def assert_only_copilot_discovery(self, access_token: str) -> None: + assert set(self.auth_connections()) <= {f"{GITHUB_API_HOST}:443"} + requests: Final = self.auth.drain() + assert { + (request.method, request.target, request.headers.get("host"), request.headers.get("authorization")) + for request in requests + } <= {("GET", "/copilot_internal/user", GITHUB_API_HOST, f"token {access_token}")}, requests + def dropped(self) -> tuple[Hangup, ...]: return tuple(self.hangups.get_nowait() for _ in range(self.hangups.qsize())) @@ -279,9 +315,8 @@ class _TunnelServer(socketserver.ThreadingTCPServer): @contextmanager def _tunnel( - destination: Wire, switches: Switches, authorities: SimpleQueue[str], hangups: SimpleQueue[Hangup] + destination: Wire, provider: Wire, switches: Switches, authorities: SimpleQueue[str], hangups: SimpleQueue[Hangup] ) -> Generator[str]: - destination_port: Final = int(destination.url.rsplit(":", 1)[1]) class Tunnel(socketserver.StreamRequestHandler): rbufsize = 0 @@ -306,20 +341,24 @@ def _tunnel( if len(request_line) < 2: return authority: Final = request_line[1] - if authority.rsplit(":", 1)[0] not in AUTH_HOSTS: + host: Final = authority.rsplit(":", 1)[0] + if host not in (*AUTH_HOSTS, COPILOT_API_HOST): self.wfile.write(b"HTTP/1.1 403 Forbidden\r\ncontent-length: 0\r\n\r\n") return - authorities.put(authority) - if switches.refuse_connect.is_set(): + is_auth: Final = host in AUTH_HOSTS + if is_auth: + authorities.put(authority) + if is_auth and switches.refuse_connect.is_set(): self.wfile.write(b"HTTP/1.1 502 Bad Gateway\r\ncontent-length: 0\r\n\r\n") return - if self.hold(authority, opened, switches.hold_connect) is None: + if is_auth and self.hold(authority, opened, switches.hold_connect) is None: return self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") - client_hello: Final = self.hold(authority, opened, switches.stall_tls) + client_hello: Final = self.hold(authority, opened, switches.stall_tls) if is_auth else b"" if client_hello is None: return self.request.settimeout(10) + destination_port: Final = int((destination if is_auth else provider).url.rsplit(":", 1)[1]) with socket.create_connection(("127.0.0.1", destination_port), timeout=10) as upstream: upstream.sendall(client_hello) outbound: Final = threading.Thread(target=_pipe, args=(self.request, upstream)) @@ -339,17 +378,21 @@ def _tunnel( @contextmanager -def device_login_peers(directory: Path) -> Generator[Peers]: - cert, key = write_self_signed_cert(directory, AUTH_HOSTS) +def device_login_peers( + directory: Path, provider: Callable[[Request], Reply] | None = None +) -> Generator[Peers]: + cert, key = write_self_signed_cert(directory, (*AUTH_HOSTS, COPILOT_API_HOST)) switches: Final = Switches(threading.Event(), threading.Event(), threading.Event(), threading.Event()) authorities: Final[SimpleQueue[str]] = SimpleQueue() hangups: Final[SimpleQueue[Hangup]] = SimpleQueue() + respond: Final = provider if provider is not None else _provider_api(rv.ResponsesVendor()) with ( - wire_server(_provider_api(rv.ResponsesVendor())) as api, + wire_server(respond) as api, + wire_server(respond, tls=server_context(cert, key)) as copilot_api, wire_server(_auth_hosts(api.url, switches.grant), tls=server_context(cert, key)) as auth, - _tunnel(auth, switches, authorities, hangups) as tunnel, + _tunnel(auth, copilot_api, switches, authorities, hangups) as tunnel, ): - yield Peers(api, auth, tunnel, cert, authorities, hangups, switches) + yield Peers(_ProviderAPI(api, copilot_api), auth, tunnel, cert, authorities, hangups, switches) def chatgpt_record( diff --git a/tests/integration/authorization/test_responses_lifecycle_during_call_guardrail.py b/tests/integration/authorization/test_responses_lifecycle_during_call_guardrail.py new file mode 100644 index 00000000000..3227caaa7f5 --- /dev/null +++ b/tests/integration/authorization/test_responses_lifecycle_during_call_guardrail.py @@ -0,0 +1,114 @@ +"""GET, compact, DELETE and cancel on /v1/responses answer 200 when a during_call guardrail is on by default +(LIT-9770): the hook used to raise `'acompact_responses' is not a valid CallTypes` and the proxy returned 500""" + +import json +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, string_value +from integration._support.process import owned_proxy +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.shares_worker_state + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MODEL: Final = "openai/gpt-5.4-mini" + + +def _input() -> list[JsonValue]: + return [ + {"role": "user", "content": f"Remember the number {uuid.uuid4().hex}."}, + {"role": "assistant", "content": "Got it."}, + ] + + +def _response(response_id: str) -> Reply: + body: Final[dict[str, JsonValue]] = { + "id": response_id, + "object": "response", + "created_at": 1, + "status": "completed", + "model": _MODEL.removeprefix("openai/"), + "output": [ + { + "type": "message", + "id": f"msg_{uuid.uuid4().hex}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Got it.", "annotations": []}], + } + ], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 12, "output_tokens": 5, "total_tokens": 17}, + } + return Reply(body=json.dumps(body).encode()) + + +def _deleted(response_id: str) -> Reply: + return Reply(body=json.dumps({"id": response_id, "object": "response.deleted", "deleted": True}).encode()) + + +def _guarded_config(path: Path) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + guardrail: Final[dict[str, JsonValue]] = { + "guardrail_name": f"content-filter-{uuid.uuid4().hex}", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "during_call", + "default_on": True, + "blocked_words": [{"keyword": "forbiddenword", "action": "BLOCK"}], + }, + } + path.write_text(yaml.safe_dump({**base, "guardrails": [guardrail]})) + return path + + +@pytest.fixture(scope="module") +def guarded(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("during-call-guardrail") + with owned_proxy(session_gateway, directory, {}, config=_guarded_config(directory / "config.yaml")) as candidate: + yield candidate + + +def _create(guarded: Gateway, provider: SharedProvider, upstream_id: str) -> str: + provider.expect(_response(upstream_id)) + created: Final = guarded.request("POST", "/v1/responses", {"model": _MODEL, "input": _input(), "store": True}) + assert created.status_code == 200, created.text + (upstream,) = provider.received() + assert upstream.target.endswith("/responses"), upstream.target + return string_value(_JSON_OBJECT.validate_json(created.content)["id"]) + + +def _lifecycle(guarded: Gateway, provider: SharedProvider, operation: str) -> tuple[httpx.Response, str]: + if operation == "compact": + provider.expect(_response(f"resp_{uuid.uuid4().hex}")) + return guarded.request( + "POST", "/v1/responses/compact", {"model": _MODEL, "input": _input()} + ), "/responses/compact" + upstream_id: Final = f"resp_{uuid.uuid4().hex}" + managed_id: Final = _create(guarded, provider, upstream_id) + provider.expect(_deleted(upstream_id) if operation == "delete" else _response(upstream_id)) + if operation == "get": + return guarded.request("GET", f"/v1/responses/{managed_id}"), f"/responses/{upstream_id}" + if operation == "delete": + return guarded.request("DELETE", f"/v1/responses/{managed_id}"), f"/responses/{upstream_id}" + return guarded.request("POST", f"/v1/responses/{managed_id}/cancel"), f"/responses/{upstream_id}/cancel" + + +@pytest.mark.parametrize("operation", ["get", "compact", "delete", "cancel"]) +def test_responses_lifecycle_answers_200_with_a_during_call_guardrail_on( + guarded: Gateway, provider: SharedProvider, operation: str +) -> None: + response, upstream_target = _lifecycle(guarded, provider, operation) + assert response.status_code == 200, response.text + (upstream,) = provider.received() + assert upstream.target.endswith(upstream_target), upstream.target diff --git a/tests/integration/database/test_skill_submissions.py b/tests/integration/database/test_skill_submissions.py index cc00bf6e359..e55e19fb110 100644 --- a/tests/integration/database/test_skill_submissions.py +++ b/tests/integration/database/test_skill_submissions.py @@ -134,6 +134,8 @@ async def test_approval_commits_version_archive_history_and_review(pending_updat assert current is not None published_manifest: Final = { **case.proposal.manifest, + "_litellm_review": {"approved_by": "integration-reviewer", "security_scan_passed": False}, + "_litellm_published_by": "integration-reviewer", "source": { **TypeAdapter(dict[str, str]).validate_python(case.submission.proposal.manifest["source"]), "url": f"https://skills.example.test/claude-code/plugins/{case.original.name}/archive/" @@ -148,17 +150,22 @@ async def test_approval_commits_version_archive_history_and_review(pending_updat ) metadata: Final = await case.publications.get_metadata(case.original.name) assert metadata is not None + assert metadata.created_by == before.created_by == case.original.created_by + assert metadata.created_at == before.created_at + assert metadata.updated_at is not None and before.updated_at is not None + assert metadata.updated_at > before.updated_at assert await case.publications.get_content(metadata) == case.proposal.content history: Final = await case.publications.versions(metadata, 10, 0) assert tuple((version.version, version.description, version.archive_available) for version in history) == ( ("1.0.1", case.proposal.description, True), ("1.0.0", case.original.description, True), ) + assert history[0].published_by == "integration-reviewer" + assert history[1].published_by == case.original.created_by for version, publication, manifest in zip( - history, (case.proposal, case.original), (published_manifest, case.original.manifest), strict=True + history, (case.proposal, case.original), (published_manifest, {**case.original.manifest, "_litellm_published_by": case.original.created_by}), strict=True ): assert version.manifest == {**manifest, "version": version.version} - assert version.published_by == publication.created_by assert await case.publications.version_content(metadata, version.id) == publication.content diff --git a/tests/integration/mcp/test_mcp_llm_endpoints.py b/tests/integration/mcp/test_mcp_llm_endpoints.py index f6743a8751a..0c97ae62d07 100644 --- a/tests/integration/mcp/test_mcp_llm_endpoints.py +++ b/tests/integration/mcp/test_mcp_llm_endpoints.py @@ -42,7 +42,6 @@ from openai.types.chat import ChatCompletionMessageParam from openai.types.responses.tool_param import Mcp from pydantic import JsonValue, TypeAdapter - pytestmark: Final = [pytest.mark.xdist_group("mcp_llm_private_gateway"), pytest.mark.shares_worker_state] @@ -589,7 +588,11 @@ def _upstream_model(surface: Surface) -> str: }[surface] -PRICES: Final[Mapping[str, JsonValue]] = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "num_retries": 0} +PRICES: Final[Mapping[str, JsonValue]] = { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "num_retries": 0, +} @contextmanager @@ -917,7 +920,9 @@ def test_an_open_server_in_another_gateway_cannot_widen_this_gateways_tools(gate register_mcp(scenario, local, "open" + uuid.uuid4().hex[:8], allow_all_keys=True) listed, called = official_client_outcomes(shared, key, "/mcp", f"{foreign_alias}-add", ADD) assert set(listed.tools) == { - f"{foreign_alias}-add", f"{foreign_alias}-multiply", f"{foreign_alias}-fail" + f"{foreign_alias}-add", + f"{foreign_alias}-multiply", + f"{foreign_alias}-fail", }, listed.tools assert called.ok and called.text == str(ADD["a"] + ADD["b"]), called assert [call["body"]["params"]["name"] for call in _peer_add_calls(foreign)] == ["add"] @@ -1528,6 +1533,7 @@ def test_streaming_responses_blocked_by_a_post_call_guardrail_bills_every_round_ one_round_rows: Final = _model_rows( one_round.headers["x-litellm-call-id"], _logs_a_failure_billing(_price(FIRST_ROUND) + FIXED_MARGIN) ) + assert _statuses(one_round_rows) == ["failure"], one_round_rows rig.wire.drain() response: Final = rig.send(key, [AUTO], stream=True, guardrails=[STREAM_BLOCK]) assert STREAM_BLOCK_DETAIL in response.text, response.text diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py index a1c5f6964bf..c9e78b1828e 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_safeguards_beta_wire.py @@ -15,6 +15,7 @@ import openai import pytest from integration._support.client import Gateway, eventually, gateway_from_environment, object_value from integration._support.database import read_rows +from integration._support.device_login import device_login_peers, write_copilot_access from integration._support.process import owned_proxy from integration._support.vertex import service_account_json from integration._support.wire import Reply, Request, Wire, wire_server @@ -665,18 +666,18 @@ def test_github_copilot_messages_forward_safeguards_without_deriving_the_beta( marker: Final = uuid.uuid4().hex token_dir: Final = tmp_path / "copilot" token_dir.mkdir() - with wire_server(anthropic_peer) as wire: - (token_dir / "api-key.json").write_text( - json.dumps({"token": "synthetic-copilot-token", "expires_at": 4102444800, "endpoints": {"api": wire.url}}) - ) + with device_login_peers(tmp_path, provider=anthropic_peer) as peers: + write_copilot_access(token_dir, "synthetic-copilot-token") with ( - owned_proxy(gateway, tmp_path, {"GITHUB_COPILOT_TOKEN_DIR": str(token_dir)}) as owned, + owned_proxy( + gateway, tmp_path, {**peers.environment(), "GITHUB_COPILOT_TOKEN_DIR": str(token_dir)} + ) as owned, owned.scenario() as scenario, ): - model: Final = scenario.model(model="github_copilot/claude-sonnet-4-5", api_base=wire.url) + model: Final = scenario.model(model="github_copilot/claude-sonnet-4-5", api_base=peers.api.url) response: Final = _post(owned, messages_body(model, marker), beta=_CLIENT_BETAS_WITH_DANGEROUS) _assert_answered(response, marker) - observed: Final = _only(wire) + observed: Final = _only(peers.api.secure) _expect_messages( observed, target="/v1/messages", auth=("authorization", "Bearer synthetic-copilot-token"), beta=None ) diff --git a/tests/integration/providers/test_device_code_login_guard_boot.py b/tests/integration/providers/test_device_code_login_guard_boot.py index 91f502179f3..384d6635bb0 100644 --- a/tests/integration/providers/test_device_code_login_guard_boot.py +++ b/tests/integration/providers/test_device_code_login_guard_boot.py @@ -67,9 +67,9 @@ class _TokenDirs: def environment(self) -> Mapping[str, str]: return MappingProxyType({"CHATGPT_TOKEN_DIR": str(self.chatgpt), "GITHUB_COPILOT_TOKEN_DIR": str(self.copilot)}) - def store_valid_tokens(self, api_url: str) -> None: + def store_valid_tokens(self) -> None: dl.write_chatgpt(self.chatgpt, dl.chatgpt_record(dl.CHATGPT_STORED)) - dl.write_copilot_key(self.copilot, dl.COPILOT_STORED_KEY, api_url) + dl.write_copilot_access(self.copilot, dl.COPILOT_ACCESS) def _token_dirs(directory: Path) -> _TokenDirs: @@ -242,16 +242,16 @@ def test_booting_without_tokens_lists_login_deployments_and_serves_only_after_to assert peers.auth_connections() == () assert dl.bearer(peers.api.drain()[-1]) == dl.CONTROL_KEY - dirs.store_valid_tokens(peers.api.url) + dirs.store_valid_tokens() _reload(owned) _every_worker_lists(owned, workers) for served_logins, name in enumerate(_LOGIN_NAMES, start=1): served: Final = _chat(_base_url(owned), key, name, uuid.uuid4().hex) assert served.status == 200, served.text _successes_logged(key, at_least=_CONTROL_CALLS_BEFORE_THE_RELOAD + served_logins) - assert peers.auth_connections() == () + peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) forwarded: Final = peers.api.drain() - assert {dl.bearer(request) for request in forwarded} == {dl.CHATGPT_STORED, dl.COPILOT_STORED_KEY} + assert {dl.bearer(request) for request in forwarded} == {dl.CHATGPT_STORED, dl.COPILOT_ACCESS} dl.write_chatgpt(dirs.chatgpt, dl.chatgpt_record(dl.CHATGPT_EXPIRED, refresh_token=dl.GOOD_REFRESH)) peers.switches.hold_connect.set() @@ -287,7 +287,7 @@ def test_a_token_removed_after_boot_turns_every_request_into_the_refusal( ) -> None: dirs: Final = _token_dirs(tmp_path) with dl.device_login_peers(tmp_path) as peers: - dirs.store_valid_tokens(peers.api.url) + dirs.store_valid_tokens() with owned_proxy_process( gateway, tmp_path, @@ -310,7 +310,7 @@ def test_a_token_removed_after_boot_turns_every_request_into_the_refusal( refused: Final = _chat(_base_url(owned), key, name, uuid.uuid4().hex) assert refused.status == 400, refused.text assert refusal in refused.message(), refused.text - assert peers.auth_connections() == () + peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) tail: Final = owned.log.read_text() assert dl.CHATGPT_USER_CODE not in tail and dl.COPILOT_USER_CODE not in tail, tail[-3000:] control: Final = _chat(_base_url(owned), key, _CONTROL, uuid.uuid4().hex) diff --git a/tests/integration/providers/test_device_code_login_guard_wire.py b/tests/integration/providers/test_device_code_login_guard_wire.py index 39e36fd95fa..9074ea83189 100644 --- a/tests/integration/providers/test_device_code_login_guard_wire.py +++ b/tests/integration/providers/test_device_code_login_guard_wire.py @@ -2,7 +2,6 @@ import asyncio import hashlib import itertools import json -import re import threading import time import uuid @@ -26,9 +25,10 @@ from integration._support.database import read_rows from integration._support.process import OwnedProxy, owned_proxy_process from integration._support.timing import assert_cut_short, timed from integration._support.wire import Request -from litellm.llms.chatgpt.authenticator import DEVICE_CODE_COOLDOWN_SECONDS from pydantic import JsonValue +from litellm.llms.chatgpt.authenticator import DEVICE_CODE_COOLDOWN_SECONDS + pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state] Provider: TypeAlias = Literal["chatgpt", "copilot"] @@ -52,7 +52,7 @@ _REFUSALS: Final[Mapping[Provider, str]] = MappingProxyType( {"chatgpt": dl.CHATGPT_REFUSAL, "copilot": dl.COPILOT_REFUSAL} ) _STORED_BEARERS: Final[Mapping[Provider, str]] = MappingProxyType( - {"chatgpt": dl.CHATGPT_STORED, "copilot": dl.COPILOT_STORED_KEY} + {"chatgpt": dl.CHATGPT_STORED, "copilot": dl.COPILOT_ACCESS} ) _CONTROL_MODEL: Final = "device-login-control" _EXTRA: Final[Mapping[str, JsonValue]] = MappingProxyType({"num_retries": 0, "cache": {"no-cache": True}}) @@ -154,7 +154,7 @@ class _Scene: if provider == "chatgpt": dl.write_chatgpt(self.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_STORED)) return - dl.write_copilot_key(self.rig.copilot_dir, dl.COPILOT_STORED_KEY, self.rig.peers.api.url) + dl.write_copilot_access(self.rig.copilot_dir, dl.COPILOT_ACCESS) def _config(peers: dl.Peers, directory: Path) -> Path: @@ -403,7 +403,7 @@ def test_missing_token_is_refused_with_the_login_instructions_before_any_auth_ho @pytest.mark.parametrize("cell", _CELLS, ids=_Cell.name) -def test_stored_token_serves_the_request_without_an_auth_host_call(scene: _Scene, cell: _Cell) -> None: +def test_stored_token_serves_the_request_without_a_device_login(scene: _Scene, cell: _Cell) -> None: scene.store_valid_token(cell.provider) call: Final = scene.call(cell) served: Final = _serve(call) @@ -412,7 +412,10 @@ def test_stored_token_serves_the_request_without_an_auth_host_call(scene: _Scene received: Final = scene.rig.peers.api.drain() (forwarded,) = _carrying(received, call.marker) assert dl.bearer(forwarded) == _STORED_BEARERS[cell.provider], _describe(received) - assert scene.rig.peers.auth_connections() == () + if cell.provider == "copilot": + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) + else: + assert scene.rig.peers.auth_connections() == () (row,) = _spend_rows(scene.key) assert row["status"] == "success", row @@ -487,7 +490,7 @@ def test_missing_token_is_refused_on_the_other_model_routes(scene: _Scene, route assert _describe(scene.rig.peers.api.drain()) == () -def test_copilot_embeddings_are_served_with_the_stored_api_key(scene: _Scene) -> None: +def test_copilot_embeddings_are_served_with_the_stored_access_token(scene: _Scene) -> None: scene.store_valid_token("copilot") served: Final = scene.post( "/v1/embeddings", {"model": "github_copilot/text-embedding-3-small", "input": "login", **_EXTRA} @@ -496,9 +499,9 @@ def test_copilot_embeddings_are_served_with_the_stored_api_key(scene: _Scene) -> (item,) = rv.ITEMS.validate_python(rv.JSON_OBJECT.validate_json(served.text)["data"]) assert item["embedding"] == [0.25, 0.5, 0.75], served.text (forwarded,) = scene.rig.peers.api.drain() - assert (forwarded.method, dl.bearer(forwarded)) == ("POST", dl.COPILOT_STORED_KEY), forwarded.target + assert (forwarded.method, dl.bearer(forwarded)) == ("POST", dl.COPILOT_ACCESS), forwarded.target assert forwarded.target.endswith("/embeddings"), forwarded.target - assert scene.rig.peers.auth_connections() == () + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) @pytest.mark.parametrize("provider", _PROVIDERS) @@ -562,71 +565,67 @@ def test_stored_token_the_provider_rejects_surfaces_its_401_without_a_login( if provider == "chatgpt": dl.write_chatgpt(scene.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_REJECTED)) else: - dl.write_copilot_key(scene.rig.copilot_dir, dl.COPILOT_REJECTED_KEY, scene.rig.peers.api.url) + dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_REJECTED_ACCESS) call: Final = scene.call(_Cell(provider, endpoint, False, "httpx")) served: Final = _serve(call) assert served.status == 401, served.text assert dl.REJECTED_DETAIL in served.text, served.text received: Final = scene.rig.peers.api.drain() assert {dl.bearer(request) for request in _carrying(received, call.marker)} == { - dl.CHATGPT_REJECTED if provider == "chatgpt" else dl.COPILOT_REJECTED_KEY + dl.CHATGPT_REJECTED if provider == "chatgpt" else dl.COPILOT_REJECTED_ACCESS }, _describe(received) - assert scene.rig.peers.auth_connections() == () + if provider == "copilot": + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_REJECTED_ACCESS) + else: + assert scene.rig.peers.auth_connections() == () @pytest.mark.parametrize("stale_key", (False, True), ids=("missing-api-key", "expired-api-key")) -def test_copilot_api_key_is_minted_once_from_the_stored_access_token(scene: _Scene, stale_key: bool) -> None: +def test_copilot_access_token_is_forwarded_without_minting_an_api_key(scene: _Scene, stale_key: bool) -> None: dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_ACCESS) if stale_key: dl.write_copilot_key(scene.rig.copilot_dir, dl.COPILOT_REJECTED_KEY, scene.rig.peers.api.url, -3600) + legacy_key: Final = scene.rig.copilot_dir / "api-key.json" + stored_key: Final = legacy_key.read_bytes() if stale_key else None call: Final = scene.call(_Cell("copilot", "chat", False, "httpx")) served: Final = _serve(call) assert served.status == 200, served.text assert set(rv.MARKER.findall(served.text)) == {call.marker}, served.text - (minted,) = scene.rig.peers.auth.drain() - assert (minted.method, minted.target, minted.headers.get("host"), minted.headers.get("authorization")) == ( - "GET", - "/copilot_internal/v2/token", - dl.GITHUB_API_HOST, - f"token {dl.COPILOT_ACCESS}", - ) + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) (forwarded,) = _carrying(scene.rig.peers.api.drain(), call.marker) - assert dl.bearer(forwarded) == dl.COPILOT_MINTED_KEY - assert json.loads((scene.rig.copilot_dir / "api-key.json").read_text())["token"] == dl.COPILOT_MINTED_KEY + assert dl.bearer(forwarded) == dl.COPILOT_ACCESS + assert (legacy_key.read_bytes() if legacy_key.exists() else None) == stored_key -def test_copilot_api_key_that_cannot_be_saved_is_minted_at_most_once_per_worker(scene: _Scene) -> None: +def test_copilot_access_token_serves_when_the_legacy_api_key_path_is_a_directory(scene: _Scene) -> None: dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_UNSAVED_KEY_ACCESS) unwritable_key_file: Final = scene.rig.copilot_dir / "api-key.json" unwritable_key_file.mkdir() first: Final = scene.call(_Cell("copilot", "chat", False, "httpx")) first_served: Final = _serve(first) assert first_served.status == 200, first_served.text - (minted,) = scene.rig.peers.auth.drain() - assert minted.headers.get("authorization") == f"token {dl.COPILOT_UNSAVED_KEY_ACCESS}" later: Final = tuple(scene.call(_Cell("copilot", "chat", False, "httpx")) for _ in range(3)) later_served: Final = tuple(_serve(call) for call in later) assert [served.status for served in later_served] == [200, 200, 200], [served.text for served in later_served] - assert len(scene.rig.peers.auth.drain()) <= 1 + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_UNSAVED_KEY_ACCESS) received: Final = scene.rig.peers.api.drain() forwarded: Final = tuple(_carrying(received, call.marker) for call in (first, *later)) assert [tuple(dl.bearer(request) for request in requests) for requests in forwarded] == [ - (dl.COPILOT_MINTED_KEY,) + (dl.COPILOT_UNSAVED_KEY_ACCESS,) ] * 4, _describe(received) assert unwritable_key_file.is_dir() def test_copilot_access_token_github_rejects_fails_the_request_without_a_device_login(scene: _Scene) -> None: dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_REJECTED_ACCESS) - served: Final = _serve(scene.call(_Cell("copilot", "chat", False, "httpx"))) - assert served.status == 400, served.text - assert "Failed to refresh API key" in _error_message(served.text), served.text + call: Final = scene.call(_Cell("copilot", "chat", False, "httpx")) + served: Final = _serve(call) + assert served.status == 401, served.text + assert dl.REJECTED_DETAIL in _error_message(served.text), served.text _failure_logged(served) - reached: Final = scene.rig.peers.auth.drain() - assert {(request.method, request.target) for request in reached} == {("GET", "/copilot_internal/v2/token")}, ( - _describe(reached) - ) - assert _describe(scene.rig.peers.api.drain()) == () + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_REJECTED_ACCESS) + (forwarded,) = _carrying(scene.rig.peers.api.drain(), call.marker) + assert (forwarded.method, dl.bearer(forwarded)) == ("POST", dl.COPILOT_REJECTED_ACCESS) def test_expired_copilot_api_key_with_no_access_token_is_refused_before_any_auth_host_call(scene: _Scene) -> None: @@ -727,7 +726,10 @@ def test_a_token_mounted_after_a_refusal_serves_without_a_restart_and_removing_i after: Final = _serve(scene.call(cell)) assert after.status == 400, after.text assert _REFUSALS[provider] in _error_message(after.text), after.text - assert scene.rig.peers.auth_connections() == () + if provider == "copilot": + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) + else: + assert scene.rig.peers.auth_connections() == () _COOLDOWN_SECONDS_LEFT: Final = 45 diff --git a/tests/integration/providers/test_gemini_interactions_wire.py b/tests/integration/providers/test_gemini_interactions_wire.py index 58a7ccb31c3..ded0718bd2b 100644 --- a/tests/integration/providers/test_gemini_interactions_wire.py +++ b/tests/integration/providers/test_gemini_interactions_wire.py @@ -1,5 +1,4 @@ import json -import uuid from pathlib import Path from typing import Final @@ -147,6 +146,7 @@ def test_interaction_forwards_native_body_to_gemini(gateway: Gateway, prefix: st "agent": None, "updated": None, "outputs": None, + "continuation_token": None, }, response.text assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions")], response.text @@ -179,6 +179,7 @@ def test_interaction_authenticates_google_key_header(gateway: Gateway, prefix: s "agent": None, "updated": None, "outputs": None, + "continuation_token": None, }, response.text assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions")], response.text @@ -218,6 +219,7 @@ def test_agent_interaction_uses_env_gemini_credentials(gateway: Gateway, tmp_pat "agent": "deep-research-pro-preview-12-2025", "updated": None, "outputs": None, + "continuation_token": None, }, response.text assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions")], response.text @@ -277,3 +279,78 @@ def test_stream_interaction_ends_without_openai_done_terminator(gateway: Gateway expected: Final = b"".join(_proxy_stream_event(payload, model) for _event_type, payload in _STREAM_EVENTS) assert response.content == expected, response.text assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions?alt=sse")] + + +_LONG_DECODE_MODEL: Final = "gemini/gemini-4-argon" +_LONG_DECODE_BODY: Final = { + "input": "prove the prime number theorem", + "generation_config": {"max_output_tokens": 120000, "thinking_level": "high"}, +} + + +def _hop(hop_id: str, status: str, text: str, token: str | None, input_tokens: int, output_tokens: int) -> dict: + return { + "id": hop_id, + "object": "interaction", + "model": "gemini-4-argon", + "status": status, + "created": "2026-11-05T00:00:00Z", + "steps": [{"type": "model_output", "content": [{"type": "text", "text": text}]}], + "usage": { + "total_input_tokens": input_tokens, + "total_output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + "initial_input_tokens": 79, + }, + "continuation_token": token, + } + + +_FIRST_HOP: Final = _hop("interaction-hop-1", "requires_continuation", "Lemma 1.", "tok-1", 79, 32768) +_SECOND_HOP: Final = _hop("interaction-hop-2", "completed", " QED.", None, 32847, 1533) + + +def test_interaction_keeps_requires_continuation_status_and_token_for_the_caller(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.target == "/v1beta/interactions" + return Reply(body=json.dumps(_FIRST_HOP).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_LONG_DECODE_MODEL, api_base=wire.url, api_key=_GEMINI_DEPLOYMENT_KEY) + key: Final = scenario.key(models=[model]) + response: Final = gateway.request( + "POST", "/v1beta/interactions", {"model": model, **_LONG_DECODE_BODY}, key=key + ) + assert response.status_code == 200, response.text + sent: Final = [json.loads(r.body) for r in wire.drain()] + + assert sent == [{"model": "gemini-4-argon", **_LONG_DECODE_BODY}], sent + payload: Final = response.json() + assert payload["id"] == "interaction-hop-1", response.text + assert payload["status"] == "requires_continuation", response.text + assert payload["continuation_token"] == "tok-1", response.text + assert payload["usage"]["initial_input_tokens"] == 79, response.text + + +def test_interaction_forwards_a_caller_continuation_token_to_resume_a_long_decode(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.target == "/v1beta/interactions" + return Reply(body=json.dumps(_SECOND_HOP).encode()) + + resume: Final = { + "generation_config": _LONG_DECODE_BODY["generation_config"], + "previous_interaction_id": "interaction-hop-1", + "continuation_token": "tok-1", + } + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_LONG_DECODE_MODEL, api_base=wire.url, api_key=_GEMINI_DEPLOYMENT_KEY) + key: Final = scenario.key(models=[model]) + response: Final = gateway.request("POST", "/v1beta/interactions", {"model": model, **resume}, key=key) + assert response.status_code == 200, response.text + sent: Final = [json.loads(r.body) for r in wire.drain()] + + assert sent == [{"model": "gemini-4-argon", **resume}], sent + payload: Final = response.json() + assert payload["id"] == "interaction-hop-2", response.text + assert payload["status"] == "completed", response.text + assert payload["continuation_token"] is None, response.text diff --git a/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py b/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py index 018c6a653d1..696253f6a98 100644 --- a/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py +++ b/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py @@ -299,14 +299,14 @@ def cooldown_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Cooldown pytest.param("chat", _BRIDGED, "pause_after_content", id="chat-bridged-to-responses"), ), ) -def test_a_deployment_whose_stream_stalls_stays_out_of_cooldown( +def test_a_deployment_whose_stream_stalls_is_cooled_down( cooldown_rig: _CooldownRig, endpoint: Endpoint, backend: str, shape: Shape ) -> None: with _peer(shape) as wire, cooldown_rig.gateway.scenario() as scenario: group, identity = _deployment(scenario, wire, backend) response: Final = _streamed(cooldown_rig.gateway, endpoint, group) error: Final = _failure_row(cooldown_rig.gateway, _ROWS, (group,)) - assert not cooled(cooldown_rig.cache, identity), (error, response.text) + assert eventually(lambda: cooled(cooldown_rig.cache, identity), bool, _ROW_SECONDS), (error, response.text) @_OWNED_PROXY diff --git a/tests/integration/providers/test_responses_minted_reasoning_replay_chaos.py b/tests/integration/providers/test_responses_minted_reasoning_replay_chaos.py index a266b20b227..0d075abbd27 100644 --- a/tests/integration/providers/test_responses_minted_reasoning_replay_chaos.py +++ b/tests/integration/providers/test_responses_minted_reasoning_replay_chaos.py @@ -18,6 +18,7 @@ import pytest import websockets import yaml from integration._support import claude_code as cc +from integration._support import device_login as dl from integration._support import responses_vendor as rv from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment from integration._support.database import read_rows @@ -365,6 +366,7 @@ async def test_worker_sigkill_mid_burst_leaves_the_sibling_dropping_the_minted_i @dataclass(frozen=True, slots=True) class _Rig: wire: Wire + copilot_wire: Wire proxy: OwnedProxy cert: Path key: Path @@ -378,12 +380,12 @@ def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: chatgpt: Final = directory / "chatgpt" copilot.mkdir() chatgpt.mkdir() - with gateway_from_environment() as gateway, wire_server(rv.ResponsesVendor().respond) as wire: - (copilot / "api-key.json").write_text( - json.dumps( - {"token": "synthetic-copilot-token", "expires_at": time.time() + 3600, "endpoints": {"api": wire.url}} - ) - ) + with ( + gateway_from_environment() as gateway, + wire_server(rv.ResponsesVendor().respond) as wire, + dl.device_login_peers(directory, provider=rv.ResponsesVendor().respond) as peers, + ): + dl.write_copilot_access(copilot, "synthetic-copilot-token") (chatgpt / "auth.json").write_text( json.dumps( { @@ -394,15 +396,15 @@ def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: ) ) overrides: Final = { + **peers.environment(), "GITHUB_COPILOT_TOKEN_DIR": str(copilot), "CHATGPT_TOKEN_DIR": str(chatgpt), "CHATGPT_API_BASE": wire.url, - "SSL_CERT_FILE": str(cert), "HTTP_PROXY": wire.url, "NO_PROXY": "127.0.0.1,localhost", } with owned_proxy_process(gateway, directory, overrides, workers=2) as owned: - yield _Rig(wire, owned, cert, key) + yield _Rig(wire, peers.api.secure, owned, cert, key) def _replay(gateway: Gateway, model: str, history: list[dict[str, JsonValue]], stream: bool) -> httpx.Response: @@ -434,13 +436,14 @@ def test_login_backed_and_foundry_deployments_forward_the_minted_item_unchanged( history: Final = rv.agents_sdk_history(marker, minted) api_base: Final = _FOUNDRY_BASE if deployment.label.startswith("azure_ai") else rig.wire.url rig.wire.drain() + rig.copilot_wire.drain() with rig.proxy.gateway.scenario() as scenario: parameters: Final[dict[str, JsonValue]] = {"model": deployment.model, "api_base": api_base} model: Final = scenario.model( **parameters, **({} if deployment.api_key is None else {"api_key": deployment.api_key}) ) response: Final = _replay(rig.proxy.gateway, model, history, stream) - received: Final = rig.wire.drain() + received: Final = (*rig.wire.drain(), *rig.copilot_wire.drain()) assert len(received) == 1, [(request.method, request.target) for request in received] target: Final = urlsplit(received[0].target) assert target.path.endswith("/responses"), received[0].target diff --git a/tests/integration/routing/test_include_fallback_errors_wire.py b/tests/integration/routing/test_include_fallback_errors_wire.py index 9075f5e41d3..2f48bc43a22 100644 --- a/tests/integration/routing/test_include_fallback_errors_wire.py +++ b/tests/integration/routing/test_include_fallback_errors_wire.py @@ -61,10 +61,10 @@ _PROVIDERS: Final = MappingProxyType( ) _STREAMING: Final = (pytest.param(False, id="non-stream"), pytest.param(True, id="stream")) _NON_BOOLEAN_FLAGS: Final = ( - pytest.param(1, True, id="int"), - pytest.param("", False, id="empty-string"), - pytest.param([], False, id="list"), - pytest.param("x" * 5120, True, id="five-kilobytes"), + pytest.param(1, id="int"), + pytest.param("", id="empty-string"), + pytest.param([], id="list"), + pytest.param("x" * 5120, id="five-kilobytes"), ) _OPENAI_UNAUTHORIZED: Final = Reply( status=401, @@ -326,10 +326,8 @@ def test_default_gateway_keeps_the_flag_off_the_wire_on_a_fallback( assert observed.attempted == "1", observed -@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS) -def test_default_gateway_keeps_a_non_boolean_flag_off_the_wire( - registered: _Registered, value: JsonValue, errors_reported: bool -) -> None: +@pytest.mark.parametrize("value", _NON_BOOLEAN_FLAGS) +def test_default_gateway_keeps_a_non_boolean_flag_off_the_wire(registered: _Registered, value: JsonValue) -> None: sent: Final = _send( registered.gateway, _PATHS["chat"], @@ -497,9 +495,9 @@ def test_exposed_chat_without_a_fallback_keeps_the_flag_off_the_wire_and_the_err assert observed.errors_header is None, observed -@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS) -def test_exposed_chat_keeps_a_non_boolean_flag_off_the_wire_and_the_errors_header_follows_its_truthiness( - exposed: _Exposed, value: JsonValue, errors_reported: bool +@pytest.mark.parametrize("value", _NON_BOOLEAN_FLAGS) +def test_exposed_chat_keeps_a_non_boolean_flag_off_the_wire_and_reports_no_errors( + exposed: _Exposed, value: JsonValue ) -> None: sent: Final = _send( exposed.proxy, @@ -515,7 +513,7 @@ def test_exposed_chat_keeps_a_non_boolean_flag_off_the_wire_and_the_errors_heade assert observed.leaks == (), observed assert observed.hit == (_EXPOSED_OPENAI_PRIMARY, _EXPOSED_OPENAI_BACKUP), observed assert observed.attempted == "1", observed - assert _errors_mention_the_outage(observed.errors_header) is errors_reported, observed + assert observed.errors_header is None, observed def test_exposed_proxy_rejects_an_unauthenticated_flagged_request_before_the_wire(exposed: _Exposed) -> None: diff --git a/tests/integration/routing/test_prompt_cache_pin_with_message_logging_off.py b/tests/integration/routing/test_prompt_cache_pin_with_message_logging_off.py new file mode 100644 index 00000000000..13bf67582b3 --- /dev/null +++ b/tests/integration/routing/test_prompt_cache_pin_with_message_logging_off.py @@ -0,0 +1,95 @@ +import collections +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Final + +import pytest +from integration._support.client import eventually, gateway_from_environment, object_value +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.redis_process import owned_redis +from integration._support.wire import Reply, Wire, wire_server +from integration.routing._prompt_cache_pin_support import ( + DEFAULT_MARK, + anthropic_deployment, + ask_turn, + chat_body, + claude_peer, + new_marker, + one_new_pin, + pin_config, + post_chat, + read_pins, + system_turn, +) +from pydantic import JsonValue, TypeAdapter +from redis import Redis + +from litellm.constants import REDACTED_BY_LITELLM + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 300) + +_GROUP: Final = "pin-logging-off" +_LEFT: Final = "pin-logging-off-left" +_RIGHT: Final = "pin-logging-off-right" +_FOLLOW_UPS: Final = 20 +_REDACTED_MESSAGES: Final = [{"role": "user", "content": REDACTED_BY_LITELLM}] +_LOGGED_EVENT: Final = TypeAdapter(dict[str, JsonValue]) + + +@dataclass(frozen=True, slots=True) +class _Rig: + owned: OwnedProxy + redis: "Redis[bytes]" + logger_sink: Wire + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("prompt-cache-pin-message-logging-off") + with ( + gateway_from_environment() as gateway, + owned_redis(directory) as cache, + wire_server(claude_peer) as left, + wire_server(claude_peer) as right, + wire_server(lambda request: Reply()) as logger_sink, + ): + config: Final = pin_config( + directory, + [anthropic_deployment(_GROUP, _LEFT, left.url), anthropic_deployment(_GROUP, _RIGHT, right.url)], + litellm_settings={ + "turn_off_message_logging": True, + "callbacks": ["integration._support.redaction_probe.probe"], + "cache": False, + "enable_redis_auth_cache": False, + }, + ) + environment: Final = { + "REDIS_HOST": cache.host, + "REDIS_PORT": str(cache.port), + "REDACTION_PROBE_URL": logger_sink.url, + } + with ( + owned_proxy_process(gateway, directory, environment, config=config, workers=1) as owned, + Redis(host=cache.host, port=cache.port, socket_timeout=2) as redis, + ): + yield _Rig(owned, redis, logger_sink) + + +def test_follow_ups_stay_on_the_first_deployment_while_the_logger_sees_redacted_messages(rig: _Rig) -> None: + before: Final = frozenset(read_pins(rig.redis)) + body: Final = chat_body(_GROUP, [system_turn(new_marker(), DEFAULT_MARK), ask_turn(new_marker())]) + + _, first_served = post_chat(rig.owned.gateway, body) + _, pin = one_new_pin(rig.redis, before) + assert pin.model_id == first_served, (pin, first_served) + + served: Final = tuple(post_chat(rig.owned.gateway, body)[1] for _ in range(_FOLLOW_UPS)) + assert served == (first_served,) * _FOLLOW_UPS, collections.Counter(served) + + eventually(rig.logger_sink.received.qsize, lambda count: count >= 1 + _FOLLOW_UPS, seconds=30) + logged: Final = tuple( + object_value(_LOGGED_EVENT.validate_json(request.body)["standard"])["messages"] + for request in rig.logger_sink.drain() + ) + assert len(logged) == 1 + _FOLLOW_UPS, len(logged) + assert all(messages == _REDACTED_MESSAGES for messages in logged), logged diff --git a/tests/integration/routing/test_web_search_deployment_filter_owned_proxy.py b/tests/integration/routing/test_web_search_deployment_filter_owned_proxy.py new file mode 100644 index 00000000000..2292ddd8611 --- /dev/null +++ b/tests/integration/routing/test_web_search_deployment_filter_owned_proxy.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import json +import uuid +from collections.abc import Iterator, Mapping +from pathlib import Path +from typing import Final + +import pytest +import yaml +from pydantic import JsonValue + +from tests.integration._support.anthropic_sse import message_json +from tests.integration._support.client import Gateway, gateway_from_environment +from tests.integration._support.process import owned_proxy +from tests.integration._support.provider import PROVIDER_URL, SharedProvider +from tests.integration._support.wire import Reply + +_ANTHROPIC_GROUP: Final = "web-search-anthropic" +_GEMINI_GROUP: Final = "web-search-gemini" +_REQUESTS: Final = 10 +_PROMPT: Final = "Reply with OK." + + +def _gemini_json() -> bytes: + return json.dumps( + { + "candidates": [ + {"content": {"parts": [{"text": "OK"}], "role": "model"}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 1, "totalTokenCount": 6}, + "modelVersion": "gemini-3.8-flash", + } + ).encode() + + +def _supported(group: str) -> str: + return f"{group}-supported" + + +def _unsupported(group: str) -> str: + return f"{group}-unsupported" + + +def _group(group: str, model: str, api_base: str) -> tuple[dict[str, JsonValue], ...]: + # The unsupported deployment is weighted so a router that keeps it in rotation picks it + # nearly every time, rather than half the time by chance + return ( + { + "model_name": group, + "litellm_params": {"model": model, "api_key": "sk-fixture", "api_base": api_base, "weight": 1}, + "model_info": {"id": _supported(group)}, + }, + { + "model_name": group, + "litellm_params": {"model": model, "api_key": "sk-fixture", "api_base": api_base, "weight": 9}, + "model_info": {"id": _unsupported(group), "supports_web_search": False}, + }, + ) + + +def _tools(tool: Mapping[str, object]) -> dict[str, object]: + return {"tools": [dict(tool)]} + + +def _config(directory: Path) -> Path: + shared: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration: Final = { + **shared, + "model_list": [ + *shared["model_list"], + *_group(_ANTHROPIC_GROUP, "anthropic/claude-haiku-5-5", PROVIDER_URL), + *_group(_GEMINI_GROUP, "gemini/gemini-3.8-flash", PROVIDER_URL), + ], + "router_settings": {**shared.get("router_settings", {}), "routing_strategy": "simple-shuffle"}, + } + path: Final = directory / "config-web-search-filter.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +@pytest.fixture(scope="module") +def web_search_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("web-search-filter") + with ( + gateway_from_environment() as shared, + owned_proxy(shared, directory, {}, config=_config(directory)) as owned, + ): + yield owned + + +@pytest.mark.parametrize( + ("path", "group", "body", "reply"), + [ + pytest.param( + "/v1/chat/completions", + _ANTHROPIC_GROUP, + _tools({"type": "web_search_20250305", "name": "web_search", "max_uses": 1}), + message_json("msg_web_search", "claude-haiku-5-5", "OK"), + id="anthropic-web_search_20250305-chat", + ), + pytest.param( + "/v1/messages", + _ANTHROPIC_GROUP, + {"max_tokens": 16, **_tools({"type": "web_search_20260301", "name": "web_search"})}, + message_json("msg_web_search", "claude-haiku-5-5", "OK"), + id="anthropic-web_search_newer-version-messages", + ), + pytest.param( + "/v1/chat/completions", + _GEMINI_GROUP, + _tools({"googleSearch": {}}), + _gemini_json(), + id="gemini-googleSearch-chat", + ), + pytest.param( + "/v1/chat/completions", + _GEMINI_GROUP, + _tools({"google_search": {}}), + _gemini_json(), + id="gemini-google_search-chat", + ), + ], +) +def test_a_web_search_request_never_reaches_a_deployment_declared_without_web_search( + web_search_proxy: Gateway, + provider: SharedProvider, + path: str, + group: str, + body: Mapping[str, JsonValue], + reply: bytes, +) -> None: + provider.expect(*(Reply(body=reply) for _ in range(_REQUESTS))) + responses: Final = tuple( + web_search_proxy.request( + "POST", + path, + {"model": group, "messages": [{"role": "user", "content": f"{_PROMPT} {uuid.uuid4().hex}"}], **body}, + ) + for _ in range(_REQUESTS) + ) + received: Final = provider.received() + assert all(response.status_code == 200 for response in responses), [response.text for response in responses] + served: Final = tuple(response.headers["x-litellm-model-id"] for response in responses) + assert served == (_supported(group),) * _REQUESTS, served + assert len(received) == _REQUESTS diff --git a/tests/integration/sdk/test_device_code_login_guard_sdk.py b/tests/integration/sdk/test_device_code_login_guard_sdk.py index 7640fe13d5d..256871faf7c 100644 --- a/tests/integration/sdk/test_device_code_login_guard_sdk.py +++ b/tests/integration/sdk/test_device_code_login_guard_sdk.py @@ -184,7 +184,7 @@ def clean(bench: _Bench) -> Iterator[_Bench]: _FIRST_LOGIN_BEARERS: Final[Mapping[Provider, str]] = MappingProxyType( - {"chatgpt": dl.CHATGPT_FIRST_LOGIN, "copilot": dl.COPILOT_MINTED_KEY} + {"chatgpt": dl.CHATGPT_FIRST_LOGIN, "copilot": dl.COPILOT_FIRST_LOGIN_ACCESS} ) @@ -205,7 +205,7 @@ def test_sync_call_on_the_main_thread_keeps_the_interactive_first_login( assert {dl.bearer(request) for request in forwarded} == {_FIRST_LOGIN_BEARERS[provider]}, [ (request.method, request.target) for request in forwarded ] - expected_files: Final = ("auth.json",) if provider == "chatgpt" else ("access-token", "api-key.json") + expected_files: Final = ("auth.json",) if provider == "chatgpt" else ("access-token",) assert clean.token_files() == expected_files diff --git a/tests/integration/sdk/test_router_include_fallback_errors_wire.py b/tests/integration/sdk/test_router_include_fallback_errors_wire.py index fb17e05b02a..a8952c8c2b5 100644 --- a/tests/integration/sdk/test_router_include_fallback_errors_wire.py +++ b/tests/integration/sdk/test_router_include_fallback_errors_wire.py @@ -36,11 +36,11 @@ _BURST: Final = 12 _CALLBACK_WINDOW: Final = 15.0 _STREAMING: Final = (pytest.param(False, id="non-stream"), pytest.param(True, id="stream")) _NON_BOOLEAN_FLAGS: Final = ( - pytest.param(1, True, id="int"), - pytest.param("", False, id="empty-string"), - pytest.param([], False, id="list"), - pytest.param("x" * 5120, True, id="five-kilobytes"), - pytest.param(False, False, id="false"), + pytest.param(1, id="int"), + pytest.param("", id="empty-string"), + pytest.param([], id="list"), + pytest.param("x" * 5120, id="five-kilobytes"), + pytest.param(False, id="false"), ) _UNAUTHORIZED: Final = Reply( status=401, @@ -263,10 +263,8 @@ def test_without_a_fallback_the_flag_still_stays_off_the_wire() -> None: assert outcome.errors == (), outcome -@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS) -def test_a_non_boolean_flag_stays_off_the_wire_and_the_errors_header_follows_its_truthiness( - value: object, errors_reported: bool -) -> None: +@pytest.mark.parametrize("value", _NON_BOOLEAN_FLAGS) +def test_a_flag_other_than_true_stays_off_the_wire_and_reports_no_errors(value: object) -> None: with wire_server(_peer) as wire: router: Final = _router(wire) outcome: Final = _outcome(wire, _complete(router, "primary", stream=False, include_fallback_errors=value)) @@ -274,7 +272,7 @@ def test_a_non_boolean_flag_stays_off_the_wire_and_the_errors_header_follows_its assert outcome.hit == ("primary", "backup"), outcome assert outcome.text == f"{_ANSWER} backup", outcome assert outcome.attempted == 1, outcome - assert (len(outcome.errors) == 1) is errors_reported, outcome + assert outcome.errors == (), outcome class _Recorder(CustomLogger): diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index 0f85f1e3993..896fc9d975a 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -16,7 +16,6 @@ deterministic stand-ins so the arithmetic under test is the only variable. import asyncio, json, time import logging -from types import MappingProxyType from typing import Final, Literal import httpx @@ -1692,17 +1691,21 @@ async def test_calculate_batch_cost_and_usage_anthropic_end_to_end(): assert result.models == ["claude-sonnet-4-5"] -def test_extract_credentials_forwards_the_trusted_model_credential_snapshot(): - """Bedrock resolves a batch's output bucket only from the immutable server-side - snapshot, never from a request param, so cost accounting on the retrieve path cannot - read the output file unless this key is forwarded. Without it the accounting raises - "S3 bucket_name is required" for a bucket the deployment has configured, and the - batch's cost is never recorded.""" - snapshot = MappingProxyType({"s3_bucket_name": "configured-bucket", "aws_region_name": "us-east-1"}) +def test_extract_credentials_forwards_the_deployment_bucket_settings() -> None: + """Cost accounting on the retrieve path reads the batch's output file from the + deployment's Bedrock buckets. Without these keys the read raises "S3 bucket_name is + required" for a bucket the deployment has configured, and the batch's cost is never + recorded.""" + params: Final = { + "s3_bucket_name": "configured-bucket", + "s3_output_bucket_name": "output-bucket", + "allow_legacy_cloud_file_ids": True, + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + } - credentials = bu._extract_file_access_credentials({"_litellm_internal_model_credentials": snapshot}) + credentials: Final = bu._extract_file_access_credentials(params) - assert credentials["_litellm_internal_model_credentials"] is snapshot + assert credentials == {key: value for key, value in params.items() if key != "model"} def test_extract_credentials_forwards_the_deployment_aws_credentials(): @@ -1736,8 +1739,6 @@ async def test_output_file_content_bedrock_reads_with_deployment_aws_credentials return type("R", (), {"content": b""})() monkeypatch.setattr(files_main, "afile_content", fake_afile_content) - snapshot = MappingProxyType({"s3_bucket_name": "configured-bucket", "aws_region_name": "us-west-2"}) - await bu._fetch_batch_output_file_content( _batch("s3://configured-bucket/litellm-batch-outputs/job-1/out.jsonl.out"), custom_llm_provider="bedrock", @@ -1746,7 +1747,7 @@ async def test_output_file_content_bedrock_reads_with_deployment_aws_credentials "aws_secret_access_key": "secret-deployment", "aws_session_token": "token-deployment", "aws_region_name": "us-west-2", - "_litellm_internal_model_credentials": snapshot, + "s3_bucket_name": "configured-bucket", "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", }, ) @@ -1757,7 +1758,7 @@ async def test_output_file_content_bedrock_reads_with_deployment_aws_credentials assert captured["aws_secret_access_key"] == "secret-deployment" assert captured["aws_session_token"] == "token-deployment" assert captured["aws_region_name"] == "us-west-2" - assert captured["_litellm_internal_model_credentials"] is snapshot + assert captured["s3_bucket_name"] == "configured-bucket" assert "model" not in captured diff --git a/tests/unit/batches/test_main.py b/tests/unit/batches/test_main.py index 5ffe6558dc3..8cd74c7d83a 100644 --- a/tests/unit/batches/test_main.py +++ b/tests/unit/batches/test_main.py @@ -732,34 +732,30 @@ def test_resolve_timeout__httpx_timeout_returns_float_read(): assert resolved == 99.0 -def test_retrieve__forwards_trusted_model_credentials_into_litellm_params(seams): +def test_retrieve__keeps_the_deployment_bucket_settings_in_litellm_params(seams: Seams) -> None: """The batch's cost is computed by reading its output file after the retrieve, and - Bedrock resolves that bucket only from this immutable snapshot. get_litellm_params has - a fixed signature that drops it, so without re-adding it here the snapshot never - reaches the logging object and cost accounting fails on a bucket that is configured.""" - snapshot = MappingProxyType({"s3_bucket_name": "configured-bucket"}) + that read finds the Bedrock bucket in these litellm_params. Dropping them fails cost + accounting on a bucket the deployment has configured.""" logging_obj = MagicMock() bm.retrieve_batch( batch_id="batch-1", custom_llm_provider="openai", litellm_logging_obj=logging_obj, - _litellm_internal_model_credentials=snapshot, + s3_bucket_name="configured-bucket", + s3_output_bucket_name="output-bucket", + allow_legacy_cloud_file_ids=True, ) litellm_params = logging_obj.update_from_kwargs.call_args.kwargs["litellm_params"] - assert litellm_params["_litellm_internal_model_credentials"] is snapshot - - -def test_retrieve__omits_trusted_model_credentials_when_not_supplied(seams): - """A retrieve with no snapshot must not invent an empty one, which would read as a - configured bucket of nothing.""" - logging_obj = MagicMock() - - bm.retrieve_batch(batch_id="batch-1", custom_llm_provider="openai", litellm_logging_obj=logging_obj) - - litellm_params = logging_obj.update_from_kwargs.call_args.kwargs["litellm_params"] - assert "_litellm_internal_model_credentials" not in litellm_params + assert { + key: litellm_params[key] + for key in ("s3_bucket_name", "s3_output_bucket_name", "allow_legacy_cloud_file_ids") + } == { + "s3_bucket_name": "configured-bucket", + "s3_output_bucket_name": "output-bucket", + "allow_legacy_cloud_file_ids": True, + } # =========================================================================== # diff --git a/tests/unit/enterprise/proxy/hooks/test_managed_files.py b/tests/unit/enterprise/proxy/hooks/test_managed_files.py index 1755ab97689..747c09b9108 100644 --- a/tests/unit/enterprise/proxy/hooks/test_managed_files.py +++ b/tests/unit/enterprise/proxy/hooks/test_managed_files.py @@ -1,8 +1,8 @@ +import asyncio import base64 import json import logging import time -import asyncio import uuid from collections.abc import Awaitable, Callable, Mapping from types import SimpleNamespace @@ -23,12 +23,13 @@ from litellm_enterprise.proxy.hooks.managed_files import ( PROXY_LiteLLMManagedFiles, _provider_file_retrieve_credentials, ) + from litellm.caching import DualCache from litellm.proxy._types import CallTypes, LiteLLM_ManagedFileTable, UserAPIKeyAuth from litellm.proxy.openai_files_endpoints.common_utils import ( BATCH_CREATE_HIDDEN_PARAM, - is_base64_encoded_unified_file_id, encode_file_id_with_model, + is_base64_encoded_unified_file_id, ) from litellm.types.llms.openai import OpenAIFileObject from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder @@ -209,6 +210,7 @@ def _marked_fallback_file_row( def _batch_response_for_output_listing() -> "LiteLLMBatch": from openai.types.batch import BatchRequestCounts + from litellm.types.utils import LiteLLMBatch response: Final = LiteLLMBatch( @@ -753,9 +755,9 @@ async def test_router_acreate_batch_only_selects_from_file_id_mapping(monkeypatc @pytest.mark.asyncio async def test_output_file_id_for_batch_retrieve(): - import litellm.proxy.proxy_server as proxy_server_module from openai.types.batch import BatchRequestCounts + import litellm.proxy.proxy_server as proxy_server_module from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -837,9 +839,9 @@ async def test_output_file_id_preserves_target_model_names_when_model_name_missi (e.g. Vertex batch retrieve), unified output_file_id should still include target_model_names from the managed input file ID. """ - import litellm.proxy.proxy_server as proxy_server_module from openai.types.batch import BatchRequestCounts + import litellm.proxy.proxy_server as proxy_server_module from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -925,9 +927,9 @@ async def test_error_file_id_for_failed_batch(): """ Test that the error_file_id is properly managed when a batch fails """ - import litellm.proxy.proxy_server as proxy_server_module from openai.types.batch import BatchRequestCounts + import litellm.proxy.proxy_server as proxy_server_module from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -1102,7 +1104,7 @@ def test_update_responses_input_with_unified_file_id(): Test that update_responses_input_with_model_file_ids correctly decodes unified file IDs and extracts llm_output_file_id from responses API input. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, ) @@ -1145,7 +1147,7 @@ def test_update_responses_input_with_regular_file_id(): Test that update_responses_input_with_model_file_ids keeps regular OpenAI file IDs unchanged. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, ) @@ -1181,7 +1183,7 @@ def test_update_responses_input_with_string_input(): """ Test that update_responses_input_with_model_file_ids returns string input unchanged. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, ) @@ -1198,7 +1200,7 @@ def test_update_responses_input_with_multiple_file_ids(): Test that update_responses_input_with_model_file_ids handles multiple file IDs (both unified and regular) in the same input. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, ) @@ -1242,7 +1244,7 @@ def test_update_responses_input_with_model_file_id_mapping(): Test that update_responses_input_with_model_file_ids correctly uses model_file_id_mapping to map managed file IDs to provider-specific file IDs. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, ) @@ -1302,7 +1304,7 @@ def test_update_responses_tools_with_model_file_id_mapping(): tools.container.file_ids were not being replaced with provider-specific file IDs, causing "string too long" errors from OpenAI. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_tools_with_model_file_ids, ) @@ -1350,7 +1352,7 @@ def test_update_responses_tools_without_mapping(): Test that update_responses_tools_with_model_file_ids keeps file IDs unchanged when no mapping is provided. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_tools_with_model_file_ids, ) @@ -1382,7 +1384,7 @@ def test_update_responses_tools_with_mixed_file_ids(): Test that update_responses_tools_with_model_file_ids correctly handles a mix of managed and regular file IDs. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_tools_with_model_file_ids, ) @@ -1978,7 +1980,6 @@ async def test_store_batch_output_file_retries_model_name_provider_fetch(): assert all( call.kwargs["custom_llm_provider"] == "bedrock" and call.kwargs["max_retries"] == 0 - and "_litellm_internal_model_credentials" not in call.kwargs for call in retrieve.call_args_list ) assert sleep_calls == [0.5, 1.0] @@ -2020,7 +2021,6 @@ async def test_store_batch_output_file_marks_model_name_fallback_after_four_tran assert all( call.kwargs["custom_llm_provider"] == "bedrock" and call.kwargs["max_retries"] == 0 - and "_litellm_internal_model_credentials" not in call.kwargs for call in retrieve.call_args_list ) assert sleep_calls == [0.5, 1.0, 2.0] @@ -3654,7 +3654,6 @@ async def test_afile_retrieve_refreshes_marked_fallback_without_router_from_mode retrieve.assert_awaited_once() assert retrieve.await_args.kwargs["custom_llm_provider"] == "bedrock" assert retrieve.await_args.kwargs["max_retries"] == 0 - assert "_litellm_internal_model_credentials" not in retrieve.await_args.kwargs updated_row: Final = managed_file_table.rows["unified-output"] assert updated_row.file_object == response assert updated_row.model_mappings == {"bedrock/model-x": "provider-output"} @@ -3698,7 +3697,6 @@ async def test_afile_retrieve_uses_model_name_when_router_does_not_know_deployme ) assert retrieve.await_args.kwargs["custom_llm_provider"] == "bedrock" assert retrieve.await_args.kwargs["max_retries"] == 0 - assert "_litellm_internal_model_credentials" not in retrieve.await_args.kwargs @pytest.mark.asyncio @@ -3826,7 +3824,6 @@ async def test_afile_retrieve_case3_uses_default_provider_for_unknown_router_dep assert retrieve.await_args.kwargs["file_id"] == "provider-output" assert retrieve.await_args.kwargs["max_retries"] == 0 assert "custom_llm_provider" not in retrieve.await_args.kwargs - assert "_litellm_internal_model_credentials" not in retrieve.await_args.kwargs @pytest.mark.asyncio @@ -3943,7 +3940,6 @@ async def test_afile_retrieve_raises_error_for_non_managed_file(): @pytest.mark.asyncio async def test_list_batches_from_managed_objects_table(): - from openai.types.batch import BatchRequestCounts from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/unit/enterprise/proxy/test_managed_files_hook.py b/tests/unit/enterprise/proxy/test_managed_files_hook.py index 942433b39fb..9ec2a67da0d 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_hook.py +++ b/tests/unit/enterprise/proxy/test_managed_files_hook.py @@ -1165,45 +1165,56 @@ async def test_should_skip_non_file_unified_id_on_output_file_id(): @pytest.mark.asyncio -async def test_afile_content_passes_trusted_model_credentials_to_router(): - """ - afile_content must hand the deployment's credential snapshot to the router - call as an immutable server-side mapping. Cloud-storage providers (Bedrock - S3) validate file ids against the bucket in that snapshot, so without it - unified-id content retrieval only works when AWS_S3_BUCKET_NAME is set. - """ - from types import MappingProxyType +async def test_afile_content_reads_from_the_deployment_bucket(monkeypatch: pytest.MonkeyPatch) -> None: + """Unified-id content retrieval must validate and fetch the s3:// file against the + deployment's s3_bucket_name, with no AWS_S3_BUCKET_NAME env set.""" + import httpx + import respx + + import litellm + from litellm import Router + + monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) + monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + router = Router( + model_list=[ + { + "model_name": "bedrock-claude", + "litellm_params": { + "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "secret", + "aws_region_name": "us-west-2", + "s3_bucket_name": "my-bucket", + }, + "model_info": {"id": "model-123"}, + } + ], + num_retries=0, + ) managed_files = _make_managed_files_instance() unified_file_id = "unified-file-id" s3_uri = "s3://my-bucket/litellm-batch-outputs/job-123/input.jsonl.out" managed_files.get_unified_file_id = AsyncMock(return_value=None) - managed_files.get_model_file_id_mapping = AsyncMock( - return_value={unified_file_id: {"model-123": s3_uri}} - ) + managed_files.get_model_file_id_mapping = AsyncMock(return_value={unified_file_id: {"model-123": s3_uri}}) - mock_router = MagicMock() - mock_router.registry.return_value.credentials = MagicMock( - return_value={ - "custom_llm_provider": "bedrock", - "s3_bucket_name": "my-bucket", - "aws_region_name": "us-west-2", - } - ) - mock_router.afile_content = AsyncMock(return_value=MagicMock()) + with respx.mock: + route = respx.get( + "https://s3.us-west-2.amazonaws.com/my-bucket/litellm-batch-outputs/job-123/input.jsonl.out" + ).mock(return_value=httpx.Response(200, content=b'{"recordId":"req-1"}')) - await managed_files.afile_content( - file_id=unified_file_id, - litellm_parent_otel_span=None, - llm_router=mock_router, - ) + response = await managed_files.afile_content( + file_id=unified_file_id, + litellm_parent_otel_span=None, + llm_router=router, + ) - call_kwargs = mock_router.afile_content.call_args.kwargs - assert call_kwargs["model"] == "model-123" - assert call_kwargs["file_id"] == s3_uri - trusted_credentials = call_kwargs["_litellm_internal_model_credentials"] - assert isinstance(trusted_credentials, MappingProxyType) - assert trusted_credentials["s3_bucket_name"] == "my-bucket" + assert route.calls[0].request.headers["Authorization"].startswith("AWS4-HMAC-SHA256") + assert response.content == b'{"recordId":"req-1"}' def _managed_deletion_file_id(provider_file_id): @@ -1273,7 +1284,6 @@ async def test_afile_delete_bedrock_uses_deployment_bucket_and_signed_s3_delete( file_id=unified_file_id, litellm_parent_otel_span=None, llm_router=router, - _litellm_internal_model_credentials={"s3_bucket_name": "request-bucket"}, ) assert len(route.calls) == 1 @@ -1299,7 +1309,6 @@ async def test_afile_delete_returns_managed_id_for_stored_provider_output(): file_id=unified_file_id, litellm_parent_otel_span=None, llm_router=router, - _litellm_internal_model_credentials={"s3_bucket_name": "request-bucket"}, ) assert response.id == unified_file_id @@ -1928,45 +1937,6 @@ async def test_post_call_hook_leaves_litellm_executed_batches_untouched(llm_batc assert response.id == original_id -@pytest.mark.asyncio -async def test_afile_delete_passes_trusted_model_credentials_to_router(): - """ - afile_delete must hand the deployment's credential snapshot to the router - call, since Bedrock validates the s3:// file id against the bucket in it. - """ - from types import MappingProxyType - - managed_files = _make_managed_files_instance() - unified_file_id = "unified-file-id" - s3_uri = "s3://my-bucket/litellm-bedrock-files/job-123/input.jsonl" - managed_files.get_unified_file_id = AsyncMock(return_value=None) - managed_files.get_model_file_id_mapping = AsyncMock(return_value={unified_file_id: {"model-123": s3_uri}}) - managed_files.delete_unified_file_id = AsyncMock(return_value=_make_file_object(unified_file_id)) - - mock_router = MagicMock() - mock_router.registry.return_value.credentials = MagicMock( - return_value={ - "custom_llm_provider": "bedrock", - "s3_bucket_name": "my-bucket", - "aws_region_name": "us-west-2", - } - ) - mock_router.afile_delete = AsyncMock(return_value=MagicMock()) - - await managed_files.afile_delete( - file_id=unified_file_id, - litellm_parent_otel_span=None, - llm_router=mock_router, - ) - - call_kwargs = mock_router.afile_delete.call_args.kwargs - assert call_kwargs["model"] == "model-123" - assert call_kwargs["file_id"] == s3_uri - trusted_credentials = call_kwargs["_litellm_internal_model_credentials"] - assert isinstance(trusted_credentials, MappingProxyType) - assert trusted_credentials["s3_bucket_name"] == "my-bucket" - - @pytest.mark.asyncio async def test_afile_delete_bedrock_unified_id_end_to_end(monkeypatch): """ @@ -2291,7 +2261,7 @@ class _FakeFilesRouter: self.delete_gate = delete_gate self.provider = provider self.upload_count = 0 - self.deletes: tuple[tuple[str, str, Mapping[str, object], Mapping[str, str]], ...] = () + self.deletes: tuple[tuple[str, str, Mapping[str, object]], ...] = () self.deletes_marked_internal: tuple[bool, ...] = () self.delete_parent_request_ids: tuple[str | None, ...] = () @@ -2328,14 +2298,13 @@ class _FakeFilesRouter: model: str, file_id: str, litellm_metadata: Mapping[str, object], - _litellm_internal_model_credentials: Mapping[str, str], litellm_call_id: str | None = None, ) -> None: if self.delete_gate is not None: await self.delete_gate.wait() if isinstance(litellm_metadata, dict): litellm_metadata.update({"model_info": {"id": model}}) - self.deletes = (*self.deletes, (model, file_id, litellm_metadata, _litellm_internal_model_credentials)) + self.deletes = (*self.deletes, (model, file_id, litellm_metadata)) self.deletes_marked_internal = (*self.deletes_marked_internal, is_internal_call.get()) self.delete_parent_request_ids = (*self.delete_parent_request_ids, internal_parent_request_id.get()) if file_id in self.gone_file_ids: @@ -2399,15 +2368,7 @@ async def test_acreate_file_keeps_the_upload_the_managed_id_carries_when_two_tar assert carried == ("wildcard-dep", "file-upload-1") assert model_mappings == {"wildcard-dep": "file-upload-1"} assert flat_ids == ["file-upload-1"] - assert router.deletes == ( - ( - "wildcard-dep", - "file-upload-2", - _REQUEST_METADATA, - {"custom_llm_provider": "openai", "api_key": "key-of-wildcard-dep"}, - ), - ) - assert isinstance(router.deletes[0][3], MappingProxyType) + assert router.deletes == (("wildcard-dep", "file-upload-2", _REQUEST_METADATA),) @pytest.mark.asyncio diff --git a/tests/unit/interactions/interactions.openapi.json b/tests/unit/interactions/interactions.openapi.json index 3830f5093cb..b67ad0aceaa 100644 --- a/tests/unit/interactions/interactions.openapi.json +++ b/tests/unit/interactions/interactions.openapi.json @@ -756,7 +756,7 @@ { "label": "list", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n res, err := client.Environments.ListEnvironments(ctx, operations.ListEnvironmentsRequest{})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.ListEnvironmentsResponse != nil {\n for _, env := range res.ListEnvironmentsResponse.Environments {\n if env.ID != \"\" {\n fmt.Println(env.ID)\n }\n }\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n res, err := client.Environments.ListEnvironments(ctx, operations.ListEnvironmentsRequest{})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.ListEnvironmentsResponse != nil {\n for _, env := range res.ListEnvironmentsResponse.Environments {\n if env.ID != \"\" {\n fmt.Println(env.ID)\n }\n }\n }\n}\n" } ] }, @@ -834,12 +834,12 @@ { "label": "create", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n req := operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n Sources: []gaos_interactions.Source{\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeInline),\n Target: genai.Ptr(\"main.py\"),\n Content: genai.Ptr(\"print('Hello, World!')\"),\n },\n },\n },\n }\n\n res, err := client.Environments.CreateEnvironment(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n req := operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n Sources: []gaos_interactions.Source{\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeInline),\n Target: genai.Ptr(\"main.py\"),\n Content: genai.Ptr(\"print('Hello, World!')\"),\n },\n },\n },\n }\n\n res, err := client.Environments.CreateEnvironment(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n" }, { "label": "copy", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.CreateEnvironment(ctx, operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n FromEnvironment: genai.Ptr(sourceEnvID),\n },\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.CreateEnvironment(ctx, operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n FromEnvironment: genai.Ptr(sourceEnvID),\n },\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n" } ] } @@ -967,12 +967,12 @@ { "label": "list_files", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"src\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n fileType := \"\"\n if file.Type != nil {\n fileType = string(*file.Type)\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, fileType, size)\n }\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"src\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n fileType := \"\"\n if file.Type != nil {\n fileType = string(*file.Type)\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, fileType, size)\n }\n }\n}\n" }, { "label": "get_file", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"main.py\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, size)\n }\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"main.py\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, size)\n }\n }\n}\n" } ] } @@ -1036,7 +1036,7 @@ { "label": "delete", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n _, err = client.Environments.DeleteEnvironment(ctx, operations.DeleteEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n fmt.Println(\"Environment deleted successfully.\")\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n _, err = client.Environments.DeleteEnvironment(ctx, operations.DeleteEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n fmt.Println(\"Environment deleted successfully.\")\n}\n" } ] }, @@ -1093,7 +1093,7 @@ { "label": "get", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.GetEnvironment(ctx, operations.GetEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.GetEnvironment(ctx, operations.GetEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n }\n}\n" } ] } @@ -1171,7 +1171,7 @@ "deep_research": { "summary": "Deep Research", "value": { - "agent": "deep-research-pro-preview-12-2025", + "agent": "deep-research-preview-04-2026", "created": "2025-11-26T12:22:47Z", "id": "v1_ChdPU0F4YWFtNkFwS2kxZThQZ05lbXdROBIXT1NBeGFhbTZBcEtpMWU4UGdOZW13UTg", "object": "interaction", @@ -1472,7 +1472,7 @@ { "label": "deep_research", "lang": "sh", - "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": \"deep-research-pro-preview-12-2025\",\n \"input\": \"Find a cure to cancer\",\n \"background\": true\n }'\n" + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": \"deep-research-preview-04-2026\",\n \"input\": \"Find a cure to cancer\",\n \"background\": true\n }'\n" }, { "label": "antigravity", @@ -1517,7 +1517,7 @@ { "label": "deep_research", "lang": "python", - "source": "from google import genai\n\nclient = genai.Client()\ninteraction = client.interactions.create(\n agent=\"deep-research-pro-preview-12-2025\",\n input=\"find a cure to cancer\",\n background=True,\n)\nprint(interaction.status)\n" + "source": "from google import genai\n\nclient = genai.Client()\ninteraction = client.interactions.create(\n agent=\"deep-research-preview-04-2026\",\n input=\"find a cure to cancer\",\n background=True,\n)\nprint(interaction.status)\n" }, { "label": "antigravity", @@ -1562,7 +1562,7 @@ { "label": "deep_research", "lang": "javascript", - "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n agent: 'deep-research-pro-preview-12-2025',\n input: 'find a cure to cancer',\n background: true,\n});\nconsole.log(interaction.status);\n" + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n agent: 'deep-research-preview-04-2026',\n input: 'find a cure to cancer',\n background: true,\n});\nconsole.log(interaction.status);\n" }, { "label": "antigravity", @@ -1607,7 +1607,7 @@ { "label": "deep_research", "lang": "java", - "source": "import com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionStatus;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateAgentInteraction params =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"deep-research-pro-preview-12-2025\"))\n .input(InteractionsInput.of(\"find a cure to cancer\"))\n .background(true)\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.status().map(InteractionStatus::value).orElse(\"\"));\n" + "source": "import com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionStatus;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateAgentInteraction params =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"deep-research-preview-04-2026\"))\n .input(InteractionsInput.of(\"find a cure to cancer\"))\n .background(true)\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.status().map(InteractionStatus::value).orElse(\"\"));\n" }, { "label": "antigravity", @@ -1652,7 +1652,7 @@ { "label": "deep_research", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"deep-research-pro-preview-12-2025\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"find a cure to cancer\")),\n Background: genai.Ptr(true),\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil {\n fmt.Println(res.Interaction.Status)\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"deep-research-preview-04-2026\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"find a cure to cancer\")),\n Background: genai.Ptr(true),\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil {\n fmt.Println(res.Interaction.Status)\n }\n}\n" }, { "label": "antigravity", @@ -3041,6 +3041,22 @@ "description": "An agent definition for the CreateAgent API.\nThis message is the target for annotation-parser-based JSON parsing.\nNew format:\n {\n \"id\": \"customer-sentinel\",\n \"base_agent\": \"\",\n \"system_instruction\": \"...\",\n \"base_environment\": { \"type\": \"remote\", \"sources\": [...] },\n \"tools\": [ {\"type\": \"code_execution\"} ]\n }", "type": "object", "properties": { + "id": { + "description": "The unique identifier for the agent.", + "type": "string" + }, + "base_agent": { + "description": "The base agent to extend.", + "type": "string" + }, + "system_instruction": { + "description": "System instruction for the agent.", + "type": "string" + }, + "description": { + "description": "Agent description for developers to quickly read and understand.", + "type": "string" + }, "agent_config": { "description": "Configuration parameters for the agent.", "oneOf": [ @@ -3060,22 +3076,6 @@ } ] }, - "id": { - "description": "The unique identifier for the agent.", - "type": "string" - }, - "base_agent": { - "description": "The base agent to extend.", - "type": "string" - }, - "system_instruction": { - "description": "System instruction for the agent.", - "type": "string" - }, - "description": { - "description": "Agent description for developers to quickly read and understand.", - "type": "string" - }, "tools": { "description": "The tools available to the agent.", "type": "array", @@ -3091,13 +3091,11 @@ "description": "The agent to interact with.", "type": "string", "enum": [ - "deep-research-pro-preview-12-2025", "deep-research-preview-04-2026", "deep-research-max-preview-04-2026", "antigravity-preview-05-2026" ], "x-speakeasy-enum-descriptions": [ - "Gemini Deep Research Agent", "Gemini Deep Research Agent", "Gemini Deep Research Max Agent", "Use the Antigravity managed agent to perform multi-step tasks that require reasoning, file operations, and tool use." @@ -3330,14 +3328,6 @@ "type": { "const": "audio" }, - "data": { - "type": "string", - "format": "byte", - "x-speakeasy-base64-input-mode": "file" - }, - "uri": { - "type": "string" - }, "mime_type": { "type": "string", "enum": [ @@ -3371,6 +3361,14 @@ "WEBM audio format" ] }, + "data": { + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "type": "string" + }, "rate": { "description": "Deprecated. Use sample_rate instead. The value is ignored.", "type": "integer", @@ -3817,32 +3815,6 @@ } ] }, - "response_format": { - "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", - "oneOf": [ - { - "$ref": "#/components/schemas/ResponseFormat" - }, - { - "title": "ResponseFormatList", - "type": "array", - "items": { - "$ref": "#/components/schemas/ResponseFormat" - } - } - ] - }, - "environment": { - "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", - "oneOf": [ - { - "$ref": "#/components/schemas/EnvironmentConfig" - }, - { - "type": "string" - } - ] - }, "id": { "description": "Required. Output only. A unique identifier for the interaction completion.", "type": "string", @@ -3858,7 +3830,8 @@ "failed", "cancelled", "incomplete", - "budget_exceeded" + "budget_exceeded", + "requires_continuation" ], "readOnly": true, "x-google-enum-deprecated": [ @@ -3868,7 +3841,8 @@ false, false, false, - true + true, + false ], "x-google-enum-descriptions": [ "The interaction is in progress.", @@ -3877,7 +3851,8 @@ "The interaction failed.", "The interaction was cancelled.", "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", - "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).", + "The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request." ] }, "created": { @@ -3909,6 +3884,21 @@ "$ref": "#/components/schemas/ResponseModality" } }, + "response_format": { + "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", + "oneOf": [ + { + "$ref": "#/components/schemas/ResponseFormat" + }, + { + "title": "ResponseFormatList", + "type": "array", + "items": { + "$ref": "#/components/schemas/ResponseFormat" + } + } + ] + }, "response_mime_type": { "description": "The mime type of the response. This is required if response_format is set.", "type": "string", @@ -3918,6 +3908,17 @@ "description": "The ID of the previous interaction, if any.", "type": "string" }, + "environment": { + "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentConfig" + }, + { + "type": "string" + } + ] + }, "environment_id": { "description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.", "type": "string", @@ -3945,6 +3946,12 @@ "type": "string" } }, + "continuation_token": { + "description": "Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, "stream": { "description": "Input only. Whether the interaction will be streamed.", "type": "boolean", @@ -4018,32 +4025,6 @@ "$ref": "#/components/schemas/GenerationConfig", "writeOnly": true }, - "response_format": { - "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", - "oneOf": [ - { - "$ref": "#/components/schemas/ResponseFormat" - }, - { - "title": "ResponseFormatList", - "type": "array", - "items": { - "$ref": "#/components/schemas/ResponseFormat" - } - } - ] - }, - "environment": { - "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", - "oneOf": [ - { - "$ref": "#/components/schemas/EnvironmentConfig" - }, - { - "type": "string" - } - ] - }, "id": { "description": "Required. Output only. A unique identifier for the interaction completion.", "type": "string", @@ -4059,7 +4040,8 @@ "failed", "cancelled", "incomplete", - "budget_exceeded" + "budget_exceeded", + "requires_continuation" ], "readOnly": true, "x-google-enum-deprecated": [ @@ -4069,7 +4051,8 @@ false, false, false, - true + true, + false ], "x-google-enum-descriptions": [ "The interaction is in progress.", @@ -4078,7 +4061,8 @@ "The interaction failed.", "The interaction was cancelled.", "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", - "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).", + "The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request." ] }, "created": { @@ -4110,6 +4094,21 @@ "$ref": "#/components/schemas/ResponseModality" } }, + "response_format": { + "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", + "oneOf": [ + { + "$ref": "#/components/schemas/ResponseFormat" + }, + { + "title": "ResponseFormatList", + "type": "array", + "items": { + "$ref": "#/components/schemas/ResponseFormat" + } + } + ] + }, "response_mime_type": { "description": "The mime type of the response. This is required if response_format is set.", "type": "string", @@ -4119,6 +4118,17 @@ "description": "The ID of the previous interaction, if any.", "type": "string" }, + "environment": { + "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentConfig" + }, + { + "type": "string" + } + ] + }, "environment_id": { "description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.", "type": "string", @@ -4146,6 +4156,12 @@ "type": "string" } }, + "continuation_token": { + "description": "Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, "stream": { "description": "Input only. Whether the interaction will be streamed.", "type": "boolean", @@ -4348,11 +4364,31 @@ "type": "string", "enum": [ "application/pdf", - "text/csv" + "text/csv", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.ms-powerpoint", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "application/vnd.oasis.opendocument.text", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel", + "application/vnd.oasis.opendocument.spreadsheet" ], "x-google-enum-descriptions": [ "PDF document format", - "CSV document format" + "CSV document format", + "PPTX presentation format", + "PPT presentation format", + "ODP presentation format", + "DOCX document format", + "DOC document format", + "DOTX document template format", + "ODT document format", + "XLSX spreadsheet format", + "XLS spreadsheet format", + "ODS spreadsheet format" ] } }, @@ -4379,6 +4415,37 @@ "type": { "const": "document" }, + "mime_type": { + "type": "string", + "enum": [ + "application/pdf", + "text/csv", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.ms-powerpoint", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "application/vnd.oasis.opendocument.text", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel", + "application/vnd.oasis.opendocument.spreadsheet" + ], + "x-google-enum-descriptions": [ + "PDF document format", + "CSV document format", + "PPTX presentation format", + "PPT presentation format", + "ODP presentation format", + "DOCX document format", + "DOC document format", + "DOTX document template format", + "ODT document format", + "XLSX spreadsheet format", + "XLS spreadsheet format", + "ODS spreadsheet format" + ] + }, "data": { "type": "string", "format": "byte", @@ -4386,17 +4453,6 @@ }, "uri": { "type": "string" - }, - "mime_type": { - "type": "string", - "enum": [ - "application/pdf", - "text/csv" - ], - "x-google-enum-descriptions": [ - "PDF document format", - "CSV document format" - ] } }, "required": [ @@ -4474,23 +4530,6 @@ "type": "string", "readOnly": true }, - "network": { - "description": "Network configuration for the environment.", - "oneOf": [ - { - "$ref": "#/components/schemas/EnvironmentNetworkEgressAllowlist" - }, - { - "type": "string", - "enum": [ - "disabled" - ], - "x-google-enum-descriptions": [ - "All network egress is blocked." - ] - } - ] - }, "sources": { "description": "Sources to be mounted into the environment.", "type": "array", @@ -4526,6 +4565,23 @@ "" ] }, + "network": { + "description": "Network configuration for the environment.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentNetworkEgressAllowlist" + }, + { + "type": "string", + "enum": [ + "disabled" + ], + "x-google-enum-descriptions": [ + "All network egress is blocked." + ] + } + ] + }, "file_count": { "description": "Output only. The number of files in the environment, output only.", "type": "string", @@ -4551,6 +4607,12 @@ "type": { "const": "remote" }, + "sources": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Source" + } + }, "network": { "description": "Network configuration for the environment.", "oneOf": [ @@ -4568,12 +4630,6 @@ } ] }, - "sources": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Source" - } - }, "environment_id": { "description": "Optional. The environment ID for the interaction. If specified, the request will\nupdate the existing environment instead of creating a new one.", "type": "string" @@ -5347,30 +5403,6 @@ "description": "Configuration parameters for model interactions.", "type": "object", "properties": { - "tool_choice": { - "description": "The tool choice configuration.", - "oneOf": [ - { - "$ref": "#/components/schemas/ToolChoiceConfig" - }, - { - "type": "string", - "enum": [ - "auto", - "any", - "none", - "validated" - ], - "x-google-enum-descriptions": [ - "Auto tool choice.", - "Any tool choice.", - "No tool choice.", - "Validated tool choice." - ], - "x-speakeasy-model-namespace": "interactions" - } - ] - }, "temperature": { "description": "Controls the randomness of the output.", "type": "number", @@ -5395,6 +5427,30 @@ "type": "string" } }, + "tool_choice": { + "description": "The tool choice configuration.", + "oneOf": [ + { + "$ref": "#/components/schemas/ToolChoiceConfig" + }, + { + "type": "string", + "enum": [ + "auto", + "any", + "none", + "validated" + ], + "x-google-enum-descriptions": [ + "Auto tool choice.", + "Any tool choice.", + "No tool choice.", + "Validated tool choice." + ], + "x-speakeasy-model-namespace": "interactions" + } + ] + }, "thinking_level": { "description": "The level of thought tokens that the model should generate.", "$ref": "#/components/schemas/ThinkingLevel" @@ -6211,14 +6267,6 @@ "type": { "const": "image" }, - "data": { - "type": "string", - "format": "byte", - "x-speakeasy-base64-input-mode": "file" - }, - "uri": { - "type": "string" - }, "mime_type": { "type": "string", "enum": [ @@ -6242,6 +6290,14 @@ "TIFF image format" ] }, + "data": { + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "type": "string" + }, "resolution": { "description": "The resolution of the media.", "$ref": "#/components/schemas/MediaResolution" @@ -6385,7 +6441,8 @@ "failed", "cancelled", "incomplete", - "budget_exceeded" + "budget_exceeded", + "requires_continuation" ], "readOnly": true, "x-google-enum-deprecated": [ @@ -6395,7 +6452,8 @@ false, false, false, - true + true, + false ], "x-google-enum-descriptions": [ "The interaction is in progress.", @@ -6404,7 +6462,8 @@ "The interaction failed.", "The interaction was cancelled.", "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", - "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).", + "The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request." ] }, "created": { @@ -6434,9 +6493,33 @@ "$ref": "#/components/schemas/GenerationConfig", "writeOnly": true }, + "system_instruction": { + "description": "System instruction for the interaction.", + "type": "string" + }, + "tools": { + "description": "A list of tool declarations the model may call during interaction.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Tool" + } + }, "input": { "$ref": "#/components/schemas/InteractionsInput" }, + "usage": { + "description": "Output only. Statistics on the interaction request's token usage.", + "$ref": "#/components/schemas/Usage", + "readOnly": true + }, + "response_modalities": { + "description": "The requested modalities of the response (TEXT, IMAGE, AUDIO).", + "type": "array", + "deprecated": true, + "items": { + "$ref": "#/components/schemas/ResponseModality" + } + }, "response_format": { "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", "oneOf": [ @@ -6452,6 +6535,15 @@ } ] }, + "response_mime_type": { + "description": "The mime type of the response. This is required if response_format is set.", + "type": "string", + "deprecated": true + }, + "previous_interaction_id": { + "description": "The ID of the previous interaction, if any.", + "type": "string" + }, "environment": { "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", "oneOf": [ @@ -6463,39 +6555,6 @@ } ] }, - "system_instruction": { - "description": "System instruction for the interaction.", - "type": "string" - }, - "tools": { - "description": "A list of tool declarations the model may call during interaction.", - "type": "array", - "items": { - "$ref": "#/components/schemas/Tool" - } - }, - "usage": { - "description": "Output only. Statistics on the interaction request's token usage.", - "$ref": "#/components/schemas/Usage", - "readOnly": true - }, - "response_modalities": { - "description": "The requested modalities of the response (TEXT, IMAGE, AUDIO).", - "type": "array", - "deprecated": true, - "items": { - "$ref": "#/components/schemas/ResponseModality" - } - }, - "response_mime_type": { - "description": "The mime type of the response. This is required if response_format is set.", - "type": "string", - "deprecated": true - }, - "previous_interaction_id": { - "description": "The ID of the previous interaction, if any.", - "type": "string" - }, "environment_id": { "description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.", "type": "string", @@ -6531,6 +6590,12 @@ }, "readOnly": true }, + "continuation_token": { + "description": "Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, "stream": { "description": "Input only. Whether the interaction will be streamed.", "type": "boolean", @@ -6813,16 +6878,16 @@ }, "readOnly": true }, - "agent": { - "description": "The agent to interact with.", - "type": "string" - }, "continuation_token": { "description": "Opaque token to resume a long decode.\nOutput: set when status is INCOMPLETE and decoding can be resumed.\nInput: pass the latest token back unchanged in CreateInteraction to\ncontinue decoding.", "type": "string", "format": "byte", "x-speakeasy-base64-input-mode": "file" }, + "agent": { + "description": "The agent to interact with.", + "type": "string" + }, "object": { "description": "Output only. The resource type.", "type": "string", @@ -6863,7 +6928,8 @@ "failed", "cancelled", "incomplete", - "budget_exceeded" + "budget_exceeded", + "requires_continuation" ], "x-google-enum-deprecated": [ false, @@ -6872,7 +6938,8 @@ false, false, false, - true + true, + false ], "x-google-enum-descriptions": [ "The interaction is in progress.", @@ -6881,7 +6948,8 @@ "The interaction failed.", "The interaction was cancelled.", "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", - "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).", + "The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request." ] }, "event_type": { @@ -7076,14 +7144,14 @@ "type": { "const": "mcp_server" }, - "url": { - "description": "The full URL for the MCPServer endpoint.\nExample: \"https://api.example.com/mcp\"", - "type": "string" - }, "name": { "description": "The name of the MCPServer.", "type": "string" }, + "url": { + "description": "The full URL for the MCPServer endpoint.\nExample: \"https://api.example.com/mcp\"", + "type": "string" + }, "headers": { "description": "Optional: Fields for authentication headers, timeouts, etc., if needed.", "type": "object", @@ -7362,7 +7430,7 @@ }, "ModelOption": { "title": "Model", - "description": "The model that will complete your prompt.\\n\\nSee [models](https://ai.google.dev/gemini-api/docs/models) for additional details.", + "description": "The model that will complete your prompt.\\n\\nSee [models](https://aistudio.google.com/docs/models) for additional details.", "type": "string", "enum": [ "gemma-4-26b-a4b-it", @@ -8695,6 +8763,20 @@ "description": "Configuration for speech recognition (transcription).", "type": "object", "properties": { + "language_codes": { + "description": "Optional. BCP-47 language codes providing hints about the languages present in the\naudio. If omitted or empty, defaults to automatic language detection.", + "type": "array", + "items": { + "type": "string" + } + }, + "custom_vocabulary": { + "description": "Optional. A list of custom vocabulary phrases to bias the speech recognition model\ntoward recognizing specific terms.", + "type": "array", + "items": { + "type": "string" + } + }, "mode": { "description": "Discriminated transcription mode options or enum.", "oneOf": [ @@ -8714,20 +8796,6 @@ } ] }, - "language_codes": { - "description": "Optional. BCP-47 language codes providing hints about the languages present in the\naudio. If omitted or empty, defaults to automatic language detection.", - "type": "array", - "items": { - "type": "string" - } - }, - "custom_vocabulary": { - "description": "Optional. A list of custom vocabulary phrases to bias the speech recognition model\ntoward recognizing specific terms.", - "type": "array", - "items": { - "type": "string" - } - }, "timestamp_granularities": { "description": "Optional. The granularity of timestamps to include in the transcription output.\nSupported values: \"word\". If empty, no timestamps are generated.", "type": "array", @@ -9332,6 +9400,11 @@ "type": "integer", "format": "int32" }, + "initial_input_tokens": { + "description": "Number of input tokens used in the first turn of this interaction.", + "type": "integer", + "format": "int32" + }, "input_tokens_by_modality": { "description": "A breakdown of input token usage by modality.", "type": "array", @@ -9497,6 +9570,10 @@ "description": "The URI of the video.", "type": "string" }, + "resolution": { + "description": "The resolution of the media.", + "$ref": "#/components/schemas/MediaResolution" + }, "processing": { "description": "How the model processes this video for understanding.", "oneOf": [ @@ -9516,10 +9593,6 @@ } ] }, - "resolution": { - "description": "The resolution of the media.", - "$ref": "#/components/schemas/MediaResolution" - }, "name": { "description": "A user-defined name for this content block. Can be referenced by the model\nin the final response.", "type": "string" @@ -9573,14 +9646,6 @@ "type": { "const": "video" }, - "data": { - "type": "string", - "format": "byte", - "x-speakeasy-base64-input-mode": "file" - }, - "uri": { - "type": "string" - }, "mime_type": { "type": "string", "enum": [ @@ -9608,6 +9673,14 @@ "JPEG 2000 video format" ] }, + "data": { + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "type": "string" + }, "resolution": { "description": "The resolution of the media.", "$ref": "#/components/schemas/MediaResolution" diff --git a/tests/unit/interactions/test_main.py b/tests/unit/interactions/test_main.py index 5af5dd7d82a..cd32eb11485 100644 --- a/tests/unit/interactions/test_main.py +++ b/tests/unit/interactions/test_main.py @@ -559,28 +559,34 @@ class TestInteractionsAcreateOffline: @pytest.mark.usefixtures("fake_provider_credentials") @pytest.mark.asyncio - async def test_acreate_gemini_keeps_continuation_token_out_of_the_request(self, respx_mock: MockRouter) -> None: + async def test_acreate_gemini_sends_a_caller_continuation_token_to_resume_a_decode( + self, respx_mock: MockRouter + ) -> None: route: Final = respx_mock.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( return_value=httpx.Response( 200, json={ "id": "interaction-offline", "object": "interaction", - "model": "gemini-3.8-flash", + "model": "gemini-4-argon", "status": "completed", "steps": [{"type": "model_output", "content": [{"type": "text", "text": "Hi"}]}], - "usage": {"input_tokens": 6, "output_tokens": 3}, + "usage": {"total_input_tokens": 6, "total_output_tokens": 3}, }, ) ) response: Final = await interactions.acreate( - model="gemini/gemini-3.8-flash", - input="Prove it", + model="gemini/gemini-4-argon", + previous_interaction_id="interaction-hop-1", continuation_token="dG9rLTE=", api_key="gemini-offline", ) - # The Interactions API answered 400 "Unknown parameter 'continuation_token'." on 2026-10-13 - assert json.loads(route.calls.last.request.content) == {"model": "gemini-3.8-flash", "input": "Prove it"} + # Google's spec (2026-11-05) takes continuation_token on CreateModelInteractionParams; a 2026-10-13 400 is gone + assert json.loads(route.calls.last.request.content) == { + "model": "gemini-4-argon", + "previous_interaction_id": "interaction-hop-1", + "continuation_token": "dG9rLTE=", + } assert response.status == "completed" @pytest.mark.usefixtures("fake_provider_credentials") @@ -746,9 +752,7 @@ def test_create_with_system_instruction_for_google_and_responses( def test_create_streaming_returns_interaction_events( respx_mock: MockRouter, ) -> None: - route: Final = respx_mock.post( - "https://generativelanguage.googleapis.com/v1beta/interactions?alt=sse" - ).mock( + route: Final = respx_mock.post("https://generativelanguage.googleapis.com/v1beta/interactions?alt=sse").mock( return_value=httpx.Response( 200, headers={"content-type": "text/event-stream"}, @@ -856,3 +860,65 @@ def test_create_streaming_hands_completed_interaction_to_success_callback( ) assert recorder.logged == (_GEMINI_COMPLETED_INTERACTION,) + + +_GEMINI_URL: Final = "https://generativelanguage.googleapis.com/v1beta/interactions" +_LONG_DECODE_REQUEST: Final = { + "model": "gemini/gemini-4-argon", + "input": "Prove the prime number theorem.", + "generation_config": {"max_output_tokens": 120000, "thinking_level": "high"}, +} +_FIRST_HOP: Final = { + "id": "hop-1", + "object": "interaction", + "model": "gemini-4-argon", + "status": "requires_continuation", + "steps": [{"type": "model_output", "content": [{"type": "text", "text": "Lemma 1."}]}], + "usage": { + "total_input_tokens": 79, + "total_output_tokens": 32668, + "total_thought_tokens": 100, + "total_tokens": 32847, + "initial_input_tokens": 79, + }, + "continuation_token": "tok-1", +} + + +@pytest.mark.usefixtures("fake_provider_credentials") +def test_create_keeps_requires_continuation_status_and_token_for_the_caller(respx_mock: MockRouter) -> None: + route: Final = respx_mock.post(_GEMINI_URL).mock(return_value=httpx.Response(200, json=_FIRST_HOP)) + + response: Final = interactions.create(**_LONG_DECODE_REQUEST, api_key="gemini-offline") + + assert route.call_count == 1 + assert response.status == "requires_continuation" + assert response.continuation_token == "tok-1" + assert response.usage == _FIRST_HOP["usage"] + + +@pytest.mark.usefixtures("fake_provider_credentials") +def test_create_sends_a_caller_continuation_token_to_resume_a_long_decode(respx_mock: MockRouter) -> None: + route: Final = respx_mock.post(_GEMINI_URL).mock( + return_value=httpx.Response( + 200, json={**_FIRST_HOP, "id": "hop-2", "status": "completed", "continuation_token": None} + ) + ) + + response: Final = interactions.create( + model="gemini/gemini-4-argon", + previous_interaction_id="hop-1", + continuation_token="tok-1", + generation_config=_LONG_DECODE_REQUEST["generation_config"], + api_key="gemini-offline", + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent == { + "model": "gemini-4-argon", + "previous_interaction_id": "hop-1", + "continuation_token": "tok-1", + "generation_config": _LONG_DECODE_REQUEST["generation_config"], + }, sent + assert response.status == "completed" + assert response.continuation_token is None diff --git a/tests/unit/interactions/test_openapi_compliance.py b/tests/unit/interactions/test_openapi_compliance.py index b6cc92e0adc..71db2cd24e3 100644 --- a/tests/unit/interactions/test_openapi_compliance.py +++ b/tests/unit/interactions/test_openapi_compliance.py @@ -2,7 +2,7 @@ OpenAPI compliance tests for Google Interactions API. Validates that our SDK requests/responses match the OpenAPI spec snapshot in -interactions.openapi.json, taken from https://ai.google.dev/static/api/interactions.openapi.json +interactions.openapi.json, taken from https://aistudio.google.com/api-reference/interactions.openapi.json (litellm/types/interactions/README.md says how to refresh it). The scheduled interactions-spec-canary workflow runs the same tests against the live spec by passing --interactions-spec-url, so a change Google makes shows up there, not in every PR. @@ -23,7 +23,7 @@ import pytest from openapi_core import OpenAPI from pydantic import BaseModel, TypeAdapter -OPENAPI_SPEC_URL = "https://ai.google.dev/static/api/interactions.openapi.json" +OPENAPI_SPEC_URL = "https://aistudio.google.com/api-reference/interactions.openapi.json" OPENAPI_SPEC_SNAPSHOT: Final = Path(__file__).with_name("interactions.openapi.json") SPEC_URL_OPTION: Final = TypeAdapter(str | None) @@ -47,10 +47,10 @@ def _requested_spec_url(request: pytest.FixtureRequest) -> str | None: def _load_openapi_spec(live_url: str | None) -> LoadedSpec: if live_url is None: return LoadedSpec(source=str(OPENAPI_SPEC_SNAPSHOT), spec=json.loads(OPENAPI_SPEC_SNAPSHOT.read_text())) - with httpx.Client(transport=httpx.HTTPTransport(retries=3), timeout=30.0) as client: + with httpx.Client(transport=httpx.HTTPTransport(retries=3), timeout=30.0, follow_redirects=True) as client: response: Final = client.get(live_url) response.raise_for_status() - return LoadedSpec(source=str(response.url), spec=response.json()) + return LoadedSpec(source=live_url, spec=response.json()) def _model_create_request_schema(spec_dict: Dict[str, Any]) -> Dict[str, Any]: diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 3e197740e98..c41073e087f 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -2814,6 +2814,24 @@ class TestConvertToModelResponseObjectAudioTranscription: assert result.usage.output_tokens == 5 assert result.usage.input_token_details.audio_tokens == 4 + def test_parse_error_keeps_the_transcript_out_of_the_exception(self): + import traceback + + from litellm.types.utils import TranscriptionResponse + + transcript: Final = "CANARYTRANSCRIPT customer account ending 2750" + + with pytest.raises(Exception, match="Invalid response object") as exc_info: + convert_to_model_response_object( + response_object={"text": transcript, "usage": {"type": "duration", "seconds": "not-a-number"}}, + model_response_object=TranscriptionResponse(), + response_type="audio_transcription", + ) + + rendered: Final = "".join(traceback.format_exception(exc_info.value)) + assert "received_args=" in rendered + assert "CANARYTRANSCRIPT" not in rendered + class TestConvertToModelResponseObjectRerank: def test_basic_rerank(self): from litellm.types.utils import RerankResponse diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_file_id_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_file_id_utils.py new file mode 100644 index 00000000000..658e3f0d3cb --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_file_id_utils.py @@ -0,0 +1,436 @@ +import os + +import pytest + +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_file_ids_from_messages, + get_format_from_file_id, + update_messages_with_model_file_ids, +) + + +UNIFIED_FILE_ID_B64 = ( + "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9wZGY7dW5pZmllZF9pZCw2YzBiNTg5MC04OTE0" + "LTQ4ZTAtYjhmNC0wYWU1ZWQzYzE0YTU7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00bztsbG1f" + "b3V0cHV0X2ZpbGVfaWQsZmlsZS1FQ0JQVzdNTDlnN1hIZHdHZ1VQWmFNO2xsbV9vdXRwdXRf" + "ZmlsZV9tb2RlbF9pZCxlMjY0NTNmOWU3NmU3OTkzNjgwZDAwNjhkOThjMWY0Y2MyMDViYmFk" + "MDk2N2EzM2M2NjQ4OTM1NjhjYTc0M2My" +) + + +def test_get_format_from_file_id(): + unified_file_id = "litellm_proxy:application/pdf;unified_id,cbbe3534-8bf8-4386-af00-f5f6b7e370bf" + + format = get_format_from_file_id(unified_file_id) + + assert format == "application/pdf" + + +def test_update_messages_with_model_file_ids(): + file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9wZGY7dW5pZmllZF9pZCxmYzdmMmVhNS0wZjUwLTQ5ZjYtODljMS03ZTZhNTRiMTIxMzg" + model_id = "my_model_id" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this recording?"}, + { + "type": "file", + "file": { + "file_id": file_id, + }, + }, + ], + }, + ] + + model_file_id_mapping = {file_id: {"my_model_id": "provider_file_id"}} + + updated_messages = update_messages_with_model_file_ids(messages, model_id, model_file_id_mapping) + + assert updated_messages == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this recording?"}, + { + "type": "file", + "file": { + "file_id": "provider_file_id", + "format": "application/pdf", + }, + }, + ], + } + ] + + +def test_get_file_ids_from_messages_skips_langchain_v1_file_block(): + """A LangChain v1 standardized file block must not crash file-id discovery.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarise this PDF"}, + # LangChain v1 shape produced by `_normalize_messages`. + # No `file` sub-dict: the discriminator is `type: "file"` but + # the payload lives on `base64`/`mime_type` siblings. + { + "type": "file", + "id": "lc_1", + "base64": "JVBERi0xLjQK", + "mime_type": "application/pdf", + "extras": {"file_format": "application/pdf"}, + }, + ], + } + ] + + assert get_file_ids_from_messages(messages) == [] + + +def test_get_file_ids_from_messages_still_extracts_from_openai_shape(): + """Well-formed OpenAI file blocks still yield their file_id.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this?"}, + {"type": "file", "file": {"file_id": "file-abc"}}, + ], + } + ] + + assert get_file_ids_from_messages(messages) == ["file-abc"] + + +def test_get_file_ids_from_messages_mixed_shapes(): + """Mixed OpenAI and non-OpenAI file blocks: extract from the former, + ignore the latter.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"file_id": "file-keep"}}, + { + "type": "file", + "id": "lc_2", + "base64": "AAA", + "mime_type": "application/pdf", + }, + ], + } + ] + + assert get_file_ids_from_messages(messages) == ["file-keep"] + + +def test_get_file_ids_from_messages_file_field_not_dict(): + """`file` set to a non-dict value (e.g. stringified payload) must not crash.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "file", "file": "unexpectedly-a-string"}, + ], + } + ] + + assert get_file_ids_from_messages(messages) == [] + + +def test_get_file_ids_from_messages_skips_bare_string_content_items(): + messages = [ + { + "role": "user", + "content": [ + "what type of file is this?", + {"type": "file", "file": {"file_id": "file-abc"}}, + ], + } + ] + + assert get_file_ids_from_messages(messages) == ["file-abc"] + + +def test_update_messages_with_model_file_ids_skips_non_openai_file_blocks(): + """`update_messages_with_model_file_ids` is also called on user content + before provider dispatch. It must tolerate non-OpenAI file blocks the same + way.""" + langchain_v1_block = { + "type": "file", + "id": "lc_3", + "base64": "AAA", + "mime_type": "application/pdf", + } + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hello"}, + langchain_v1_block, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-1", {}) + + # Messages pass through unchanged when there is no `file` sub-dict to remap. + assert updated == messages + + +def test_update_messages_with_model_file_ids_decodes_unified_id_when_mapping_empty(): + """When the mapping is empty (e.g. multi-replica cache miss), the function + must decode the base64-encoded unified file id and substitute the embedded + llm_output_file_id — mirroring the Responses-API sibling.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this recording?"}, + { + "type": "file", + "file": { + "file_id": UNIFIED_FILE_ID_B64, + "format": "audio/wav", + }, + }, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "any-model-id", {}) + + assert updated[0]["content"][1]["file"]["file_id"] == "file-ECBPW7ML9g7XHdwGgUPZaM" + # Customer-supplied format is preserved (this is the field whose absence + # the misleading error message used to complain about). + assert updated[0]["content"][1]["file"]["format"] == "audio/wav" + + +def test_update_messages_with_model_file_ids_mapping_takes_precedence_over_decode(): + """When both mapping and decode would resolve, the mapping must win + (preserves per-deployment routing precision).""" + mapping = {UNIFIED_FILE_ID_B64: {"model-A": "mapped-provider-file-id"}} + messages = [ + { + "role": "user", + "content": [ + { + "type": "file", + "file": { + "file_id": UNIFIED_FILE_ID_B64, + "format": "application/pdf", + }, + }, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", mapping) + + assert updated[0]["content"][0]["file"]["file_id"] == "mapped-provider-file-id" + + +def test_update_messages_with_model_file_ids_non_unified_passes_through(): + """A raw provider id (e.g. gs:// URI or a random string) must be left + untouched when the mapping doesn't resolve it. The decode fallback must + not corrupt non-unified ids.""" + raw_id = "gs://my-bucket/uploads/abc-123.wav" + messages = [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"file_id": raw_id, "format": "audio/wav"}}, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", {}) + + assert updated[0]["content"][0]["file"]["file_id"] == raw_id + + +def test_update_messages_with_model_file_ids_mapping_miss_falls_back_to_decode(): + """A mapping that exists but doesn't contain this file_id should still + trigger the decode fallback — covers the case where the hook resolved + *some* ids but not this one.""" + other_id = "some-other-file-id" + mapping = {other_id: {"model-A": "other-provider-id"}} + messages = [ + { + "role": "user", + "content": [ + { + "type": "file", + "file": {"file_id": UNIFIED_FILE_ID_B64, "format": "audio/wav"}, + }, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", mapping) + + assert updated[0]["content"][0]["file"]["file_id"] == "file-ECBPW7ML9g7XHdwGgUPZaM" + + +def test_update_messages_with_model_file_ids_tolerates_non_dict_content_items(): + """Content list items aren't always dicts. text_completion forwards + token-ids (list of ints, or list of list of ints for batch) through + this path. The function must skip non-dict items instead of indexing + into them.""" + messages_token_ids = [{"role": "user", "content": [15496, 995]}] + messages_token_ids_batch = [{"role": "user", "content": [[15496, 995], [9906, 0]]}] + + # Both should pass through unchanged without raising. + assert update_messages_with_model_file_ids(messages_token_ids, "model-A", {}) == messages_token_ids + assert update_messages_with_model_file_ids(messages_token_ids_batch, "model-A", {}) == messages_token_ids_batch + + +class TestExtractFileDataBareStr: + """``extract_file_data`` used to accept bare ``str`` values and ``open()`` + them server-side. When the helper runs inside a proxy request handler the + value is attacker-controlled, so the open() call was a textbook arbitrary + local file read. Lock the new contract: bare ``str`` is rejected with a + clear migration message; ``pathlib.Path`` is still accepted for SDK + ergonomics because it's a Python-level type that HTTP form values can't + fabricate.""" + + def test_rejects_bare_str(self): + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, + ) + + with pytest.raises(ValueError, match="does not accept bare str inputs"): + extract_file_data("/etc/passwd") + + def test_accepts_pathlib_path(self): + import tempfile + from pathlib import Path + + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, + ) + + content = b"hello" + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f: + f.write(content) + tmp_path = Path(f.name) + + try: + extracted = extract_file_data(tmp_path) + assert extracted.get("content") == content + finally: + os.unlink(str(tmp_path)) + + def test_accepts_bytes(self): + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, + ) + + extracted = extract_file_data(b"raw bytes content") + assert extracted.get("content") == b"raw bytes content" + + def test_accepts_tuple(self): + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, + ) + + extracted = extract_file_data(("foo.txt", b"raw bytes content")) + assert extracted.get("filename") == "foo.txt" + assert extracted.get("content") == b"raw bytes content" + + +def _xlitellm_encoded(raw_id: str, model: str) -> str: + from litellm.proxy.openai_files_endpoints.common_utils import ( + encode_file_id_with_model, + ) + + return encode_file_id_with_model(raw_id, model) + + +def test_update_messages_with_model_file_ids_decodes_xlitellm_encoded_id(): + """x-litellm-model upload returns `file-;model,)>`. + Without decoding, the encoded id leaks to upstream OpenAI and errors as + 'Files [...] were not found'. Decode it back to raw provider id.""" + raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" + encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this."}, + {"type": "file", "file": {"file_id": encoded_id}}, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", {}) + + assert updated[0]["content"][1]["file"]["file_id"] == raw_id + + +def test_update_responses_input_with_model_file_ids_decodes_xlitellm_encoded_id(): + """Same bug on /v1/responses path. Without decoding the encoded id (>64 + chars), OpenAI rejects with 'string too long. Expected ... maximum length + 64'.""" + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + update_responses_input_with_model_file_ids, + ) + + raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" + encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") + input_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Summarize."}, + {"type": "input_file", "file_id": encoded_id}, + ], + } + ] + + updated = update_responses_input_with_model_file_ids(input_items) + + assert updated[0]["content"][1]["file_id"] == raw_id + + +@pytest.mark.parametrize("unusable_file_id", [[1], {"a": 1}]) +def test_update_responses_input_with_model_file_ids_keeps_a_non_string_file_id_as_sent(unusable_file_id): + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + update_responses_input_with_model_file_ids, + ) + + encoded_id = _xlitellm_encoded("file-ExTuCawUqxEMjVFK6xwR9B", "gpt-5.1") + input_items = [ + { + "role": "user", + "content": [ + {"type": "input_file", "file_id": encoded_id}, + {"type": "input_file", "file_id": unusable_file_id}, + ], + } + ] + + updated = update_responses_input_with_model_file_ids(input_items, "model-A", {encoded_id: {"model-A": "file-on-a"}}) + + assert [item["file_id"] for item in updated[0]["content"]] == ["file-on-a", unusable_file_id] + + +def test_update_messages_xlitellm_decode_does_not_override_mapping(): + """If the call-site already resolved a provider id via the mapping, that + wins. The new decode fallback runs only when no mapping match.""" + raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" + encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") + mapping = {encoded_id: {"model-A": "provider-explicit-id"}} + messages = [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"file_id": encoded_id}}, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", mapping) + + assert updated[0]["content"][0]["file"]["file_id"] == "provider-explicit-id" diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_json_schema_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_json_schema_utils.py new file mode 100644 index 00000000000..19951c7efad --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_json_schema_utils.py @@ -0,0 +1,801 @@ +import functools +import json +import sys +from collections.abc import Mapping +from typing import Final + +import pytest + +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_grammar_branch_count, + schema_has_open_object, + schema_has_unresolvable_ref, + schema_property_count, +) + +_ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' +_ARTIFACT_DATA_ID_PATTERN: Final = r"^(?!\.\.?(?:\/|$))[A-Za-z0-9_\-.~:@+]{1,200}$" + +class TestFlattenTopLevelSchemaCombinators: + def _customer_anyof_schema(self): + return { + "type": "object", + "anyOf": [ + { + "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, + "required": ["id", "enabled"], + }, + { + "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, + "required": ["id", "schedule"], + }, + ], + "properties": {"id": {"type": "string"}}, + "required": ["id"], + } + + def test_merges_anyof_branches_into_object_schema(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + result = flatten_top_level_schema_combinators(self._customer_anyof_schema()) + + assert "anyOf" not in result + assert result["type"] == "object" + assert set(result["properties"]) == {"id", "enabled", "schedule"} + assert result["properties"]["enabled"] == {"type": "boolean"} + assert result["required"] == ["id"] + + def test_typeless_anyof_of_object_branches_gets_intersected_required(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "anyOf": [ + { + "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, + "required": ["id", "enabled"], + }, + { + "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, + "required": ["id", "schedule"], + }, + ] + } + + result = flatten_top_level_schema_combinators(schema) + + assert result["type"] == "object" + assert "anyOf" not in result + assert result["required"] == ["id"] + + def test_allof_required_is_the_union_of_branches(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "allOf": [ + {"properties": {"id": {"type": "string"}}, "required": ["id"]}, + {"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}, + ], + } + + result = flatten_top_level_schema_combinators(schema) + + assert "allOf" not in result + assert result["required"] == ["enabled", "id"] + assert set(result["properties"]) == {"id", "enabled"} + + def test_top_level_schema_wins_property_collisions(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "anyOf": [ + {"properties": {"id": {"type": "integer"}}}, + {"properties": {"id": {"type": "number"}}}, + ], + "properties": {"id": {"type": "string"}}, + } + + result = flatten_top_level_schema_combinators(schema) + + assert result["properties"]["id"] == {"type": "string"} + + def test_drops_openai_rejected_scalar_keys_on_object_schema(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "properties": {"id": {"type": "string"}}, + "enum": [{"id": "a"}], + "const": {"id": "a"}, + "not": {"required": ["other"]}, + } + + result = flatten_top_level_schema_combinators(schema) + + assert "enum" not in result + assert "const" not in result + assert "not" not in result + assert result["properties"] == {"id": {"type": "string"}} + + def test_resolves_local_ref_branches_from_defs(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "anyOf": [{"$ref": "#/$defs/Enable"}, {"$ref": "#/$defs/Schedule"}], + "$defs": { + "Enable": { + "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, + "required": ["id", "enabled"], + }, + "Schedule": { + "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, + "required": ["id", "schedule"], + }, + }, + } + + result = flatten_top_level_schema_combinators(schema) + + assert "anyOf" not in result + assert result["type"] == "object" + assert set(result["properties"]) == {"id", "enabled", "schedule"} + assert result["required"] == ["id"] + assert "$defs" in result + + def test_flattens_nested_combinator_branch_from_definitions(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "oneOf": [ + {"$ref": "#/definitions/Toggle"}, + {"allOf": [{"properties": {"schedule": {"type": "string"}}, "required": ["schedule"]}]}, + ], + "definitions": {"Toggle": {"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}}, + } + + result = flatten_top_level_schema_combinators(schema) + + assert "oneOf" not in result + assert set(result["properties"]) == {"enabled", "schedule"} + assert "required" not in result + + def test_unresolvable_ref_branch_leaves_schema_untouched(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "anyOf": [{"$ref": "https://example.com/schemas/automation.json"}], + "properties": {"id": {"type": "string"}}, + } + + assert flatten_top_level_schema_combinators(schema) is schema + + def test_self_referencing_ref_branch_leaves_schema_untouched(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "anyOf": [{"$ref": "#/$defs/Node"}], + "$defs": {"Node": {"type": "object", "anyOf": [{"$ref": "#/$defs/Node"}]}}, + } + + assert flatten_top_level_schema_combinators(schema) is schema + + @pytest.mark.parametrize("boolean_branch", [True, False]) + def test_boolean_branch_leaves_schema_untouched(self, boolean_branch): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "anyOf": [boolean_branch, {"properties": {"id": {"type": "string"}}, "required": ["id"]}], + } + + assert flatten_top_level_schema_combinators(schema) is schema + + def test_root_required_is_combined_with_branch_required(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + allof_schema = { + "type": "object", + "required": ["id"], + "properties": {"id": {"type": "string"}}, + "allOf": [{"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}], + } + anyof_schema = { + "type": "object", + "required": ["id"], + "properties": {"id": {"type": "string"}}, + "anyOf": [ + {"properties": {"name": {"type": "string"}, "a": {"type": "string"}}, "required": ["name", "a"]}, + {"properties": {"name": {"type": "string"}, "b": {"type": "string"}}, "required": ["name", "b"]}, + ], + } + + assert flatten_top_level_schema_combinators(allof_schema)["required"] == ["enabled", "id"] + assert flatten_top_level_schema_combinators(anyof_schema)["required"] == ["id", "name"] + + def test_repeated_refs_are_expanded_once(self): + import time + + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + fan_out, chain_length = 8, 8 + schema = { + "type": "object", + "anyOf": [{"$ref": "#/$defs/Level0"}], + "$defs": { + **{ + f"Level{level}": {"anyOf": [{"$ref": f"#/$defs/Level{level + 1}"}] * fan_out} + for level in range(chain_length) + }, + f"Level{chain_length}": {"type": "object", "properties": {"id": {"type": "string"}}}, + }, + } + + started = time.perf_counter() + result = flatten_top_level_schema_combinators(schema) + + assert time.perf_counter() - started < 5 + assert "anyOf" not in result + assert result["properties"] == {"id": {"type": "string"}} + + def test_nesting_past_the_depth_cap_leaves_schema_untouched(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + def nested(levels): + leaf = {"type": "object", "properties": {"id": {"type": "string"}}} + return functools.reduce(lambda inner, _: {"type": "object", "anyOf": [inner]}, range(levels), leaf) + + shallow, deep = nested(20), nested(40) + + assert "anyOf" not in flatten_top_level_schema_combinators(shallow) + assert flatten_top_level_schema_combinators(deep) is deep + + @pytest.mark.parametrize( + "branches", + [ + [{"required": ["enabled"]}, {"required": ["schedule"]}], + [{"type": "object", "required": ["enabled"]}, {"type": "object", "required": ["schedule"]}], + ], + ) + def test_typeless_root_with_properties_flattens_branches_without_properties(self, branches): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}, "schedule": {"type": "string"}}, + "required": ["id"], + "anyOf": branches, + } + + result = flatten_top_level_schema_combinators(schema) + + assert "anyOf" not in result + assert result["type"] == "object" + assert set(result["properties"]) == {"id", "enabled", "schedule"} + assert result["required"] == ["id"] + + def test_typeless_root_flattens_typed_object_branches_without_properties(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "anyOf": [ + {"type": "object", "properties": {"id": {"type": "string"}}}, + {"type": "object", "required": ["id"]}, + ] + } + + result = flatten_top_level_schema_combinators(schema) + + assert "anyOf" not in result + assert result["type"] == "object" + assert result["properties"] == {"id": {"type": "string"}} + assert "required" not in result + + def test_non_object_union_passes_through_unchanged(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = {"anyOf": [{"type": "string"}, {"type": "number"}]} + + assert flatten_top_level_schema_combinators(schema) is schema + + def test_schema_without_rejected_keys_is_returned_as_is(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = {"type": "object", "properties": {"nested": {"anyOf": [{"type": "string"}, {"type": "null"}]}}} + + assert flatten_top_level_schema_combinators(schema) is schema + + def test_input_schema_is_never_mutated(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = self._customer_anyof_schema() + snapshot = json.loads(json.dumps(schema)) + + flatten_top_level_schema_combinators(schema) + + assert schema == snapshot + + +class TestToolWithSanitizedParameters: + def _anyof_tool(self): + return { + "type": "function", + "function": { + "name": "automation_update", + "description": "Update an automation", + "parameters": { + "type": "object", + "anyOf": [ + { + "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, + "required": ["id", "enabled"], + }, + { + "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, + "required": ["id", "schedule"], + }, + ], + "properties": {"id": {"type": "string"}}, + "required": ["id"], + }, + }, + } + + def test_flattens_anyof_parameters_into_new_tool(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, + ) + + tool = self._anyof_tool() + result = tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) + + assert result is not tool + parameters = result["function"]["parameters"] + assert "anyOf" not in parameters + assert parameters["type"] == "object" + assert set(parameters["properties"]) == {"id", "enabled", "schedule"} + assert parameters["required"] == ["id"] + assert result["function"]["name"] == "automation_update" + assert tool == self._anyof_tool() + + def test_clean_parameters_return_the_same_tool_object(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, + ) + + tool = { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + }, + } + + assert tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) is tool + + def test_pattern_only_sanitizer_drops_the_regex_and_keeps_the_union(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + tool_with_sanitized_parameters, + ) + + tool = self._anyof_tool() + tool["function"]["parameters"]["properties"]["id"]["pattern"] = _ARTIFACT_FIELD_PATTERN + + result = tool_with_sanitized_parameters(tool, drop_non_python_regex_patterns) + + parameters = result["function"]["parameters"] + assert parameters["properties"]["id"] == {"type": "string"} + assert parameters["anyOf"] == self._anyof_tool()["function"]["parameters"]["anyOf"] + assert tool["function"]["parameters"]["properties"]["id"]["pattern"] == _ARTIFACT_FIELD_PATTERN + + @pytest.mark.parametrize( + "tool", + [ + {"type": "function"}, + {"type": "function", "function": "not-a-dict"}, + {"type": "function", "function": {"name": "no_params"}}, + {"type": "function", "function": {"name": "bad_params", "parameters": "not-a-dict"}}, + ], + ) + def test_non_dict_function_or_parameters_return_the_same_tool_object(self, tool): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, + ) + + assert tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) is tool + + def test_sanitizes_the_input_schema_of_an_anthropic_tool(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + tool_with_sanitized_parameters, + ) + + tool = { + "name": "ArtifactData", + "description": "Read a shared database", + "input_schema": { + "type": "object", + "properties": {"doc_id": {"type": "string", "pattern": _ARTIFACT_DATA_ID_PATTERN}}, + }, + } + + result = tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) + + assert result == { + "name": "ArtifactData", + "description": "Read a shared database", + "input_schema": {"type": "object", "properties": {"doc_id": {"type": "string"}}}, + } + assert tool["input_schema"]["properties"]["doc_id"]["pattern"] == _ARTIFACT_DATA_ID_PATTERN + + def test_returns_the_same_anthropic_tool_when_its_schema_has_nothing_to_drop(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + tool_with_sanitized_parameters, + ) + + tool = {"name": "Read", "input_schema": {"type": "object", "properties": {"path": {"type": "string"}}}} + + assert tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) is tool + + +def _regex_schema(pattern): + return { + "type": "object", + "properties": { + "field": {"type": "string", "pattern": pattern}, + "writes": { + "type": "array", + "items": {"properties": {"doc_id": {"type": "string", "pattern": pattern}}}, + }, + "query": {"anyOf": [{"type": "string", "pattern": pattern}, {"type": "null"}]}, + "pair": {"type": "array", "prefixItems": [{"type": "string", "pattern": pattern}]}, + "extra": {"type": "object", "additionalProperties": {"type": "string", "pattern": pattern}}, + "tagged": { + "type": "object", + "patternProperties": {pattern: {"type": "string"}, "^x_": {"type": "integer"}}, + }, + }, + "$defs": {"segment": {"type": "string", "pattern": pattern}}, + "required": ["field"], + } + + +class TestDropNonPythonRegexPatterns: + """Claude Code's Artifact tool declares ECMA-262 ``\\p{..}`` escapes that OpenAI's + validator, which compiles ``pattern`` values and ``patternProperties`` keys with + Python ``re``, refuses as "not a 'regex'".""" + + def test_drops_every_regex_python_re_rejects_from_every_schema_position(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + schema = _regex_schema(_ARTIFACT_FIELD_PATTERN) + + result = drop_non_python_regex_patterns(schema) + + assert '"pattern"' not in json.dumps(result) + properties = result["properties"] + assert properties["field"] == {"type": "string"} + assert properties["writes"]["items"]["properties"]["doc_id"] == {"type": "string"} + assert properties["query"]["anyOf"] == [{"type": "string"}, {"type": "null"}] + assert properties["pair"]["prefixItems"] == [{"type": "string"}] + assert properties["extra"]["additionalProperties"] == {"type": "string"} + assert properties["tagged"]["patternProperties"] == {"^x_": {"type": "integer"}} + assert result["$defs"]["segment"] == {"type": "string"} + assert result["required"] == ["field"] + assert schema == _regex_schema(_ARTIFACT_FIELD_PATTERN) + + def test_keeps_regexes_python_re_compiles_and_returns_the_same_object(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + schema = _regex_schema(r'^(?!__.*__$)[^"\\./[\]]{1,200}$') + + assert drop_non_python_regex_patterns(schema) is schema + + def test_pattern_keys_inside_data_positions_are_not_regexes(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + schema = { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "template": {"type": "object", "default": {"pattern": _ARTIFACT_FIELD_PATTERN}}, + "samples": {"type": "array", "examples": [{"pattern": _ARTIFACT_FIELD_PATTERN}]}, + "fixed": {"const": {"pattern": _ARTIFACT_FIELD_PATTERN}}, + "vendor": {"type": "string", "x-litellm": {"pattern": _ARTIFACT_FIELD_PATTERN}}, + }, + "required": ["pattern"], + } + + assert drop_non_python_regex_patterns(schema) is schema + + def test_regex_nested_past_what_python_re_can_parse_is_dropped_not_raised(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + schema = { + "type": "object", + "properties": {"deep": {"type": "string", "pattern": "(" * 2000 + "a" + ")" * 2000}}, + } + + assert drop_non_python_regex_patterns(schema)["properties"]["deep"] == {"type": "string"} + + def test_walks_schemas_deeper_than_the_interpreter_recursion_limit(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + depth = sys.getrecursionlimit() + leaf = {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN} + schema = functools.reduce( + lambda inner, _: {"type": "object", "properties": {"child": inner}}, range(depth), leaf + ) + + result = drop_non_python_regex_patterns(schema) + + assert functools.reduce(lambda node, _: node["properties"]["child"], range(depth), result) == {"type": "string"} + assert functools.reduce(lambda node, _: node["properties"]["child"], range(depth), schema) is leaf + + def test_leaves_levels_past_the_json_nesting_limit_alone(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + leaf = {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN} + schema = functools.reduce( + lambda inner, _: {"type": "object", "properties": {"child": inner}}, range(1100), leaf + ) + + assert drop_non_python_regex_patterns(schema) is schema + + +class TestDropLookaroundRegexPatterns: + """Kimi K3 and Grok 4.6/4.7 on Bedrock Converse reject every tool schema regex that + uses a lookaround assertion, Claude Code's ``ArtifactData`` ``pattern`` included.""" + + @pytest.mark.parametrize( + "pattern", + [r"^(?!x).*$", r"^(?=.*a).*$", r"^.*(?\w+)$", r"^(?i)abc$", r"^[^\p{Cc}\p{Cf}]{1,200}$"], + ids=["plain", "non-capturing-group", "named-group", "inline-flag", "non-python-without-lookaround"], + ) + def test_keeps_regexes_without_lookaround_and_returns_the_same_object(self, pattern): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + ) + + schema = _regex_schema(pattern) + + assert drop_lookaround_regex_patterns(schema) is schema + + def test_lookaround_inside_data_positions_is_not_a_regex(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + ) + + schema = { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "template": {"type": "object", "default": {"pattern": _ARTIFACT_DATA_ID_PATTERN}}, + "hint": {"type": "string", "description": "ids match " + _ARTIFACT_DATA_ID_PATTERN}, + }, + "required": ["pattern"], + } + + assert drop_lookaround_regex_patterns(schema) is schema + + +@pytest.mark.parametrize( + ("dropper", "patterns"), + [ + ("drop_non_python_regex_patterns", (_ARTIFACT_FIELD_PATTERN, r"^\p{L}+$")), + ("drop_lookaround_regex_patterns", (_ARTIFACT_DATA_ID_PATTERN, r"^(?=.*[a-z])\w+$")), + ], + ids=["non-python", "lookaround"], +) +class TestDroppedPatternPropertiesKeepTheirNamesAllowed: + """Dropping a ``patternProperties`` key from an object closed by ``additionalProperties: + false`` must not ban the names that key allowed: its value schema takes over as the + object's ``additionalProperties``.""" + + @staticmethod + def _drop(dropper): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + drop_non_python_regex_patterns, + ) + + return { + "drop_non_python_regex_patterns": drop_non_python_regex_patterns, + "drop_lookaround_regex_patterns": drop_lookaround_regex_patterns, + }[dropper] + + def test_closed_object_takes_the_dropped_value_schema(self, dropper, patterns): + schema = { + "type": "object", + "patternProperties": {patterns[0]: {"type": "string", "pattern": patterns[0]}}, + "additionalProperties": False, + } + + assert self._drop(dropper)(schema) == { + "type": "object", + "patternProperties": {}, + "additionalProperties": {"type": "string"}, + } + + def test_closed_object_losing_two_entries_accepts_either_value_schema(self, dropper, patterns): + schema = { + "type": "object", + "patternProperties": { + patterns[0]: {"type": "string"}, + patterns[1]: {"type": "integer"}, + "^x_": {"type": "boolean"}, + }, + "additionalProperties": False, + } + + assert self._drop(dropper)(schema) == { + "type": "object", + "patternProperties": {"^x_": {"type": "boolean"}}, + "additionalProperties": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, + } + + def test_object_with_its_own_additional_properties_schema_keeps_it(self, dropper, patterns): + schema = { + "type": "object", + "patternProperties": {patterns[0]: {"type": "string"}}, + "additionalProperties": {"type": "integer"}, + } + + assert self._drop(dropper)(schema) == { + "type": "object", + "patternProperties": {}, + "additionalProperties": {"type": "integer"}, + } + + +@pytest.mark.parametrize( + ("schema", "is_open"), + [ + ({"type": "object", "properties": {"a": {"type": "string"}}}, False), + ({"type": "object", "properties": {}}, False), + ({"type": "object", "additionalProperties": False}, False), + ({"type": "string"}, False), + ({"type": "object"}, True), + ({"type": ["object", "null"]}, True), + ({"type": "object", "properties": {"a": {"type": "string"}}, "additionalProperties": True}, True), + ({"type": "object", "additionalProperties": {"type": "integer"}}, True), + ({"type": "object", "properties": {}, "patternProperties": {"^x": {"type": "string"}}}, True), + ({"type": "object", "properties": {"inner": {"anyOf": [{"type": "object"}, {"type": "null"}]}}}, True), + ({"type": "array", "items": {"type": "object", "additionalProperties": {"type": "string"}}}, True), + ], +) +def test_schema_has_open_object_flags_any_node_that_accepts_unlisted_keys(schema, is_open): + assert schema_has_open_object(schema) is is_open + + +def _object_with(properties: Mapping[str, object], **defs: Mapping[str, object]) -> Mapping[str, object]: + return {"type": "object", "properties": properties, **defs} + + +@pytest.mark.parametrize( + ("schema", "unresolvable"), + [ + (_object_with({"a": {"type": "string"}}), False), + (_object_with({"a": {"$ref": "#/$defs/A"}, "b": {"$ref": "#/$defs/A"}}, **{"$defs": {"A": {"type": "string"}}}), False), + (_object_with({"a": {"$ref": "#/definitions/A"}}, definitions={"A": {"$ref": "#/definitions/B"}, "B": {"type": "integer"}}), False), + (_object_with({"a": {"$ref": "/$defs/A"}}, **{"$defs": {"A": {"items": {"$ref": "/$defs/B"}}, "B": {"type": "string"}}}), False), + (_object_with({"a": {"$ref": "#/$defs/Missing"}}, **{"$defs": {"A": {"type": "string"}}}), True), + (_object_with({"a": {"$ref": "other.json#/A"}}), True), + (_object_with({"parent": {"$ref": "#"}}), True), + (_object_with({"a": {"$ref": "#/$defs/A"}}, **{"$defs": {"A": {"type": "array", "items": {"$ref": "#/$defs/A"}}}}), True), + (_object_with({"a": {"$ref": "/$defs/A"}}, **{"$defs": {"A": {"type": "array", "items": {"$ref": "/$defs/A"}}}}), True), + ( + _object_with( + {"a": {"$ref": "#/$defs/A"}}, + **{"$defs": {"A": {"properties": {"b": {"$ref": "#/$defs/B"}}}, "B": {"properties": {"a": {"$ref": "#/$defs/A"}}}}}, + ), + True, + ), + ], + ids=["no-refs", "shared-def", "chained-definitions", "root-path-defs", "missing-def", "external", "root", "self-cycle", "root-path-self-cycle", "mutual-cycle"], +) +def test_schema_has_unresolvable_ref_flags_refs_that_cannot_be_inlined(schema, unresolvable): + assert schema_has_unresolvable_ref(schema) is unresolvable + + +@pytest.mark.parametrize( + ("schema", "properties", "branches"), + [ + (_object_with({"a": {"type": "string"}, "b": {"type": "string"}, "c": {"type": "string"}}, required=["a"]), 3, 8), + ( + _object_with( + {"x": {"$ref": "#/$defs/Pair"}, "y": {"$ref": "#/$defs/Pair"}}, + required=["x", "y"], + **{"$defs": {"Pair": _object_with({"l": {"type": "string"}, "r": {"type": "string"}}, required=["l"])}}, + ), + 4, + 7, + ), + (_object_with({"inner": _object_with({"z": {"type": "string"}})}, required=["inner"]), 2, 4), + ], + ids=["flat", "shared-def-counted-once", "nested"], +) +def test_schema_size_counts_every_written_object_once(schema, properties, branches): + assert schema_property_count(schema) == properties + assert schema_grammar_branch_count(schema) == branches diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index a17deedf7a9..390da500c32 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1,51 +1,24 @@ -import base64 import copy -import functools import json -import os -import sys from collections.abc import Mapping from typing import Final, cast import pytest from litellm.litellm_core_utils.prompt_templates.common_utils import ( - ENCRYPTED_REASONING_SIGNATURE_PREFIX, TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, TOOL_RESULT_MEDIA_BOUNDARY, TOOL_RESULT_MEDIA_PLACEHOLDER, add_system_prompt_to_messages, - cache_control_as_prompt_cache_breakpoint, - converse_redacted_content, - encrypted_content_from_signature, - encrypted_reasoning_signature, - get_file_ids_from_messages, - get_format_from_file_id, get_semantic_cache_prompt_from_messages, get_str_from_messages, handle_any_messages_to_chat_completion_str_messages_conversion, handle_messages_with_content_list_to_str_conversion, hoist_images_from_tool_messages, - is_encrypted_reasoning_block, merge_consecutive_system_messages, - native_redacted_thinking_data, - parse_tool_call_arguments, - prompt_cache_breakpoint_for_wire, - responses_reasoning_items_from_thinking_blocks, - salvage_concatenated_tool_arguments, - schema_grammar_branch_count, - schema_has_open_object, - schema_has_unresolvable_ref, - schema_property_count, - split_concatenated_json_objects, - split_embedding_media_for_tpm, - split_media_for_tpm, - strip_encrypted_reasoning_from_messages, strip_litellm_response_only_fields, system_messages_first, - update_messages_with_model_file_ids, - with_prompt_cache_breakpoint, ) from litellm.types.llms.openai import ChatCompletionAudioObject, ChatCompletionFileObject, ChatCompletionImageObject @@ -53,53 +26,6 @@ _ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\ _ARTIFACT_DATA_ID_PATTERN: Final = r"^(?!\.\.?(?:\/|$))[A-Za-z0-9_\-.~:@+]{1,200}$" -def test_get_format_from_file_id(): - unified_file_id = "litellm_proxy:application/pdf;unified_id,cbbe3534-8bf8-4386-af00-f5f6b7e370bf" - - format = get_format_from_file_id(unified_file_id) - - assert format == "application/pdf" - - -def test_update_messages_with_model_file_ids(): - file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9wZGY7dW5pZmllZF9pZCxmYzdmMmVhNS0wZjUwLTQ5ZjYtODljMS03ZTZhNTRiMTIxMzg" - model_id = "my_model_id" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is in this recording?"}, - { - "type": "file", - "file": { - "file_id": file_id, - }, - }, - ], - }, - ] - - model_file_id_mapping = {file_id: {"my_model_id": "provider_file_id"}} - - updated_messages = update_messages_with_model_file_ids(messages, model_id, model_file_id_mapping) - - assert updated_messages == [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is in this recording?"}, - { - "type": "file", - "file": { - "file_id": "provider_file_id", - "format": "application/pdf", - }, - }, - ], - } - ] - - def test_handle_any_messages_to_chat_completion_str_messages_conversion_list(): # Test with list of messages messages = [ @@ -218,413 +144,6 @@ def test_convert_prefix_message_to_non_prefix_messages(): # ── split_concatenated_json_objects tests ── -def test_split_concatenated_json_single_object(): - """A single valid JSON object is returned as a one-element list.""" - result = split_concatenated_json_objects('{"location": "Boston"}') - assert result == [{"location": "Boston"}] - - -def test_split_concatenated_json_multiple_objects(): - """ - Multiple JSON objects concatenated without separators are split correctly. - This is the exact pattern from issue #20543 where Bedrock Claude Sonnet 4.5 - returns concatenated JSON in a single tool call arguments string. - """ - raw = ( - '{"command": ["curl", "-i", "http://localhost:9009"]}' - '{"command": ["curl", "-i", "http://localhost:9009/robots.txt"]}' - '{"command": ["curl", "-i", "http://localhost:9009/sitemap.xml"]}' - ) - result = split_concatenated_json_objects(raw) - assert len(result) == 3 - assert result[0] == {"command": ["curl", "-i", "http://localhost:9009"]} - assert result[1] == {"command": ["curl", "-i", "http://localhost:9009/robots.txt"]} - assert result[2] == {"command": ["curl", "-i", "http://localhost:9009/sitemap.xml"]} - - -def test_split_concatenated_json_with_whitespace(): - """Objects separated by whitespace are handled correctly.""" - raw = '{"a": 1} {"b": 2}\n{"c": 3}' - result = split_concatenated_json_objects(raw) - assert len(result) == 3 - assert result[0] == {"a": 1} - assert result[1] == {"b": 2} - assert result[2] == {"c": 3} - - -def test_split_concatenated_json_empty_string(): - """Empty or whitespace-only strings return an empty list.""" - assert split_concatenated_json_objects("") == [] - assert split_concatenated_json_objects(" ") == [] - - -def test_split_concatenated_json_non_dict_value(): - """Non-dict JSON values (e.g. arrays, strings) are replaced with {}.""" - result = split_concatenated_json_objects("[1, 2, 3]") - assert result == [{}] - - -def test_split_concatenated_json_wholly_invalid_returns_empty(): - """ - Wholly unparseable JSON degrades to an empty list instead of raising. - - Regression for https://github.com/BerriAI/litellm/issues/18667: a raise - here propagated out of `_convert_to_bedrock_tool_call_invoke` and turned - every replayed conversation into a 500. - """ - assert split_concatenated_json_objects("not json at all") == [] - - -def test_split_concatenated_json_malformed_object_returns_empty(): - """ - A single malformed object (missing comma between keys) degrades to an - empty list rather than raising `Expecting ',' delimiter`. - """ - assert split_concatenated_json_objects('{"location": "Boston" "unit": "celsius"}') == [] - - -def test_split_concatenated_json_salvages_prefix_before_truncated_tail(): - """ - Complete objects parsed before an unparseable/truncated tail are kept; - only the bad tail is discarded. - """ - result = split_concatenated_json_objects('{"a": 1}{"b": 2}{"c":') - assert result == [{"a": 1}, {"b": 2}] - - -def test_parse_tool_call_arguments_rejects_concatenated_json() -> None: - with pytest.raises(ValueError, match="Failed to parse tool call arguments"): - parse_tool_call_arguments('{"a":1}{"b":2}') - - -def _distinct_json_objects(count: int) -> str: - return "".join(json.dumps({"n": index}, separators=(",", ":")) for index in range(count)) - - -@pytest.mark.parametrize( - ("raw", "expected"), - ( - ('{"a":1}{"b":2}', ({"a": 1}, {"b": 2})), - ('{"a":1}{"a":1}{"a":1}', ({"a": 1},)), - ('{"a":1}{"a":1}{"b":2}', ({"a": 1}, {"a": 1}, {"b": 2})), - (_distinct_json_objects(8), tuple({"n": index} for index in range(8))), - (_distinct_json_objects(9), ()), - (_distinct_json_objects(9) + " junk", ()), - ('{"a":1}' * 7 + '{"b":2}', tuple({"a": 1} for _ in range(7)) + ({"b": 2},)), - ('{"a":1}' * 8 + '{"b":2}', ()), - ('{"a":1}' * 5000, ({"a": 1},)), - ('{"a":1}' * 20, ({"a": 1},)), - ('{"a":1}{"b":', ()), - ('0{"x":1}', ()), - ('{"x":1}0', ()), - ('[1]{"x":1}', ()), - ('{"a":1}{"b":2}}', ()), - ('{"a":1} junk', ()), - ), -) -def test_salvage_concatenated_tool_arguments(raw: str, expected: tuple[dict[str, object], ...]) -> None: - assert salvage_concatenated_tool_arguments(raw) == expected - - -# --------------------------------------------------------------------------- -# Regression tests for non-OpenAI file content blocks. -# -# `type: "file"` is a public content-block discriminator. Several producers -# (LangChain v1, provider-native shapes, custom user code) emit blocks with -# `type: "file"` but without the OpenAI Chat Completions `file` sub-dict. -# The discovery helpers below are used unconditionally inside -# `AnthropicConfig.validate_environment`, so any crash there surfaces as a -# `500 InternalServerError` before the request is even dispatched. -# --------------------------------------------------------------------------- - - -def test_get_file_ids_from_messages_skips_langchain_v1_file_block(): - """A LangChain v1 standardized file block must not crash file-id discovery.""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "summarise this PDF"}, - # LangChain v1 shape produced by `_normalize_messages`. - # No `file` sub-dict: the discriminator is `type: "file"` but - # the payload lives on `base64`/`mime_type` siblings. - { - "type": "file", - "id": "lc_1", - "base64": "JVBERi0xLjQK", - "mime_type": "application/pdf", - "extras": {"file_format": "application/pdf"}, - }, - ], - } - ] - - assert get_file_ids_from_messages(messages) == [] - - -def test_get_file_ids_from_messages_still_extracts_from_openai_shape(): - """Well-formed OpenAI file blocks still yield their file_id.""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "what is this?"}, - {"type": "file", "file": {"file_id": "file-abc"}}, - ], - } - ] - - assert get_file_ids_from_messages(messages) == ["file-abc"] - - -def test_get_file_ids_from_messages_mixed_shapes(): - """Mixed OpenAI and non-OpenAI file blocks: extract from the former, - ignore the latter.""" - messages = [ - { - "role": "user", - "content": [ - {"type": "file", "file": {"file_id": "file-keep"}}, - { - "type": "file", - "id": "lc_2", - "base64": "AAA", - "mime_type": "application/pdf", - }, - ], - } - ] - - assert get_file_ids_from_messages(messages) == ["file-keep"] - - -def test_get_file_ids_from_messages_file_field_not_dict(): - """`file` set to a non-dict value (e.g. stringified payload) must not crash.""" - messages = [ - { - "role": "user", - "content": [ - {"type": "file", "file": "unexpectedly-a-string"}, - ], - } - ] - - assert get_file_ids_from_messages(messages) == [] - - -def test_get_file_ids_from_messages_skips_bare_string_content_items(): - messages = [ - { - "role": "user", - "content": [ - "what type of file is this?", - {"type": "file", "file": {"file_id": "file-abc"}}, - ], - } - ] - - assert get_file_ids_from_messages(messages) == ["file-abc"] - - -def test_update_messages_with_model_file_ids_skips_non_openai_file_blocks(): - """`update_messages_with_model_file_ids` is also called on user content - before provider dispatch. It must tolerate non-OpenAI file blocks the same - way.""" - langchain_v1_block = { - "type": "file", - "id": "lc_3", - "base64": "AAA", - "mime_type": "application/pdf", - } - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "hello"}, - langchain_v1_block, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-1", {}) - - # Messages pass through unchanged when there is no `file` sub-dict to remap. - assert updated == messages - - -# Reusable fixture (decodes to: litellm_proxy:application/pdf;unified_id,...; -# target_model_names,gpt-4o;llm_output_file_id,file-ECBPW7ML9g7XHdwGgUPZaM; -# llm_output_file_model_id,...) -UNIFIED_FILE_ID_B64 = ( - "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9wZGY7dW5pZmllZF9pZCw2YzBiNTg5MC04OTE0" - "LTQ4ZTAtYjhmNC0wYWU1ZWQzYzE0YTU7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00bztsbG1f" - "b3V0cHV0X2ZpbGVfaWQsZmlsZS1FQ0JQVzdNTDlnN1hIZHdHZ1VQWmFNO2xsbV9vdXRwdXRf" - "ZmlsZV9tb2RlbF9pZCxlMjY0NTNmOWU3NmU3OTkzNjgwZDAwNjhkOThjMWY0Y2MyMDViYmFk" - "MDk2N2EzM2M2NjQ4OTM1NjhjYTc0M2My" -) - - -def test_update_messages_with_model_file_ids_decodes_unified_id_when_mapping_empty(): - """When the mapping is empty (e.g. multi-replica cache miss), the function - must decode the base64-encoded unified file id and substitute the embedded - llm_output_file_id — mirroring the Responses-API sibling.""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is in this recording?"}, - { - "type": "file", - "file": { - "file_id": UNIFIED_FILE_ID_B64, - "format": "audio/wav", - }, - }, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "any-model-id", {}) - - assert updated[0]["content"][1]["file"]["file_id"] == "file-ECBPW7ML9g7XHdwGgUPZaM" - # Customer-supplied format is preserved (this is the field whose absence - # the misleading error message used to complain about). - assert updated[0]["content"][1]["file"]["format"] == "audio/wav" - - -def test_update_messages_with_model_file_ids_mapping_takes_precedence_over_decode(): - """When both mapping and decode would resolve, the mapping must win - (preserves per-deployment routing precision).""" - mapping = {UNIFIED_FILE_ID_B64: {"model-A": "mapped-provider-file-id"}} - messages = [ - { - "role": "user", - "content": [ - { - "type": "file", - "file": { - "file_id": UNIFIED_FILE_ID_B64, - "format": "application/pdf", - }, - }, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-A", mapping) - - assert updated[0]["content"][0]["file"]["file_id"] == "mapped-provider-file-id" - - -def test_update_messages_with_model_file_ids_non_unified_passes_through(): - """A raw provider id (e.g. gs:// URI or a random string) must be left - untouched when the mapping doesn't resolve it. The decode fallback must - not corrupt non-unified ids.""" - raw_id = "gs://my-bucket/uploads/abc-123.wav" - messages = [ - { - "role": "user", - "content": [ - {"type": "file", "file": {"file_id": raw_id, "format": "audio/wav"}}, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-A", {}) - - assert updated[0]["content"][0]["file"]["file_id"] == raw_id - - -def test_update_messages_with_model_file_ids_mapping_miss_falls_back_to_decode(): - """A mapping that exists but doesn't contain this file_id should still - trigger the decode fallback — covers the case where the hook resolved - *some* ids but not this one.""" - other_id = "some-other-file-id" - mapping = {other_id: {"model-A": "other-provider-id"}} - messages = [ - { - "role": "user", - "content": [ - { - "type": "file", - "file": {"file_id": UNIFIED_FILE_ID_B64, "format": "audio/wav"}, - }, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-A", mapping) - - assert updated[0]["content"][0]["file"]["file_id"] == "file-ECBPW7ML9g7XHdwGgUPZaM" - - -def test_update_messages_with_model_file_ids_tolerates_non_dict_content_items(): - """Content list items aren't always dicts. text_completion forwards - token-ids (list of ints, or list of list of ints for batch) through - this path. The function must skip non-dict items instead of indexing - into them.""" - messages_token_ids = [{"role": "user", "content": [15496, 995]}] - messages_token_ids_batch = [{"role": "user", "content": [[15496, 995], [9906, 0]]}] - - # Both should pass through unchanged without raising. - assert update_messages_with_model_file_ids(messages_token_ids, "model-A", {}) == messages_token_ids - assert update_messages_with_model_file_ids(messages_token_ids_batch, "model-A", {}) == messages_token_ids_batch - - -class TestExtractFileDataBareStr: - """``extract_file_data`` used to accept bare ``str`` values and ``open()`` - them server-side. When the helper runs inside a proxy request handler the - value is attacker-controlled, so the open() call was a textbook arbitrary - local file read. Lock the new contract: bare ``str`` is rejected with a - clear migration message; ``pathlib.Path`` is still accepted for SDK - ergonomics because it's a Python-level type that HTTP form values can't - fabricate.""" - - def test_rejects_bare_str(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - with pytest.raises(ValueError, match="does not accept bare str inputs"): - extract_file_data("/etc/passwd") - - def test_accepts_pathlib_path(self): - import tempfile - from pathlib import Path - - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - content = b"hello" - with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f: - f.write(content) - tmp_path = Path(f.name) - - try: - extracted = extract_file_data(tmp_path) - assert extracted.get("content") == content - finally: - os.unlink(str(tmp_path)) - - def test_accepts_bytes(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - extracted = extract_file_data(b"raw bytes content") - assert extracted.get("content") == b"raw bytes content" - - def test_accepts_tuple(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - extracted = extract_file_data(("foo.txt", b"raw bytes content")) - assert extracted.get("filename") == "foo.txt" - assert extracted.get("content") == b"raw bytes content" - - class TestUnpackLegacyDefs: """Cover the public ``unpack_legacy_defs`` helper directly so the no-op branches (non-dict input, schema with no legacy/OpenAPI defs) are exercised @@ -873,21 +392,25 @@ def test_hoist_images_from_tool_messages_bare_data_uri_string_passes_through(): [ ( {"type": "image_url", "image_url": {"url": DATA_URI_PNG}}, - TOOL_RESULT_IMAGE_PLACEHOLDER, TOOL_RESULT_IMAGE_BOUNDARY, + TOOL_RESULT_IMAGE_PLACEHOLDER, + TOOL_RESULT_IMAGE_BOUNDARY, ), ( {"type": "file", "file": {"file_data": "data:application/pdf;base64,cGRm", "filename": "report.pdf"}}, - TOOL_RESULT_MEDIA_PLACEHOLDER, TOOL_RESULT_MEDIA_BOUNDARY, + TOOL_RESULT_MEDIA_PLACEHOLDER, + TOOL_RESULT_MEDIA_BOUNDARY, ), ( {"type": "input_audio", "input_audio": {"data": "c291bmQ=", "format": "wav"}}, - TOOL_RESULT_MEDIA_PLACEHOLDER, TOOL_RESULT_MEDIA_BOUNDARY, + TOOL_RESULT_MEDIA_PLACEHOLDER, + TOOL_RESULT_MEDIA_BOUNDARY, ), ], ) def test_hoist_tool_media_preserves_payload_and_call( part: ChatCompletionImageObject | ChatCompletionFileObject | ChatCompletionAudioObject, - placeholder: str, boundary: str, + placeholder: str, + boundary: str, ) -> None: messages: Final = [ _assistant_tool_call_msg("call_1"), @@ -898,7 +421,8 @@ def test_hoist_tool_media_preserves_payload_and_call( result: Final = hoist_images_from_tool_messages(messages) assert result == [ - messages[0], {**messages[1], "content": placeholder}, + messages[0], + {**messages[1], "content": placeholder}, {"role": "user", "content": [{"type": "text", "text": boundary}, part]}, ] assert messages == original @@ -1062,104 +586,6 @@ class TestCustomToolFormatShapeConversion: # --- x-litellm-model upload-path decoding (litellm #29830) ------------------- -def _xlitellm_encoded(raw_id: str, model: str) -> str: - from litellm.proxy.openai_files_endpoints.common_utils import ( - encode_file_id_with_model, - ) - - return encode_file_id_with_model(raw_id, model) - - -def test_update_messages_with_model_file_ids_decodes_xlitellm_encoded_id(): - """x-litellm-model upload returns `file-;model,)>`. - Without decoding, the encoded id leaks to upstream OpenAI and errors as - 'Files [...] were not found'. Decode it back to raw provider id.""" - raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" - encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Summarize this."}, - {"type": "file", "file": {"file_id": encoded_id}}, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-A", {}) - - assert updated[0]["content"][1]["file"]["file_id"] == raw_id - - -def test_update_responses_input_with_model_file_ids_decodes_xlitellm_encoded_id(): - """Same bug on /v1/responses path. Without decoding the encoded id (>64 - chars), OpenAI rejects with 'string too long. Expected ... maximum length - 64'.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - update_responses_input_with_model_file_ids, - ) - - raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" - encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") - input_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "Summarize."}, - {"type": "input_file", "file_id": encoded_id}, - ], - } - ] - - updated = update_responses_input_with_model_file_ids(input_items) - - assert updated[0]["content"][1]["file_id"] == raw_id - - - -@pytest.mark.parametrize("unusable_file_id", [[1], {"a": 1}]) -def test_update_responses_input_with_model_file_ids_keeps_a_non_string_file_id_as_sent(unusable_file_id): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - update_responses_input_with_model_file_ids, - ) - - encoded_id = _xlitellm_encoded("file-ExTuCawUqxEMjVFK6xwR9B", "gpt-5.1") - input_items = [ - { - "role": "user", - "content": [ - {"type": "input_file", "file_id": encoded_id}, - {"type": "input_file", "file_id": unusable_file_id}, - ], - } - ] - - updated = update_responses_input_with_model_file_ids( - input_items, "model-A", {encoded_id: {"model-A": "file-on-a"}} - ) - - assert [item["file_id"] for item in updated[0]["content"]] == ["file-on-a", unusable_file_id] - -def test_update_messages_xlitellm_decode_does_not_override_mapping(): - """If the call-site already resolved a provider id via the mapping, that - wins. The new decode fallback runs only when no mapping match.""" - raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" - encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") - mapping = {encoded_id: {"model-A": "provider-explicit-id"}} - messages = [ - { - "role": "user", - "content": [ - {"type": "file", "file": {"file_id": encoded_id}}, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-A", mapping) - - assert updated[0]["content"][0]["file"]["file_id"] == "provider-explicit-id" - - def test_drop_tool_reference_parts_keeps_text_parts(): from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_tool_reference_parts_from_tool_messages, @@ -1259,719 +685,6 @@ class TestSystemMessagesFirst: assert system_messages_first(messages) == messages -class TestFlattenTopLevelSchemaCombinators: - def _customer_anyof_schema(self): - return { - "type": "object", - "anyOf": [ - { - "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, - "required": ["id", "enabled"], - }, - { - "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, - "required": ["id", "schedule"], - }, - ], - "properties": {"id": {"type": "string"}}, - "required": ["id"], - } - - def test_merges_anyof_branches_into_object_schema(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - result = flatten_top_level_schema_combinators(self._customer_anyof_schema()) - - assert "anyOf" not in result - assert result["type"] == "object" - assert set(result["properties"]) == {"id", "enabled", "schedule"} - assert result["properties"]["enabled"] == {"type": "boolean"} - assert result["required"] == ["id"] - - def test_typeless_anyof_of_object_branches_gets_intersected_required(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "anyOf": [ - { - "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, - "required": ["id", "enabled"], - }, - { - "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, - "required": ["id", "schedule"], - }, - ] - } - - result = flatten_top_level_schema_combinators(schema) - - assert result["type"] == "object" - assert "anyOf" not in result - assert result["required"] == ["id"] - - def test_allof_required_is_the_union_of_branches(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "allOf": [ - {"properties": {"id": {"type": "string"}}, "required": ["id"]}, - {"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}, - ], - } - - result = flatten_top_level_schema_combinators(schema) - - assert "allOf" not in result - assert result["required"] == ["enabled", "id"] - assert set(result["properties"]) == {"id", "enabled"} - - def test_top_level_schema_wins_property_collisions(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "anyOf": [ - {"properties": {"id": {"type": "integer"}}}, - {"properties": {"id": {"type": "number"}}}, - ], - "properties": {"id": {"type": "string"}}, - } - - result = flatten_top_level_schema_combinators(schema) - - assert result["properties"]["id"] == {"type": "string"} - - def test_drops_openai_rejected_scalar_keys_on_object_schema(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "properties": {"id": {"type": "string"}}, - "enum": [{"id": "a"}], - "const": {"id": "a"}, - "not": {"required": ["other"]}, - } - - result = flatten_top_level_schema_combinators(schema) - - assert "enum" not in result - assert "const" not in result - assert "not" not in result - assert result["properties"] == {"id": {"type": "string"}} - - def test_resolves_local_ref_branches_from_defs(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "anyOf": [{"$ref": "#/$defs/Enable"}, {"$ref": "#/$defs/Schedule"}], - "$defs": { - "Enable": { - "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, - "required": ["id", "enabled"], - }, - "Schedule": { - "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, - "required": ["id", "schedule"], - }, - }, - } - - result = flatten_top_level_schema_combinators(schema) - - assert "anyOf" not in result - assert result["type"] == "object" - assert set(result["properties"]) == {"id", "enabled", "schedule"} - assert result["required"] == ["id"] - assert "$defs" in result - - def test_flattens_nested_combinator_branch_from_definitions(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "oneOf": [ - {"$ref": "#/definitions/Toggle"}, - {"allOf": [{"properties": {"schedule": {"type": "string"}}, "required": ["schedule"]}]}, - ], - "definitions": {"Toggle": {"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}}, - } - - result = flatten_top_level_schema_combinators(schema) - - assert "oneOf" not in result - assert set(result["properties"]) == {"enabled", "schedule"} - assert "required" not in result - - def test_unresolvable_ref_branch_leaves_schema_untouched(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "anyOf": [{"$ref": "https://example.com/schemas/automation.json"}], - "properties": {"id": {"type": "string"}}, - } - - assert flatten_top_level_schema_combinators(schema) is schema - - def test_self_referencing_ref_branch_leaves_schema_untouched(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "anyOf": [{"$ref": "#/$defs/Node"}], - "$defs": {"Node": {"type": "object", "anyOf": [{"$ref": "#/$defs/Node"}]}}, - } - - assert flatten_top_level_schema_combinators(schema) is schema - - @pytest.mark.parametrize("boolean_branch", [True, False]) - def test_boolean_branch_leaves_schema_untouched(self, boolean_branch): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "anyOf": [boolean_branch, {"properties": {"id": {"type": "string"}}, "required": ["id"]}], - } - - assert flatten_top_level_schema_combinators(schema) is schema - - def test_root_required_is_combined_with_branch_required(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - allof_schema = { - "type": "object", - "required": ["id"], - "properties": {"id": {"type": "string"}}, - "allOf": [{"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}], - } - anyof_schema = { - "type": "object", - "required": ["id"], - "properties": {"id": {"type": "string"}}, - "anyOf": [ - {"properties": {"name": {"type": "string"}, "a": {"type": "string"}}, "required": ["name", "a"]}, - {"properties": {"name": {"type": "string"}, "b": {"type": "string"}}, "required": ["name", "b"]}, - ], - } - - assert flatten_top_level_schema_combinators(allof_schema)["required"] == ["enabled", "id"] - assert flatten_top_level_schema_combinators(anyof_schema)["required"] == ["id", "name"] - - def test_repeated_refs_are_expanded_once(self): - import time - - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - fan_out, chain_length = 8, 8 - schema = { - "type": "object", - "anyOf": [{"$ref": "#/$defs/Level0"}], - "$defs": { - **{ - f"Level{level}": {"anyOf": [{"$ref": f"#/$defs/Level{level + 1}"}] * fan_out} - for level in range(chain_length) - }, - f"Level{chain_length}": {"type": "object", "properties": {"id": {"type": "string"}}}, - }, - } - - started = time.perf_counter() - result = flatten_top_level_schema_combinators(schema) - - assert time.perf_counter() - started < 5 - assert "anyOf" not in result - assert result["properties"] == {"id": {"type": "string"}} - - def test_nesting_past_the_depth_cap_leaves_schema_untouched(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - def nested(levels): - leaf = {"type": "object", "properties": {"id": {"type": "string"}}} - return functools.reduce(lambda inner, _: {"type": "object", "anyOf": [inner]}, range(levels), leaf) - - shallow, deep = nested(20), nested(40) - - assert "anyOf" not in flatten_top_level_schema_combinators(shallow) - assert flatten_top_level_schema_combinators(deep) is deep - - @pytest.mark.parametrize( - "branches", - [ - [{"required": ["enabled"]}, {"required": ["schedule"]}], - [{"type": "object", "required": ["enabled"]}, {"type": "object", "required": ["schedule"]}], - ], - ) - def test_typeless_root_with_properties_flattens_branches_without_properties(self, branches): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}, "schedule": {"type": "string"}}, - "required": ["id"], - "anyOf": branches, - } - - result = flatten_top_level_schema_combinators(schema) - - assert "anyOf" not in result - assert result["type"] == "object" - assert set(result["properties"]) == {"id", "enabled", "schedule"} - assert result["required"] == ["id"] - - def test_typeless_root_flattens_typed_object_branches_without_properties(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "anyOf": [ - {"type": "object", "properties": {"id": {"type": "string"}}}, - {"type": "object", "required": ["id"]}, - ] - } - - result = flatten_top_level_schema_combinators(schema) - - assert "anyOf" not in result - assert result["type"] == "object" - assert result["properties"] == {"id": {"type": "string"}} - assert "required" not in result - - def test_non_object_union_passes_through_unchanged(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = {"anyOf": [{"type": "string"}, {"type": "number"}]} - - assert flatten_top_level_schema_combinators(schema) is schema - - def test_schema_without_rejected_keys_is_returned_as_is(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = {"type": "object", "properties": {"nested": {"anyOf": [{"type": "string"}, {"type": "null"}]}}} - - assert flatten_top_level_schema_combinators(schema) is schema - - def test_input_schema_is_never_mutated(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = self._customer_anyof_schema() - snapshot = json.loads(json.dumps(schema)) - - flatten_top_level_schema_combinators(schema) - - assert schema == snapshot - - -class TestToolWithSanitizedParameters: - def _anyof_tool(self): - return { - "type": "function", - "function": { - "name": "automation_update", - "description": "Update an automation", - "parameters": { - "type": "object", - "anyOf": [ - { - "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, - "required": ["id", "enabled"], - }, - { - "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, - "required": ["id", "schedule"], - }, - ], - "properties": {"id": {"type": "string"}}, - "required": ["id"], - }, - }, - } - - def test_flattens_anyof_parameters_into_new_tool(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_combinators_and_drop_non_python_regex_patterns, - tool_with_sanitized_parameters, - ) - - tool = self._anyof_tool() - result = tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) - - assert result is not tool - parameters = result["function"]["parameters"] - assert "anyOf" not in parameters - assert parameters["type"] == "object" - assert set(parameters["properties"]) == {"id", "enabled", "schedule"} - assert parameters["required"] == ["id"] - assert result["function"]["name"] == "automation_update" - assert tool == self._anyof_tool() - - def test_clean_parameters_return_the_same_tool_object(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_combinators_and_drop_non_python_regex_patterns, - tool_with_sanitized_parameters, - ) - - tool = { - "type": "function", - "function": { - "name": "lookup", - "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, - }, - } - - assert tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) is tool - - def test_pattern_only_sanitizer_drops_the_regex_and_keeps_the_union(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - tool_with_sanitized_parameters, - ) - - tool = self._anyof_tool() - tool["function"]["parameters"]["properties"]["id"]["pattern"] = _ARTIFACT_FIELD_PATTERN - - result = tool_with_sanitized_parameters(tool, drop_non_python_regex_patterns) - - parameters = result["function"]["parameters"] - assert parameters["properties"]["id"] == {"type": "string"} - assert parameters["anyOf"] == self._anyof_tool()["function"]["parameters"]["anyOf"] - assert tool["function"]["parameters"]["properties"]["id"]["pattern"] == _ARTIFACT_FIELD_PATTERN - - @pytest.mark.parametrize( - "tool", - [ - {"type": "function"}, - {"type": "function", "function": "not-a-dict"}, - {"type": "function", "function": {"name": "no_params"}}, - {"type": "function", "function": {"name": "bad_params", "parameters": "not-a-dict"}}, - ], - ) - def test_non_dict_function_or_parameters_return_the_same_tool_object(self, tool): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_combinators_and_drop_non_python_regex_patterns, - tool_with_sanitized_parameters, - ) - - assert tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) is tool - - def test_sanitizes_the_input_schema_of_an_anthropic_tool(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - tool_with_sanitized_parameters, - ) - - tool = { - "name": "ArtifactData", - "description": "Read a shared database", - "input_schema": { - "type": "object", - "properties": {"doc_id": {"type": "string", "pattern": _ARTIFACT_DATA_ID_PATTERN}}, - }, - } - - result = tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) - - assert result == { - "name": "ArtifactData", - "description": "Read a shared database", - "input_schema": {"type": "object", "properties": {"doc_id": {"type": "string"}}}, - } - assert tool["input_schema"]["properties"]["doc_id"]["pattern"] == _ARTIFACT_DATA_ID_PATTERN - - def test_returns_the_same_anthropic_tool_when_its_schema_has_nothing_to_drop(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - tool_with_sanitized_parameters, - ) - - tool = {"name": "Read", "input_schema": {"type": "object", "properties": {"path": {"type": "string"}}}} - - assert tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) is tool - - -def _regex_schema(pattern): - return { - "type": "object", - "properties": { - "field": {"type": "string", "pattern": pattern}, - "writes": { - "type": "array", - "items": {"properties": {"doc_id": {"type": "string", "pattern": pattern}}}, - }, - "query": {"anyOf": [{"type": "string", "pattern": pattern}, {"type": "null"}]}, - "pair": {"type": "array", "prefixItems": [{"type": "string", "pattern": pattern}]}, - "extra": {"type": "object", "additionalProperties": {"type": "string", "pattern": pattern}}, - "tagged": { - "type": "object", - "patternProperties": {pattern: {"type": "string"}, "^x_": {"type": "integer"}}, - }, - }, - "$defs": {"segment": {"type": "string", "pattern": pattern}}, - "required": ["field"], - } - - -class TestDropNonPythonRegexPatterns: - """Claude Code's Artifact tool declares ECMA-262 ``\\p{..}`` escapes that OpenAI's - validator, which compiles ``pattern`` values and ``patternProperties`` keys with - Python ``re``, refuses as "not a 'regex'".""" - - def test_drops_every_regex_python_re_rejects_from_every_schema_position(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - schema = _regex_schema(_ARTIFACT_FIELD_PATTERN) - - result = drop_non_python_regex_patterns(schema) - - assert '"pattern"' not in json.dumps(result) - properties = result["properties"] - assert properties["field"] == {"type": "string"} - assert properties["writes"]["items"]["properties"]["doc_id"] == {"type": "string"} - assert properties["query"]["anyOf"] == [{"type": "string"}, {"type": "null"}] - assert properties["pair"]["prefixItems"] == [{"type": "string"}] - assert properties["extra"]["additionalProperties"] == {"type": "string"} - assert properties["tagged"]["patternProperties"] == {"^x_": {"type": "integer"}} - assert result["$defs"]["segment"] == {"type": "string"} - assert result["required"] == ["field"] - assert schema == _regex_schema(_ARTIFACT_FIELD_PATTERN) - - def test_keeps_regexes_python_re_compiles_and_returns_the_same_object(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - schema = _regex_schema(r'^(?!__.*__$)[^"\\./[\]]{1,200}$') - - assert drop_non_python_regex_patterns(schema) is schema - - def test_pattern_keys_inside_data_positions_are_not_regexes(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - schema = { - "type": "object", - "properties": { - "pattern": {"type": "string"}, - "template": {"type": "object", "default": {"pattern": _ARTIFACT_FIELD_PATTERN}}, - "samples": {"type": "array", "examples": [{"pattern": _ARTIFACT_FIELD_PATTERN}]}, - "fixed": {"const": {"pattern": _ARTIFACT_FIELD_PATTERN}}, - "vendor": {"type": "string", "x-litellm": {"pattern": _ARTIFACT_FIELD_PATTERN}}, - }, - "required": ["pattern"], - } - - assert drop_non_python_regex_patterns(schema) is schema - - def test_regex_nested_past_what_python_re_can_parse_is_dropped_not_raised(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - schema = { - "type": "object", - "properties": {"deep": {"type": "string", "pattern": "(" * 2000 + "a" + ")" * 2000}}, - } - - assert drop_non_python_regex_patterns(schema)["properties"]["deep"] == {"type": "string"} - - def test_walks_schemas_deeper_than_the_interpreter_recursion_limit(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - depth = sys.getrecursionlimit() - leaf = {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN} - schema = functools.reduce( - lambda inner, _: {"type": "object", "properties": {"child": inner}}, range(depth), leaf - ) - - result = drop_non_python_regex_patterns(schema) - - assert functools.reduce(lambda node, _: node["properties"]["child"], range(depth), result) == {"type": "string"} - assert functools.reduce(lambda node, _: node["properties"]["child"], range(depth), schema) is leaf - - def test_leaves_levels_past_the_json_nesting_limit_alone(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - leaf = {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN} - schema = functools.reduce( - lambda inner, _: {"type": "object", "properties": {"child": inner}}, range(1100), leaf - ) - - assert drop_non_python_regex_patterns(schema) is schema - - -class TestDropLookaroundRegexPatterns: - """Kimi K3 and Grok 4.6/4.7 on Bedrock Converse reject every tool schema regex that - uses a lookaround assertion, Claude Code's ``ArtifactData`` ``pattern`` included.""" - - @pytest.mark.parametrize( - "pattern", - [r"^(?!x).*$", r"^(?=.*a).*$", r"^.*(?\w+)$", r"^(?i)abc$", r"^[^\p{Cc}\p{Cf}]{1,200}$"], - ids=["plain", "non-capturing-group", "named-group", "inline-flag", "non-python-without-lookaround"], - ) - def test_keeps_regexes_without_lookaround_and_returns_the_same_object(self, pattern): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - ) - - schema = _regex_schema(pattern) - - assert drop_lookaround_regex_patterns(schema) is schema - - def test_lookaround_inside_data_positions_is_not_a_regex(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - ) - - schema = { - "type": "object", - "properties": { - "pattern": {"type": "string"}, - "template": {"type": "object", "default": {"pattern": _ARTIFACT_DATA_ID_PATTERN}}, - "hint": {"type": "string", "description": "ids match " + _ARTIFACT_DATA_ID_PATTERN}, - }, - "required": ["pattern"], - } - - assert drop_lookaround_regex_patterns(schema) is schema - - -@pytest.mark.parametrize( - ("dropper", "patterns"), - [ - ("drop_non_python_regex_patterns", (_ARTIFACT_FIELD_PATTERN, r"^\p{L}+$")), - ("drop_lookaround_regex_patterns", (_ARTIFACT_DATA_ID_PATTERN, r"^(?=.*[a-z])\w+$")), - ], - ids=["non-python", "lookaround"], -) -class TestDroppedPatternPropertiesKeepTheirNamesAllowed: - """Dropping a ``patternProperties`` key from an object closed by ``additionalProperties: - false`` must not ban the names that key allowed: its value schema takes over as the - object's ``additionalProperties``.""" - - @staticmethod - def _drop(dropper): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - drop_non_python_regex_patterns, - ) - - return { - "drop_non_python_regex_patterns": drop_non_python_regex_patterns, - "drop_lookaround_regex_patterns": drop_lookaround_regex_patterns, - }[dropper] - - def test_closed_object_takes_the_dropped_value_schema(self, dropper, patterns): - schema = { - "type": "object", - "patternProperties": {patterns[0]: {"type": "string", "pattern": patterns[0]}}, - "additionalProperties": False, - } - - assert self._drop(dropper)(schema) == { - "type": "object", - "patternProperties": {}, - "additionalProperties": {"type": "string"}, - } - - def test_closed_object_losing_two_entries_accepts_either_value_schema(self, dropper, patterns): - schema = { - "type": "object", - "patternProperties": { - patterns[0]: {"type": "string"}, - patterns[1]: {"type": "integer"}, - "^x_": {"type": "boolean"}, - }, - "additionalProperties": False, - } - - assert self._drop(dropper)(schema) == { - "type": "object", - "patternProperties": {"^x_": {"type": "boolean"}}, - "additionalProperties": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, - } - - def test_object_with_its_own_additional_properties_schema_keeps_it(self, dropper, patterns): - schema = { - "type": "object", - "patternProperties": {patterns[0]: {"type": "string"}}, - "additionalProperties": {"type": "integer"}, - } - - assert self._drop(dropper)(schema) == { - "type": "object", - "patternProperties": {}, - "additionalProperties": {"type": "integer"}, - } - - class TestRequestContainsImageContent: """One detector for every dialect that reaches pre-routing hooks untranslated.""" @@ -2025,134 +738,6 @@ class TestRequestContainsImageContent: assert request_contains_image_content([{"role": "user", "content": [nested]}]) is False -class TestEncryptedReasoningReplay: - """Regression for https://github.com/BerriAI/litellm/issues/40288.""" - - def test_signature_round_trips_the_encrypted_content(self): - assert encrypted_content_from_signature(encrypted_reasoning_signature("gAAAA_bytes")) == "gAAAA_bytes" - - @pytest.mark.parametrize( - "signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7] - ) - def test_anything_else_is_not_encrypted_content(self, signature): - assert encrypted_content_from_signature(signature) is None - - def test_encrypted_thinking_block_replays_its_own_item(self): - items = responses_reasoning_items_from_thinking_blocks( - [{"type": "thinking", "thinking": "Plan.", "signature": encrypted_reasoning_signature("gAAAA_1")}] - ) - assert items == ( - { - "type": "reasoning", - "summary": [{"type": "summary_text", "text": "Plan."}], - "encrypted_content": "gAAAA_1", - }, - ) - - def test_encrypted_redacted_block_replays_with_an_empty_summary(self): - items = responses_reasoning_items_from_thinking_blocks( - [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_1")}] - ) - assert items == ({"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_1"},) - - def test_plain_blocks_collapse_into_one_summary_item_around_encrypted_ones(self): - items = responses_reasoning_items_from_thinking_blocks( - [ - {"type": "thinking", "thinking": "A.", "signature": None}, - {"type": "thinking", "thinking": "B.", "signature": ""}, - {"type": "thinking", "thinking": "C.", "signature": encrypted_reasoning_signature("gAAAA_c")}, - {"type": "redacted_thinking", "data": "anthropic-minted-opaque-data"}, - {"type": "thinking", "thinking": "D."}, - ] - ) - assert items == ( - { - "type": "reasoning", - "summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}], - }, - {"type": "reasoning", "summary": [{"type": "summary_text", "text": "C."}], "encrypted_content": "gAAAA_c"}, - {"type": "reasoning", "summary": [{"type": "summary_text", "text": "D."}]}, - ) - assert all("id" not in item for item in items) - - def test_blocks_without_text_or_encrypted_content_produce_nothing(self): - assert responses_reasoning_items_from_thinking_blocks([{"type": "thinking", "thinking": ""}]) == () - assert responses_reasoning_items_from_thinking_blocks([]) == () - - @pytest.mark.parametrize( - ("block", "expected"), - [ - ({"type": "thinking", "thinking": "x", "signature": encrypted_reasoning_signature("g")}, True), - ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("g")}, True), - ({"type": "thinking", "thinking": "x", "signature": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), - ({"type": "redacted_thinking", "data": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), - ({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}, False), - ({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, False), - ({"type": "text", "text": encrypted_reasoning_signature("g")}, False), - ("not a block", False), - ], - ) - def test_is_encrypted_reasoning_block(self, block, expected): - assert is_encrypted_reasoning_block(block) is expected - - def test_strip_drops_every_bridge_tagged_block_and_leaves_no_unsigned_thinking_behind(self): - assistant_content = [ - {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, - {"type": "thinking", "thinking": "packed by the bridge", "signature": encrypted_reasoning_signature("g1")}, - {"type": "redacted_thinking", "data": encrypted_reasoning_signature("g2")}, - {"type": "thinking", "thinking": "", "signature": encrypted_reasoning_signature("g3")}, - {"type": "text", "text": "answer"}, - ] - messages = [ - {"role": "user", "content": "question"}, - {"role": "assistant", "content": assistant_content}, - {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}, - ] - - strip_encrypted_reasoning_from_messages(messages) - - assert messages[1]["content"] is assistant_content - assert assistant_content == [ - {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, - {"type": "text", "text": "answer"}, - ] - assert all(block["signature"] for block in assistant_content if block["type"] == "thinking") - assert messages[0] == {"role": "user", "content": "question"} - assert messages[2] == {"role": "user", "content": [{"type": "text", "text": "follow-up"}]} - - def test_strip_uses_predicate_to_keep_selected_encrypted_blocks(self): - kept_signature = encrypted_reasoning_signature("keep") - stripped_signature = encrypted_reasoning_signature("strip") - content = [ - {"type": "thinking", "thinking": "keep", "signature": kept_signature}, - {"type": "thinking", "thinking": "strip", "signature": stripped_signature}, - ] - messages = [{"role": "assistant", "content": content}] - - strip_encrypted_reasoning_from_messages(messages, should_strip=lambda block: block.get("thinking") == "strip") - - assert messages[0]["content"] is content - assert content == [{"type": "thinking", "thinking": "keep", "signature": kept_signature}] - - @pytest.mark.parametrize( - "messages", - [ - "not a list", - None, - [{"role": "user", "content": None}], - [{"role": "user", "content": "plain string"}], - ["not a message"], - [{"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}]}], - ], - ) - def test_strip_leaves_history_without_bridge_reasoning_untouched(self, messages): - before = copy.deepcopy(messages) - - strip_encrypted_reasoning_from_messages(messages) - - assert messages == before - - class TestMergeConsecutiveSystemMessages: def test_merges_each_run_of_string_system_messages_with_a_blank_line(self): messages = [ @@ -2337,132 +922,6 @@ def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_w assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages) -_MALFORMED_PROMPT_CACHE_BREAKPOINTS: Final = ( - "yes", - 1, - ["explicit"], - "", - {}, - {"mode": "bogus"}, - {"mode": "bogus", "ttl": "30m"}, - {1: "explicit"}, -) -_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS: Final = ( - "string", - "int", - "list", - "empty-string", - "empty-object", - "bogus-mode", - "bogus-mode-with-ttl", - "non-string-key", -) -_VALID_PROMPT_CACHE_BREAKPOINTS: Final = ({"mode": "explicit"}, {"mode": "explicit", "ttl": "30m"}) - - -@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) -def test_prompt_cache_breakpoint_for_wire_passes_a_missing_marker_through(drop_params: bool) -> None: - assert prompt_cache_breakpoint_for_wire(None, drop_params) is None - - -@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) -def test_prompt_cache_breakpoint_for_wire_drops_a_malformed_marker_under_drop_params(marker: object) -> None: - assert prompt_cache_breakpoint_for_wire(marker, True) is None - - -@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) -def test_prompt_cache_breakpoint_for_wire_forwards_a_malformed_marker_without_drop_params(marker: object) -> None: - assert prompt_cache_breakpoint_for_wire(marker, False) is marker - - -@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) -@pytest.mark.parametrize("marker", _VALID_PROMPT_CACHE_BREAKPOINTS, ids=("explicit", "explicit-30m")) -def test_prompt_cache_breakpoint_for_wire_keeps_a_valid_marker(marker: dict[str, str], drop_params: bool) -> None: - assert prompt_cache_breakpoint_for_wire(marker, drop_params) == marker - - -def test_prompt_cache_breakpoint_for_wire_strips_an_unknown_key_under_drop_params() -> None: - assert prompt_cache_breakpoint_for_wire({"mode": "explicit", "note": "kept"}, True) == {"mode": "explicit"} - - -@pytest.mark.parametrize( - "marker", - ({"mode": "explicit", "ttl": "1h"}, {"mode": "explicit", "ttl": "1h", "note": "kept"}), - ids=("bad-ttl", "bad-ttl-and-unknown-key"), -) -def test_prompt_cache_breakpoint_for_wire_drops_an_unsupported_ttl_and_keeps_the_mode_under_drop_params( - marker: dict[str, str], -) -> None: - assert prompt_cache_breakpoint_for_wire(marker, True) == {"mode": "explicit"} - assert prompt_cache_breakpoint_for_wire(marker, False) is marker - - -def test_cache_control_is_replaced_by_an_explicit_prompt_cache_breakpoint_on_the_same_block() -> None: - marked: Final = {"type": "text", "text": "rules", "cache_control": {"type": "ephemeral", "ttl": "1h"}} - - assert cache_control_as_prompt_cache_breakpoint(marked) == { - "type": "text", - "text": "rules", - "prompt_cache_breakpoint": {"mode": "explicit"}, - } - - -def test_a_block_without_cache_control_is_returned_untouched() -> None: - unmarked: Final = {"type": "text", "text": "rules"} - - assert cache_control_as_prompt_cache_breakpoint(unmarked) is unmarked - - -def test_a_breakpoint_the_client_wrote_survives_a_message_level_mark_ttl_included() -> None: - block: Final = { - "type": "input_text", - "text": "rules", - "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, - } - - assert with_prompt_cache_breakpoint(block, {"mode": "explicit"}) is block - - -def test_a_breakpoint_the_client_wrote_survives_a_per_part_cache_control() -> None: - both: Final = { - "type": "input_text", - "text": "rules", - "cache_control": {"type": "ephemeral"}, - "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, - } - - assert cache_control_as_prompt_cache_breakpoint(both) == { - "type": "input_text", - "text": "rules", - "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, - } - - -_NATIVE_REDACTED_DATA: Final = "EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIwxtE3rAFBa8cr3qpP" -_CONVERSE_REDACTED_DATA: Final = base64.b64encode(_NATIVE_REDACTED_DATA.encode()).decode() - - -@pytest.mark.parametrize( - "data, native", - [ - pytest.param(_NATIVE_REDACTED_DATA, _NATIVE_REDACTED_DATA, id="claude-native"), - pytest.param(_CONVERSE_REDACTED_DATA, _NATIVE_REDACTED_DATA, id="converse-wrapped-legacy-history"), - pytest.param( - encrypted_reasoning_signature("gAAAA_1"), encrypted_reasoning_signature("gAAAA_1"), id="openai-encrypted" - ), - pytest.param("", "", id="blank"), - pytest.param(base64.b64encode(b"not base64!").decode(), base64.b64encode(b"not base64!").decode(), id="b64-once"), - ], -) -def test_native_redacted_thinking_data_only_unwraps_the_converse_base64_layer(data: str, native: str) -> None: - assert native_redacted_thinking_data(data) == native - - -@pytest.mark.parametrize("data", [_NATIVE_REDACTED_DATA, _CONVERSE_REDACTED_DATA], ids=["native", "legacy-wrapped"]) -def test_converse_redacted_content_wraps_claude_native_data_exactly_once(data: str) -> None: - assert converse_redacted_content(data) == _CONVERSE_REDACTED_DATA - - def test_strip_litellm_response_only_fields_drops_nested_tool_call_provider_fields() -> None: message: Final = { "role": "assistant", @@ -2490,78 +949,6 @@ def test_strip_litellm_response_only_fields_drops_nested_tool_call_provider_fiel assert function["arguments"] == '{"city": "Paris"}' -@pytest.mark.parametrize( - ("schema", "is_open"), - [ - ({"type": "object", "properties": {"a": {"type": "string"}}}, False), - ({"type": "object", "properties": {}}, False), - ({"type": "object", "additionalProperties": False}, False), - ({"type": "string"}, False), - ({"type": "object"}, True), - ({"type": ["object", "null"]}, True), - ({"type": "object", "properties": {"a": {"type": "string"}}, "additionalProperties": True}, True), - ({"type": "object", "additionalProperties": {"type": "integer"}}, True), - ({"type": "object", "properties": {}, "patternProperties": {"^x": {"type": "string"}}}, True), - ({"type": "object", "properties": {"inner": {"anyOf": [{"type": "object"}, {"type": "null"}]}}}, True), - ({"type": "array", "items": {"type": "object", "additionalProperties": {"type": "string"}}}, True), - ], -) -def test_schema_has_open_object_flags_any_node_that_accepts_unlisted_keys(schema, is_open): - assert schema_has_open_object(schema) is is_open - - -def _object_with(properties: Mapping[str, object], **defs: Mapping[str, object]) -> Mapping[str, object]: - return {"type": "object", "properties": properties, **defs} - - -@pytest.mark.parametrize( - ("schema", "unresolvable"), - [ - (_object_with({"a": {"type": "string"}}), False), - (_object_with({"a": {"$ref": "#/$defs/A"}, "b": {"$ref": "#/$defs/A"}}, **{"$defs": {"A": {"type": "string"}}}), False), - (_object_with({"a": {"$ref": "#/definitions/A"}}, definitions={"A": {"$ref": "#/definitions/B"}, "B": {"type": "integer"}}), False), - (_object_with({"a": {"$ref": "/$defs/A"}}, **{"$defs": {"A": {"items": {"$ref": "/$defs/B"}}, "B": {"type": "string"}}}), False), - (_object_with({"a": {"$ref": "#/$defs/Missing"}}, **{"$defs": {"A": {"type": "string"}}}), True), - (_object_with({"a": {"$ref": "other.json#/A"}}), True), - (_object_with({"parent": {"$ref": "#"}}), True), - (_object_with({"a": {"$ref": "#/$defs/A"}}, **{"$defs": {"A": {"type": "array", "items": {"$ref": "#/$defs/A"}}}}), True), - (_object_with({"a": {"$ref": "/$defs/A"}}, **{"$defs": {"A": {"type": "array", "items": {"$ref": "/$defs/A"}}}}), True), - ( - _object_with( - {"a": {"$ref": "#/$defs/A"}}, - **{"$defs": {"A": {"properties": {"b": {"$ref": "#/$defs/B"}}}, "B": {"properties": {"a": {"$ref": "#/$defs/A"}}}}}, - ), - True, - ), - ], - ids=["no-refs", "shared-def", "chained-definitions", "root-path-defs", "missing-def", "external", "root", "self-cycle", "root-path-self-cycle", "mutual-cycle"], -) -def test_schema_has_unresolvable_ref_flags_refs_that_cannot_be_inlined(schema, unresolvable): - assert schema_has_unresolvable_ref(schema) is unresolvable - - -@pytest.mark.parametrize( - ("schema", "properties", "branches"), - [ - (_object_with({"a": {"type": "string"}, "b": {"type": "string"}, "c": {"type": "string"}}, required=["a"]), 3, 8), - ( - _object_with( - {"x": {"$ref": "#/$defs/Pair"}, "y": {"$ref": "#/$defs/Pair"}}, - required=["x", "y"], - **{"$defs": {"Pair": _object_with({"l": {"type": "string"}, "r": {"type": "string"}}, required=["l"])}}, - ), - 4, - 7, - ), - (_object_with({"inner": _object_with({"z": {"type": "string"}})}, required=["inner"]), 2, 4), - ], - ids=["flat", "shared-def-counted-once", "nested"], -) -def test_schema_size_counts_every_written_object_once(schema, properties, branches): - assert schema_property_count(schema) == properties - assert schema_grammar_branch_count(schema) == branches - - @pytest.mark.parametrize("part_type", ["image_url", "input_image"]) @pytest.mark.parametrize( "fields", @@ -2572,7 +959,8 @@ def test_schema_size_counts_every_written_object_once(schema, properties, branch ], ) def test_shared_image_url_agrees_with_responses_normalization( - part_type: str, fields: Mapping[str, object], + part_type: str, + fields: Mapping[str, object], ) -> None: from litellm.litellm_core_utils.prompt_templates.common_utils import as_openai_image_part from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig @@ -2584,121 +972,6 @@ def test_shared_image_url_agrees_with_responses_normalization( assert as_openai_image_part(normalized) == expected -@pytest.mark.parametrize( - "part,projected,media", - [ - ( - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}}, - {"type": "image_url", "image_url": {"url": "", "detail": "high"}}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}}, - ), - ( - {"type": "input_image", "image_url": "https://example.com/image.png", "detail": "low"}, - {"type": "input_image", "image_url": "", "detail": "low"}, - {"type": "image_url", "image_url": {"url": "https://example.com/image.png", "detail": "low"}}, - ), - ( - {"type": "computer_screenshot", "image_url": "data:image/png;base64,IMAGE"}, - {"type": "computer_screenshot", "image_url": ""}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, - ), - ( - {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "IMAGE"}}, - {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, - ), - ( - {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, - {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}}, - {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, - ), - ( - {"type": "file", "file": {"file_data": "IMAGE", "format": "image/png", "filename": "image.png"}}, - {"type": "file", "file": {"file_data": "", "format": "image/png", "filename": "image.png"}}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, - ), - ( - {"type": "input_file", "file_data": "data:audio/wav;base64,AUDIO", "filename": "audio.wav"}, - {"type": "input_file", "file_data": "", "filename": "audio.wav"}, - {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, - ), - ], -) -def test_tpm_projection_blanks_only_declared_media_transport( - part: Mapping[str, object], projected: Mapping[str, object], media: Mapping[str, object], -) -> None: - text: Final = {"type": "text", "text": "data:image/png;base64,IMAGE"} - request: Final = [{"role": "user", "content": [text, {**part, "cache_control": {"type": "ephemeral"}}]}] - original: Final = copy.deepcopy(request) - assert split_media_for_tpm(request) == ( - [{"role": "user", "content": [text, {**projected, "cache_control": {"type": "ephemeral"}}]}], - (media,), - ) - assert request == original - - -def test_tpm_projection_retains_nested_and_native_history_around_media() -> None: - image: Final = {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}} - empty_image: Final = {"type": "image_url", "image_url": {"url": "", "detail": "high"}} - document: Final = {"type": "document", "title": "keep", "source": {"type": "content", "content": [image]}} - empty_document: Final = {**document, "source": {"type": "content", "content": [empty_image]}} - tool: Final = {"type": "tool_result", "tool_use_id": "call", "content": [document, {"type": "text", "text": "keep"}]} - empty_tool: Final = {**tool, "content": [empty_document, {"type": "text", "text": "keep"}]} - request: Final = [ - {"role": "user", "content": [tool]}, - {"type": "function_call_output", "call_id": "call", "output": [image]}, - {"type": "custom_tool_call_output", "call_id": "custom", "output": [image]}, - {"type": "computer_call_output", "call_id": "computer", "output": { - "type": "computer_screenshot", "image_url": "data:image/png;base64,IMAGE", - }}, - {"type": "image_generation_call", "id": "ig", "result": "IMAGE", "status": "completed"}, - {"type": "code_interpreter_call", "id": "ci", "code": "keep()", "outputs": [ - {"type": "logs", "logs": "keep"}, {"type": "image", "url": "https://example.com/image.png"}, - ]}, - ] - original: Final = copy.deepcopy(request) - projected, media = split_media_for_tpm(request) - assert projected == [ - {"role": "user", "content": [empty_tool]}, - {"type": "function_call_output", "call_id": "call", "output": [empty_image]}, - {"type": "custom_tool_call_output", "call_id": "custom", "output": [empty_image]}, - {"type": "computer_call_output", "call_id": "computer", "output": { - "type": "computer_screenshot", "image_url": "", - }}, - {"type": "image_generation_call", "id": "ig", "result": "", "status": "completed"}, - {"type": "code_interpreter_call", "id": "ci", "code": "keep()", "outputs": [ - {"type": "logs", "logs": "keep"}, {"type": "image", "url": ""}, - ]}, - ] - assert media == ( - image, image, image, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, - {"type": "image_url", "image_url": {"url": "data:application/octet-stream;base64,IMAGE", "detail": "auto"}}, - {"type": "image_url", "image_url": {"url": "https://example.com/image.png", "detail": "auto"}}, - ) - assert request == original - - -def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None: - image: Final = {"type": "input_image", "image_url": "data:image/png;base64,OPAQUE"} - request: Final = [ - {"role": "user", "content": json.dumps(image)}, - {"type": "custom_tool_call_output", "call_id": "call", "output": json.dumps(image)}, - {"type": "function_call", "call_id": "call", "name": "keep", "arguments": {"content": [image]}}, - {"type": "file_search_call", "results": [{"text": "keep", "content": [image]}]}, - {"type": "image_generation_call", "id": "ig", "status": "in_progress", "result": None}, - {"role": "user", "content": [ - {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "PDF"}}, - {"type": "document", "source": {"type": "text", "data": json.dumps(image)}}, - {"type": "file", "file": {"format": "application/pdf", "file_data": "PDF"}}, - {"type": "text", "text": "keep", "metadata": {"content": [image]}}, - ]}, - ] - projected, media = split_media_for_tpm(request) - assert projected is request - assert media == () - - def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None: messages: Final = [ {"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]}, @@ -2710,22 +983,3 @@ def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_unt assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}] assert messages == messages_before - - -@pytest.mark.parametrize("mime", ["application/pdf", "video/mp4", "audio/wav", "image/png"]) -def test_embedding_media_split_holds_at_most_one_copy_of_a_large_payload(mime: str) -> None: - import gc - import tracemalloc - - uri: Final = f"data:{mime};base64," + base64.b64encode(bytes(3 * 1024 * 1024)).decode("ascii") - split_embedding_media_for_tpm(f"data:{mime};base64,AAAA") - gc.collect() - tracemalloc.start() - try: - _, media = split_embedding_media_for_tpm(uri) - _, peak = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - - assert media - assert peak < 1.5 * len(uri) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index d65f094971a..bb88f632073 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1,4 +1,6 @@ -import asyncio, base64, importlib, uuid +import asyncio +import base64 +import importlib import json import logging import os @@ -6,8 +8,10 @@ import re import sys import time import types +import uuid from collections.abc import Callable, Mapping, Sequence from copy import deepcopy +from datetime import datetime from typing import Final, List, TypeAlias, cast from unittest.mock import MagicMock, patch @@ -16,62 +20,49 @@ import pytest import respx import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + get_completion_messages, +) from litellm.litellm_core_utils.prompt_templates.factory import ( - BEDROCK_DOCUMENT_PLACEHOLDER_TEXT, _EMPTY_TEXT_PLACEHOLDER, + BEDROCK_DOCUMENT_PLACEHOLDER_TEXT, + THOUGHT_SIGNATURE_SEPARATOR, BedrockConverseMessagesProcessor, BedrockImageProcessor, _bedrock_converse_messages_pt, _bedrock_tools_pt, - _rename_duplicate_bedrock_document_names, _convert_to_bedrock_tool_call_invoke, - sanitize_anthropic_tool_use_id, _convert_to_bedrock_tool_call_result, + _rename_duplicate_bedrock_document_names, anthropic_messages_pt, - convert_to_anthropic_tool_result, - convert_to_gemini_tool_call_result, - convert_to_ollama_image, - encode_tool_call_id_with_signature, - function_call_prompt, - get_thought_signature_from_tool, - get_tool_calls_from_response, - make_valid_bedrock_tool_name, - ollama_pt, - parse_mime_type, - sanitize_messages_for_tool_calling, - THOUGHT_SIGNATURE_SEPARATOR, -) -from litellm.types.llms.openai import ChatCompletionAssistantMessage, ChatCompletionToolMessage -from litellm.types.utils import ChatCompletionMessageToolCall, Function -from litellm.utils import ( - _invalidate_model_cost_lowercase_map, - function_setup, - Rules, - validate_and_fix_openai_messages, -) -from datetime import datetime -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_pt, claude_2_1_pt, convert_generic_image_chunk_to_openai_image_obj, convert_to_anthropic_image_obj, convert_to_anthropic_tool_invoke, + convert_to_anthropic_tool_result, + convert_to_gemini_tool_call_result, + convert_to_ollama_image, convert_url_to_base64, create_anthropic_image_param, + encode_tool_call_id_with_signature, + function_call_prompt, + get_thought_signature_from_tool, + get_tool_calls_from_response, has_tool_with_name, llama_2_chat_pt, + make_valid_bedrock_tool_name, + ollama_pt, + parse_mime_type, prompt_factory, + response_schema_prompt, + sanitize_anthropic_tool_use_id, + sanitize_messages_for_tool_calling, ) -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, - get_completion_messages, ) -from litellm.llms.vertex_ai.gemini.transformation import gemini_convert_messages_with_history -from litellm.types.llms.bedrock import BedrockConverseReasoningContentBlock -from litellm.types.llms.openai import AllMessageValues -from litellm.litellm_core_utils.prompt_templates.factory import response_schema_prompt from litellm.llms.anthropic.completion.transformation import AnthropicTextConfig from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig from litellm.llms.codestral.completion.handler import CodestralTextCompletion @@ -79,10 +70,19 @@ from litellm.llms.ollama.completion.transformation import OllamaConfig from litellm.llms.petals.completion import handler as petals_handler from litellm.llms.predibase.chat.transformation import PredibaseConfig from litellm.llms.sagemaker.completion.transformation import SagemakerConfig +from litellm.llms.vertex_ai.gemini.transformation import gemini_convert_messages_with_history from litellm.llms.vllm.completion import handler as vllm_handler from litellm.llms.watsonx.common_utils import _convert_watsonx_messages_core -from litellm.types.utils import ModelResponse, TextCompletionResponse - +from litellm.types.llms.bedrock import BedrockConverseReasoningContentBlock +from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionToolMessage +from litellm.types.utils import ChatCompletionMessageToolCall, Function, ModelResponse, TextCompletionResponse +from litellm.utils import ( + Rules, + _invalidate_model_cost_lowercase_map, + function_setup, + validate_and_fix_openai_messages, +) +from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome READ_FILE: Final[list[object]] = [{"name": "read_file", "description": "Read the contents of a file."}] USER_TURN: Final[dict[str, object]] = {"role": "user", "content": "Read the file."} @@ -3460,8 +3460,8 @@ def test_bedrock_tools_unpack_defs_no_oom_with_nested_refs(): This test creates a schema with multiple nested $defs that reference each other to verify the fix prevents memory explosion while still correctly resolving refs. """ - import sys import copy + import sys from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt @@ -7852,7 +7852,7 @@ def test_anthropic_messages_pt_with_bash_tool_result_in_provider_specific_fields def test_parse_tool_call_arguments_valid_json(): """Test that valid JSON is parsed correctly.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -7862,7 +7862,7 @@ def test_parse_tool_call_arguments_valid_json(): def test_parse_tool_call_arguments_empty_input(): """Test that None/empty input returns empty dict.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -7872,7 +7872,7 @@ def test_parse_tool_call_arguments_empty_input(): def test_parse_tool_call_arguments_malformed_json(): """Test that malformed JSON raises ValueError with context.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -7918,7 +7918,7 @@ def test_convert_to_anthropic_tool_invoke_malformed_json(): def test_attempt_json_repair_missing_closing_brace(): """Repair JSON truncated with a missing closing brace (issue #22312).""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7934,7 +7934,7 @@ def test_attempt_json_repair_missing_closing_brace(): def test_attempt_json_repair_missing_bracket_and_brace(): """Repair JSON truncated with both missing ] and }.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7946,7 +7946,7 @@ def test_attempt_json_repair_missing_bracket_and_brace(): def test_attempt_json_repair_trailing_comma(): """Repair JSON with a trailing comma before missing close.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7958,7 +7958,7 @@ def test_attempt_json_repair_trailing_comma(): def test_attempt_json_repair_returns_none_for_unterminated_string(): """Cannot repair an unterminated string — returns None.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7967,7 +7967,7 @@ def test_attempt_json_repair_returns_none_for_unterminated_string(): def test_attempt_json_repair_returns_none_for_valid_json(): """Valid JSON has no unmatched brackets — returns None (no repair needed).""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7976,7 +7976,7 @@ def test_attempt_json_repair_returns_none_for_valid_json(): def test_attempt_json_repair_returns_none_for_empty(): """Empty / whitespace input returns None.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7986,7 +7986,7 @@ def test_attempt_json_repair_returns_none_for_empty(): def test_attempt_json_repair_interleaved_nesting(): """Repair JSON with interleaved {} and [] nesting.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7998,7 +7998,7 @@ def test_attempt_json_repair_interleaved_nesting(): def test_attempt_json_repair_deeply_nested(): """Repair deeply nested truncated JSON.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -8010,7 +8010,7 @@ def test_attempt_json_repair_deeply_nested(): def test_parse_tool_call_arguments_whitespace_only(): """Whitespace-only input returns empty dict.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -8020,7 +8020,7 @@ def test_parse_tool_call_arguments_whitespace_only(): def test_parse_tool_call_arguments_non_object_json(): """Non-object JSON (list, string, number) is returned as-is (no wrapping).""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -8030,7 +8030,7 @@ def test_parse_tool_call_arguments_non_object_json(): def test_parse_tool_call_arguments_repairs_truncated_json(): """parse_tool_call_arguments should repair truncated JSON instead of raising.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -8041,7 +8041,7 @@ def test_parse_tool_call_arguments_repairs_truncated_json(): def test_parse_tool_call_arguments_still_raises_for_unrepairable(): """parse_tool_call_arguments raises ValueError when repair also fails.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py index d7696c00dee..34d525946b3 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py @@ -12,13 +12,15 @@ from typing import Final import pytest import litellm -from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( CONVERTED_SYSTEM_NOTE, anthropic_system_blocks, place_mid_conversation_system, split_leading_system_run, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_reasoning_signature, +) def _roles(messages: object) -> list[str]: diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_prompt_cache_breakpoint_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_prompt_cache_breakpoint_utils.py new file mode 100644 index 00000000000..9dd50368529 --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_prompt_cache_breakpoint_utils.py @@ -0,0 +1,113 @@ +from typing import Final + +import pytest + +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + cache_control_as_prompt_cache_breakpoint, + prompt_cache_breakpoint_for_wire, + with_prompt_cache_breakpoint, +) + +_MALFORMED_PROMPT_CACHE_BREAKPOINTS: Final = ( + "yes", + 1, + ["explicit"], + "", + {}, + {"mode": "bogus"}, + {"mode": "bogus", "ttl": "30m"}, + {1: "explicit"}, +) + + +_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS: Final = ( + "string", + "int", + "list", + "empty-string", + "empty-object", + "bogus-mode", + "bogus-mode-with-ttl", + "non-string-key", +) + + +_VALID_PROMPT_CACHE_BREAKPOINTS: Final = ({"mode": "explicit"}, {"mode": "explicit", "ttl": "30m"}) + + +@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) +def test_prompt_cache_breakpoint_for_wire_passes_a_missing_marker_through(drop_params: bool) -> None: + assert prompt_cache_breakpoint_for_wire(None, drop_params) is None + + +@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) +def test_prompt_cache_breakpoint_for_wire_drops_a_malformed_marker_under_drop_params(marker: object) -> None: + assert prompt_cache_breakpoint_for_wire(marker, True) is None + + +@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) +def test_prompt_cache_breakpoint_for_wire_forwards_a_malformed_marker_without_drop_params(marker: object) -> None: + assert prompt_cache_breakpoint_for_wire(marker, False) is marker + + +@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) +@pytest.mark.parametrize("marker", _VALID_PROMPT_CACHE_BREAKPOINTS, ids=("explicit", "explicit-30m")) +def test_prompt_cache_breakpoint_for_wire_keeps_a_valid_marker(marker: dict[str, str], drop_params: bool) -> None: + assert prompt_cache_breakpoint_for_wire(marker, drop_params) == marker + + +def test_prompt_cache_breakpoint_for_wire_strips_an_unknown_key_under_drop_params() -> None: + assert prompt_cache_breakpoint_for_wire({"mode": "explicit", "note": "kept"}, True) == {"mode": "explicit"} + + +@pytest.mark.parametrize( + "marker", + ({"mode": "explicit", "ttl": "1h"}, {"mode": "explicit", "ttl": "1h", "note": "kept"}), + ids=("bad-ttl", "bad-ttl-and-unknown-key"), +) +def test_prompt_cache_breakpoint_for_wire_drops_an_unsupported_ttl_and_keeps_the_mode_under_drop_params( + marker: dict[str, str], +) -> None: + assert prompt_cache_breakpoint_for_wire(marker, True) == {"mode": "explicit"} + assert prompt_cache_breakpoint_for_wire(marker, False) is marker + + +def test_cache_control_is_replaced_by_an_explicit_prompt_cache_breakpoint_on_the_same_block() -> None: + marked: Final = {"type": "text", "text": "rules", "cache_control": {"type": "ephemeral", "ttl": "1h"}} + + assert cache_control_as_prompt_cache_breakpoint(marked) == { + "type": "text", + "text": "rules", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + + +def test_a_block_without_cache_control_is_returned_untouched() -> None: + unmarked: Final = {"type": "text", "text": "rules"} + + assert cache_control_as_prompt_cache_breakpoint(unmarked) is unmarked + + +def test_a_breakpoint_the_client_wrote_survives_a_message_level_mark_ttl_included() -> None: + block: Final = { + "type": "input_text", + "text": "rules", + "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, + } + + assert with_prompt_cache_breakpoint(block, {"mode": "explicit"}) is block + + +def test_a_breakpoint_the_client_wrote_survives_a_per_part_cache_control() -> None: + both: Final = { + "type": "input_text", + "text": "rules", + "cache_control": {"type": "ephemeral"}, + "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, + } + + assert cache_control_as_prompt_cache_breakpoint(both) == { + "type": "input_text", + "text": "rules", + "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, + } diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_reasoning_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_reasoning_utils.py new file mode 100644 index 00000000000..840497777ad --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_reasoning_utils.py @@ -0,0 +1,171 @@ +import base64 +import copy +from typing import Final + +import pytest + +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + ENCRYPTED_REASONING_SIGNATURE_PREFIX, + converse_redacted_content, + encrypted_content_from_signature, + encrypted_reasoning_signature, + is_encrypted_reasoning_block, + native_redacted_thinking_data, + responses_reasoning_items_from_thinking_blocks, + strip_encrypted_reasoning_from_messages, +) + + +class TestEncryptedReasoningReplay: + """Regression for https://github.com/BerriAI/litellm/issues/40288.""" + + def test_signature_round_trips_the_encrypted_content(self): + assert encrypted_content_from_signature(encrypted_reasoning_signature("gAAAA_bytes")) == "gAAAA_bytes" + + @pytest.mark.parametrize( + "signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7] + ) + def test_anything_else_is_not_encrypted_content(self, signature): + assert encrypted_content_from_signature(signature) is None + + def test_encrypted_thinking_block_replays_its_own_item(self): + items = responses_reasoning_items_from_thinking_blocks( + [{"type": "thinking", "thinking": "Plan.", "signature": encrypted_reasoning_signature("gAAAA_1")}] + ) + assert items == ( + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Plan."}], + "encrypted_content": "gAAAA_1", + }, + ) + + def test_encrypted_redacted_block_replays_with_an_empty_summary(self): + items = responses_reasoning_items_from_thinking_blocks( + [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_1")}] + ) + assert items == ({"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_1"},) + + def test_plain_blocks_collapse_into_one_summary_item_around_encrypted_ones(self): + items = responses_reasoning_items_from_thinking_blocks( + [ + {"type": "thinking", "thinking": "A.", "signature": None}, + {"type": "thinking", "thinking": "B.", "signature": ""}, + {"type": "thinking", "thinking": "C.", "signature": encrypted_reasoning_signature("gAAAA_c")}, + {"type": "redacted_thinking", "data": "anthropic-minted-opaque-data"}, + {"type": "thinking", "thinking": "D."}, + ] + ) + assert items == ( + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}], + }, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "C."}], "encrypted_content": "gAAAA_c"}, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "D."}]}, + ) + assert all("id" not in item for item in items) + + def test_blocks_without_text_or_encrypted_content_produce_nothing(self): + assert responses_reasoning_items_from_thinking_blocks([{"type": "thinking", "thinking": ""}]) == () + assert responses_reasoning_items_from_thinking_blocks([]) == () + + @pytest.mark.parametrize( + ("block", "expected"), + [ + ({"type": "thinking", "thinking": "x", "signature": encrypted_reasoning_signature("g")}, True), + ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("g")}, True), + ({"type": "thinking", "thinking": "x", "signature": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), + ({"type": "redacted_thinking", "data": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), + ({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}, False), + ({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, False), + ({"type": "text", "text": encrypted_reasoning_signature("g")}, False), + ("not a block", False), + ], + ) + def test_is_encrypted_reasoning_block(self, block, expected): + assert is_encrypted_reasoning_block(block) is expected + + def test_strip_drops_every_bridge_tagged_block_and_leaves_no_unsigned_thinking_behind(self): + assistant_content = [ + {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, + {"type": "thinking", "thinking": "packed by the bridge", "signature": encrypted_reasoning_signature("g1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("g2")}, + {"type": "thinking", "thinking": "", "signature": encrypted_reasoning_signature("g3")}, + {"type": "text", "text": "answer"}, + ] + messages = [ + {"role": "user", "content": "question"}, + {"role": "assistant", "content": assistant_content}, + {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}, + ] + + strip_encrypted_reasoning_from_messages(messages) + + assert messages[1]["content"] is assistant_content + assert assistant_content == [ + {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, + {"type": "text", "text": "answer"}, + ] + assert all(block["signature"] for block in assistant_content if block["type"] == "thinking") + assert messages[0] == {"role": "user", "content": "question"} + assert messages[2] == {"role": "user", "content": [{"type": "text", "text": "follow-up"}]} + + def test_strip_uses_predicate_to_keep_selected_encrypted_blocks(self): + kept_signature = encrypted_reasoning_signature("keep") + stripped_signature = encrypted_reasoning_signature("strip") + content = [ + {"type": "thinking", "thinking": "keep", "signature": kept_signature}, + {"type": "thinking", "thinking": "strip", "signature": stripped_signature}, + ] + messages = [{"role": "assistant", "content": content}] + + strip_encrypted_reasoning_from_messages(messages, should_strip=lambda block: block.get("thinking") == "strip") + + assert messages[0]["content"] is content + assert content == [{"type": "thinking", "thinking": "keep", "signature": kept_signature}] + + @pytest.mark.parametrize( + "messages", + [ + "not a list", + None, + [{"role": "user", "content": None}], + [{"role": "user", "content": "plain string"}], + ["not a message"], + [{"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}]}], + ], + ) + def test_strip_leaves_history_without_bridge_reasoning_untouched(self, messages): + before = copy.deepcopy(messages) + + strip_encrypted_reasoning_from_messages(messages) + + assert messages == before + + +_NATIVE_REDACTED_DATA: Final = "EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIwxtE3rAFBa8cr3qpP" + + +_CONVERSE_REDACTED_DATA: Final = base64.b64encode(_NATIVE_REDACTED_DATA.encode()).decode() + + +@pytest.mark.parametrize( + "data, native", + [ + pytest.param(_NATIVE_REDACTED_DATA, _NATIVE_REDACTED_DATA, id="claude-native"), + pytest.param(_CONVERSE_REDACTED_DATA, _NATIVE_REDACTED_DATA, id="converse-wrapped-legacy-history"), + pytest.param( + encrypted_reasoning_signature("gAAAA_1"), encrypted_reasoning_signature("gAAAA_1"), id="openai-encrypted" + ), + pytest.param("", "", id="blank"), + pytest.param(base64.b64encode(b"not base64!").decode(), base64.b64encode(b"not base64!").decode(), id="b64-once"), + ], +) +def test_native_redacted_thinking_data_only_unwraps_the_converse_base64_layer(data: str, native: str) -> None: + assert native_redacted_thinking_data(data) == native + + +@pytest.mark.parametrize("data", [_NATIVE_REDACTED_DATA, _CONVERSE_REDACTED_DATA], ids=["native", "legacy-wrapped"]) +def test_converse_redacted_content_wraps_claude_native_data_exactly_once(data: str) -> None: + assert converse_redacted_content(data) == _CONVERSE_REDACTED_DATA diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_tool_call_json_repair.py b/tests/unit/litellm_core_utils/prompt_templates/test_tool_call_json_repair.py new file mode 100644 index 00000000000..75f4e686137 --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_tool_call_json_repair.py @@ -0,0 +1,117 @@ +import json + +import pytest + +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + parse_tool_call_arguments, + salvage_concatenated_tool_arguments, + split_concatenated_json_objects, +) + + +def test_split_concatenated_json_single_object(): + """A single valid JSON object is returned as a one-element list.""" + result = split_concatenated_json_objects('{"location": "Boston"}') + assert result == [{"location": "Boston"}] + + +def test_split_concatenated_json_multiple_objects(): + """ + Multiple JSON objects concatenated without separators are split correctly. + This is the exact pattern from issue #20543 where Bedrock Claude Sonnet 4.5 + returns concatenated JSON in a single tool call arguments string. + """ + raw = ( + '{"command": ["curl", "-i", "http://localhost:9009"]}' + '{"command": ["curl", "-i", "http://localhost:9009/robots.txt"]}' + '{"command": ["curl", "-i", "http://localhost:9009/sitemap.xml"]}' + ) + result = split_concatenated_json_objects(raw) + assert len(result) == 3 + assert result[0] == {"command": ["curl", "-i", "http://localhost:9009"]} + assert result[1] == {"command": ["curl", "-i", "http://localhost:9009/robots.txt"]} + assert result[2] == {"command": ["curl", "-i", "http://localhost:9009/sitemap.xml"]} + + +def test_split_concatenated_json_with_whitespace(): + """Objects separated by whitespace are handled correctly.""" + raw = '{"a": 1} {"b": 2}\n{"c": 3}' + result = split_concatenated_json_objects(raw) + assert len(result) == 3 + assert result[0] == {"a": 1} + assert result[1] == {"b": 2} + assert result[2] == {"c": 3} + + +def test_split_concatenated_json_empty_string(): + """Empty or whitespace-only strings return an empty list.""" + assert split_concatenated_json_objects("") == [] + assert split_concatenated_json_objects(" ") == [] + + +def test_split_concatenated_json_non_dict_value(): + """Non-dict JSON values (e.g. arrays, strings) are replaced with {}.""" + result = split_concatenated_json_objects("[1, 2, 3]") + assert result == [{}] + + +def test_split_concatenated_json_wholly_invalid_returns_empty(): + """ + Wholly unparseable JSON degrades to an empty list instead of raising. + + Regression for https://github.com/BerriAI/litellm/issues/18667: a raise + here propagated out of `_convert_to_bedrock_tool_call_invoke` and turned + every replayed conversation into a 500. + """ + assert split_concatenated_json_objects("not json at all") == [] + + +def test_split_concatenated_json_malformed_object_returns_empty(): + """ + A single malformed object (missing comma between keys) degrades to an + empty list rather than raising `Expecting ',' delimiter`. + """ + assert split_concatenated_json_objects('{"location": "Boston" "unit": "celsius"}') == [] + + +def test_split_concatenated_json_salvages_prefix_before_truncated_tail(): + """ + Complete objects parsed before an unparseable/truncated tail are kept; + only the bad tail is discarded. + """ + result = split_concatenated_json_objects('{"a": 1}{"b": 2}{"c":') + assert result == [{"a": 1}, {"b": 2}] + + +def test_parse_tool_call_arguments_rejects_concatenated_json() -> None: + with pytest.raises(ValueError, match="Failed to parse tool call arguments"): + parse_tool_call_arguments('{"a":1}{"b":2}') + + +def _distinct_json_objects(count: int) -> str: + return "".join(json.dumps({"n": index}, separators=(",", ":")) for index in range(count)) + + +@pytest.mark.parametrize( + ("raw", "expected"), + ( + ('{"a":1}{"b":2}', ({"a": 1}, {"b": 2})), + ('{"a":1}{"a":1}{"a":1}', ({"a": 1},)), + ('{"a":1}{"a":1}{"b":2}', ({"a": 1}, {"a": 1}, {"b": 2})), + (_distinct_json_objects(8), tuple({"n": index} for index in range(8))), + (_distinct_json_objects(9), ()), + (_distinct_json_objects(9) + " junk", ()), + ('{"a":1}' * 7 + '{"b":2}', tuple({"a": 1} for _ in range(7)) + ({"b": 2},)), + ('{"a":1}' * 8 + '{"b":2}', ()), + ('{"a":1}' * 5000, ({"a": 1},)), + ('{"a":1}' * 20, ({"a": 1},)), + ('{"a":1}{"b":', ()), + ('0{"x":1}', ()), + ('{"x":1}0', ()), + ('[1]{"x":1}', ()), + ('{"a":1}{"b":2}}', ()), + ('{"a":1} junk', ()), + ), +) +def test_salvage_concatenated_tool_arguments(raw: str, expected: tuple[dict[str, object], ...]) -> None: + assert salvage_concatenated_tool_arguments(raw) == expected diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_tpm_media_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_tpm_media_utils.py new file mode 100644 index 00000000000..81a688e71e2 --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_tpm_media_utils.py @@ -0,0 +1,146 @@ +import base64 +import copy +import json +from collections.abc import Mapping +from typing import Final + +import pytest + +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + split_embedding_media_for_tpm, + split_media_for_tpm, +) + + +@pytest.mark.parametrize( + "part,projected,media", + [ + ( + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}}, + {"type": "image_url", "image_url": {"url": "", "detail": "high"}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}}, + ), + ( + {"type": "input_image", "image_url": "https://example.com/image.png", "detail": "low"}, + {"type": "input_image", "image_url": "", "detail": "low"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png", "detail": "low"}}, + ), + ( + {"type": "computer_screenshot", "image_url": "data:image/png;base64,IMAGE"}, + {"type": "computer_screenshot", "image_url": ""}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, + ), + ( + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "IMAGE"}}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, + ), + ( + {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, + {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}}, + {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, + ), + ( + {"type": "file", "file": {"file_data": "IMAGE", "format": "image/png", "filename": "image.png"}}, + {"type": "file", "file": {"file_data": "", "format": "image/png", "filename": "image.png"}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, + ), + ( + {"type": "input_file", "file_data": "data:audio/wav;base64,AUDIO", "filename": "audio.wav"}, + {"type": "input_file", "file_data": "", "filename": "audio.wav"}, + {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, + ), + ], +) +def test_tpm_projection_blanks_only_declared_media_transport( + part: Mapping[str, object], projected: Mapping[str, object], media: Mapping[str, object], +) -> None: + text: Final = {"type": "text", "text": "data:image/png;base64,IMAGE"} + request: Final = [{"role": "user", "content": [text, {**part, "cache_control": {"type": "ephemeral"}}]}] + original: Final = copy.deepcopy(request) + assert split_media_for_tpm(request) == ( + [{"role": "user", "content": [text, {**projected, "cache_control": {"type": "ephemeral"}}]}], + (media,), + ) + assert request == original + + +def test_tpm_projection_retains_nested_and_native_history_around_media() -> None: + image: Final = {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}} + empty_image: Final = {"type": "image_url", "image_url": {"url": "", "detail": "high"}} + document: Final = {"type": "document", "title": "keep", "source": {"type": "content", "content": [image]}} + empty_document: Final = {**document, "source": {"type": "content", "content": [empty_image]}} + tool: Final = {"type": "tool_result", "tool_use_id": "call", "content": [document, {"type": "text", "text": "keep"}]} + empty_tool: Final = {**tool, "content": [empty_document, {"type": "text", "text": "keep"}]} + request: Final = [ + {"role": "user", "content": [tool]}, + {"type": "function_call_output", "call_id": "call", "output": [image]}, + {"type": "custom_tool_call_output", "call_id": "custom", "output": [image]}, + {"type": "computer_call_output", "call_id": "computer", "output": { + "type": "computer_screenshot", "image_url": "data:image/png;base64,IMAGE", + }}, + {"type": "image_generation_call", "id": "ig", "result": "IMAGE", "status": "completed"}, + {"type": "code_interpreter_call", "id": "ci", "code": "keep()", "outputs": [ + {"type": "logs", "logs": "keep"}, {"type": "image", "url": "https://example.com/image.png"}, + ]}, + ] + original: Final = copy.deepcopy(request) + projected, media = split_media_for_tpm(request) + assert projected == [ + {"role": "user", "content": [empty_tool]}, + {"type": "function_call_output", "call_id": "call", "output": [empty_image]}, + {"type": "custom_tool_call_output", "call_id": "custom", "output": [empty_image]}, + {"type": "computer_call_output", "call_id": "computer", "output": { + "type": "computer_screenshot", "image_url": "", + }}, + {"type": "image_generation_call", "id": "ig", "result": "", "status": "completed"}, + {"type": "code_interpreter_call", "id": "ci", "code": "keep()", "outputs": [ + {"type": "logs", "logs": "keep"}, {"type": "image", "url": ""}, + ]}, + ] + assert media == ( + image, image, image, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, + {"type": "image_url", "image_url": {"url": "data:application/octet-stream;base64,IMAGE", "detail": "auto"}}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png", "detail": "auto"}}, + ) + assert request == original + + +def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None: + image: Final = {"type": "input_image", "image_url": "data:image/png;base64,OPAQUE"} + request: Final = [ + {"role": "user", "content": json.dumps(image)}, + {"type": "custom_tool_call_output", "call_id": "call", "output": json.dumps(image)}, + {"type": "function_call", "call_id": "call", "name": "keep", "arguments": {"content": [image]}}, + {"type": "file_search_call", "results": [{"text": "keep", "content": [image]}]}, + {"type": "image_generation_call", "id": "ig", "status": "in_progress", "result": None}, + {"role": "user", "content": [ + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "PDF"}}, + {"type": "document", "source": {"type": "text", "data": json.dumps(image)}}, + {"type": "file", "file": {"format": "application/pdf", "file_data": "PDF"}}, + {"type": "text", "text": "keep", "metadata": {"content": [image]}}, + ]}, + ] + projected, media = split_media_for_tpm(request) + assert projected is request + assert media == () + + +@pytest.mark.parametrize("mime", ["application/pdf", "video/mp4", "audio/wav", "image/png"]) +def test_embedding_media_split_holds_at_most_one_copy_of_a_large_payload(mime: str) -> None: + import gc + import tracemalloc + + uri: Final = f"data:{mime};base64," + base64.b64encode(bytes(3 * 1024 * 1024)).decode("ascii") + split_embedding_media_for_tpm(f"data:{mime};base64,AAAA") + gc.collect() + tracemalloc.start() + try: + _, media = split_embedding_media_for_tpm(uri) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert media + assert peak < 1.5 * len(uri) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 3a4231443dd..95bc8af7270 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -8,9 +8,9 @@ from typing import Final from unittest.mock import MagicMock, patch import httpx -from httpx import Headers import pytest import respx +from httpx import Headers from pydantic import BaseModel, ConfigDict import litellm @@ -24,20 +24,22 @@ from litellm.constants import ( RESPONSE_FORMAT_TOOL_NAME, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) -from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_reasoning_signature, +) from litellm.llms.anthropic.chat import ModelResponseIterator -from litellm.llms.anthropic.common_utils import AnthropicError, process_anthropic_headers from litellm.llms.anthropic.chat.transformation import AnthropicConfig -from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema +from litellm.llms.anthropic.common_utils import AnthropicError, process_anthropic_headers from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig -from litellm.llms.bedrock.claude_platform.transformation import BedrockClaudePlatformConfig +from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) +from litellm.llms.bedrock.claude_platform.transformation import BedrockClaudePlatformConfig from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( VertexAIAnthropicConfig, ) @@ -9067,10 +9069,11 @@ async def test_anthropic_api_max_completion_tokens(model: str): def test_anthropic_tool_cache_control(): - from litellm.utils import return_raw_request - from litellm.types.utils import CallTypes import json + from litellm.types.utils import CallTypes + from litellm.utils import return_raw_request + tool_content = "Result: 4. " * 1000 messages = [ {"role": "user", "content": "Calculate 2+2"}, diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index cc366dcde4c..fb4f19a862c 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -11,17 +11,16 @@ import pytest import respx import litellm - - - from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_PLACEHOLDER, - encrypted_reasoning_signature, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, _bedrock_converse_messages_pt, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_reasoning_signature, +) from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, AnthropicAdapter, @@ -3302,7 +3301,6 @@ def test_translate_openai_response_to_anthropic_input_tokens_no_cache(): """ Regression test: input_tokens should equal prompt_tokens when there are no cached tokens. """ - from litellm.types.utils import PromptTokensDetailsWrapper # Create OpenAI format response without cached tokens usage = Usage( diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py index cd6d2d9be96..4b52f7fbef1 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py @@ -1,4 +1,4 @@ -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.messages.native.transformation import ( diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py index 4626f76c8fc..ffda84f32b7 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -19,7 +19,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../. import litellm from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.messages.native.streaming_iterator import aclose_superseded_attempt diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index ece0cf5d423..dbb8c925fa0 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -20,6 +20,8 @@ from litellm.constants import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.messages.responses_adapters.transformation import ( diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 99e6438c723..c97585e02c2 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -1745,7 +1745,7 @@ class TestAnthropicThinkingSignatureSelfHeal: def test_strip_keeps_encrypted_reasoning_blocks_for_the_responses_bridge(self): """The /v1/messages handler runs this before dispatch, so the bridge must still see the replay.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.common_utils import ( @@ -1766,7 +1766,7 @@ class TestAnthropicThinkingSignatureSelfHeal: def test_strip_encrypted_reasoning_drops_only_the_bridge_tagged_blocks(self): """A session resumed on an Anthropic model replays reasoning only OpenAI can verify.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.common_utils import ( diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index f93968eb2e6..8571a0c3cbc 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -1,7 +1,11 @@ -import asyncio, httpx, importlib, json, os +import asyncio +import importlib +import json +import os import sys from typing import Final +import httpx import pytest from pydantic import TypeAdapter @@ -9,19 +13,21 @@ sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) ) -import litellm -from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY -from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config -from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig -from litellm.utils import _invalidate_model_cost_lowercase_map, get_optional_params from datetime import datetime -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.router import Router +from unittest.mock import AsyncMock + from openai.types.chat import ChatCompletionMessage from openai.types.chat.chat_completion import ChatCompletion, Choice from respx import MockRouter + +import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY +from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config +from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig +from litellm.router import Router +from litellm.utils import _invalidate_model_cost_lowercase_map, get_optional_params from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from unittest.mock import AsyncMock _MAPPED_PARAMS: Final = TypeAdapter(dict[str, object]) _SUPPORTED_PARAMS: Final = TypeAdapter(list[str]) @@ -505,7 +511,7 @@ async def test_azure_gpt5_async_transform_drops_tool_choice_without_tools() -> N def test_transform_request_strips_litellm_format_from_managed_file_id(): import base64 - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_messages_with_model_file_ids, ) diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index d66ba3f324e..65f8400e53c 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -477,6 +477,7 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "aget_responses", "adelete_responses", "acancel_responses", + "acompact_responses", "aresponses_websocket", "alist_input_items", "acreate_fine_tuning_job", diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index 93f6e237fc7..053574d7dd6 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -2047,8 +2047,9 @@ def _invoke_chat_params( ) @pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) def test_invoke_uses_opt_in_native_output_when_the_opt_in_drove_native_output( - local_model_cost_map, model, thinking_params + local_model_cost_map, model, thinking_params, monkeypatch ): + monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False) optional_params: Final = _invoke_chat_params( model, None, {"response_format": _CITY_RESPONSE_FORMAT, **thinking_params} ) @@ -2083,6 +2084,15 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co ) +def test_invoke_uses_default_native_output_for_schema_retry(local_model_cost_map: None) -> None: + model: Final = "us.anthropic.claude-opus-5" + optional_params: Final = _invoke_chat_params( + model, None, {"response_format": _CITY_RESPONSE_FORMAT, "thinking": {"type": "disabled"}} + ) + + assert AmazonAnthropicClaudeConfig().uses_native_output(model, optional_params, {}) + + @pytest.mark.parametrize( ("forced_tool_use", "map_flag", "model_info", "thinking_params", "native"), [ diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index b8eb5858fda..b37d6977c10 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -5244,6 +5244,44 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only assert ("toolConfig" in request) is not native +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("model", "thinking_params", "native"), + [ + ("us.anthropic.claude-opus-5", None, True), + ("us.anthropic.claude-opus-5", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-opus-5", {"thinking": {"type": "disabled"}}, True), + ("us.anthropic.claude-opus-4-8", None, True), + ("us.anthropic.claude-opus-4-8", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-opus-4-8", {"thinking": {"type": "disabled"}}, True), + ("us-gov.anthropic.claude-opus-5", None, False), + ("us-gov.anthropic.claude-opus-4-8", None, False), + ], + ids=[ + "opus-5-unset", + "opus-5-adaptive", + "opus-5-disabled", + "opus-4-8-unset", + "opus-4-8-adaptive", + "opus-4-8-disabled", + "opus-5-us-gov", + "opus-4-8-us-gov", + ], +) +def test_opus_5_and_4_8_cost_map_defaults_use_native_output_across_thinking_modes( + model: str, thinking_params: Mapping[str, object] | None, native: bool +) -> None: + _, request = _converse_request_for_json_schema(model, False, thinking_params) + + if native: + assert request["outputConfig"]["textFormat"]["type"] == "json_schema" + assert "toolConfig" not in request + return + + assert "outputConfig" not in request + assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] + + def _converse_capital_answer(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, @@ -11599,7 +11637,10 @@ _OUTPUT_CONFIG_FORMAT: Final = { ids=["adaptive", "reasoning_effort", "budget", "output_config_format"], ) @pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) -def test_uses_opt_in_native_output_when_the_opt_in_drove_native_output(local_model_cost_map, model, request_params): +def test_uses_opt_in_native_output_when_the_opt_in_drove_native_output( + local_model_cost_map, monkeypatch, model, request_params +): + monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False) optional_params: Final = _converse_params(model, None, request_params) assert AmazonConverseConfig().uses_opt_in_native_output(model, optional_params, {"model_info": {"id": "d1"}}) @@ -11630,7 +11671,10 @@ def test_uses_opt_in_native_output_is_false_when_native_output_did_not_come_from assert not AmazonConverseConfig().uses_opt_in_native_output(model, optional_params, {"model_info": model_info}) -def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_inference_profile(local_model_cost_map): +def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_inference_profile( + local_model_cost_map, monkeypatch +): + monkeypatch.setitem(litellm.model_cost["us.anthropic.claude-opus-4-8"], "supports_native_structured_output", False) profile: Final = "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id" litellm_params: Final = {"base_model": "bedrock/us.anthropic.claude-opus-4-8"} optional_params: Final = _converse_params( @@ -11698,8 +11742,9 @@ _BEDROCK_SCHEMA_REFUSAL: Final = "output_config.format.schema: pattern is too co ) @pytest.mark.respx(assert_all_called=True) def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_output( - caller_params: dict, expected_calls: int, respx_mock: respx.Router, local_model_cost_map: None + caller_params: dict, expected_calls: int, respx_mock: respx.Router, local_model_cost_map: None, monkeypatch ) -> None: + monkeypatch.setitem(litellm.model_cost["us.anthropic.claude-opus-4-8"], "supports_native_structured_output", False) route: Final = respx_mock.post(url__regex=r"/converse$").mock( return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL}) ) @@ -11722,6 +11767,61 @@ def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_ assert all("outputConfig" not in json.loads(call.request.content) for call in route.calls[1:]) +@pytest.mark.respx(assert_all_called=True) +def test_a_default_native_schema_rejection_retries_with_the_json_tool( + respx_mock: respx.Router, local_model_cost_map: None +) -> None: + route: Final = respx_mock.post(url__regex=r"/converse$").mock( + return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL}) + ) + + with pytest.raises(litellm.BadRequestError, match=re.escape(_BEDROCK_SCHEMA_REFUSAL)): + completion( + model="bedrock/converse/us.anthropic.claude-opus-5", + messages=[{"role": "user", "content": "What is the capital of France?"}], + response_format=CAPITAL_RESPONSE_FORMAT, + thinking={"type": "disabled"}, + num_retries=0, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + client=HTTPHandler(), + ) + + assert route.call_count == 2 + first_request: Final = json.loads(route.calls[0].request.content) + retry_request: Final = json.loads(route.calls[1].request.content) + assert "outputConfig" in first_request and "toolConfig" not in first_request + assert "toolConfig" in retry_request and "outputConfig" not in retry_request + + +@pytest.mark.respx(assert_all_called=True) +def test_an_unforceable_tool_native_schema_rejection_retries_with_the_json_tool( + respx_mock: respx.Router, local_model_cost_map: None +) -> None: + route: Final = respx_mock.post(url__regex=r"/converse$").mock( + return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL}) + ) + + with pytest.raises(litellm.BadRequestError, match=re.escape(_BEDROCK_SCHEMA_REFUSAL)): + completion( + model="bedrock/converse/anthropic.claude-fable-5-1", + messages=[{"role": "user", "content": "What is the capital of France?"}], + response_format=CAPITAL_RESPONSE_FORMAT, + num_retries=0, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + client=HTTPHandler(), + ) + + assert route.call_count == 2 + first_request: Final = json.loads(route.calls[0].request.content) + retry_request: Final = json.loads(route.calls[1].request.content) + assert "outputConfig" in first_request and "toolConfig" not in first_request + assert "toolConfig" in retry_request and "outputConfig" not in retry_request + + CONTROL_PLANE_ENDPOINT: Final = "https://bedrock-fips.us-east-1.amazonaws.com" diff --git a/tests/unit/llms/bedrock/chat/test_invoke_handler.py b/tests/unit/llms/bedrock/chat/test_invoke_handler.py index 4424f7640a3..2faa089e1c3 100644 --- a/tests/unit/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/unit/llms/bedrock/chat/test_invoke_handler.py @@ -1214,7 +1214,7 @@ async def test_converse_json_schema_stream_sends_claude_prose_only_when_claude_s transport: Final = httpx.MockTransport(handle_request) request: Final = { - "model": "bedrock/us.anthropic.claude-opus-5", + "model": "bedrock/anthropic.claude-opus-5", "messages": [{"role": "user", "content": "What is the LCM of 24 and 120? Reply in JSON."}], "response_format": { "type": "json_schema", diff --git a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py index 9ee8c14f5d6..81832d7d8bf 100644 --- a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py +++ b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py @@ -13,10 +13,33 @@ _MODEL: Final = "us.anthropic.claude-opus-4-8" _FALLBACK: Final = NativeOutputFallback( optional_params={"tools": "json_tool_call"}, litellm_params={"model_info": {"opted_out": True}} ) +_NATIVE_RETRY_WITH_STRICT_TOOLS: Final = NativeOutputFallback( + optional_params={ + "tools": ( + {"function": {"name": "json_schema", "strict": True}}, + {"function": {"name": "custom", "strict": True}}, + ) + }, + litellm_params={ + "model_info": { + "supports_native_structured_output": False, + "supports_bedrock_native_structured_output": False, + } + }, +) +_COMPOSED_FALLBACK: Final = NativeOutputFallback( + optional_params={"tools": [{"function": {"name": "json_schema"}}, {"function": {"name": "custom"}}]}, + litellm_params=_NATIVE_RETRY_WITH_STRICT_TOOLS.litellm_params, +) _REJECTION: Final = BedrockError( status_code=400, message='{"message":"The model returned the following errors: output_config.format: Extra inputs are not permitted"}', ) +_GRAMMAR_REJECTION: Final = BedrockError( + status_code=400, + message='{"message":"The model returned the following errors: The compiled grammar is too large, which would cause ' + 'performance issues. Simplify your tool schemas or reduce the number of strict tools."}', +) _SCHEMA_REFUSAL: Final = BedrockError( status_code=400, message='{"message":"output_config.format.schema: pattern is too complex for structured output"}' ) @@ -58,8 +81,15 @@ _MODES: Final = pytest.mark.parametrize("asynchronous", [False, True], ids=["syn @_MODES -def test_a_native_output_rejection_is_sent_again_once_with_the_fallback_params(asynchronous: bool) -> None: - bedrock: Final = ScriptedBedrock(errors=(_REJECTION, None), asynchronous=asynchronous) +@pytest.mark.parametrize( + "rejection", + [_REJECTION, _GRAMMAR_REJECTION], + ids=["format_not_permitted", "compiled_grammar_too_large"], +) +def test_a_native_output_rejection_is_sent_again_once_with_the_fallback_params( + asynchronous: bool, rejection: BedrockError +) -> None: + bedrock: Final = ScriptedBedrock(errors=(rejection, None), asynchronous=asynchronous) response: Final = _call(bedrock, _FALLBACK) @@ -77,6 +107,19 @@ def test_a_second_native_output_rejection_reaches_the_caller(asynchronous: bool) assert len(bedrock.sent) == 2 +@_MODES +def test_a_grammar_rejection_after_a_native_retry_reaches_the_caller_without_strict_tools_fallback( + asynchronous: bool, +) -> None: + bedrock: Final = ScriptedBedrock(errors=(_GRAMMAR_REJECTION, _GRAMMAR_REJECTION), asynchronous=asynchronous) + + with pytest.raises(BedrockError) as raised: + _resolve(call_with_native_output_fallback(bedrock, lambda: _FALLBACK, _MODEL)) + + assert raised.value is _GRAMMAR_REJECTION + assert bedrock.sent == (None, _FALLBACK) + + @_MODES @pytest.mark.parametrize( ("error", "fallback"), @@ -155,13 +198,16 @@ _OTHER_400S_WITH_A_STRICT_TOOL: Final = tuple( def _call_with_strict_tools_fallback( - bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, strict_tools_fallback: NativeOutputFallback | None + bedrock: ScriptedBedrock, + fallback: NativeOutputFallback | None, + strict_tools_fallback: NativeOutputFallback | None, + model: str = _MODEL, ) -> object: return _resolve( call_with_native_output_fallback( bedrock, None if fallback is None else lambda: fallback, - _MODEL, + model, strict_tools_fallback=None if strict_tools_fallback is None else lambda: strict_tools_fallback, ) ) @@ -223,3 +269,18 @@ def test_each_rejection_is_sent_again_with_its_own_fallback_when_both_are_set( _call_with_strict_tools_fallback(bedrock, _FALLBACK, _STRICT_FALLBACK) assert bedrock.sent == (None, retried_with) + + +@_MODES +def test_a_strict_tools_rejection_after_native_retry_composes_both_fallbacks(asynchronous: bool) -> None: + bedrock: Final = ScriptedBedrock( + errors=(_GRAMMAR_REJECTION, _GRAMMAR_REJECTION, None), + asynchronous=asynchronous, + ) + + response: Final = _call_with_strict_tools_fallback( + bedrock, _NATIVE_RETRY_WITH_STRICT_TOOLS, _STRICT_FALLBACK, "us.anthropic.claude-sonnet-4-6" + ) + + assert isinstance(response, ModelResponse) and response.id == "answer-3" + assert bedrock.sent == (None, _NATIVE_RETRY_WITH_STRICT_TOOLS, _COMPOSED_FALLBACK) diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py b/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py index 5c078affffc..2b76b54e905 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py @@ -1,12 +1,15 @@ import base64 import os from types import MappingProxyType -from unittest.mock import MagicMock, patch +from typing import Final +from unittest.mock import patch +import httpx import pytest import litellm.files.main as files_main from litellm.llms.bedrock.files.handler import BedrockFilesHandler +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.utils import SpecialEnums @@ -123,88 +126,67 @@ class TestBedrockFilesHandler: configured_bucket_name="safe-bucket", ) - def test_should_not_trust_request_s3_bucket_name_for_expected_bucket(self): - with patch.dict(os.environ, {"AWS_S3_BUCKET_NAME": "safe-bucket"}): - assert ( - self.handler._get_configured_s3_bucket_name( - {"s3_bucket_name": "attacker-bucket"} - ) - == "safe-bucket" - ) + def test_call_s3_bucket_name_wins_over_env_bucket(self): + with patch.dict(os.environ, {"AWS_S3_BUCKET_NAME": "env-bucket"}): + assert self.handler._get_configured_s3_bucket_name({"s3_bucket_name": "call-bucket"}) == "call-bucket" - def test_should_trust_proxy_config_s3_bucket_name_for_expected_bucket(self): - trusted_credentials = MappingProxyType({"s3_bucket_name": "safe-bucket"}) + def test_env_bucket_is_used_when_the_call_names_none(self): + with patch.dict(os.environ, {"AWS_S3_BUCKET_NAME": "env-bucket"}): + assert self.handler._get_configured_s3_bucket_name({}) == "env-bucket" - with patch.dict(os.environ, {}, clear=True): - assert ( - self.handler._get_configured_s3_bucket_name( - { - "s3_bucket_name": "attacker-bucket", - "_litellm_internal_model_credentials": trusted_credentials, - } - ) - == "safe-bucket" - ) - - def test_should_not_trust_user_supplied_internal_credentials_dict(self): + def test_missing_bucket_in_call_and_env_is_rejected(self): with patch.dict(os.environ, {}, clear=True): with pytest.raises(ValueError, match="S3 bucket_name is required"): - self.handler._get_configured_s3_bucket_name( - { - "_litellm_internal_model_credentials": { - "s3_bucket_name": "attacker-bucket" - } - } - ) - - def test_should_require_server_s3_bucket_name(self): - with patch.dict(os.environ, {}, clear=True): - with pytest.raises(ValueError, match="S3 bucket_name is required"): - self.handler._get_configured_s3_bucket_name( - {"s3_bucket_name": "attacker-bucket"} - ) + self.handler._get_configured_s3_bucket_name({}) -def test_should_forward_trusted_model_credentials_to_bedrock_provider_config(): - trusted_credentials = MappingProxyType({"s3_bucket_name": "safe-bucket"}) - mock_response = MagicMock() +def _echo_request_url_client() -> HTTPHandler: + return HTTPHandler( + client=httpx.Client( + transport=httpx.MockTransport(lambda request: httpx.Response(200, content=str(request.url).encode())) + ) + ) - with patch.object( - files_main.base_llm_http_handler, - "retrieve_file_content", - return_value=mock_response, - ) as mock_retrieve_file_content: - response = files_main.file_content( - file_id="s3://safe-bucket/litellm-bedrock-files/file.jsonl", + +_SDK_AWS_CREDENTIALS: Final = MappingProxyType( + { + "aws_access_key_id": "AKIASDKCALLER", + "aws_secret_access_key": "sdk-caller-secret", + "aws_region_name": "us-west-2", + } +) + + +def test_sdk_file_content_reads_from_the_bucket_passed_to_the_call(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) + monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) + + response: Final = files_main.file_content( + file_id="s3://sdk-bucket/litellm-bedrock-files/batch.jsonl", + custom_llm_provider="bedrock", + s3_bucket_name="sdk-bucket", + client=_echo_request_url_client(), + **_SDK_AWS_CREDENTIALS, + ) + + assert httpx.URL(response.content.decode()).path == "/sdk-bucket/litellm-bedrock-files/batch.jsonl" + + +def test_sdk_file_content_rejects_a_file_outside_the_bucket_passed_to_the_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) + monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) + + with pytest.raises(Exception, match="configured storage bucket"): + files_main.file_content( + file_id="s3://other-bucket/litellm-bedrock-files/batch.jsonl", custom_llm_provider="bedrock", - _litellm_internal_model_credentials=trusted_credentials, + s3_bucket_name="sdk-bucket", + client=_echo_request_url_client(), + **_SDK_AWS_CREDENTIALS, ) - assert response is mock_response - litellm_params = mock_retrieve_file_content.call_args.kwargs["litellm_params"] - assert litellm_params["_litellm_internal_model_credentials"] is trusted_credentials - assert "s3_bucket_name" not in litellm_params - - -def test_should_forward_trusted_model_credentials_to_retrieve_provider_config(): - trusted_credentials = MappingProxyType({"allow_legacy_cloud_file_ids": True}) - mock_response = MagicMock() - - with patch.object( - files_main.base_llm_http_handler, - "retrieve_file", - return_value=mock_response, - ) as mock_retrieve_file: - response = files_main.file_retrieve( - file_id="gs://safe-bucket/private/file.jsonl", - custom_llm_provider="vertex_ai", - _litellm_internal_model_credentials=trusted_credentials, - ) - - assert response is mock_response - litellm_params = mock_retrieve_file.call_args.kwargs["litellm_params"] - assert litellm_params["_litellm_internal_model_credentials"] is trusted_credentials - @pytest.mark.asyncio async def test_afile_content_assumes_role_with_external_id(monkeypatch): @@ -249,7 +231,7 @@ async def test_afile_content_assumes_role_with_external_id(monkeypatch): return FakeS3Client() optional_params = { - "_litellm_internal_model_credentials": MappingProxyType({"s3_bucket_name": "safe-bucket"}), + "s3_bucket_name": "safe-bucket", "aws_region_name": "us-east-1", "aws_access_key_id": "AKIAFILESDOWNLOADCALLER", "aws_secret_access_key": "pod-caller-secret", @@ -285,7 +267,7 @@ async def test_afile_content_builds_the_s3_client_with_the_s3_pair_when_it_diffe return {"Body": FakeS3Body()} optional_params = { - "_litellm_internal_model_credentials": MappingProxyType({"s3_bucket_name": "safe-bucket"}), + "s3_bucket_name": "safe-bucket", "aws_region_name": "us-east-1", "aws_access_key_id": "AKIABEDROCKONLY", "aws_secret_access_key": "bedrock-only-secret", diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 512e1f325f4..93bda5b444d 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -9,7 +9,6 @@ from contextlib import AsyncExitStack, closing from datetime import datetime, timezone from io import BytesIO from pathlib import Path -from types import MappingProxyType from typing import Final from unittest.mock import MagicMock from urllib.parse import unquote, urlparse @@ -2047,7 +2046,7 @@ class TestBedrockBatchAnthropicRowParams: @pytest.mark.parametrize( ("model", "native_output"), - [("bedrock/us.anthropic.claude-opus-4-8", False), (MODEL, True)], + [("bedrock/us.anthropic.claude-sonnet-5-5", False), (MODEL, True)], ids=["opt_in_native_output_model", "native_output_model"], ) def test_thinking_json_schema_rows_use_native_output_only_on_models_that_always_take_it( @@ -2351,7 +2350,7 @@ class TestBedrockFileContentTransformation: monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) - litellm_params: Final = _trusted_bucket_snapshot( + litellm_params: Final = _deployment_bucket_params( s3_bucket_name="in-bucket/pfx", s3_output_bucket_name="out-bucket/outpfx", ) @@ -2541,7 +2540,7 @@ class TestBedrockFileContentTransformation: def test_transform_file_content_request_requires_configured_bucket( self, monkeypatch ): - """Without a server-configured bucket (env or snapshot), the request must fail + """Without a bucket (on the call or in env), the request must fail before any S3 call rather than guessing a bucket from the file id.""" from litellm.llms.bedrock.files.transformation import BedrockFilesConfig @@ -2566,30 +2565,24 @@ class TestBedrockFileContentTransformation: litellm_params=self._litellm_params(), ) - def _trusted(self, **deployment_litellm_params) -> dict: - """Build the trusted snapshot the way the proxy does: deployment - litellm_params funneled through ``CredentialLiteLLMParams`` (the strict - allowlist ``get_deployment_credentials_with_provider`` applies) before - retrieval ever sees them. Injecting a raw ``MappingProxyType`` would - bypass that filter and hide whether a bucket field actually survives - into the snapshot in production.""" - from types import MappingProxyType - + def _deployment_params(self, **deployment_litellm_params) -> dict: + """Deployment litellm_params funneled through ``CredentialLiteLLMParams`` + (the strict allowlist ``get_deployment_credentials_with_provider`` applies) + the way the proxy hands them to retrieval, so a bucket field dropped from + that allowlist fails these tests.""" from litellm.types.router import CredentialLiteLLMParams - snapshot = CredentialLiteLLMParams(**deployment_litellm_params).model_dump( + credentials = CredentialLiteLLMParams(**deployment_litellm_params).model_dump( exclude_none=True ) - params = self._litellm_params() - params["_litellm_internal_model_credentials"] = MappingProxyType(snapshot) - return params + return {**self._litellm_params(), **credentials} def test_retrieves_from_distinct_output_bucket(self, monkeypatch): """Batch outputs can land in a separate s3_output_bucket_name. Retrieval must validate the file id against the output bucket too, not just the input bucket, or the very outputs the feature serves are unreachable. - The snapshot is built through the production credential filter, so this - fails if s3_output_bucket_name is dropped from that allowlist.""" + The params go through the production credential filter, so this fails + if s3_output_bucket_name is dropped from that allowlist.""" from litellm.llms.bedrock.files.transformation import BedrockFilesConfig monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) @@ -2600,7 +2593,7 @@ class TestBedrockFileContentTransformation: "file_id": "s3://out-bucket/litellm-batch-outputs/job/in.jsonl.out" }, optional_params={}, - litellm_params=self._trusted( + litellm_params=self._deployment_params( s3_bucket_name="in-bucket", s3_output_bucket_name="out-bucket" ), ) @@ -2611,8 +2604,8 @@ class TestBedrockFileContentTransformation: ) def test_output_bucket_falls_back_to_env(self, monkeypatch): - """The output bucket resolves from AWS_S3_OUTPUT_BUCKET_NAME when not in - the trusted snapshot, mirroring the input-bucket env fallback.""" + """The output bucket resolves from AWS_S3_OUTPUT_BUCKET_NAME when the + call has none, mirroring the input-bucket env fallback.""" from litellm.llms.bedrock.files.transformation import BedrockFilesConfig monkeypatch.setenv("AWS_S3_BUCKET_NAME", "in-bucket") @@ -2644,7 +2637,7 @@ class TestBedrockFileContentTransformation: "file_id": "s3://in-bucket/litellm-batch-outputs/job/in.jsonl.out" }, optional_params={}, - litellm_params=self._trusted( + litellm_params=self._deployment_params( s3_bucket_name="in-bucket", s3_output_bucket_name="out-bucket" ), ) @@ -2669,7 +2662,7 @@ class TestBedrockFileContentTransformation: "file_id": "s3://other-bucket/litellm-batch-outputs/job/x.jsonl.out" }, optional_params={}, - litellm_params=self._trusted( + litellm_params=self._deployment_params( s3_bucket_name="in-bucket", s3_output_bucket_name="out-bucket" ), ) @@ -2694,22 +2687,16 @@ class TestBedrockFileContentTransformation: litellm_params=self._litellm_params(), ) - def test_bucket_resolved_from_trusted_model_credentials(self, monkeypatch): - """Per-model s3_bucket_name must be honored via the server-side credential snapshot.""" - from types import MappingProxyType - + def test_bucket_resolved_from_call_s3_bucket_name(self, monkeypatch): + """An s3_bucket_name on the call is honored without AWS_S3_BUCKET_NAME.""" from litellm.llms.bedrock.files.transformation import BedrockFilesConfig monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) - litellm_params = self._litellm_params() - litellm_params["_litellm_internal_model_credentials"] = MappingProxyType( - {"s3_bucket_name": "my-bucket"} - ) url, _ = BedrockFilesConfig().transform_file_content_request( file_content_request={"file_id": self.S3_URI}, optional_params={}, - litellm_params=litellm_params, + litellm_params={**self._litellm_params(), "s3_bucket_name": "my-bucket"}, ) assert url == self.EXPECTED_URL @@ -3226,13 +3213,13 @@ def _bedrock_s3_params() -> dict: } -def _trusted_bucket_snapshot(**deployment_litellm_params) -> dict: - from types import MappingProxyType - +def _deployment_bucket_params(**deployment_litellm_params: object) -> dict[str, object]: from litellm.types.router import CredentialLiteLLMParams - snapshot = CredentialLiteLLMParams(**deployment_litellm_params).model_dump(exclude_none=True) - return {**_bedrock_s3_params(), "_litellm_internal_model_credentials": MappingProxyType(snapshot)} + credentials: Final = CredentialLiteLLMParams.model_validate(deployment_litellm_params).model_dump( + exclude_none=True + ) + return {**_bedrock_s3_params(), **credentials} class TestBedrockFileDeletionTransformation: @@ -3287,7 +3274,7 @@ class TestBedrockFileDeletionTransformation: assert url == self.EXPECTED_URL - def test_transform_delete_file_request_uses_trusted_snapshot_bucket(self, monkeypatch): + def test_transform_delete_file_request_uses_call_bucket(self, monkeypatch): from litellm.llms.bedrock.files.transformation import BedrockFilesConfig monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) @@ -3295,7 +3282,7 @@ class TestBedrockFileDeletionTransformation: url, _ = BedrockFilesConfig().transform_delete_file_request( file_id=self.S3_URI, optional_params={}, - litellm_params=_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + litellm_params=_deployment_bucket_params(s3_bucket_name="my-bucket"), ) assert url == self.EXPECTED_URL @@ -3595,7 +3582,7 @@ class TestBedrockFileListTransformation: "GET", f"{url}?list-type=2&prefix=LLM%20AI%20Projects%2Flitellm-b", signed_headers ) - def test_transform_list_files_request_uses_trusted_snapshot_bucket(self, monkeypatch): + def test_transform_list_files_request_uses_call_bucket(self, monkeypatch): from litellm.llms.bedrock.files.transformation import BedrockFilesConfig monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) @@ -3603,7 +3590,7 @@ class TestBedrockFileListTransformation: url, params = BedrockFilesConfig().transform_list_files_request( purpose=None, optional_params={}, - litellm_params=_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + litellm_params=_deployment_bucket_params(s3_bucket_name="my-bucket"), ) assert url == self.BUCKET_URL @@ -3696,7 +3683,7 @@ class TestBedrockFileListTransformation: assert _sent_signature(request.headers) == _s3_signature_for("GET", str(request.url), request.headers) assert [file.id for file in files] == list(self.BATCH_IDS) - def test_file_list_uses_trusted_snapshot_bucket_without_env(self, monkeypatch): + def test_file_list_uses_call_bucket_without_env(self, monkeypatch): import httpx import respx @@ -3711,7 +3698,7 @@ class TestBedrockFileListTransformation: files = litellm.file_list( custom_llm_provider="bedrock", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket"), ) assert route.called @@ -3766,7 +3753,7 @@ class TestBedrockFileListTransformation: monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) - litellm_params = _trusted_bucket_snapshot( + litellm_params = _deployment_bucket_params( s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket/team-a" ) @@ -3852,7 +3839,7 @@ class TestBedrockFileListTransformation: monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) - litellm_params = _trusted_bucket_snapshot(s3_output_bucket_name="my-output-bucket") + litellm_params = _deployment_bucket_params(s3_output_bucket_name="my-output-bucket") url, params = BedrockFilesConfig().transform_list_files_request( purpose="batch_output", optional_params={}, litellm_params=litellm_params @@ -3879,7 +3866,7 @@ class TestBedrockFileListTransformation: raw_response=httpx.Response(200, content=self.OUTPUT_BUCKET_LISTING), logging_obj=MagicMock(), litellm_params={ - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), LIST_FILES_PURPOSE_PARAM: "batch_output", }, ) @@ -3903,7 +3890,7 @@ class TestBedrockFileListTransformation: files = litellm.file_list( custom_llm_provider="bedrock", purpose="batch_output", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), ) assert route.called @@ -3930,7 +3917,7 @@ class TestBedrockFileListTransformation: files = litellm.file_list( custom_llm_provider="bedrock", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), ) assert (input_route.call_count, output_route.call_count) == (1, 1) @@ -3962,7 +3949,7 @@ class TestBedrockFileListTransformation: files = litellm.file_list( custom_llm_provider="bedrock", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), ) requested_urls = [str(call.request.url) for call in respx.calls] @@ -3975,7 +3962,7 @@ class TestBedrockFileListTransformation: assert [file.id for file in files] == [*self.PAGED_IDS, self.OUTPUT_BUCKET_ID] @pytest.mark.parametrize( - ("purpose", "bucket_snapshot"), + ("purpose", "bucket_params"), [ pytest.param(None, {"s3_bucket_name": "my-bucket"}, id="outputs-share-the-input-bucket"), pytest.param( @@ -3986,7 +3973,7 @@ class TestBedrockFileListTransformation: ], ) def test_file_list_leaves_the_output_bucket_alone_unless_an_unfiltered_list_needs_it( - self, monkeypatch, purpose, bucket_snapshot + self, monkeypatch, purpose, bucket_params ): import httpx import respx @@ -4003,7 +3990,7 @@ class TestBedrockFileListTransformation: ) files = litellm.file_list( - custom_llm_provider="bedrock", purpose=purpose, **_trusted_bucket_snapshot(**bucket_snapshot) + custom_llm_provider="bedrock", purpose=purpose, **_deployment_bucket_params(**bucket_params) ) assert (input_route.call_count, output_route.call_count) == (1, 0) @@ -4019,7 +4006,7 @@ class TestBedrockFileListTransformation: monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) - litellm_params = _trusted_bucket_snapshot(s3_bucket_name="my-bucket", s3_output_bucket_name="my-bucket/out") + litellm_params = _deployment_bucket_params(s3_bucket_name="my-bucket", s3_output_bucket_name="my-bucket/out") config = BedrockFilesConfig() config.transform_list_files_request(purpose=None, optional_params={}, litellm_params=litellm_params) litellm_params.pop(S3_SIGNED_REQUEST_HEADERS_PARAM) @@ -4130,7 +4117,7 @@ class TestBedrockFileListTransformation: custom_llm_provider="bedrock", purpose="batch", timeout=12, - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket"), ) self._assert_paged_listing(first_page, last_page, files) @@ -4152,7 +4139,7 @@ class TestBedrockFileListTransformation: custom_llm_provider="bedrock", purpose="batch", timeout=12, - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket"), ) self._assert_paged_listing(first_page, last_page, files) @@ -4209,7 +4196,7 @@ class TestBedrockFileListTransformation: files = litellm.file_list( custom_llm_provider="bedrock", purpose="batch", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket"), ) self._assert_capped_listing(route, files) @@ -4230,7 +4217,7 @@ class TestBedrockFileListTransformation: files = await litellm.afile_list( custom_llm_provider="bedrock", purpose="batch", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket"), ) self._assert_capped_listing(route, files) @@ -4321,7 +4308,7 @@ def test_transform_file_content_request_signs_with_the_s3_pair_from_litellm_para litellm_params = { **_SPLIT_IDENTITY_PARAMS, - "_litellm_internal_model_credentials": MappingProxyType({"s3_bucket_name": "safe-bucket"}), + "s3_bucket_name": "safe-bucket", } BedrockFilesConfig().transform_file_content_request( file_content_request={"file_id": "s3://safe-bucket/litellm-bedrock-files-model-id-abc.jsonl"}, diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 07fabbf2393..ff4e3650464 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -4016,7 +4016,10 @@ _NATIVE_OUTPUT_REJECTION: Final = { "message": "The model returned the following errors: compilation exceeded limits: Schema is too complex." } _STALE_SIGNATURE_ERROR: Final = {"message": "messages.1.content.0: Invalid `signature` in `thinking` block"} -_OPTED_OUT: Final = {"supports_bedrock_native_structured_output": False} +_OPTED_OUT: Final = { + "supports_native_structured_output": False, + "supports_bedrock_native_structured_output": False, +} _CONSTRAINED_SCHEMA: Final = { "type": "object", "properties": { @@ -4158,21 +4161,21 @@ async def test_bedrock_messages_native_output_rejection_and_signature_400_each_g @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize( - ("model", "model_info", "error"), + ("model", "model_info", "error", "should_retry"), [ - ("us.anthropic.claude-opus-4-8", None, {"message": "messages.0.content: Field required"}), - ("us.anthropic.claude-sonnet-4-6", None, _NATIVE_OUTPUT_REJECTION), - ("us.anthropic.claude-opus-4-8", _OPTED_OUT, _NATIVE_OUTPUT_REJECTION), + ("us.anthropic.claude-opus-4-8", None, {"message": "messages.0.content: Field required"}, False), + ("us.anthropic.claude-sonnet-4-6", None, _NATIVE_OUTPUT_REJECTION, True), + ("us.anthropic.claude-opus-4-8", _OPTED_OUT, _NATIVE_OUTPUT_REJECTION, False), ], - ids=["other_400", "native_model", "deployment_opt_out"], + ids=["other_400", "native_rejection", "deployment_opt_out"], ) -async def test_bedrock_messages_400_is_not_retried_unless_it_rejects_opt_in_native_output( - model: str, model_info: Mapping[str, object] | None, error: Mapping[str, object] +async def test_bedrock_messages_retries_only_native_output_rejections( + model: str, model_info: Mapping[str, object] | None, error: Mapping[str, object], should_retry: bool ) -> None: bedrock, raised = await _send_bedrock_messages(model, model_info, [(400, error), (200, _ANTHROPIC_MESSAGE)]) - assert isinstance(raised, litellm.BadRequestError) - assert len(bedrock.bodies) == 1 + assert (raised is None) is should_retry + assert len(bedrock.bodies) == (2 if should_retry else 1) @pytest.mark.usefixtures("local_model_cost_map") diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 80715105b7f..d8e9726358c 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -13,6 +13,7 @@ from litellm.llms.bedrock.common_utils import( bedrock_strict_tools_fallback, deployment_model_info, extract_model_name_from_bedrock_arn, + for_bedrock_json_tool_retry, get_bedrock_base_model, get_bedrock_cross_region_inference_regions, is_bedrock_native_output_rejection, @@ -2328,6 +2329,10 @@ def test_bedrock_capability_model_reads_base_model_only_for_application_profile_ _FORMAT_REJECTION: Final = "The model returned the following errors: output_config.format: Extra inputs are not permitted" _COMPILE_REJECTION: Final = "The model returned the following errors: compilation exceeded limits: Schema is too complex." +_GRAMMAR_REJECTION: Final = ( + "The model returned the following errors: The compiled grammar is too large, which would cause performance issues. " + "Simplify your tool schemas or reduce the number of strict tools." +) _SCHEMA_PATTERN_REJECTION: Final = ( "The model returned the following errors: output_config.format.schema: Unsupported regex feature in pattern " "field: pattern is too complex for structured output: reduce the {n,m} upper bound, narrow the character class " @@ -2340,6 +2345,7 @@ _SCHEMA_PATTERN_REJECTION: Final = ( [ (400, f'{{"message":"{_FORMAT_REJECTION}"}}', True), (400, f'{{"message":"{_COMPILE_REJECTION}"}}', True), + (400, _GRAMMAR_REJECTION, True), (400, f'{{"message":"{_SCHEMA_PATTERN_REJECTION}"}}', True), (400, "Malformed input request: #: extraneous key [output_config] is not permitted", False), (400, "The model returned the following errors: messages.0.content: Field required", False), @@ -2349,6 +2355,7 @@ _SCHEMA_PATTERN_REJECTION: Final = ( ids=[ "format_not_permitted", "schema_too_complex", + "compiled_grammar_too_large", "schema_pattern_too_complex", "other_key_not_permitted", "other_400", @@ -2365,7 +2372,7 @@ def test_is_bedrock_native_output_rejection_matches_only_the_400s_that_reject_na @pytest.mark.parametrize( ("model", "model_info", "expected"), [ - ("us.anthropic.claude-opus-4-8", None, True), + ("us.anthropic.claude-opus-4-8", None, False), ("global.anthropic.claude-sonnet-5", None, True), ("us-gov.anthropic.claude-opus-4-8", None, False), ("us.anthropic.claude-opus-4-8", {"supports_bedrock_native_structured_output": False}, False), @@ -2389,9 +2396,43 @@ def test_without_bedrock_native_output_opt_in_turns_the_opt_in_off_and_keeps_the "us.anthropic.claude-opus-4-8", model_info=deployment_model_info(turned_off) ) assert turned_off["api_base"] == "https://bedrock.example" - assert deployment_model_info(turned_off)["id"] == "deployment-1" + assert deployment_model_info(turned_off) == { + "id": "deployment-1", + "supports_bedrock_native_structured_output": False, + } assert litellm_params["model_info"] == {"id": "deployment-1"} + native_default: Final = without_bedrock_native_output_opt_in( + {"model_info": {"supports_native_structured_output": True}} + ) + assert deployment_model_info(native_default) == { + "supports_native_structured_output": True, + "supports_bedrock_native_structured_output": False, + } + + +def test_for_bedrock_json_tool_retry_disables_both_native_output_flags(local_model_cost_map: None) -> None: + litellm_params: Final = { + "model_info": { + "id": "deployment-1", + "supports_native_structured_output": True, + "supports_bedrock_native_structured_output": True, + } + } + + retry_params: Final = for_bedrock_json_tool_retry(litellm_params) + + assert deployment_model_info(retry_params) == { + "id": "deployment-1", + "supports_native_structured_output": False, + "supports_bedrock_native_structured_output": False, + } + assert litellm_params["model_info"] == { + "id": "deployment-1", + "supports_native_structured_output": True, + "supports_bedrock_native_structured_output": True, + } + def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_the_service_as_a_list(): import litellm.llms.bedrock.common_utils as mod diff --git a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py index 0951ba6ba92..d298142fe4b 100644 --- a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py +++ b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py @@ -1,11 +1,13 @@ import io import os import pathlib +import traceback from typing import Final from unittest.mock import MagicMock import httpx import pytest +from pydantic import ValidationError from litellm.llms.base_llm.audio_transcription.transformation import AudioTranscriptionRequestData from litellm.llms.deepgram.audio_transcription.transformation import ( @@ -588,31 +590,33 @@ def test_transform_response_maps_words_to_openai_word_timestamps(words: object, [{"word": "hello", "start": 0.0}], ], ) -def test_transform_response_wraps_malformed_words_with_the_raw_body(words: object): +def test_transform_response_keeps_the_transcript_out_of_malformed_words_errors(words: object): raw_response = httpx.Response(200, json=_deepgram_payload({"transcript": "hello", "words": words}, {})) with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) - assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + assert "hello" not in "".join(traceback.format_exception(exc_info.value)) @pytest.mark.parametrize( - ("payload", "error"), + ("payload", "failing_lookup_error"), [ - ({"results": None}, "'NoneType' object is not subscriptable"), - (_deepgram_payload({"transcript": "hello", "words": 5}, {}), "object of type 'int' has no len()"), - ( - {**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, - "'NoneType' object is not subscriptable", - ), + ({"results": None}, TypeError), + (_deepgram_payload({"transcript": "hello", "words": 5}, {}), TypeError), + ({**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, TypeError), + (_deepgram_payload({"transcript": ["hello"]}, {}), ValidationError), ], ) -def test_transform_response_wraps_the_error_of_the_failing_lookup_in_a_malformed_body(payload: object, error: str): +def test_transform_response_wraps_the_error_of_the_failing_lookup_without_the_body( + payload: object, failing_lookup_error: type[Exception] +): raw_response: Final = httpx.Response(200, json=payload) with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) - assert exc_info.type is ValueError - assert str(exc_info.value) == f"Error transforming Deepgram response: {error}\nResponse: {raw_response.text}" + failing_lookup: Final = exc_info.value.__context__ + assert isinstance(failing_lookup, failing_lookup_error) + assert str(exc_info.value) == f"Error transforming Deepgram response: {failing_lookup}" + assert "hello" not in "".join(traceback.format_exception(exc_info.value)) diff --git a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py index f55644eccae..2f5754b3765 100644 --- a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py +++ b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py @@ -1,9 +1,11 @@ +import traceback from collections.abc import Iterator from typing import Final import httpx import pytest import respx +from pydantic import ValidationError import litellm from litellm.caching.llm_caching_handler import LLMClientCache @@ -109,24 +111,35 @@ def test_transform_audio_transcription_response_without_words_key_has_no_word_li @pytest.mark.parametrize( "payload", [ - ["not", "an", "object"], - "plain text", - 7, - {"text": ["not", "text"]}, - {"text": "t", "words": None}, - {"text": "t", "words": 7}, - {"text": "t", "words": "not a list"}, - {"text": "t", "words": ["not an object"]}, - {"text": "t", "words": [{"type": "word", "text": "ok"}, None]}, + ["sensitive", "an", "object"], + "sensitive text", + {"text": ["sensitive", "text"]}, + {"text": "sensitive", "words": None}, + {"text": "sensitive", "words": 7}, + {"text": "sensitive", "words": "not a list"}, + {"text": "sensitive", "words": ["not an object"]}, + {"text": "sensitive", "words": [{"type": "word", "text": "ok"}, None]}, ], ) -def test_transform_audio_transcription_response_wraps_malformed_payloads_with_the_raw_body(payload: object): +def test_transform_audio_transcription_response_keeps_the_body_out_of_malformed_payload_errors(payload: object): raw_response = httpx.Response(200, json=payload) with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info: ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response) - assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) + + +def test_transform_audio_transcription_response_names_the_failing_lookup_of_a_malformed_payload(): + raw_response = httpx.Response(200, json={"text": "sensitive", "words": [None]}) + + with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info: + ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response) + + failing_lookup: Final = exc_info.value.__context__ + assert isinstance(failing_lookup, ValidationError) + assert str(exc_info.value) == f"Error transforming ElevenLabs response: {failing_lookup}" + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) class TestElevenLabsAudioTranscription: diff --git a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py index 08084c8fac0..a9ef468525f 100644 --- a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py +++ b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py @@ -1,8 +1,10 @@ import base64 import json +import traceback import httpx import pytest +import respx import litellm from litellm.llms.gemini.audio_transcription.transformation import ( @@ -241,6 +243,21 @@ class TestTransformResponse: with pytest.raises(GeminiError, match="non-JSON"): config.transform_audio_transcription_response(raw) + def test_non_completed_status_error_omits_response_body(self, config): + payload = json.loads(json.dumps(COMPLETED_RESPONSE)) + payload["status"] = "incomplete" + payload["steps"][0]["content"][0]["text"] = "sensitive transcript text" + with pytest.raises(GeminiError) as excinfo: + config.transform_audio_transcription_response(make_response(payload)) + assert "status=incomplete" in str(excinfo.value) + assert "sensitive transcript text" not in str(excinfo.value) + + def test_non_json_error_omits_response_body(self, config): + raw = httpx.Response(200, text="sensitive body", request=httpx.Request("POST", "https://example.test")) + with pytest.raises(GeminiError) as excinfo: + config.transform_audio_transcription_response(raw) + assert "sensitive body" not in str(excinfo.value) + def test_word_without_offsets_survives(self, config): payload = json.loads(json.dumps(COMPLETED_RESPONSE)) payload["steps"][0]["content"][0]["annotations"] = [{"type": "word_info", "text": "Hello"}] @@ -249,6 +266,66 @@ class TestTransformResponse: assert response.get("duration") is None + +class TestNonCompletedInteractionThroughRouter: + @pytest.fixture + def router(self, monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return litellm.Router( + model_list=[ + { + "model_name": "transcribe", + "litellm_params": {"model": "gemini/gemini-3.5-transcribe", "api_key": "test-gemini-key"}, + } + ], + num_retries=2, + ) + + def _interaction(self, status): + payload = json.loads(json.dumps(COMPLETED_RESPONSE)) + payload["status"] = status + payload["steps"][0]["content"][0]["text"] = "sensitive transcript text" + return payload + + @pytest.mark.asyncio + @respx.mock + async def test_retry_recovers_an_incomplete_interaction(self, router): + route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + side_effect=[ + httpx.Response(200, json=self._interaction("incomplete")), + httpx.Response(200, json=COMPLETED_RESPONSE), + ] + ) + response = await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert route.call_count == 2 + assert response.text == COMPLETED_RESPONSE["steps"][0]["content"][0]["text"] + + @pytest.mark.asyncio + @respx.mock + @pytest.mark.parametrize("status", ["incomplete", "failed", "cancelled", "in_progress", None]) + async def test_non_completed_interaction_retries_then_fails_without_transcript(self, router, status): + route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + return_value=httpx.Response(200, json=self._interaction(status)) + ) + with pytest.raises(litellm.APIConnectionError) as excinfo: + await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert route.call_count == 3 + assert f"status={status}" in str(excinfo.value) + assert "sensitive transcript text" not in str(excinfo.value) + + @pytest.mark.asyncio + @respx.mock + async def test_response_field_of_unexpected_type_keeps_transcript_out_of_the_error(self, router): + payload = self._interaction("incomplete") + payload["steps"][0]["content"] = "sensitive transcript text" + respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + return_value=httpx.Response(200, json=payload) + ) + with pytest.raises(litellm.APIConnectionError) as excinfo: + await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert "sensitive transcript text" not in "".join(traceback.format_exception(excinfo.value)) + + class TestSubtitleSynthesisThroughHandler: def _transform(self, config, response_format): from unittest.mock import Mock diff --git a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py index b475cf7e3ff..494b1fe34ad 100644 --- a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py +++ b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py @@ -3,12 +3,15 @@ from unittest.mock import MagicMock import httpx import litellm +import pytest +from pydantic import ValidationError from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, ) from litellm.llms.mistral.audio_transcription.transformation import ( MistralAudioTranscriptionConfig, + MistralAudioTranscriptionException, ) from litellm.types.utils import TranscriptionResponse from litellm.utils import ProviderConfigManager @@ -191,3 +194,22 @@ def test_mistral_audio_transcription_response_transform_empty(): assert isinstance(response, TranscriptionResponse) assert response.text == "" + + +def test_mistral_audio_transcription_response_transform_keeps_non_json_body_out_of_the_error(): + raw_response = httpx.Response(200, text='{"text": "sensitive') + + with pytest.raises(MistralAudioTranscriptionException, match="non-JSON") as exc_info: + MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) + + +@pytest.mark.parametrize("text", [["sensitive"], {"value": "sensitive"}]) +def test_mistral_audio_transcription_response_transform_keeps_a_malformed_text_field_out_of_the_error(text: object): + raw_response = httpx.Response(200, json={"text": text, "segments": [{"text": "sensitive"}]}) + + with pytest.raises(ValidationError) as exc_info: + MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) diff --git a/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py b/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py index 2dc24b1b313..339d62ef7ea 100644 --- a/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py +++ b/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py @@ -5,8 +5,10 @@ and transform_audio_transcription_response. import io import json +import traceback from unittest.mock import MagicMock +import httpx import pytest from litellm.llms.openai.transcriptions.whisper_transformation import ( @@ -104,3 +106,10 @@ class TestWhisperTransformResponse: is_json=False, ) ) + + def test_json_object_without_transcription_fields_keeps_the_body_out_of_the_error(self): + config = OpenAIWhisperAudioTranscriptionConfig() + raw_response = httpx.Response(200, json={"segments": [{"text": "sensitive"}]}) + with pytest.raises(ValueError, match="Invalid response format") as exc_info: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) diff --git a/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py index 422658932e6..ed8ec88e209 100644 --- a/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py +++ b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py @@ -3,6 +3,7 @@ import pytest from pydantic import ValidationError from litellm.llms.ovhcloud.audio_transcription.transformation import OVHCloudAudioTranscriptionConfig +from litellm.llms.ovhcloud.utils import OVHCloudException from litellm.types.utils import TranscriptionResponse @@ -98,3 +99,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies(payloa _transform(payload) assert "spoken secret" not in str(exc_info.value) + + +def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error(): + raw_response = httpx.Response(200, text='{"text": "spoken secret') + + with pytest.raises(OVHCloudException, match="non-JSON") as exc_info: + OVHCloudAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "spoken secret" not in str(exc_info.value) diff --git a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py index f313f8bd873..16e60a15539 100644 --- a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py +++ b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py @@ -204,6 +204,17 @@ def test_scaleway_transform_response_raises_typed_exception_on_non_json(): ) +def test_scaleway_transform_response_keeps_non_json_body_out_of_the_error(): + raw_response = httpx.Response( + 200, text='{"text": "sensitive', headers={"content-type": "application/json"} + ) + + with pytest.raises(ScalewayAudioTranscriptionException, match="non-JSON") as exc_info: + ScalewayAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) + + def test_scaleway_transform_response_returns_plain_text_for_non_json_content_type(): """When Scaleway responds with text/srt/vtt (response_format="text" etc.), the content-type is not application/json — return the body as plain text diff --git a/tests/unit/llms/test_file_content_block.py b/tests/unit/llms/test_file_content_block.py index 4ab4ef8b857..297b3b6f7d6 100644 --- a/tests/unit/llms/test_file_content_block.py +++ b/tests/unit/llms/test_file_content_block.py @@ -19,14 +19,16 @@ import pytest import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_file_ids_from_messages, migrate_file_to_image_url, - update_messages_with_model_file_ids, ) from litellm.litellm_core_utils.prompt_templates.factory import ( BedrockConverseMessagesProcessor, anthropic_process_openai_file_message, ) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_file_ids_from_messages, + update_messages_with_model_file_ids, +) from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.vertex_ai.gemini.transformation import ( diff --git a/tests/unit/llms/test_file_search_responses.py b/tests/unit/llms/test_file_search_responses.py index b2cbcd4a6dd..424aaad1fd1 100644 --- a/tests/unit/llms/test_file_search_responses.py +++ b/tests/unit/llms/test_file_search_responses.py @@ -13,13 +13,13 @@ Coverage: import base64 import logging -from typing import Any, Dict, List, Optional from importlib import import_module +from typing import Any, Dict, List, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( _decode_vector_store_ids_in_tools, update_responses_tools_with_model_file_ids, ) @@ -476,6 +476,7 @@ class TestManagedFilesVectorStoreAccess: from litellm_enterprise.proxy.hooks.managed_files import ( PROXY_LiteLLMManagedFiles as ManagedFiles, ) + from litellm.proxy._types import CallTypes # If call_type is acompletion, the vector_store check branch isn't reached. diff --git a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py index 5898d933941..02b7c0ae71a 100644 --- a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py +++ b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse import httpx import pytest - +from pydantic import ValidationError import litellm from litellm.llms.vertex_ai.audio_transcription.transformation import ( @@ -208,6 +208,18 @@ class TestTransformResponse: response = config.transform_audio_transcription_response(raw_response) assert response.text == "" + def test_non_json_error_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, text='{"results": [{"alternatives": [{"transcript": "sensitive') + with pytest.raises(VertexAIError, match="non-JSON") as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + + def test_field_of_unexpected_type_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, json={"results": "sensitive transcript"}) + with pytest.raises(ValidationError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + def test_fractional_billed_duration(self, config): raw_response = httpx.Response( status_code=200, diff --git a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py index 82ea034f91b..532e013c3d7 100644 --- a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py +++ b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py @@ -4,6 +4,7 @@ import os import httpx import pytest +from pydantic import ValidationError import litellm from litellm.llms.vertex_ai.audio_transcription.gemini_transcribe_transformation import ( @@ -255,6 +256,20 @@ class TestTransformResponse: with pytest.raises(VertexAIError, match="non-JSON"): config.transform_audio_transcription_response(raw_response) + def test_non_json_error_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, text='{"candidates": [{"content": {"parts": [{"text": "sensitive') + with pytest.raises(VertexAIError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + + def test_field_of_unexpected_type_omits_response_body(self, config): + raw_response = httpx.Response( + status_code=200, json={"candidates": [{"content": {"parts": "sensitive transcript"}}]} + ) + with pytest.raises(ValidationError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + class TestValidateEnvironment: def test_sets_oauth_headers(self): diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py index 39310836bf2..7d60de73718 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py @@ -3,7 +3,6 @@ Test Vertex AI files handler functionality """ import re -from types import MappingProxyType import pytest from unittest.mock import AsyncMock, patch @@ -62,15 +61,14 @@ class TestVertexAIFilesHandler: configured_bucket_name="test-bucket", ) - def test_extract_bucket_and_object_from_file_id_allows_trusted_legacy_flag(self): + def test_extract_bucket_and_object_from_file_id_allows_legacy_flag_on_the_call(self): file_id = "gs%3A%2F%2Ftest-bucket%2Ftest-file.txt" - trusted_credentials = MappingProxyType({"allow_legacy_cloud_file_ids": True}) bucket_name, object_path = self.handler._extract_bucket_and_object_from_file_id( file_id=file_id, configured_bucket_name="test-bucket", litellm_params={ - "_litellm_internal_model_credentials": trusted_credentials, + "allow_legacy_cloud_file_ids": True, }, ) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 842500a7b27..c6b6bca2014 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -6,8 +6,8 @@ Includes tests for Vertex AI batch output transformation to OpenAI format. import json import urllib.parse from collections.abc import Mapping, Sequence -from typing import Final from types import MappingProxyType +from typing import Final from urllib.parse import parse_qs, urlparse import httpx @@ -72,46 +72,24 @@ class TestParseGcsUri: litellm_params={"gcs_bucket_name": "my-bucket"}, ) - def test_should_reject_request_supplied_legacy_flag(self, config): - with pytest.raises(ValueError, match="LiteLLM-managed"): - config._parse_gcs_uri( - "gs://my-bucket/private/object.txt", - litellm_params={ - "gcs_bucket_name": "my-bucket", - "allow_legacy_cloud_file_ids": True, - }, - ) - - def test_should_allow_legacy_object_path_with_trusted_server_flag(self, config): - trusted_credentials = MappingProxyType({"allow_legacy_cloud_file_ids": True}) + def test_should_allow_legacy_object_path_when_the_call_sets_the_flag(self, config): bucket, encoded = config._parse_gcs_uri( "gs://my-bucket/private/object.txt", litellm_params={ "gcs_bucket_name": "my-bucket", - "_litellm_internal_model_credentials": trusted_credentials, + "allow_legacy_cloud_file_ids": True, }, ) assert bucket == "my-bucket" assert encoded == urllib.parse.quote("private/object.txt", safe="") - def test_should_reject_user_supplied_legacy_flag_snapshot(self, config): - with pytest.raises(ValueError, match="LiteLLM-managed"): - config._parse_gcs_uri( - "gs://my-bucket/private/object.txt", - litellm_params={ - "gcs_bucket_name": "my-bucket", - "_litellm_internal_model_credentials": {"allow_legacy_cloud_file_ids": True}, - }, - ) - def test_should_keep_configured_prefix_for_legacy_object_path(self, config): - trusted_credentials = MappingProxyType({"allow_legacy_cloud_file_ids": True}) bucket, encoded = config._parse_gcs_uri( "gs://my-bucket/team-a/private/object.txt", litellm_params={ "gcs_bucket_name": "my-bucket/team-a", - "_litellm_internal_model_credentials": trusted_credentials, + "allow_legacy_cloud_file_ids": True, }, ) @@ -119,13 +97,12 @@ class TestParseGcsUri: assert encoded == urllib.parse.quote("team-a/private/object.txt", safe="") def test_should_reject_legacy_object_outside_configured_prefix(self, config): - trusted_credentials = MappingProxyType({"allow_legacy_cloud_file_ids": True}) with pytest.raises(ValueError, match="configured storage prefix"): config._parse_gcs_uri( "gs://my-bucket/team-b/private/object.txt", litellm_params={ "gcs_bucket_name": "my-bucket/team-a", - "_litellm_internal_model_credentials": trusted_credentials, + "allow_legacy_cloud_file_ids": True, }, ) diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 5de4d6424c5..0539a939bb5 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -1980,7 +1980,6 @@ def test_file_data_field_order(): def test_file_data_field_order_gcs_urls(): """Test that GCS URLs also maintain correct field order.""" - import json from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media @@ -2066,7 +2065,7 @@ def test_extract_file_data_with_path_object(): import tempfile from pathlib import Path - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) @@ -2105,7 +2104,7 @@ def test_extract_file_data_with_pathlib_path(): import tempfile from pathlib import Path - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) @@ -2129,7 +2128,7 @@ def test_extract_file_data_with_pathlib_path(): def test_extract_file_data_with_tuple_format(): """Test that tuple format (with explicit content_type) still works correctly.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) @@ -2152,7 +2151,7 @@ def test_extract_file_data_fallback_to_octet_stream(): import tempfile from pathlib import Path - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) diff --git a/tests/unit/llms/vertex_ai/test_vertex.py b/tests/unit/llms/vertex_ai/test_vertex.py index 7e0baaf9e10..9503182cad7 100644 --- a/tests/unit/llms/vertex_ai/test_vertex.py +++ b/tests/unit/llms/vertex_ai/test_vertex.py @@ -6,9 +6,9 @@ import sys from types import SimpleNamespace from typing import Final -from dotenv import load_dotenv import httpx import respx +from dotenv import load_dotenv from google.oauth2.credentials import Credentials import litellm.litellm_core_utils @@ -23,7 +23,6 @@ import pytest import litellm from litellm import Router, get_optional_params from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media -from litellm.types.llms.vertex_ai import BlobType def encode_image_to_base64(image_path): @@ -110,7 +109,6 @@ def test_completion_pydantic_obj_2(): def test_build_vertex_schema(): - import json from litellm.llms.vertex_ai.common_utils import build_vertex_schema @@ -1278,7 +1276,9 @@ def test_process_gemini_media(): def test_get_image_mime_type_from_url(): """Test MIME type inference for remote media URLs""" - from litellm.litellm_core_utils.prompt_templates.common_utils import get_image_mime_type_from_url + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_image_mime_type_from_url, + ) # Test JPEG images assert ( @@ -1350,7 +1350,7 @@ def test_vertex_embedding_url(model, expected_url): assert endpoint == "predict" -from unittest.mock import Mock, patch +from unittest.mock import patch import pytest diff --git a/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py b/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py index b6e5007c456..c9f6f566ad6 100644 --- a/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py +++ b/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py @@ -4,6 +4,7 @@ Tests for IBM WatsonX Audio Transcription. Validates the WatsonX transcription response transformation. """ +import traceback from unittest.mock import MagicMock import httpx @@ -120,17 +121,21 @@ def test_transform_audio_transcription_response_copies_every_field_except_model( assert {key: response[key] for key in expected_extras} == expected_extras -@pytest.mark.parametrize("payload", [{}, {"model": "whisper"}, {"duration": 2.0}, {"text": None, "usage": None}]) -def test_transform_audio_transcription_response_without_text_or_usage_reports_the_body( +@pytest.mark.parametrize( + "payload", + [ + {"model": "sensitive"}, + {"duration": 2.0, "segments": [{"text": "sensitive"}]}, + {"text": None, "usage": None, "note": "sensitive"}, + ], +) +def test_transform_audio_transcription_response_without_text_or_usage_keeps_the_body_out_of_the_error( payload: dict[str, object], ) -> None: with pytest.raises(ValueError, match="Invalid response format") as exc_info: _transform_transcription_response(payload) - assert exc_info.value.args == ( - "Invalid response format. Received response does not match the expected format. Got: ", - payload, - ) + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) @pytest.mark.parametrize("payload", [["not", "an", "object"], "plain text", 7, True]) @@ -141,3 +146,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies_withou _transform_transcription_response(payload) assert "input_value" not in str(exc_info.value) + + +def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error() -> None: + raw_response = httpx.Response(200, text='{"text": "sensitive') + + with pytest.raises(ValueError, match="Error transforming response to json") as exc_info: + IBMWatsonXAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) diff --git a/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py b/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py index e2e3fc3d4dc..3505116819b 100644 --- a/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py +++ b/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py @@ -1,3 +1,5 @@ +import traceback + import httpx import pytest @@ -146,6 +148,23 @@ def test_transform_response_raises_on_error_status(): assert "Incorrect API key provided" in exc.value.message +@pytest.mark.parametrize( + "content", + [ + b'{"text": ["sensitive"]}', + b'{"text": "ok", "words": [{"text": ["sensitive"]}]}', + b'{"text": "sensitive', + ], +) +def test_transform_response_keeps_a_malformed_body_out_of_the_error(content): + raw = httpx.Response(200, content=content, request=httpx.Request("POST", "https://api.x.ai/v1/stt")) + + with pytest.raises(XAIAudioTranscriptionError, match="Error parsing xAI response") as exc: + CONFIG.transform_audio_transcription_response(raw_response=raw) + + assert "sensitive" not in "".join(traceback.format_exception(exc.value)) + + def test_transcription_routes_to_xai_stt(monkeypatch): monkeypatch.delenv("XAI_API_KEY", raising=False) monkeypatch.setattr(litellm, "xai_key", None) diff --git a/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py b/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py index 7e5b3a7abd7..a1be416f4cb 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py +++ b/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py @@ -74,6 +74,7 @@ def _make_mock_prisma(): record.description = data.get("description") record.manifest_json = data.get("manifest_json", "{}") record.enabled = data.get("enabled", True) + record.created_by = data.get("created_by") record.created_at = data.get("created_at") record.updated_at = data.get("updated_at") store[data["name"]] = record @@ -221,7 +222,7 @@ async def test_metadata_edits_preserve_uploaded_source_and_reject_a_changed_http await table.create(data={"name": _UPLOAD_NAME, "manifest_json": json.dumps({"source": _UPLOAD_SOURCE})}) request: Final = UpdatePluginRequest(source=_UPLOAD_SOURCE, description="Updated description", version="1.0.0") response: Final = await update_plugin(uploads=_uploads(), plugin_name=_UPLOAD_NAME, request=request, user_api_key_dict=_USER) - expected: Final = {"name": _UPLOAD_NAME, "source": _UPLOAD_SOURCE, "description": "Updated description", "version": "1.0.0"} + expected: Final = {"name": _UPLOAD_NAME, "source": _UPLOAD_SOURCE, "description": "Updated description", "version": "1.0.0", "_litellm_published_by": _USER.user_id} assert response.plugin.description == "Updated description" assert await _read_stored_manifest(_UPLOAD_NAME) == expected with pytest.raises(HTTPException) as error: @@ -354,7 +355,7 @@ async def _register_public_and_private_plugins() -> None: async def _listed_names(user: UserAPIKeyAuth) -> set[str]: - response = await list_plugins(user_api_key_dict=user) + response = await list_plugins(uploads=_uploads(), user_api_key_dict=user) return {plugin.name for plugin in response.plugins} @@ -474,7 +475,7 @@ async def test_update_plugin_propagates_installation_preference_to_get_and_list( ) plugin = await get_plugin(plugin_name=name, user_api_key_dict=_USER) - listed_plugin = (await list_plugins(user_api_key_dict=_USER)).plugins[0] + listed_plugin = (await list_plugins(uploads=_uploads(), user_api_key_dict=_USER)).plugins[0] assert response.plugin.installation_preference == "auto_install" assert plugin["installation_preference"] == "auto_install" @@ -514,7 +515,7 @@ async def test_stored_unknown_installation_preference_is_not_served(stored_value ) marketplace = json.loads((await get_marketplace(request=MagicMock())).body) - listed_plugin = (await list_plugins(user_api_key_dict=_USER)).plugins[0] + listed_plugin = (await list_plugins(uploads=_uploads(), user_api_key_dict=_USER)).plugins[0] plugin = await get_plugin(plugin_name=name, user_api_key_dict=_USER) assert marketplace["plugins"] == [{"name": name, "source": _ARCHIVE_SOURCE}] @@ -1288,7 +1289,7 @@ async def test_skill_upload_creates_metadata_and_bytes_atomically_and_rejects_du "url": f"http://test/claude-code/plugins/{_UPLOAD_NAME}/archive/{digest}.zip", "sha256": digest, } - assert json.loads(stored["manifest_json"]) == {**metadata, "source": source} + assert json.loads(stored["manifest_json"]) == {**metadata, "source": source, "_litellm_published_by": stored["created_by"]} assert stored == { "name": _UPLOAD_NAME, "description": "My notes", "version": "2.0.0", "manifest_json": stored["manifest_json"], "created_by": litellm.proxy.proxy_server.litellm_proxy_admin_name, diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 928c73d4b04..8651d522ac8 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -5311,6 +5311,124 @@ def test_get_end_user_id_from_request_body_backwards_compatibility(): assert get_end_user_id_from_request_body(request_body) == expected_end_user_id +_BUCKET_ALLOWLIST: Final = {"allowed_client_side_s3_buckets": ["team-batch-bucket"]} +_ENDPOINT_ALLOWLIST: Final = {"allowed_client_side_s3_endpoint_urls": ["https://s3.us-west-2.amazonaws.com"]} + + +def _batches_body_is_safe(body: dict[str, object], general_settings: dict[str, object]) -> bool: + return is_request_body_safe( + request_body=body, + general_settings=general_settings, + llm_router=None, + model="", + route="/v1/batches", + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("s3_bucket_name", "team-batch-bucket"), + ("s3_output_bucket_name", "team-batch-bucket"), + ("allow_legacy_cloud_file_ids", True), + ("s3_endpoint_url", "https://s3.us-west-2.amazonaws.com"), + ], +) +def test_storage_setting_in_request_body_is_rejected_without_an_allowlist(field: str, value: object) -> None: + with pytest.raises(ValueError, match="Rejected Request") as error: + _batches_body_is_safe({field: value}, {}) + + assert field in str(error.value) + + +@pytest.mark.parametrize("field", ["s3_bucket_name", "s3_output_bucket_name"]) +def test_bucket_on_the_proxy_allowlist_is_accepted(field: str) -> None: + assert _batches_body_is_safe({field: "team-batch-bucket"}, _BUCKET_ALLOWLIST) is True + + +@pytest.mark.parametrize("field", ["s3_bucket_name", "s3_output_bucket_name"]) +def test_bucket_off_the_proxy_allowlist_is_rejected_and_names_the_setting(field: str) -> None: + with pytest.raises(ValueError, match=field) as error: + _batches_body_is_safe({field: "someone-elses-bucket"}, _BUCKET_ALLOWLIST) + + assert "general_settings.allowed_client_side_s3_buckets" in str(error.value) + + +@pytest.mark.parametrize( + "general_settings,accepted_body,rejected_field", + [ + ( + {**_BUCKET_ALLOWLIST, "allowed_client_side_s3_endpoint_urls": None}, + {"s3_bucket_name": "team-batch-bucket"}, + "s3_endpoint_url", + ), + ( + {**_ENDPOINT_ALLOWLIST, "allowed_client_side_s3_buckets": None}, + {"s3_endpoint_url": "https://s3.us-west-2.amazonaws.com"}, + "s3_bucket_name", + ), + ], +) +def test_allowlist_left_null_in_config_is_treated_as_empty( + general_settings: dict[str, object], accepted_body: dict[str, object], rejected_field: str +) -> None: + assert _batches_body_is_safe(accepted_body, general_settings) is True + with pytest.raises(ValueError, match=f"{rejected_field} is not allowed"): + _batches_body_is_safe({rejected_field: "team-batch-bucket"}, general_settings) + + +def test_endpoint_on_the_proxy_allowlist_is_accepted() -> None: + assert _batches_body_is_safe({"s3_endpoint_url": "https://s3.us-west-2.amazonaws.com"}, _ENDPOINT_ALLOWLIST) is True + + +def test_endpoint_off_the_proxy_allowlist_is_rejected_and_names_the_setting() -> None: + with pytest.raises(ValueError, match="s3_endpoint_url") as error: + _batches_body_is_safe({"s3_endpoint_url": "https://attacker.example"}, _ENDPOINT_ALLOWLIST) + + assert "general_settings.allowed_client_side_s3_endpoint_urls" in str(error.value) + + +def test_bucket_allowlist_admits_neither_an_endpoint_nor_the_legacy_file_flag() -> None: + with pytest.raises(ValueError, match="s3_endpoint_url"): + _batches_body_is_safe( + {"s3_bucket_name": "team-batch-bucket", "s3_endpoint_url": "team-batch-bucket"}, + _BUCKET_ALLOWLIST, + ) + with pytest.raises(ValueError, match="allow_legacy_cloud_file_ids"): + _batches_body_is_safe( + {"s3_bucket_name": "team-batch-bucket", "allow_legacy_cloud_file_ids": True}, + _BUCKET_ALLOWLIST, + ) + + +def test_endpoint_allowlist_does_not_admit_a_bucket() -> None: + with pytest.raises(ValueError, match="s3_bucket_name"): + _batches_body_is_safe({"s3_bucket_name": "https://s3.us-west-2.amazonaws.com"}, _ENDPOINT_ALLOWLIST) + + +def test_bucket_nested_in_extra_body_is_checked_against_the_allowlist() -> None: + assert _batches_body_is_safe({"extra_body": {"s3_bucket_name": "team-batch-bucket"}}, _BUCKET_ALLOWLIST) is True + with pytest.raises(ValueError, match="s3_bucket_name"): + _batches_body_is_safe({"extra_body": {"s3_bucket_name": "someone-elses-bucket"}}, _BUCKET_ALLOWLIST) + + +def test_proxy_wide_client_side_credentials_still_admit_any_bucket() -> None: + assert ( + _batches_body_is_safe( + {"s3_bucket_name": "any-bucket", "allow_legacy_cloud_file_ids": True}, + {"allow_client_side_credentials": True}, + ) + is True + ) + + +def test_other_blocked_fields_do_not_point_at_the_storage_allowlist() -> None: + with pytest.raises(ValueError, match="api_base") as error: + _batches_body_is_safe({"api_base": "https://attacker.example"}, _BUCKET_ALLOWLIST) + + assert "allowed_client_side" not in str(error.value) + + def _request_with_content_length(content_length: int) -> Request: return Request( { diff --git a/tests/unit/proxy/batches_endpoints/test_endpoints.py b/tests/unit/proxy/batches_endpoints/test_endpoints.py index 24f7f128b0a..b5da6f31eb4 100644 --- a/tests/unit/proxy/batches_endpoints/test_endpoints.py +++ b/tests/unit/proxy/batches_endpoints/test_endpoints.py @@ -1723,11 +1723,6 @@ async def test_retrieve__unified_batch_id_routes_to_router(retrieve_harness): assert retrieve_harness.router_aretrieve.call_count == 1 retrieve_harness.litellm_aretrieve.assert_not_called() - # Credentials are resolved for the deployment behind the unified id so the batch's - # output file can be read for cost accounting. This id resolves to nothing here, and - # the retrieve must still serve the batch rather than fail on the lookup. - retrieve_harness.creds_resolver.assert_called_once_with("gpt-4o-mini", form="with_provider") - # router receives the (still-encoded) batch id verbatim - this layer does # not decode it for the unified path. assert retrieve_harness.router_kwargs() == {"batch_id": "batch-unified-blob"} diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index eee5d96511b..09c76b2dd65 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -362,17 +362,19 @@ class TestCheckBatchCost: assert update_data["status"] == "complete" @pytest.mark.asyncio - async def test_output_fetch_passes_deployment_credentials_as_trusted_snapshot( - self, check_batch_cost_instance, mock_prisma_client, mock_llm_router + async def test_output_fetch_reads_from_the_deployment_bucket( + self, check_batch_cost_instance, mock_prisma_client, mock_llm_router, monkeypatch ): - """Bedrock resolves the output bucket ONLY from the immutable snapshot kwarg. + """Without the deployment's s3_bucket_name the cost poller cannot read a Bedrock + batch's output file, and every completed Bedrock batch stays unbilled.""" + import httpx - Spreading the credentials as plain kwargs is not enough: get_litellm_params drops - s3_bucket_name, so without _litellm_internal_model_credentials the cost poller - cannot read the output file and every completed Bedrock batch stays unbilled. - """ - from types import MappingProxyType - from unittest.mock import patch + import litellm + + monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) + monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1) mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock() @@ -386,7 +388,7 @@ class TestCheckBatchCost: mock_response = MagicMock() mock_response.status = "completed" - mock_response.output_file_id = "file-output-123" + mock_response.output_file_id = "s3://configured-batch-bucket/litellm-batch-outputs/job-1/input.jsonl.out" mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}' mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response) @@ -395,6 +397,8 @@ class TestCheckBatchCost: "custom_llm_provider": "bedrock", "s3_bucket_name": "configured-batch-bucket", "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "secret", } ) @@ -404,12 +408,10 @@ class TestCheckBatchCost: mock_deployment.model_info.model_dump.return_value = {} mock_llm_router.registry.return_value.deployment = MagicMock(return_value=mock_deployment) - mock_file_content = MagicMock() - mock_file_content.content = b'{"recordId":"req-1"}' - decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;" with ( + respx.mock(assert_all_called=False) as s3, patch( "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id", side_effect=[decoded_id, None], @@ -422,11 +424,6 @@ class TestCheckBatchCost: "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id", return_value="batch-456", ), - patch( - "litellm.files.main.afile_content", - new_callable=AsyncMock, - return_value=mock_file_content, - ) as mock_afile_content, patch( "litellm.batches.batch_utils.get_file_content_as_dictionary", return_value=[{"recordId": "req-1"}], @@ -447,17 +444,15 @@ class TestCheckBatchCost: mock_logging_obj = MagicMock() mock_logging_obj.async_success_handler = AsyncMock() mock_logging_cls.return_value = mock_logging_obj + s3.get(url__regex=r"^https://s3\.us-east-1\.amazonaws\.com/").mock( + return_value=httpx.Response(200, content=b'{"recordId":"req-1"}') + ) await check_batch_cost_instance.check_batch_cost() - mock_afile_content.assert_awaited() - passed_kwargs = mock_afile_content.await_args[1] - snapshot = passed_kwargs.get("_litellm_internal_model_credentials") - assert snapshot is not None, "cost poller must pass the trusted credential snapshot" - assert isinstance(snapshot, MappingProxyType), ( - "snapshot must be a MappingProxyType; a plain dict is rejected by get_configured_s3_bucket_name" - ) - assert snapshot["s3_bucket_name"] == "configured-batch-bucket" + assert [str(call.request.url) for call in s3.calls] == [ + "https://s3.us-east-1.amazonaws.com/configured-batch-bucket/litellm-batch-outputs/job-1/input.jsonl.out" + ] @pytest.mark.asyncio async def test_poller_prices_with_deployment_registered_batch_rates( diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index a72772df492..0884d0bbbed 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,7 +1,6 @@ import _hashlib from collections.abc import Mapping from contextlib import nullcontext -from typing import Final from unittest.mock import Mock import pytest @@ -17,30 +16,32 @@ from litellm.proxy.common_utils.fips import ( ) -def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: - real_new: Final = _hashlib.new - - def new(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: - if name == "md5" and md5_refused: - raise _hashlib.UnsupportedDigestmodError("unsupported hash type") - return real_new(name, data, usedforsecurity=usedforsecurity) - +def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, scrypt_refused: bool) -> None: monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) - monkeypatch.setattr(_hashlib, "new", new) + if scrypt_refused: + monkeypatch.setattr(_hashlib, "scrypt", Mock(side_effect=ValueError("[digital envelope routines] unsupported"))) + else: + monkeypatch.setattr(_hashlib, "scrypt", Mock(return_value=b"\0" * 16)) @pytest.mark.parametrize( - "fips_flag, md5_refused, enforces", + "fips_flag, scrypt_refused, enforces", [(1, True, True), (0, True, False), (1, False, False)], - ids=["fips", "fips-flag-off", "md5-allowed"], + ids=["fips", "fips-flag-off", "scrypt-allowed"], ) -def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5( - monkeypatch: pytest.MonkeyPatch, fips_flag: int, md5_refused: bool, enforces: bool +def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_scrypt( + monkeypatch: pytest.MonkeyPatch, fips_flag: int, scrypt_refused: bool, enforces: bool ) -> None: - _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused) + _fake_openssl(monkeypatch, fips_flag=fips_flag, scrypt_refused=scrypt_refused) assert openssl_enforces_fips() is enforces +def test_openssl_enforces_fips_while_md5_is_still_served_for_non_security_use(monkeypatch: pytest.MonkeyPatch) -> None: + _fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True) + monkeypatch.setattr(_hashlib, "new", Mock(return_value=Mock())) + assert openssl_enforces_fips() is True + + def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delattr(_hashlib, "get_fips_mode") assert openssl_enforces_fips() is False @@ -113,7 +114,7 @@ def test_enforce_fips_mode( cryptography_fips: bool, refusal: str | None, ) -> None: - _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) + _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), scrypt_refused=openssl_fips) monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): diff --git a/tests/unit/proxy/discovery_endpoints/test_skill_hub_endpoints.py b/tests/unit/proxy/discovery_endpoints/test_skill_hub_endpoints.py index 8127fc80bab..7912df3939b 100644 --- a/tests/unit/proxy/discovery_endpoints/test_skill_hub_endpoints.py +++ b/tests/unit/proxy/discovery_endpoints/test_skill_hub_endpoints.py @@ -196,25 +196,25 @@ def _assert_portable(response: httpx.Response) -> None: assert stat.S_IMODE(archive.getinfo("scripts/check.sh").external_attr >> 16) == 0o755 -@pytest.mark.parametrize("key, expected", ((MASTER_KEY, 7), (PLAIN_KEY, None))) -async def test_catalog_counts_are_admin_only_and_batched_for_the_page( - hub: HubClient, key: str, expected: int | None, +@pytest.mark.parametrize("route", (f"{HUB}?limit=1", "/claude-code/plugins")) +@pytest.mark.parametrize("key", (MASTER_KEY, PLAIN_KEY)) +async def test_catalog_counts_are_batched_for_the_page( + hub: HubClient, key: str, route: str, ) -> None: assert (await _upload(hub, _zip(FILES))).status_code == 200 record: Final = _stored_upload(hub) hub.plugins.find_many.return_value = [record, record.model_copy(update={"id": "other", "name": "zzz-other"})] hub.database.query_raw.reset_mock() hub.database.query_raw.return_value = [{"id": record.id, "download_count": 7}] - response: Final = await hub.client.get(f"{HUB}?limit=1", headers=_headers(key)) + response: Final = await hub.client.get(route, headers=_headers(key)) assert response.status_code == 200, response.text - assert [(item["id"], item["download_count"]) for item in response.json()["skills"]] == [(record.id, expected)] - if expected is None: - hub.database.query_raw.assert_not_awaited() - else: - assert hub.database.query_raw.await_count == 1 - assert hub.database.query_raw.await_args.args[1] == (record.id,) - assert (await hub.client.get(f"{HUB}?offset=2", headers=_headers(key))).json()["skills"] == [] - assert hub.database.query_raw.await_count == 1 + items: Final = response.json()["plugins" if route == "/claude-code/plugins" else "skills"] + expected: Final = {record.id: 7, "other": 0} if route == "/claude-code/plugins" else {record.id: 7} + assert {item["id"]: item["download_count"] for item in items} == expected + assert hub.database.query_raw.await_count == 1 + assert set(hub.database.query_raw.await_args.args[1]) == set(expected) + assert (await hub.client.get(f"{HUB}?offset=2", headers=_headers(key))).json()["skills"] == [] + assert hub.database.query_raw.await_count == 1 @pytest.mark.parametrize("operation", ("upload", "import")) @@ -806,3 +806,98 @@ async def test_configured_scan_persists_result_without_changing_manual_approval( assert (await hub.client.get(path, headers=_headers(MASTER_KEY))).json() == before approved: Final = await hub.client.put(path + "/approve", headers=_headers(MASTER_KEY)) assert approved.status_code == 200 and approved.json()["submission"]["status"] == "approved" + + +async def test_catalog_filters_metadata_before_pagination_and_preserves_tag_facets(hub: HubClient) -> None: + hub.plugins.find_many.return_value = [ + prisma_models.LiteLLM_ClaudeCodePluginTable( + id=name, name=name, enabled=True, anonymous_download_count=0, created_by="author@example.test", + created_at=datetime(2026, 1, 2), updated_at=datetime(2026, 2, 3), + manifest_json=json.dumps({ + "source": {"source": "github", "repo": "example/skills"}, + "keywords": tags, + "_litellm_published_by": "publisher@example.test", + **({"_litellm_review": {"approved_by": "reviewer@example.test", "security_scan_passed": scanned}} if approved else {}), + }), + ) + for name, tags, approved, scanned in ( + ("a-other", ["support"], False, False), + ("b-review", ["engineering"], True, False), + ("c-review", ["engineering"], True, True), + ) + ] + response = await hub.client.get(HUB, headers=_headers(MASTER_KEY), params={ + "query": " REVIEWER ", "tag": "ENGINEERING", "limit": 1, "offset": 1, + }) + assert response.status_code == 200 + body = response.json() + assert body["count"] == 2 + assert body["tags"] == ["engineering", "support"] + assert [skill["name"] for skill in body["skills"]] == ["c-review"] + skill = body["skills"][0] + assert skill["created_by"] == "author@example.test" + assert skill["published_by"] == "publisher@example.test" + assert skill["approved_by"] == "reviewer@example.test" + assert skill["created_at"].startswith("2026-01-02") + assert skill["updated_at"].startswith("2026-02-03") + assert skill["security_scan_passed"] is True + scanned = await hub.client.get(HUB, headers=_headers(MASTER_KEY), params={"security_scan_passed": "true"}) + assert [skill["name"] for skill in scanned.json()["skills"]] == ["c-review"] + unscanned = await hub.client.get(HUB, headers=_headers(MASTER_KEY), params={"security_scan_passed": "false"}) + assert [skill["name"] for skill in unscanned.json()["skills"]] == ["a-other", "b-review"] + assert unscanned.json()["skills"][0]["approved_by"] is None + + +@pytest.mark.parametrize("has_issues", (False, True)) +async def test_admin_approval_does_not_override_scan_result(pending_hub: HubClient, has_issues: bool) -> None: + hub: Final = pending_hub + pending: Final = await _upload(hub, _zip(FILES), MEMBER_KEY, version=None) + receipt: Final = pending.json() + submission: Final = hub.database.litellm_skillsubmission.rows[0] + digest: Final = hashlib.sha256(submission.archive_content.decode()).hexdigest() + await hub.database.litellm_skillsubmission.update( + where={"id": submission.id}, data={"scan_json": json.dumps({ + "id": "scan", "requested_by": "admin", "started_at": "2026-01-01T00:00:00Z", + "expires_at": "2026-01-01T00:03:00Z", "status": "completed", "archive_sha256": digest, + "report": {"scanner_version": "test", "scan_mode": "static-only", "complete": True, + "risk_score": 0, "findings": [{"rule_id": "rule", "severity": "LOW", + "message": "Issue", "file": "SKILL.md"}] if has_issues else []}, + })}, + ) + approved: Final = await hub.client.put( + f"/skill-hub/submissions/{receipt['submission_id']}/approve", headers=_headers(MASTER_KEY), + ) + assert approved.status_code == 200 + skill: Final = approved.json()["skill"] + assert skill["security_scan_passed"] is not has_issues + assert skill["created_by"] == MEMBER_KEY + assert skill["approved_by"] == approved.json()["submission"]["reviewed_by"] + assert skill["published_by"] == skill["approved_by"] + _stored_upload(hub) + listed: Final = (await hub.client.get(HUB, headers=_headers(MASTER_KEY))).json()["skills"][0] + assert listed["security_scan_passed"] is not has_issues + assert listed["published_by"] == skill["published_by"] + + +@pytest.mark.parametrize("field,order,expected", [ + ("name", "asc", "a"), ("name", "desc", "c"), + ("download_count", "desc", "a"), ("download_count", "asc", "b"), + ("updated_at", "desc", "c"), ("updated_at", "asc", "b"), +]) +async def test_catalog_sorts_all_matches_before_pagination( + hub: HubClient, field: str, order: str, expected: str, +) -> None: + hub.plugins.find_many.return_value = [ + prisma_models.LiteLLM_ClaudeCodePluginTable( + id=name, name=name, enabled=True, anonymous_download_count=0, updated_at=datetime(2026, 1, day), + manifest_json=json.dumps({"source": {"source": "github", "repo": "example/skills"}}), + ) for name, day in (("a", 2), ("b", 1), ("c", 3)) + ] + hub.database.query_raw.return_value = [ + {"id": name, "download_count": count} for name, count in (("a", 8), ("b", 0), ("c", 3)) + ] + response = await hub.client.get(HUB, headers=_headers(PLAIN_KEY), params={ + "sort_by": field, "sort_order": order, "limit": 1, "offset": 0, + }) + assert response.status_code == 200, response.text + assert [skill["name"] for skill in response.json()["skills"]] == [expected] diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 84f4d7a4173..de3ced0c583 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -3291,3 +3291,21 @@ async def test_pass_through_guardrail_text_is_the_client_body_without_the_inject texts: Final = [call["inputs"]["texts"] for call in guardrail.apply_calls] assert [[json.loads(text) for text in call_texts] for call_texts in texts] == [[body]], texts assert data["litellm_metadata"]["user_api_key_user_email"] == "someone@example.com", data["litellm_metadata"] + + +@pytest.mark.asyncio +async def test_moderation_hook_lets_responses_compact_through_without_a_translation() -> None: + handler: Final = UnifiedLLMGuardrails() + guardrail: Final = RecordingGuardrail() + data: Final = { + "guardrail_to_apply": guardrail, + "model": "gpt-5.4-mini", + "input": [{"role": "user", "content": "Remember the number 42."}], + } + + result: Final = await handler.async_moderation_hook( + data=data, user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), call_type="acompact_responses" + ) + + assert result is data + assert guardrail.apply_calls == [] diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 0c7292674a6..892c891f20c 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -1939,6 +1939,7 @@ async def test_callback_handles_every_status_the_interactions_api_can_return(): assert released == { "completed", "requires_action", + "requires_continuation", "failed", "cancelled", "incomplete", diff --git a/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index 180362f45f1..94e0f91ecd1 100644 --- a/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -411,10 +411,11 @@ async def test_create_user_without_groups_defers_to_default_team(mocker: MockerF @pytest.mark.asyncio -async def test_create_user_with_groups_keeps_idp_teams(mocker: MockerFixture, monkeypatch): +@pytest.mark.parametrize("user_id", ["new-user", " employee-42 "]) +async def test_create_user_with_groups_keeps_idp_teams(mocker: MockerFixture, monkeypatch, user_id: str): scim_user = SCIMUser( schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], - userName="new-user", + userName=user_id, emails=[SCIMUserEmail(value="new@example.com")], groups=[SCIMUserGroup(value="idp-team")], ) @@ -428,10 +429,12 @@ async def test_create_user_with_groups_keeps_idp_teams(mocker: MockerFixture, mo await create_user(user=scim_user) assert new_user_mock.call_args.kwargs["data"].teams == ["idp-team"] + assert new_user_mock.call_args.kwargs["data"].user_id == user_id @pytest.mark.asyncio -async def test_create_user_if_not_exists_defers_to_default_team(mocker: MockerFixture, monkeypatch): +@pytest.mark.parametrize("user_id", ["group-user", " employee-42 "]) +async def test_create_user_if_not_exists_defers_to_default_team(mocker: MockerFixture, monkeypatch, user_id: str): monkeypatch.setattr( "litellm.default_internal_user_params", {"teams": [{"team_id": "default-team", "max_budget_in_team": 25}]}, @@ -439,14 +442,15 @@ async def test_create_user_if_not_exists_defers_to_default_team(mocker: MockerFi ) new_user_mock = mocker.patch( # test-quality-ok: new_user is imported inside the helper, not injectable "litellm.proxy.management_endpoints.internal_user_endpoints.new_user", - AsyncMock(return_value=NewUserResponse(user_id="group-user", key="k")), + AsyncMock(return_value=NewUserResponse(user_id=user_id, key="k")), ) - created = await _create_user_if_not_exists(user_id="group-user") + created = await _create_user_if_not_exists(user_id=user_id) assert created is not None assert new_user_mock.call_args.kwargs["data"].teams is None assert new_user_mock.call_args.kwargs["user_api_key_dict"] == UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + assert new_user_mock.call_args.kwargs["data"].user_id == user_id @pytest.mark.asyncio diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index cee0c90f80c..b96a5b606eb 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -13,6 +13,7 @@ import httpx import pytest from fastapi import HTTPException from fastapi.testclient import TestClient +from pydantic import ValidationError from pytest_mock import MockerFixture from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -48,6 +49,51 @@ from tests.unit.proxy.management_endpoints.jwt_key_mapping_doubles import ( client = TestClient(app) + +@pytest.mark.parametrize("user_id", ["", " \t\n", "user\x00id", 123, True, [], {}]) +def test_new_user_rejects_invalid_custom_id(user_id: object) -> None: + with pytest.raises(ValidationError) as exc: + NewUserRequest.model_validate({"user_id": user_id}) + assert exc.value.errors()[0]["loc"] == ("user_id",) + + +@pytest.mark.parametrize("user_id", [None, "employee-42", "社員@example.test", " employee-42 "]) +def test_new_user_normalizes_custom_id_and_preserves_auto_generation(user_id: str | None) -> None: + request: Final = NewUserRequest(user_id=user_id) + assert request.user_id == (user_id.strip() if user_id is not None else None) + + +@pytest.mark.asyncio +async def test_new_user_returns_conflict_when_id_is_taken_after_duplicate_check(mocker: MockerFixture) -> None: + from prisma.errors import UniqueViolationError + + from litellm.proxy import proxy_server + from litellm.proxy.utils import PrismaClient + + table: Final = SimpleNamespace( + find_first=AsyncMock(return_value=None), + count=AsyncMock(return_value=0), + create=AsyncMock(side_effect=UniqueViolationError( + {"user_facing_error": {"meta": {"target": ["user_id"]}}}, + message="Unique constraint failed on user_id", + )), + upsert=AsyncMock(return_value=LiteLLM_UserTable(user_id="employee-42")), + ) + mocker.patch("prisma.Prisma", return_value=SimpleNamespace(litellm_usertable=table)) + database: Final = PrismaClient( + database_url="postgresql://unused:unused@localhost/unused", + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + mocker.patch("litellm.proxy.proxy_server.prisma_client", database) + with pytest.raises(ProxyException) as exc: + await new_user( + data=NewUserRequest(user_id="employee-42", auto_create_key=False, send_invite_email=False), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert str(exc.value.code) == "409" + assert "employee-42" in exc.value.message + + if TYPE_CHECKING: from litellm.caching.redis_cache import RedisCache from litellm.proxy.utils import PrismaClient @@ -4862,7 +4908,9 @@ async def test_user_new_persists_model_max_budget(monkeypatch, model_max_budget, models = [] class _FakePrisma: - async def insert_data(self, data, table_name): + async def insert_data( + self, data: Mapping[str, object], table_name: str, *, allow_existing_user: bool = True + ) -> _FakeUserRow | SimpleNamespace: if table_name == "user": captured["user_data"] = dict(data) return _FakeUserRow() diff --git a/tests/unit/proxy/management_endpoints/test_ui_sso.py b/tests/unit/proxy/management_endpoints/test_ui_sso.py index ba78fefd653..6ccb019b937 100644 --- a/tests/unit/proxy/management_endpoints/test_ui_sso.py +++ b/tests/unit/proxy/management_endpoints/test_ui_sso.py @@ -22,6 +22,7 @@ from litellm.proxy._types import ( LiteLLM_JWTAuth, LiteLLM_UserTable, NewUserResponse, + SSOUserDefinedValues, ProxyException, ) from litellm.proxy.auth.handle_jwt import JWTHandler @@ -52,6 +53,62 @@ from litellm.types.proxy.management_endpoints.ui_sso import ( from tests._master_key import MASTER_KEY +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("billable_users", "premium", "admitted"), + [(4, False, True), (5, False, False), (26, False, False), (26, True, True)], +) +async def test_sso_auto_creation_enforces_free_user_cap_through_new_user( + monkeypatch: pytest.MonkeyPatch, billable_users: int, premium: bool, admitted: bool +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.utils import PrismaClient + + row: Final = LiteLLM_UserTable(user_id="new-sso-user", user_email="new-sso-user@example.test") + table: Final = SimpleNamespace( + find_first=AsyncMock(return_value=None), + find_unique=AsyncMock(return_value=None), + count=AsyncMock(side_effect=lambda where=None: billable_users if where is None else 0), + create=AsyncMock(return_value=row), + upsert=AsyncMock(return_value=row), + ) + transaction: Final = MagicMock() + transaction.__aenter__ = AsyncMock( + return_value=SimpleNamespace(litellm_usertable=table, execute_raw=AsyncMock(), query_raw=AsyncMock()) + ) + transaction.__aexit__ = AsyncMock(return_value=None) + monkeypatch.setattr( + "prisma.Prisma", + lambda **kwargs: SimpleNamespace(litellm_usertable=table, tx=MagicMock(return_value=transaction)), + ) + database: Final = PrismaClient( + database_url="postgresql://unused:unused@localhost/unused", + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + monkeypatch.setattr(proxy_server, "prisma_client", database) + monkeypatch.setattr(proxy_server, "premium_user", premium) + monkeypatch.setattr("asyncio.sleep", AsyncMock()) + values: Final[SSOUserDefinedValues] = { + "models": [], + "user_id": row.user_id, + "user_email": row.user_email, + "user_role": "internal_user_viewer", + "max_budget": None, + "budget_duration": None, + } + if admitted: + response: Final = await insert_sso_user(CustomOpenID(id=row.user_id, team_ids=[]), values) + assert response.user_id == row.user_id + assert table.create.await_count + table.upsert.await_count == 1 + else: + with pytest.raises(ProxyException) as error: + await insert_sso_user(CustomOpenID(id=row.user_id, team_ids=[]), values) + assert str(error.value.code) == "403" + assert "SSO for more than 5 users" in error.value.message + table.create.assert_not_awaited() + table.upsert.assert_not_awaited() + + def _sso_http_handler(route: Callable[[httpx.Request], httpx.Response] | None = None) -> AsyncHTTPHandler: """litellm's SSO httpx client answered in-process, by ``route`` or by the active respx mock.""" handler = AsyncHTTPHandler() @@ -1281,7 +1338,8 @@ async def test_upsert_sso_user_fills_user_alias_for_existing_user(): @pytest.mark.asyncio -async def test_insert_sso_user_sets_user_alias_from_display_name(): +@pytest.mark.parametrize("user_id", ["S-1-5-21-adfs-user", " employee-42 "]) +async def test_insert_sso_user_sets_user_alias_from_display_name(user_id: str): """ A newly created SSO user is inserted with the IdP display name as user_alias. """ @@ -1290,7 +1348,7 @@ async def test_insert_sso_user_sets_user_alias_from_display_name(): from litellm.proxy.management_endpoints.ui_sso import insert_sso_user sso_result = CustomOpenID( - id="S-1-5-21-adfs-user", + id=user_id, email="jane.doe@example.com", first_name="Jane", last_name="Doe", @@ -1300,7 +1358,7 @@ async def test_insert_sso_user_sets_user_alias_from_display_name(): ) user_defined_values: SSOUserDefinedValues = { "models": [], - "user_id": "S-1-5-21-adfs-user", + "user_id": user_id, "user_email": "jane.doe@example.com", "max_budget": None, "user_role": "internal_user", @@ -1309,12 +1367,14 @@ async def test_insert_sso_user_sets_user_alias_from_display_name(): with patch( "litellm.proxy.management_endpoints.ui_sso.new_user", - return_value=NewUserResponse(user_id="S-1-5-21-adfs-user", key="sk-xxxxx", teams=None), + return_value=NewUserResponse(user_id=user_id, key="sk-xxxxx", teams=None), ) as mock_new_user: await insert_sso_user(result_openid=sso_result, user_defined_values=user_defined_values) new_user_request = mock_new_user.call_args.kwargs["data"] - assert new_user_request.user_id == "S-1-5-21-adfs-user" + assert new_user_request.user_id == user_id + assert new_user_request.sso_user_id == user_id + assert new_user_request.metadata == {"auth_provider": "generic"} assert new_user_request.user_email == "jane.doe@example.com" assert new_user_request.user_alias == "Doe, Jane" diff --git a/tests/unit/proxy/management_helpers/test_bulk_user_creation.py b/tests/unit/proxy/management_helpers/test_bulk_user_creation.py index cf1ac61c93a..2b966a8ebeb 100644 --- a/tests/unit/proxy/management_helpers/test_bulk_user_creation.py +++ b/tests/unit/proxy/management_helpers/test_bulk_user_creation.py @@ -191,6 +191,19 @@ async def _run(prisma, users, caller=ADMIN, license=None, generate_key=_no_keys) ) +@pytest.mark.asyncio +@pytest.mark.parametrize("user_id", [None, "", " employee-42 ", " \t"]) +async def test_bulk_keeps_existing_user_id_generation_and_whitespace_behavior(user_id: str | None) -> None: + prisma: Final = _FakePrisma() + response: Final = await _run(prisma, [{"user_id": user_id}]) + result: Final = response.data[0] + assert result.success + assert result.user_id + assert result.user_id in prisma.db.litellm_usertable.rows + if user_id: + assert result.user_id == user_id + + @pytest.mark.asyncio async def test_creates_users_and_team_membership_in_every_store(): prisma = _FakePrisma(teams=[_team("t1", [Member(user_id="existing", role="admin")]), _team("t2")]) diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index 238f1fa6495..b3323de3de5 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -396,74 +396,6 @@ async def test_a_caller_that_handed_off_accounting_still_leaves_the_marker_alone assert data["status"] == "complete" -# =========================================================================== # -# add_internal_model_credentials - the snapshot that lets a completed -# batch's output file be read, and therefore its cost be recorded -# =========================================================================== # - - -def test_add_internal_model_credentials_attaches_an_immutable_snapshot(): - """Cost accounting for a completed batch reads its output file, and Bedrock resolves - that bucket only from this snapshot. It must be immutable so nothing downstream can - redirect the bucket that managed file ids are validated against.""" - from litellm.proxy.openai_files_endpoints.common_utils import ( - add_internal_model_credentials, - ) - - router = route_entry_points_to_stubs(MagicMock()) - router.get_deployment_credentials_with_provider = MagicMock( - return_value={"s3_bucket_name": "configured-bucket", "aws_region_name": "us-east-1"} - ) - data = {"batch_id": "unified-batch-id"} - - add_internal_model_credentials(data=data, llm_router=router, model_id="deployment-1") - - snapshot = data["_litellm_internal_model_credentials"] - assert snapshot["s3_bucket_name"] == "configured-bucket" - assert isinstance(snapshot, MappingProxyType) - with pytest.raises(TypeError): - snapshot["s3_bucket_name"] = "attacker-bucket" - router.get_deployment_credentials_with_provider.assert_called_once_with(model_id="deployment-1") - - -@pytest.mark.parametrize( - "model_id, credentials", - [(None, {"s3_bucket_name": "b"}), ("deployment-1", None)], - ids=["no-model-id", "deployment-has-no-credentials"], -) -def test_add_internal_model_credentials_is_a_noop_without_a_resolvable_deployment(model_id, credentials): - """An unroutable batch must be left alone rather than given an empty snapshot, which - would look like a configured bucket of nothing.""" - from litellm.proxy.openai_files_endpoints.common_utils import ( - add_internal_model_credentials, - ) - - router = route_entry_points_to_stubs(MagicMock()) - router.get_deployment_credentials_with_provider = MagicMock(return_value=credentials) - data = {"batch_id": "unified-batch-id"} - - add_internal_model_credentials(data=data, llm_router=router, model_id=model_id) - - assert "_litellm_internal_model_credentials" not in data - - -def test_add_internal_model_credentials_survives_a_failing_deployment_lookup(): - """The snapshot only enables cost accounting, so a batch whose deployment no longer - resolves, which happens when a model group is removed while batches are in flight, - must still be retrievable rather than failing the request on the lookup.""" - from litellm.proxy.openai_files_endpoints.common_utils import ( - add_internal_model_credentials, - ) - - router = route_entry_points_to_stubs(MagicMock()) - router.get_deployment_credentials_with_provider = MagicMock(side_effect=KeyError("deployment-gone")) - data = {"batch_id": "unified-batch-id"} - - add_internal_model_credentials(data=data, llm_router=router, model_id="deployment-gone") - - assert data == {"batch_id": "unified-batch-id"} - - from openai.types.batch import BatchRequestCounts from litellm.proxy.openai_files_endpoints.common_utils import ( diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py index ff63e37bcd8..27a65a91c41 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py @@ -4128,28 +4128,28 @@ def test_require_managed_files_rejects_raw_provider_file_id( mock_call.assert_not_called() -def test_get_file_content_model_routed_attaches_trusted_model_credentials(monkeypatch): - """A managed batch output id routes by model, and that branch must build the snapshot. - - The managed-files pre-call hook sets data["model"] for any id carrying - llm_output_file_id, so batch output retrieval always takes the model-routed branch - and never reaches managed_files_obj.afile_content. Bedrock resolves its output - bucket only from _litellm_internal_model_credentials, so without the snapshot every - Bedrock batch output retrieval fails with "S3 bucket_name is required". - """ +def test_get_file_content_model_routed_reads_from_the_deployment_bucket( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A managed batch output id routes by model, so Bedrock batch output retrieval only + works when the deployment's s3_bucket_name reaches the SDK. Without it every + retrieval fails with "S3 bucket_name is required".""" import base64 - from types import MappingProxyType import litellm.proxy.proxy_server as ps from litellm.proxy._types import LitellmUserRoles from litellm.types.utils import SpecialEnums + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = Router( model_list=[ { "model_name": "anthropic.batch.claude-4.5-haiku", "litellm_params": { "model": "bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "secret", "aws_region_name": "us-east-1", "s3_bucket_name": "configured-batch-bucket", }, @@ -4173,27 +4173,11 @@ def test_get_file_content_model_routed_attaches_trusted_model_credentials(monkey monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) - # One frozen snapshot per call rather than one dict merged across calls, so a second - # invocation is visible instead of silently overwriting the first. - calls: list[MappingProxyType] = [] - - async def _mock_router_afile_content(**kwargs): - calls.append(MappingProxyType(dict(kwargs))) - return HttpxBinaryResponseContent( - response=httpx.Response( - status_code=200, - content=b'{"recordId":"req-1"}', - headers={"content-type": "application/octet-stream"}, - ) - ) - - monkeypatch.setattr(router, "afile_content", _mock_router_afile_content) - unified_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format( "application/jsonl", "unified-output-id", "anthropic.batch.claude-4.5-haiku", - "llm_output_file_id,s3://configured-batch-bucket/out/batch.jsonl", + "s3://configured-batch-bucket/litellm-batch-outputs/job/output.jsonl", "bedrock-batch-deployment-id", ) encoded_id = base64.urlsafe_b64encode(unified_id.encode()).decode().rstrip("=") @@ -4204,21 +4188,22 @@ def test_get_file_content_model_routed_attaches_trusted_model_credentials(monkey user_id="test-user", ) try: - response = client.get( - f"/v1/files/{encoded_id}/content", - headers={"Authorization": "Bearer test-key", "custom-llm-provider": "bedrock"}, - ) + with respx.mock(assert_all_called=False) as s3: + s3_read: Final = s3.get(url__regex=r"https://.*amazonaws\.com/.*").mock( + return_value=httpx.Response(200, content=b'{"recordId":"req-1"}') + ) + response = client.get( + f"/v1/files/{encoded_id}/content", + headers={"Authorization": "Bearer test-key", "custom-llm-provider": "bedrock"}, + ) finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) assert response.status_code == 200, response.text - assert len(calls) == 1, f"expected exactly one routed retrieval, got {len(calls)}" - snapshot = calls[0].get("_litellm_internal_model_credentials") - assert snapshot is not None, "model-routed branch must attach the trusted credential snapshot" - assert isinstance( - snapshot, MappingProxyType - ), "snapshot must be a MappingProxyType; a plain dict is rejected by get_configured_s3_bucket_name" - assert snapshot["s3_bucket_name"] == "configured-batch-bucket" + assert response.content == b'{"recordId":"req-1"}' + assert [call.request.url.path for call in s3_read.calls] == [ + "/configured-batch-bucket/litellm-batch-outputs/job/output.jsonl" + ] def _unified_managed_file_id() -> str: @@ -5471,16 +5456,11 @@ def test_create_file_path_traversal_filename_rejected_before_forwarding(monkeypa assert forwarded_calls == [] -def test_list_files_target_model_names_passes_trusted_bedrock_credentials( - mocker: MockerFixture, monkeypatch -): - """ - GET /v1/files?target_model_names= must hand the deployment's - immutable credential snapshot to litellm.afile_list, since Bedrock resolves - the S3 bucket to list from that snapshot rather than from request params. - """ - from types import MappingProxyType - +def test_list_files_target_model_names_passes_the_deployment_bucket( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + """GET /v1/files?target_model_names= lists the deployment's bucket, + so its s3_bucket_name must reach litellm.afile_list.""" import litellm.proxy.proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -5532,9 +5512,7 @@ def test_list_files_target_model_names_passes_trusted_bedrock_credentials( assert response.status_code == 200, response.text assert captured_kwargs["custom_llm_provider"] == "bedrock" assert captured_kwargs["purpose"] == "batch" - trusted_credentials = captured_kwargs["_litellm_internal_model_credentials"] - assert isinstance(trusted_credentials, MappingProxyType) - assert trusted_credentials["s3_bucket_name"] == "my-bucket" + assert captured_kwargs["s3_bucket_name"] == "my-bucket" proxy_logging_obj.post_call_failure_hook.assert_not_called() diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index dd3ffad4727..d1afcf80ec9 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -1,3 +1,5 @@ +from litellm.proxy.deps.repositories import get_plugin_upload_repository +from litellm.repositories.plugin_upload_repository import PluginUploadRepository import asyncio import json import re @@ -1804,7 +1806,6 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference: from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import register_plugin - from litellm.repositories.plugin_upload_repository import PluginUploadRepository from litellm.types.proxy.claude_code_endpoints import RegisterPluginRequest def create(data: Mapping[str, object]) -> LiteLLM_ClaudeCodePluginTable: @@ -1826,6 +1827,7 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference: prisma.db.litellm_claudecodepluginversion.create = AsyncMock() app: Final = FastAPI() app.include_router(router) + app.dependency_overrides[get_plugin_upload_repository] = lambda: PluginUploadRepository(prisma) with patch("litellm.proxy.proxy_server.prisma_client", prisma): asyncio.run( @@ -1843,8 +1845,10 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference: user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), ) ) + prisma.db.query_raw.return_value = [{"id": "plugin-id", "download_count": 12}] response: Final = TestClient(app).get("/public/skill_hub") + assert response.json()["plugins"][0]["download_count"] == 12 assert response.status_code == 200 assert [(p["name"], p["installation_preference"]) for p in response.json()["plugins"]] == [ ("security-review", preference) @@ -1854,6 +1858,7 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference: _INTERNAL_PLUGIN: Final = SimpleNamespace( id="plugin-id", name="deploy-helper", + created_by=None, enabled=True, created_at=None, updated_at=None, @@ -1874,8 +1879,11 @@ def _skill_hub_response(find_many: AsyncMock, general_settings: Mapping[str, obj table.find_many = find_many prisma: Final = MagicMock() prisma.db.litellm_claudecodeplugintable = table + prisma.writer_db = prisma.db + prisma.db.query_raw = AsyncMock(return_value=[]) app: Final = FastAPI() app.include_router(router) + app.dependency_overrides[get_plugin_upload_repository] = lambda: PluginUploadRepository(prisma) with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), @@ -1909,6 +1917,7 @@ def test_public_skill_hub_500_body_carries_no_exception_text() -> None: def test_public_skill_hub_without_a_database_answers_db_not_connected() -> None: app: Final = FastAPI() app.include_router(router) + app.dependency_overrides[get_plugin_upload_repository] = lambda: None with ( patch("litellm.proxy.proxy_server.prisma_client", None), diff --git a/tests/unit/proxy/test_model_based_routing_files_batches.py b/tests/unit/proxy/test_model_based_routing_files_batches.py index f9bc07212a2..f586dadffc4 100644 --- a/tests/unit/proxy/test_model_based_routing_files_batches.py +++ b/tests/unit/proxy/test_model_based_routing_files_batches.py @@ -6,8 +6,6 @@ Tests the model-based routing ID encoding/decoding used by the batch and file proxy endpoints. """ -from types import MappingProxyType - from litellm.proxy.openai_files_endpoints.common_utils import ( decode_model_from_file_id, encode_file_id_with_model, @@ -54,40 +52,24 @@ class TestEncodeFileIdWithModel: class TestPrepareDataWithCredentials: - def test_preserves_trusted_internal_credentials_snapshot(self): - data = {"file_id": "file-abc"} + def test_deployment_bucket_settings_override_the_request_and_reach_the_sdk_as_kwargs(self): + data = {"file_id": "file-abc", "s3_bucket_name": "request-bucket"} credentials = { "custom_llm_provider": "bedrock", - "s3_bucket_name": "safe-bucket", + "s3_bucket_name": "deployment-bucket", + "s3_output_bucket_name": "deployment-output-bucket", + "allow_legacy_cloud_file_ids": True, } - prepare_data_with_credentials( - data=data, - credentials=credentials, - include_internal_credentials=True, - ) + prepare_data_with_credentials(data=data, credentials=credentials, file_id="decoded-file-id") - assert data["s3_bucket_name"] == "safe-bucket" - assert "custom_llm_provider" not in data - assert isinstance( - data["_litellm_internal_model_credentials"], type(MappingProxyType({})) - ) - assert ( - data["_litellm_internal_model_credentials"]["s3_bucket_name"] - == "safe-bucket" - ) - - def test_does_not_add_internal_credentials_by_default(self): - data = {"file_id": "file-abc"} - credentials = { - "custom_llm_provider": "bedrock", - "s3_bucket_name": "safe-bucket", + assert data == { + "file_id": "decoded-file-id", + "s3_bucket_name": "deployment-bucket", + "s3_output_bucket_name": "deployment-output-bucket", + "allow_legacy_cloud_file_ids": True, } - prepare_data_with_credentials(data=data, credentials=credentials) - - assert "_litellm_internal_model_credentials" not in data - class TestRoundTrip: """Tests for encode -> decode round-trip integrity.""" diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 71f816ea183..cb903fb134a 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -1522,6 +1522,37 @@ def test_raise_if_required_body_param_missing_rejects_missing_param(route_type, assert exc_info.value.message == f"{route}: Missing required parameter: '{param}'." +@pytest.mark.parametrize( + "data", + [ + {"model": "gemini-4-argon", "input": "prove the prime number theorem"}, + {"model": "gemini-4-argon", "previous_interaction_id": "v1_abc", "continuation_token": "tok-1"}, + {"model": "gemini-4-argon", "input": "prove it again", "continuation_token": "tok-1"}, + ], +) +def test_create_interaction_accepts_input_or_a_continuation_token(data: dict[str, object]) -> None: + from litellm.proxy.route_llm_request import raise_if_required_body_param_missing + + assert raise_if_required_body_param_missing(route_type="acreate_interaction", data=data, llm_router=None) is None + + +def test_create_interaction_without_input_or_continuation_token_names_input() -> None: + from litellm.proxy.route_llm_request import ( + ProxyMissingRequiredParamError, + raise_if_required_body_param_missing, + ) + + with pytest.raises(ProxyMissingRequiredParamError) as exc_info: + raise_if_required_body_param_missing( + route_type="acreate_interaction", + data={"model": "gemini-4-argon", "previous_interaction_id": "v1_abc"}, + llm_router=None, + ) + + assert exc_info.value.param == "input" + assert exc_info.value.message == "/interactions: Missing required parameter: 'input'." + + @pytest.mark.parametrize( "data, param", [ diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py b/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py index e335b08f040..8603de0c3a1 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py @@ -18,8 +18,10 @@ from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest from fastapi import HTTPException +from prisma.errors import UniqueViolationError from litellm._service_logger import ServiceTypes from litellm.models.user import LiteLLM_UserTable @@ -192,6 +194,62 @@ async def test_insert_data_team_serializes_members(prisma_client: PrismaClient) assert create_payload["team_id"] == "t1" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata", "message"), + [ + ({"target": ["user_id"]}, "User with id requested-user already exists"), + ({"target": ["sso_user_id"]}, "User with this sso_user_id already exists"), + (None, "User conflicts with an existing unique value"), + ({"target": ["unknown_field"]}, "User conflicts with an existing unique value"), + ], +) +async def test_user_conflict_names_constraint_and_does_not_retry( + prisma_client: PrismaClient, + metadata: Mapping[str, object] | None, + message: str, +) -> None: + create: Final = AsyncMock(side_effect=UniqueViolationError({"user_facing_error": {"meta": metadata}})) + prisma_client.db.litellm_usertable.create = create + with pytest.raises(HTTPException) as error: + await prisma_client.insert_data( + data={"user_id": "requested-user", "sso_user_id": "sso-subject"}, + table_name="user", allow_existing_user=False, + ) + assert error.value.status_code == 409 + assert error.value.detail == {"error": message} + assert create.await_count == 1 + + +@pytest.mark.asyncio +async def test_user_upsert_recovers_from_concurrent_creation( + prisma_client: PrismaClient, monkeypatch: pytest.MonkeyPatch +) -> None: + row: Final = LiteLLM_UserTable(user_id="requested-user") + conflict: Final = UniqueViolationError({"user_facing_error": {"meta": {"target": ["user_id"]}}}) + prisma_client.db.litellm_usertable.upsert = AsyncMock(side_effect=[conflict, row]) + monkeypatch.setattr("asyncio.sleep", AsyncMock()) + + result: Final = await prisma_client.insert_data(data={"user_id": row.user_id}, table_name="user") + + assert result is row + + +@pytest.mark.asyncio +async def test_user_insert_still_retries_transient_connection_failure( + prisma_client: PrismaClient, monkeypatch: pytest.MonkeyPatch +) -> None: + row: Final = LiteLLM_UserTable(user_id="requested-user") + create: Final = AsyncMock(side_effect=[httpx.ConnectError("connection unavailable"), row]) + prisma_client.db.litellm_usertable.create = create + monkeypatch.setattr("asyncio.sleep", AsyncMock()) + result: Final = await prisma_client.insert_data( + data={"user_id": row.user_id}, table_name="user", allow_existing_user=False + ) + assert result is row + assert create.await_count == 2 + + @pytest.mark.asyncio async def test_insert_data_user_organization_fk_raises_400( prisma_client: PrismaClient, diff --git a/tests/unit/repositories/test_plugin_publication.py b/tests/unit/repositories/test_plugin_publication.py index 0a784553d09..7a96b0fd5a7 100644 --- a/tests/unit/repositories/test_plugin_publication.py +++ b/tests/unit/repositories/test_plugin_publication.py @@ -16,6 +16,7 @@ from litellm.models.plugin_upload import ( PluginPublication, PluginPublicationBase, PluginPublicationError, + PluginReview, PublishedPlugin, ) from litellm.repositories.plugin_publication import ( @@ -144,6 +145,44 @@ async def test_first_legacy_update_preserves_its_original_snapshot( } assert replacement.kwargs["data"] == { "plugin_id": current.id, "version": result.version, "description": "replacement", - "manifest_json": json.dumps({"source": source, "version": result.version}), + "manifest_json": json.dumps({"source": source, "version": result.version, "_litellm_published_by": "admin"}), "published_at": result.updated_at, "published_by": "admin", "content": None, "archive_available": False, } + + +@pytest.mark.parametrize("same_source", (True, False)) +@pytest.mark.parametrize("review", [None, PluginReview("reviewer@example.test", False)]) +@pytest.mark.asyncio +async def test_publication_preserves_only_trusted_review_for_unchanged_source( + publication_tx: MagicMock, review: PluginReview | None, same_source: bool, +) -> None: + original = publication_tx.litellm_claudecodeplugintable.find_unique.return_value + stored_review = {"approved_by": "original-reviewer", "security_scan_passed": True} + stored_manifest = {**json.loads(original.manifest_json), "_litellm_review": stored_review} + current = original.model_copy(update={"manifest_json": json.dumps(stored_manifest)}) + publication_tx.litellm_claudecodeplugintable.find_unique.return_value = current + + async def update(where: Mapping[str, str], data: Mapping[str, object]) -> LiteLLM_ClaudeCodePluginTable: + return current.model_copy(update=dict(data)) + + publication_tx.litellm_claudecodeplugintable.update.side_effect = update + result = await publish_plugin( + cast(PluginPublicationTransaction, publication_tx), + PluginPublication("skill", None, "updated", { + "_litellm_review": {"approved_by": "forged", "security_scan_passed": True}, + "source": stored_manifest["source"] if same_source else {"source": "github", "repo": "example/skills"}, + }, "author", review=review), + PluginPublicationBase(current.id, current.updated_at), + ) + assert isinstance(result, PublishedPlugin) + manifest = json.loads(publication_tx.litellm_claudecodeplugintable.update.call_args.kwargs["data"]["manifest_json"]) + assert manifest["_litellm_published_by"] == (review.approved_by if review else "author") + expected = ( + {"approved_by": review.approved_by, "security_scan_passed": review.security_scan_passed} if review + else stored_review if same_source else None + ) + assert manifest.get("_litellm_review") == expected + assert (None if result.review is None else { + "approved_by": result.review.approved_by, "security_scan_passed": result.review.security_scan_passed, + }) == expected + assert json.loads(publication_tx.litellm_claudecodepluginversion.create.call_args.kwargs["data"]["manifest_json"]).get("_litellm_review") == expected diff --git a/tests/unit/repositories/test_skill_submission_repository.py b/tests/unit/repositories/test_skill_submission_repository.py new file mode 100644 index 00000000000..7187cdbe514 --- /dev/null +++ b/tests/unit/repositories/test_skill_submission_repository.py @@ -0,0 +1,31 @@ +import json + +import pytest + +from litellm.repositories.skill_submission_repository import scan_passed_for_archive + + +@pytest.mark.parametrize("status,digest,report,expected", [ + ("completed", "current", {"complete": True, "risk_score": 0, "findings": []}, True), + ("running", "current", {"complete": True, "risk_score": 0, "findings": []}, False), + ("failed", "current", {"complete": True, "risk_score": 0, "findings": []}, False), + ("completed", "previous", {"complete": True, "risk_score": 0, "findings": []}, False), + ("completed", "current", {"complete": False, "risk_score": 0, "findings": []}, False), + ("completed", "current", None, False), + ("completed", "current", {"complete": True, "risk_score": 90, "findings": []}, False), + ("completed", "current", {"complete": True, "risk_score": 0, "findings": [{"rule_id": "rule", "severity": "LOW", "message": "Issue", "file": "SKILL.md"}]}, False), + ("completed", "current", {"complete": True, "risk_score": 0}, False), +]) +def test_scan_pass_requires_clean_complete_report_for_published_archive( + status: str, digest: str, report: dict[str, object] | None, expected: bool, +) -> None: + assert scan_passed_for_archive(json.dumps({ + "id": "scan", "requested_by": "admin", "started_at": "2026-01-01T00:00:00Z", + "expires_at": "2026-01-01T00:03:00Z", "status": status, "archive_sha256": digest, + "report": None if report is None else {"scanner_version": "test", "scan_mode": "static-only", **report}, + }), "current") is expected + + +@pytest.mark.parametrize("record", [None, "", "{}", "null", "not-json"]) +def test_missing_or_invalid_scan_is_not_reported_as_passed(record: str | None) -> None: + assert scan_passed_for_archive(record, "current") is False diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index 037004ab999..a39bc09cee6 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -2722,7 +2722,7 @@ async def test_stream_auto_execute_bills_a_follow_up_cut_after_streamed_text_wit @pytest.mark.asyncio -async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_keeps_both_rounds_on_the_success_row( +async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_keeps_both_rounds_on_one_failure_row( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ): from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -2743,10 +2743,22 @@ async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_keeps_both_r if not ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, blocked): ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) await logging_obj.async_failure_handler(blocked, "blocked by output-policy") - await _settle(lambda: len(recorder.logged) > 0 and len(recorder.failed) > 0) + await _settle(lambda: len(recorder.failed) > 0) - _assert_one_stream_row_bills_both_rounds_once(recorder.logged, _final_response(events)) - assert [row["response_cost"] for row in recorder.failed] == [0.0] + assert recorder.logged == () + assert len(recorder.failed) == 1 + row: Final = recorder.failed[0] + charged_once: Final = _round_cost(_ROUND_ONE_BODY) + _round_cost(_ROUND_TWO_BODY) - _FIXED_MARGIN + assert row["status"] == "failure" + assert (row["prompt_tokens"], row["completion_tokens"]) == (71 + 149, 19 + 5) + assert row["response_cost"] == pytest.approx(charged_once) + assert row["cost_breakdown"]["total_cost"] == pytest.approx(charged_once) + client: Final = _final_response(events) + assert "message" in _output_types(client.output) + assert "function_call" not in _output_types(client.output) + assert client.usage is not None + assert (client.usage.input_tokens, client.usage.output_tokens) == (71 + 149, 19 + 5) + assert client.usage.cost == pytest.approx(charged_once) @pytest.mark.asyncio diff --git a/tests/unit/router_rust/test_walk.py b/tests/unit/router_rust/test_walk.py new file mode 100644 index 00000000000..2f2febea3eb --- /dev/null +++ b/tests/unit/router_rust/test_walk.py @@ -0,0 +1,49 @@ +import pytest + +from litellm.router_rust.invoke.walk import WalkMetadata + + +def _no_base_model(_model_id: str) -> str | None: + return None + + +@pytest.mark.parametrize("field", ["metadata", "litellm_metadata"]) +def test_hop_kwargs_keep_the_requests_key_labels_under_the_walk_stamps(field: str) -> None: + """The fallback loggers (prometheus' fallback counters) read the caller's key and team off + the hop's metadata, so a hop keeps the request's bucket and lays its own stamps over it.""" + request = { + "model": "primary", + field: { + "user_api_key_hash": "hashed-key", + "user_api_key_alias": "key-alias", + "user_api_key_team_id": "team-id", + "user_api_key_team_alias": "team-alias", + "attempted_fallbacks": "stale", + }, + } + walk = WalkMetadata(request, field, _no_base_model) + walk.hop({"parent": 0, "scope": 1, "from": "primary", "to": "fallback", "depth": 1, "max_fallbacks": 5}) + + kwargs = walk.hop_kwargs({"scope": 1, "to": "fallback"}) + + assert kwargs["model"] == "fallback" + metadata = kwargs[field] + assert isinstance(metadata, dict) + assert metadata["user_api_key_hash"] == "hashed-key" + assert metadata["user_api_key_alias"] == "key-alias" + assert metadata["user_api_key_team_id"] == "team-id" + assert metadata["user_api_key_team_alias"] == "team-alias" + assert metadata["model_group"] == "fallback" + assert metadata["original_model_group"] == "primary" + assert metadata["attempted_fallbacks"] == 1 + assert request[field]["attempted_fallbacks"] == "stale" + assert "model_group" not in request[field] + + +def test_hop_kwargs_without_request_metadata_hold_the_walk_stamps() -> None: + walk = WalkMetadata({"model": "primary"}, "metadata", _no_base_model) + walk.hop({"parent": 0, "scope": 1, "from": "primary", "to": "fallback", "depth": 1, "max_fallbacks": 5}) + + metadata = walk.hop_kwargs({"scope": 1, "to": "fallback"})["metadata"] + + assert metadata == {"original_model_group": "primary", "model_group": "fallback", "attempted_fallbacks": 1} diff --git a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index 9487e4d6321..5169642bc43 100644 --- a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -1480,3 +1480,58 @@ async def test_a_pin_written_without_tools_is_read_when_tools_is_an_empty_list() ) assert filtered == [deployments[0]] + + +class _LoggedMessagesProbe(CustomLogger): + def __init__(self) -> None: + self.logged_messages: list[object] = [] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.logged_messages.append((kwargs["standard_logging_object"]["messages"], kwargs["messages"])) + + +@pytest.mark.asyncio +async def test_pin_is_written_with_message_logging_off_while_loggers_still_see_redacted_messages( + monkeypatch, local_model_cost_map +): + """ + `turn_off_message_logging` redacted the shared success-event payload before the check read it, so the + pin was keyed off `redacted-by-litellm`, never written, and a cacheable conversation bounced across the + group. The pin must land whatever the flag says, while a real logger alongside still sees only the + redacted messages. + """ + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False) + probe = _LoggedMessagesProbe() + monkeypatch.setattr(litellm, "callbacks", [probe]) + router = litellm.Router( + model_list=[ + { + "model_name": MODEL_GROUP_ALIAS, + "litellm_params": {"model": "anthropic/claude-opus-4-6", "api_key": "sk-fake"}, + "model_info": {"id": model_id}, + } + for model_id in ("dep-1", "dep-2") + ], + optional_pre_call_checks=["prompt_caching"], + ) + messages = _messages(word_count=5000) + + first = await router.acompletion(model=MODEL_GROUP_ALIAS, messages=messages, mock_response="ok") + served_by = first._hidden_params["model_id"] + + affinity_key = PromptCachingCache.get_prompt_caching_cache_key(messages, None) + pin = await _eventually(lambda: router.prompt_caching_pin_cache.get_cache(key=affinity_key)) + assert pin == {"model_id": served_by}, pin + + subsequent = [ + (await router.acompletion(model=MODEL_GROUP_ALIAS, messages=messages, mock_response="ok"))._hidden_params[ + "model_id" + ] + for _ in range(4) + ] + assert subsequent == [served_by] * 4 + + await _eventually(lambda: len(probe.logged_messages) == 5) + redacted = [{"role": "user", "content": "redacted-by-litellm"}] + assert probe.logged_messages == [(redacted, redacted)] * 5, probe.logged_messages diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 8ef683c0aee..6a35c8a9602 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -36,6 +36,7 @@ from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments, cast_exception_status_to_int, get_cooldown_deployments, + is_response_lookup_not_found, mark_advisor_orchestration_failure, set_cooldown_deployments, should_cooldown_based_on_allowed_fails_policy, @@ -3565,3 +3566,25 @@ def test_the_refusal_names_each_cooled_deployment_some_cooldown_state_routes_the with pytest.raises(RouterRateLimitError) as refusal: _select(_tenant_router(EVERY_TENANT, **BOTH_ON), "async", "tenants", copy.deepcopy(request_kwargs)) assert {deployment["model_id"] for deployment in refusal.value.cooldown_deployments or ()} == reachable + + +@pytest.mark.parametrize( + ("status_code", "call_type", "expected"), + [ + (404, "aget_responses", True), + (404, "adelete_responses", True), + (404, "acancel_responses", True), + (404, "alist_input_items", True), + (404, "get_responses", True), + (404, "aresponses", False), + (404, "acompact_responses", False), + (404, "acompletion", False), + (500, "aget_responses", False), + (429, "alist_input_items", False), + ], +) +def test_is_response_lookup_not_found(status_code: int, call_type: str, expected: bool) -> None: + error: Final = litellm.NotFoundError(message="gone", llm_provider="openai", model="gpt-4o-mini") + error.status_code = status_code + + assert is_response_lookup_not_found(error, {"call_type": call_type}) is expected diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 0de10a64f2e..42b21692a8f 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -285,6 +285,35 @@ class TestFilterWebSearchDeployments: ) assert result == sample_deployments + @pytest.mark.parametrize( + "tool", + [ + {"type": "web_search_20250305", "name": "web_search", "max_uses": 1}, + {"type": "web_search_20260301", "name": "web_search"}, + {"googleSearch": {}}, + {"google_search": {}}, + ], + ids=["anthropic_versioned", "anthropic_newer_version", "gemini_camel_case", "gemini_snake_case"], + ) + def test_a_provider_native_web_search_tool_filters_unsupported( + self, sample_deployments: list[dict[str, object]], tool: dict[str, object] + ) -> None: + result: Final = filter_web_search_deployments(sample_deployments, {"tools": [{"type": "function"}, tool]}) + assert [d["model_info"]["id"] for d in result] == ["deployment-1", "deployment-2"] + + @pytest.mark.parametrize( + "tool", + [ + {"type": "function", "function": {"name": "web_search"}}, + {"codeExecution": {}}, + ], + ids=["function_named_web_search", "gemini_code_execution"], + ) + def test_a_tool_that_is_no_web_search_keeps_every_deployment( + self, sample_deployments: list[dict[str, object]], tool: dict[str, object] + ) -> None: + assert filter_web_search_deployments(sample_deployments, {"tools": [tool]}) == sample_deployments + def test_web_search_preview_filters_unsupported(self, sample_deployments): """web_search_preview type should also trigger filtering""" request_kwargs = {"tools": [{"type": "web_search_preview"}]} diff --git a/tests/unit/test_azure_ad_token_credential_resolution.py b/tests/unit/test_azure_ad_token_credential_resolution.py index f6f47b3f4c4..b1abc1d3c1e 100644 --- a/tests/unit/test_azure_ad_token_credential_resolution.py +++ b/tests/unit/test_azure_ad_token_credential_resolution.py @@ -149,11 +149,11 @@ class TestRouterCredentialResolutionS3OutputBucket: """Same strict-dump trap as azure_ad_token (#30235), for Bedrock batch file retrieval (#26335). Bedrock batch outputs land in a per-model ``s3_output_bucket_name`` when it differs from the input bucket. The - file-content retrieval path validates a file id against the buckets in the - trusted credential snapshot, and that snapshot is built by round-tripping - the deployment's ``litellm_params`` through ``CredentialLiteLLMParams``. If - the field is undeclared it is dropped, so the output bucket never reaches - retrieval and output-bucket file ids are rejected as foreign.""" + file-content retrieval path validates a file id against the buckets the + proxy passes along, and those are built by round-tripping the deployment's + ``litellm_params`` through ``CredentialLiteLLMParams``. If the field is + undeclared it is dropped, so the output bucket never reaches retrieval and + output-bucket file ids are rejected as foreign.""" def test_credentials_preserve_s3_output_bucket_name(self): from litellm import Router @@ -185,6 +185,33 @@ class TestRouterCredentialResolutionS3OutputBucket: ) assert credentials.get("s3_bucket_name") == "in-bucket" + def test_credentials_preserve_allow_legacy_cloud_file_ids(self): + """An admin who sets allow_legacy_cloud_file_ids on a deployment expects + legacy (pre-managed-prefix) file ids in its bucket to stay readable + through the proxy's model-routed file endpoints.""" + from litellm import Router + + deployment_id = "bedrock-batch-legacy-ids-fixed-uuid" + router = Router( + model_list=[ + { + "model_name": "bedrock-batch-legacy-ids", + "litellm_params": { + "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + "s3_bucket_name": "in-bucket", + "allow_legacy_cloud_file_ids": True, + "aws_region_name": "us-west-2", + }, + "model_info": {"id": deployment_id}, + } + ] + ) + + credentials = router.get_deployment_credentials_with_provider(model_id=deployment_id) + + assert credentials is not None + assert credentials.get("allow_legacy_cloud_file_ids") is True + def test_credentials_without_output_bucket_unaffected(self): """A deployment that configures only the input bucket keeps it and does not gain a phantom output bucket in the resolved credentials.""" diff --git a/tests/unit/test_router/core/test_construction_precall_characterization.py b/tests/unit/test_router/core/test_construction_precall_characterization.py index 2985819cb5b..f58d9a35110 100644 --- a/tests/unit/test_router/core/test_construction_precall_characterization.py +++ b/tests/unit/test_router/core/test_construction_precall_characterization.py @@ -13,6 +13,11 @@ from collections.abc import Callable, Generator, Mapping from typing import Final import pytest +import redis +import redis.asyncio +from fakeredis import FakeRedis, FakeServer +from fakeredis.aioredis import FakeConnection +from redis.asyncio import BlockingConnectionPool import litellm from litellm._logging import verbose_router_logger @@ -23,6 +28,7 @@ from litellm.router_api import ChangeSet, router_init_args from litellm.router_core import RouterCore from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck +from litellm.scheduler import FlowItem from litellm.types.router import ( AlertingConfig, AllowedFailsPolicy, @@ -620,6 +626,41 @@ def test_applying_a_redis_cache_attaches_it_to_the_routing_state_and_the_pins(ma assert core.registry().plugins("prompt_caching_pins").redis_cache is redis_cache +@pytest.fixture +def shared_redis(monkeypatch: pytest.MonkeyPatch) -> FakeServer: + server: Final = FakeServer() + + class Pool(BlockingConnectionPool): + def __init__(self, **kwargs: object) -> None: # kwargs-ok: forwarded to the redis pool + super().__init__(connection_class=FakeConnection, server=server, **kwargs) + + monkeypatch.setattr(redis, "Redis", FakeRedis) + monkeypatch.setattr(redis.asyncio, "BlockingConnectionPool", Pool) + return server + + +async def test_two_workers_on_one_redis_share_the_priority_queue(make_core: MakeCore, shared_redis: FakeServer) -> None: + first: Final = _one(make_core, redis_host="scheduler-shared", redis_port=6399) + second: Final = _one(make_core, redis_host="scheduler-shared", redis_port=6399) + await first.scheduler.add_request(FlowItem(priority=0, request_id="urgent", model_name="g")) # pyright: ignore[reportAttributeAccessIssue] + waiter: Final = FlowItem(priority=5, request_id="waiter", model_name="g") + await second.scheduler.add_request(waiter) # pyright: ignore[reportAttributeAccessIssue] + + assert not await second.scheduler.poll(waiter, []) # pyright: ignore[reportAttributeAccessIssue] + await first.scheduler.remove_request("urgent", "g") # pyright: ignore[reportAttributeAccessIssue] + assert await second.scheduler.poll(waiter, []) # pyright: ignore[reportAttributeAccessIssue] + + +async def test_without_redis_each_worker_keeps_its_own_queue(make_core: MakeCore) -> None: + first: Final = _one(make_core) + second: Final = _one(make_core) + await first.scheduler.add_request(FlowItem(priority=0, request_id="urgent", model_name="g")) # pyright: ignore[reportAttributeAccessIssue] + waiter: Final = FlowItem(priority=5, request_id="waiter", model_name="g") + await second.scheduler.add_request(waiter) # pyright: ignore[reportAttributeAccessIssue] + + assert await second.scheduler.poll(waiter, []) # pyright: ignore[reportAttributeAccessIssue] + + def test_cache_responses_installs_a_local_response_cache_on_the_host(make_core: MakeCore, host: ScriptedHost) -> None: core: Final = _one(make_core, cache_responses=True) assert isinstance(host.cache, litellm.Cache) diff --git a/tests/unit/test_router/test_router_provider_endpoints.py b/tests/unit/test_router/test_router_provider_endpoints.py index a78228b8658..baf3e94f1f0 100644 --- a/tests/unit/test_router/test_router_provider_endpoints.py +++ b/tests/unit/test_router/test_router_provider_endpoints.py @@ -18,6 +18,7 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.proxy._types import UserAPIKeyAuth +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.router_callbacks.track_deployment_metrics import ( get_deployment_successes_for_current_minute, ) @@ -921,6 +922,43 @@ async def test_video_lookup_404_never_cools_down_the_deployment_that_answered(re assert await router.cooldown_cache.async_get_active_cooldowns(["dep-a", "dep-b"], parent_otel_span=None) == [] +@pytest.mark.parametrize( + ("endpoint", "method", "path"), + [ + ("aget_responses", "GET", ""), + ("adelete_responses", "DELETE", ""), + ("acancel_responses", "POST", "/cancel"), + ("alist_input_items", "GET", "/input_items"), + ], +) +@pytest.mark.asyncio +async def test_response_lookup_404_never_cools_down_the_deployment_that_answered( + respx_mock: respx.MockRouter, endpoint: str, method: str, path: str +) -> None: + respx_mock.route(method=method, url=f"https://api.openai.com/v1/responses/resp_gone{path}").respond( + 404, json={"error": {"code": "NotFound", "message": "Response not found"}} + ) + router: Final = Router( + model_list=[ + { + "model_name": "gpt", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": api_key}, + "model_info": {"id": deployment_id}, + } + for api_key, deployment_id in (("key-a", "dep-a"), ("key-b", "dep-b")) + ], + num_retries=0, + allowed_fails=1, + ) + response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id("openai", "dep-b", "resp_gone") + + for _ in range(3): + with pytest.raises(litellm.NotFoundError): + await getattr(router, endpoint)(response_id=response_id) + + assert await router.cooldown_cache.async_get_active_cooldowns(["dep-a", "dep-b"], parent_otel_span=None) == [] + + @pytest.mark.asyncio async def test_chat_404_still_cools_down_the_deployment(respx_mock: respx.MockRouter) -> None: respx_mock.post("https://api.openai.com/v1/chat/completions").respond( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 9e860f574b8..4f2c4fc1e3f 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -104,6 +104,7 @@ from litellm.utils import ( _snapshot_exception_for_hook, async_post_call_failure_deployment_hook, async_post_call_success_deployment_hook, + async_pre_call_deployment_hook, calculate_max_parallel_requests, client, get_non_default_completion_params, @@ -11655,3 +11656,25 @@ def test_get_response_string_joins_the_text_parts_of_a_completed_responses_event ) assert get_response_string(ResponseCompletedEvent(type="response.completed", response=response)) == "Hello world" + + +class _RecordingPreCallDeploymentHook(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.seen_call_types: tuple[CallTypes | None, ...] = () + + async def async_pre_call_deployment_hook( + self, kwargs: dict[str, object], call_type: CallTypes | None + ) -> dict[str, object] | None: + self.seen_call_types = (*self.seen_call_types, call_type) + return None + + +@pytest.mark.asyncio +async def test_pre_call_deployment_hook_hands_responses_compact_its_call_type(monkeypatch: pytest.MonkeyPatch) -> None: + hook: Final = _RecordingPreCallDeploymentHook() + monkeypatch.setattr(litellm, "callbacks", [hook]) + + await async_pre_call_deployment_hook({"model": "gpt-5.4-mini"}, "acompact_responses") + + assert hook.seen_call_types == (CallTypes.acompact_responses,) diff --git a/tests/unit/test_utils_get_model_info.py b/tests/unit/test_utils_get_model_info.py index 302a7d60cdd..d08742be4af 100644 --- a/tests/unit/test_utils_get_model_info.py +++ b/tests/unit/test_utils_get_model_info.py @@ -332,6 +332,18 @@ def _cross_region_profiles( ) +_BEDROCK_PROFILE_ONLY_NATIVE_OUTPUT_MODELS: Final = frozenset( + { + "anthropic.claude-opus-5-5", + "anthropic.claude-fable-5-1", + "anthropic.claude-sonnet-5", + "anthropic.claude-haiku-5-5", + "anthropic.claude-opus-5", + "anthropic.claude-opus-4-8", + } +) + + def test_get_model_info_bedrock_cross_region_capability_parity(): """ Cross-region inference profiles carry litellm_provider "bedrock_converse", so the @@ -362,6 +374,13 @@ def test_get_model_info_bedrock_cross_region_capability_parity(): if _govcloud_may_lag(k, cap): assert base_value or not v[cap], f"{cap} is on {k} but not on {base_model_key}" continue + if ( + cap == "supports_native_structured_output" + and base_model_key in _BEDROCK_PROFILE_ONLY_NATIVE_OUTPUT_MODELS + and v[cap] is True + ): + assert base_value is False, f"{cap} is already enabled on bare model {base_model_key}" + continue assert ( v[cap] == base_value ), f"{cap} is {v[cap]} on {k} but {base_value} on {base_model_key}" diff --git a/tests/unit/types/test_utils.py b/tests/unit/types/test_utils.py index 32479c62372..bdbcbb698d4 100644 --- a/tests/unit/types/test_utils.py +++ b/tests/unit/types/test_utils.py @@ -1,7 +1,12 @@ import pytest from litellm.litellm_core_utils.initialize_dynamic_callback_params import validate_excluded_services_value -from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VALUES, CaptureMessageContent, captures_span_content +from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VALUES, + CallTypes, + CaptureMessageContent, + captures_span_content, +) @pytest.mark.parametrize("value", ["", " ", "auth", "none,redis", "redis,", ",postgres"]) @@ -45,3 +50,8 @@ def test_capture_message_content_constructor_accepts_any_string() -> None: ) def test_captures_span_content(mode: str | None, captures: bool) -> None: assert captures_span_content(mode) is captures + + +@pytest.mark.parametrize("call_type", ["compact_responses", "acompact_responses"]) +def test_responses_compact_call_types_round_trip_their_public_names(call_type: str) -> None: + assert CallTypes(call_type).value == call_type diff --git a/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx index ec88a69670e..fe4692192d8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx @@ -1,14 +1,16 @@ "use client"; +import { useTranslation } from "react-i18next"; import { Page, PageTabs, PageTabsList, PageTabsTrigger, PageTabsContent } from "@/components/shared/Page"; import ModelDeprecationsPanel from "@/components/ModelDeprecationAlerts/ModelDeprecationsPanel"; export default function AlertsPage() { + const { t } = useTranslation(); return ( - Model deprecations + {t("alerts.page.tabs.modelDeprecations")} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx index 359fb12dd95..9a260bdfc65 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx @@ -1,7 +1,8 @@ +import userEvent from "@testing-library/user-event"; import { act, fireEvent, screen, waitFor, within } from "@testing-library/react"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import type { OnUrlUpdateFunction } from "nuqs/adapters/testing"; -import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import { chooseSelectOption, renderWithProviders, testQueryClient } from "@/../tests/test-utils"; import { setGlobalLitellmHeaderName } from "@/components/networking"; import type { SkillHubSkill, SkillSubmission } from "@/components/claude_code_plugins/types"; import { toast } from "@/lib/toast"; @@ -61,6 +62,7 @@ const pending: SkillSubmission = { const fetchMock = vi.fn(); const queryDefaults = testQueryClient.getDefaultOptions(); let catalog: SkillHubSkill[]; +let catalogRequests: URL[]; let submissions: SkillSubmission[]; let pageRequests: { limit: number; offset: number }[]; let writes: { path: string; method: string; body: RequestInit["body"] }[]; @@ -161,6 +163,7 @@ beforeEach(() => { testQueryClient.clear(); vi.clearAllMocks(); catalog = [published]; + catalogRequests = []; submissions = [pending]; pageRequests = []; writes = []; @@ -180,14 +183,17 @@ beforeEach(() => { return respondToWrite(path, body); } if (path === "/skill-hub/skills") { + catalogRequests.push(url); const limit = Number(url.searchParams.get("limit")); const offset = Number(url.searchParams.get("offset")); const skills = catalog.slice(offset, offset + limit); - return Response.json({ + const result = { + tags: ["review", "engineering"], skills, count: skills.length, next_offset: offset + limit < catalog.length ? offset + limit : null, - }); + }; + return Response.json(result); } if (path === `/skill-hub/skills/${published.name}`) return Response.json(catalog.find((skill) => skill.name === published.name)); @@ -250,7 +256,7 @@ it.each(["current", "retained"])("refreshes published counts after a %s download catalog = [{ ...published, download_count: 1 }]; fireEvent.click(button); await waitFor(() => expect(button).toBeEnabled()); - fireEvent.click(screen.getByText("Skills")); + fireEvent.click(screen.getByText("Skills Hub")); expect(await screen.findByRole("cell", { name: "1" })).toBeInTheDocument(); }); @@ -266,7 +272,7 @@ it("paginates the neutral catalog and refreshes detail for a recreated skill", a renderWithProviders(); fireEvent.click(await screen.findByText(published.name)); expect(await screen.findByRole("button", { name: "Edit Skill" })).toBeInTheDocument(); - fireEvent.click(screen.getByText("Skills")); + fireEvent.click(screen.getByText("Skills Hub")); catalog = [ { ...published, id: "filler", name: "catalog-0" }, ...catalog.slice(1), @@ -281,7 +287,7 @@ it("paginates the neutral catalog and refreshes detail for a recreated skill", a fireEvent.click(screen.getByText(published.name)); expect(await screen.findByRole("button", { name: "Edit Skill" })).toBeInTheDocument(); expect(screen.getByText("Recreated instructions")).toBeInTheDocument(); - fireEvent.click(screen.getByText("Skills")); + fireEvent.click(screen.getByText("Skills Hub")); fireEvent.click(screen.getByRole("button", { name: "Previous" })); await waitFor(() => expect(screen.getByRole("button", { name: "Previous" })).toBeDisabled()); }); @@ -394,7 +400,7 @@ it("allows a member submission, explains approval and refreshes their pending li await waitFor(() => expect(screen.getByRole("tab", { name: "Published skills" })).toHaveAttribute("aria-selected", "true"), ); - fireEvent.click(screen.getByRole("button", { name: "+ Add Skill" })); + fireEvent.click(screen.getByRole("button", { name: "+ Register your skill" })); fireEvent.click(screen.getByRole("radio", { name: "Import URL" })); fireEvent.change(screen.getByRole("textbox", { name: "Source URL" }), { target: { value: "https://github.com/example/new-skill" }, @@ -593,10 +599,10 @@ it("refreshes detail after a review conflict and removes actions when another re it("keeps an effective Admin viewer read-only while allowing full submission inspection", async () => { renderWithProviders(); - expect(screen.getByRole("button", { name: "+ Add Skill" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "+ Register your skill" })).toBeDisabled(); fireEvent.click(await screen.findByText(published.name)); expect(screen.queryByRole("button", { name: "Edit Skill" })).not.toBeInTheDocument(); - fireEvent.click(screen.getByText("Skills")); + fireEvent.click(screen.getByText("Skills Hub")); fireEvent.click(screen.getByRole("tab", { name: "Submissions" })); fireEvent.click(await screen.findByRole("button", { name: "View details" })); expect(await screen.findByText("Currently published")).toBeInTheDocument(); @@ -643,7 +649,7 @@ it.each(["role", "read-only"])("does not reuse a published admin snapshot after expect(screen.queryByText("Approved instructions")).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Edit Skill" })).not.toBeInTheDocument(); expect(await screen.findByRole("alert")).toHaveTextContent("Could not refresh published skills"); - expect(screen.queryByRole("columnheader", { name: "Downloads" })).not.toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Downloads" })).toBeInTheDocument(); }); it("does not reuse admin review metadata when the same account loses its role and detail refresh fails", async () => { @@ -762,3 +768,27 @@ it.each([200, 409])("refreshes the running scan after POST %s and ignores an old fireEvent.click(screen.getByRole("button", { name: "Approve and publish" })); expect(await screen.findByText("Approved")).toBeInTheDocument(); }); + +it("applies tag and scan filters across the catalog and resets pagination when searching", async () => { + const user = userEvent.setup(); + catalog = [published, { ...published, id: "second", name: "second-skill" }]; + renderWithProviders(, { + searchParams: { skills_page: "2", skills_page_size: "1" }, + }); + await waitFor(() => expect(catalogRequests.at(-1)?.searchParams.get("offset")).toBe("1")); + await screen.findByRole("cell", { name: "Approved instructions" }); + await user.click(screen.getByRole("button", { name: "Downloads" })); + await waitFor(() => expect(catalogRequests.at(-1)?.searchParams.get("sort_by")).toBe("download_count")); + expect(catalogRequests.at(-1)?.searchParams.get("offset")).toBe("0"); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by tag" }), "engineering"); + await waitFor(() => expect(catalogRequests.at(-1)?.searchParams.get("tag")).toBe("engineering")); + expect(catalogRequests.at(-1)?.searchParams.get("offset")).toBe("0"); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Security scan" }), "Security scan: Yes"); + await waitFor(() => expect(catalogRequests.at(-1)?.searchParams.get("security_scan_passed")).toBe("true")); + fireEvent.change(screen.getByRole("textbox", { name: /Search skills/ }), { + target: { value: "reviewer@example.test" }, + }); + await waitFor(() => expect(catalogRequests.at(-1)?.searchParams.get("query")).toBe("reviewer@example.test")); + expect(catalogRequests.at(-1)?.searchParams.get("tag")).toBe("engineering"); + expect(catalogRequests.at(-1)?.searchParams.get("security_scan_passed")).toBe("true"); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.test.tsx index a0e785268ea..dbdf5ca86d7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.test.tsx @@ -87,7 +87,16 @@ describe("ClaudeCodePluginsPanel loading state", () => { resolveFetch({ skills: [], count: 0, next_offset: null }); expect(await screen.findByText("table-loaded")).toBeInTheDocument(); - expect(mockListSkillHubSkills).toHaveBeenCalledWith("sk-test", { limit: 50, offset: 0 }); + const expectedQuery = { + limit: 50, + offset: 0, + query: "", + tag: "", + security_scan_passed: undefined, + sort_by: "name", + sort_order: "asc", + }; + expect(mockListSkillHubSkills).toHaveBeenCalledWith("sk-test", expectedQuery); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx index 6efa511ce49..6497c0922d2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx @@ -1,3 +1,6 @@ +import type { SkillListQuery } from "@/components/claude_code_plugins/types"; +import { SkillFilters } from "@/components/claude_code_plugins/SkillFilters"; +import { useSkillFilters, skillSortQuery } from "@/components/claude_code_plugins/skillCatalog"; import React, { useState } from "react"; import { useQuery, useQueryClient, type QueryKey } from "@tanstack/react-query"; import { Trans, useTranslation } from "react-i18next"; @@ -21,7 +24,6 @@ import { toast } from "@/lib/toast"; import { isPendingSkill, type SkillHubSkill, type SkillMutationResponse } from "@/components/claude_code_plugins/types"; import { useUrlTab } from "@/hooks/useUrlTab"; import SkillSubmissions from "./SkillSubmissions"; -import { useUrlTableState, type UrlTableStateOptions } from "@/components/shared/DataTable/useUrlTableState"; interface ClaudeCodePluginsPanelProps { accessToken: string | null; @@ -29,14 +31,7 @@ interface ClaudeCodePluginsPanelProps { isViewOnly?: boolean; } -const CATALOG_STATE: UrlTableStateOptions = { - keyPrefix: "skills_", - defaultPageSize: 50, - maxPageSize: 100, - sortFields: ["name"], - defaultSort: { id: "name", desc: false }, - filterColumns: [], -}; +const scanResultFilter = (scan: string): boolean | undefined => (scan === "all" ? undefined : scan === "yes"); const SKILLS_TABS = ["skills", "submissions"] as const; @@ -117,11 +112,20 @@ const SkillsPanelContent: React.FC = ({ const { readOnly, canWrite, isAdmin, adminView } = skillPermissions(accessToken, userRole, isViewOnly); const queryScope = ["skill-hub", accessToken, userRole, readOnly] as const; const showForm = Boolean(isAddModalVisible || editingSkill) && canWrite; - const { pagination, onPaginationChange } = useUrlTableState(CATALOG_STATE); + const filters = useSkillFilters(); + const { pagination, onPaginationChange, sorting, onSortingChange } = filters; const { pageIndex, pageSize } = pagination; + const catalogQuery: SkillListQuery & { limit: number; offset: number } = { + limit: pageSize, + offset: pageIndex * pageSize, + query: filters.search, + tag: filters.tag, + security_scan_passed: scanResultFilter(filters.scan), + ...skillSortQuery(sorting), + }; const plugins = useQuery({ - queryKey: [...queryScope, "published", pageIndex, pageSize], - queryFn: () => listSkillHubSkills(accessToken!, { limit: pageSize, offset: pageIndex * pageSize }), + queryKey: [...queryScope, "published", catalogQuery], + queryFn: () => listSkillHubSkills(accessToken!, catalogQuery), enabled: Boolean(accessToken), }); const pluginsList = plugins.data?.skills ?? []; @@ -193,6 +197,9 @@ const SkillsPanelContent: React.FC = ({ +
+ +
{plugins.isError && (
{t("skills.panel.loadFailed")} @@ -204,6 +211,9 @@ const SkillsPanelContent: React.FC = ({ { [12345, "12,345"], ] as const)("renders headers and download count %s", (download_count, display) => { render(); - for (const header of ["Skill Name", "Version", "Description", "Domain", "Public", "Updated at", "Downloads"]) { + for (const header of [ + "Skill Name", + "Tags", + "Created by", + "Approved by", + "Created", + "Last updated", + "Security scan", + "Downloads", + ]) { expect(screen.getByText(header)).toBeInTheDocument(); } - expect(within(screen.getByRole("row", { name: /newer-skill/ })).getByText(display)).toBeInTheDocument(); + expect(within(screen.getByRole("row", { name: /newer-skill/ })).getAllByText(display)[0]).toBeInTheDocument(); }); it("should display the empty state when data is empty", () => { render(); - expect(screen.getByText("No skills found")).toBeInTheDocument(); - }); - - it("should sort by name ascending by default", () => { - render(); - const rows = screen.getAllByRole("row").slice(1); - expect(within(rows[0]).getByText("newer-skill")).toBeInTheDocument(); - expect(within(rows[1]).getByText("older-skill")).toBeInTheDocument(); + expect(screen.getByText("No skills registered yet")).toBeInTheDocument(); }); it("should call onPluginClick with the plugin ID when the skill name is clicked", async () => { @@ -89,7 +93,8 @@ describe("PluginTable", () => { it("should show whether the skill is public", () => { render(); expect(screen.getByText("Yes")).toBeInTheDocument(); - expect(screen.getByText("No")).toBeInTheDocument(); + const privateRow = screen.getByRole("row", { name: /older-skill/ }); + expect(within(privateRow).getAllByText("No")).toHaveLength(2); }); it("should delete a skill through the actions menu when admin", async () => { @@ -111,7 +116,7 @@ describe("PluginTable", () => { it("should hide the delete action for non-admins but keep copy available", async () => { const user = userEvent.setup(); render(); - expect(screen.queryByRole("columnheader", { name: "Downloads" })).not.toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Downloads" })).toBeInTheDocument(); await user.click(screen.getByTestId("plugin-actions-newer-skill")); expect(await screen.findByTestId("plugin-action-copy")).toBeInTheDocument(); expect(screen.queryByTestId("plugin-action-delete")).not.toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTable.tsx index db61f80c690..943b34c7023 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTable.tsx @@ -1,8 +1,8 @@ "use client"; -import { SortingState } from "@tanstack/react-table"; +import { type OnChangeFn, type SortingState } from "@tanstack/react-table"; import { Inbox } from "lucide-react"; -import React, { useMemo, useState } from "react"; +import React, { useMemo } from "react"; import { useTranslation } from "react-i18next"; import { DataTable } from "@/components/shared/DataTable"; @@ -12,30 +12,43 @@ import { getPluginTableColumns } from "./PluginTableColumns"; interface PluginTableProps { pluginsList: SkillHubSkill[]; + sorting: SortingState; + onSortingChange: OnChangeFn; isLoading: boolean; + filtered?: boolean; onDeleteClick: (pluginName: string, displayName: string) => void; isAdmin: boolean; onPluginClick: (pluginId: string) => void; } -const DEFAULT_SORTING: SortingState = [{ id: "name", desc: false }]; - -function EmptyState() { +function EmptyState({ filtered }: { filtered: boolean }) { const { t } = useTranslation(); return (
-
{t("skills.table.empty.title")}
-
{t("skills.table.empty.description")}
+
+ {t(filtered ? "aiHub.skillHubDashboard.emptyState.noMatchTitle" : "skills.table.empty.title")} +
+
+ {t(filtered ? "aiHub.skillHubDashboard.emptyState.noMatchBody" : "skills.table.empty.description")} +
); } -const PluginTable: React.FC = ({ pluginsList, isLoading, onDeleteClick, isAdmin, onPluginClick }) => { +const PluginTable: React.FC = ({ + pluginsList, + sorting, + onSortingChange, + isLoading, + filtered = false, + onDeleteClick, + isAdmin, + onPluginClick, +}) => { const { t } = useTranslation(); - const [sorting, setSorting] = useState(DEFAULT_SORTING); const columns = useMemo( () => getPluginTableColumns({ isAdmin, onPluginClick, onDeleteClick }), @@ -48,12 +61,12 @@ const PluginTable: React.FC = ({ pluginsList, isLoading, onDel paginationMode="none" columns={columns} getRowId={(plugin, index) => plugin.id || String(index)} - sortingMode="client" + sortingMode="server" sorting={sorting} - onSortingChange={setSorting} + onSortingChange={onSortingChange} isLoading={isLoading} loadingMessage={t("skills.table.loading")} - noDataMessage={} + noDataMessage={} size="compact" /> ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTableColumns.tsx index f2055b77969..c838ea5deda 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTableColumns.tsx @@ -1,11 +1,13 @@ "use client"; +import { getSkillMetadataColumns } from "@/components/claude_code_plugins/SkillMetadataColumns"; + import { ColumnDef } from "@tanstack/react-table"; import { Copy, MoreHorizontal, Trash2 } from "lucide-react"; import { useTranslation } from "react-i18next"; import { DataTableSortHeader } from "@/components/shared/DataTable"; -import { DateCell, IdentityCell, StatusBadge } from "@/components/shared/table_cells"; +import { IdentityCell, StatusBadge } from "@/components/shared/table_cells"; import i18n from "@/i18n/i18n"; import { SkillHubSkill } from "@/components/claude_code_plugins/types"; import { buttonVariants } from "@/components/ui/button"; @@ -143,31 +145,7 @@ export const getPluginTableColumns = ({ /> ), }, - { - id: "updated_at", - accessorKey: "updated_at", - sortingFn: "datetime", - meta: { title: i18n.t("skills.table.columns.updatedAt") }, - header: ({ column }) => , - size: 160, - enableSorting: true, - cell: ({ row }) => , - }, - ...(isAdmin - ? ([ - { - id: "download_count", - accessorKey: "download_count", - meta: { title: i18n.t("skills.table.columns.downloads") }, - header: i18n.t("skills.table.columns.downloads"), - size: 120, - enableSorting: false, - cell: ({ row }) => ( - {row.original.download_count?.toLocaleString(i18n.language) ?? "—"} - ), - }, - ] satisfies ColumnDef[]) - : []), + ...getSkillMetadataColumns(), { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/add_plugin_form.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/add_plugin_form.integration.test.tsx index 778bd8e309a..9547db39ca1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/add_plugin_form.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/add_plugin_form.integration.test.tsx @@ -15,7 +15,7 @@ const file = new File(["skill archive"], "personal-skill.zip", { type: "applicat const chooseFile = (selected = file) => fireEvent.change(screen.getByLabelText("Skill ZIP file"), { target: { files: [selected] } }); const change = (label: string, value: string) => fireEvent.change(screen.getByLabelText(label), { target: { value } }); -const submit = () => fireEvent.click(screen.getByRole("button", { name: "Add Skill" })); +const submit = () => fireEvent.click(screen.getByRole("button", { name: "Register skill" })); const openImport = () => { renderWithProviders(); fireEvent.click(screen.getByRole("radio", { name: "Import URL" })); diff --git a/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.integration.test.tsx b/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.integration.test.tsx new file mode 100644 index 00000000000..8f97f63cd2f --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.integration.test.tsx @@ -0,0 +1,31 @@ +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it } from "vitest"; +import { renderWithProviders } from "@/../tests/test-utils"; +import type { Plugin } from "@/components/claude_code_plugins/types"; +import SkillHubDashboard from "./SkillHubDashboard"; + +const skills: Plugin[] = [ + { id: "alpha", name: "alpha", source: {}, category: "Writing", domain: "Support", enabled: false, download_count: 1 }, + { + id: "beta", + name: "beta", + source: {}, + category: "Analysis", + domain: "Engineering", + enabled: true, + download_count: 7, + }, +]; + +describe("Skills Hub sorting", () => { + it.each(["Category", "Domain", "Status", "Downloads"])("sorts by %s through the table header", async (header) => { + const user = userEvent.setup(); + renderWithProviders(); + expect(screen.getAllByRole("row")[1]).toHaveTextContent("alpha"); + await user.click(screen.getByRole("button", { name: header, exact: true })); + await waitFor(() => expect(screen.getAllByRole("row")[1]).toHaveTextContent("beta")); + await user.click(screen.getByRole("button", { name: header, exact: true })); + await waitFor(() => expect(screen.getAllByRole("row")[1]).toHaveTextContent("alpha")); + }); +}); diff --git a/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx b/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx index 00f2da9171d..3099da3476e 100644 --- a/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx @@ -1,15 +1,21 @@ import React, { useMemo, useState } from "react"; -import { SortingState } from "@tanstack/react-table"; -import { Inbox, Search, X } from "lucide-react"; +import { Inbox } from "lucide-react"; import { useTranslation } from "react-i18next"; import { Plugin } from "@/components/claude_code_plugins/types"; import { DataTable } from "@/components/shared/DataTable"; import { getSkillHubTableColumns } from "@/components/AIHub/SkillHubTableColumns"; import SkillDetail, { legacySkillDetail } from "@/components/claude_code_plugins/skill_detail"; -import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; +import { SkillFilters } from "@/components/claude_code_plugins/SkillFilters"; +import { + useSkillFilters, + matchesSkillFilters, + skillTags, + SKILL_SORT_FIELDS, +} from "@/components/claude_code_plugins/skillCatalog"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; const ALL_DOMAINS = "__all_domains__"; +const DASHBOARD_SORT_FIELDS = [...SKILL_SORT_FIELDS, "category", "domain", "enabled"]; interface SkillHubDashboardProps { skills: Plugin[]; @@ -50,10 +56,9 @@ const SkillHubDashboard: React.FC = ({ onPublishSuccess, }) => { const { t } = useTranslation(); - const [search, setSearch] = useState(""); + const filters = useSkillFilters(DASHBOARD_SORT_FIELDS); const [domainFilter, setDomainFilter] = useState(undefined); const [selectedSkill, setSelectedSkill] = useState(null); - const [sorting, setSorting] = useState([{ id: "name", desc: false }]); // Derived stats const totalSkills = skills.length; @@ -69,19 +74,10 @@ const SkillHubDashboard: React.FC = ({ if (domainFilter) { result = result.filter((s) => (s.domain || "General") === domainFilter); } - if (search.trim()) { - const q = search.toLowerCase(); - result = result.filter( - (s) => - s.name.toLowerCase().includes(q) || - s.description?.toLowerCase().includes(q) || - s.domain?.toLowerCase().includes(q) || - s.namespace?.toLowerCase().includes(q) || - s.keywords?.some((k) => k.toLowerCase().includes(q)), - ); - } - return result; - }, [skills, search, domainFilter]); + return result.filter((skill) => + matchesSkillFilters(skill, { search: filters.search, tag: filters.tag, scan: filters.scan }), + ); + }, [skills, filters.search, filters.tag, filters.scan, domainFilter]); const columns = useMemo(() => getSkillHubTableColumns({ onSkillClick: setSelectedSkill }), []); @@ -93,7 +89,7 @@ const SkillHubDashboard: React.FC = ({ [domains, t], ); - const hasActiveFilter = search.trim().length > 0 || domainFilter != null; + const hasActiveFilter = filters.active || domainFilter != null; if (selectedSkill) { return ( @@ -131,7 +127,7 @@ const SkillHubDashboard: React.FC = ({

{publicPage ? t("aiHub.skillHubDashboard.allPublicSkills") : t("aiHub.skillHubDashboard.allSkills")}

-
+
- - - - - setSearch(e.target.value)} - /> - {search !== "" && ( - - setSearch("")} - > - - - - )} - +
= ({ columns={columns} getRowId={(skill, index) => skill.id || String(index)} sortingMode="client" - sorting={sorting} - onSortingChange={setSorting} + sorting={filters.sorting} + onSortingChange={filters.onSortingChange} isLoading={isLoading} loadingMessage={t("aiHub.skillHubDashboard.loading")} noDataMessage={} diff --git a/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx index 766c4243c45..76ba9f6e592 100644 --- a/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx @@ -1,5 +1,7 @@ "use client"; +import { getSkillMetadataColumns } from "@/components/claude_code_plugins/SkillMetadataColumns"; + import { ColumnDef } from "@tanstack/react-table"; import { Copy, ExternalLink, Info, MoreHorizontal } from "lucide-react"; import { useTranslation } from "react-i18next"; @@ -159,6 +161,7 @@ export const getSkillHubTableColumns = ({ onSkillClick }: SkillHubTableColumnsDe /> ), }, + ...getSkillMetadataColumns(), { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, diff --git a/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts b/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts index fceee39e943..d1f14137577 100644 --- a/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts +++ b/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts @@ -59,7 +59,7 @@ describe("visibleMenuGroups", () => { }); expect(flattenNavItems(visible).find(({ route }) => route === "skills")).toMatchObject({ - label: "Skills", + label: "Skills Hub", route: "skills", }); }, diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.integration.test.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.integration.test.tsx new file mode 100644 index 00000000000..e89211596f3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CreateUserButton.integration.test.tsx @@ -0,0 +1,111 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { CreateUserButton } from "./CreateUserButton"; + +const createUser = vi.fn<(body: Record) => Response>(); +const unexpectedRequest = vi.fn((path: string) => + Response.json({ error: `Unexpected request ${path}` }, { status: 500 }), +); + +beforeEach(() => { + unexpectedRequest.mockClear(); + createUser.mockReset(); + createUser.mockImplementation((body) => Response.json({ user_id: body.user_id ?? "generated-id" })); + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const path = new URL(input instanceof Request ? input.url : String(input), "http://localhost").pathname; + if (path === "/user/new") return createUser(JSON.parse(String(init?.body))); + if (path === "/models") return Response.json({ data: [] }); + if (path === "/sso/get/ui_settings") return Response.json({ SSO_ENABLED: true }); + if (path === "/v2/team/list") return Response.json({ teams: [], total: 0, total_pages: 0 }); + return unexpectedRequest(path); + }), + ); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + expect(unexpectedRequest).not.toHaveBeenCalled(); +}); + +const renderForm = (isEmbedded: boolean, onUserCreated = vi.fn()) => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + render( + + + , + ); + return onUserCreated; +}; + +describe("custom User ID invitation", () => { + it.each([true, false])("sends the custom ID from the form (embedded=%s)", async (embedded) => { + const user = userEvent.setup(); + const onCreated = renderForm(embedded); + if (!embedded) await user.click(screen.getByRole("button", { name: "+ Invite User" })); + await user.click(screen.getByRole("checkbox", { name: "Supply custom User ID" })); + fireEvent.change(screen.getByRole("textbox", { name: "User ID", exact: true }), { + target: { value: " employee-42 " }, + }); + fireEvent.change(screen.getByRole("textbox", { name: "User Email" }), { + target: { value: "employee@example.test" }, + }); + await user.click(screen.getByRole("button", { name: embedded ? "Create User" : "Invite User", exact: true })); + await waitFor(() => + expect(createUser).toHaveBeenCalledWith( + expect.objectContaining({ + user_id: "employee-42", + user_email: "employee@example.test", + auto_create_key: false, + }), + ), + ); + expect(createUser.mock.calls[0][0]).not.toHaveProperty("supply_user_id"); + if (embedded) { + await waitFor(() => expect(onCreated).toHaveBeenCalledWith("employee-42")); + expect(screen.getByRole("checkbox", { name: "Supply custom User ID" })).not.toBeChecked(); + expect(screen.queryByRole("textbox", { name: "User ID", exact: true })).not.toBeInTheDocument(); + } + }); + + it("blocks whitespace and clears the custom ID when automatic generation is selected", async () => { + const user = userEvent.setup(); + renderForm(true); + await user.click(screen.getByRole("checkbox", { name: "Supply custom User ID" })); + fireEvent.change(screen.getByRole("textbox", { name: "User ID", exact: true }), { target: { value: " " } }); + await user.click(screen.getByRole("button", { name: "Create User" })); + expect(await screen.findByText("Enter a User ID")).toBeInTheDocument(); + expect(createUser).not.toHaveBeenCalled(); + fireEvent.change(screen.getByRole("textbox", { name: "User ID", exact: true }), { + target: { value: "discard-me" }, + }); + await user.click(screen.getByRole("checkbox", { name: "Supply custom User ID" })); + await user.click(screen.getByRole("button", { name: "Create User" })); + await waitFor(() => expect(createUser).toHaveBeenCalledWith(expect.objectContaining({ user_id: null }))); + expect(createUser.mock.calls[0][0]).not.toHaveProperty("supply_user_id"); + }); + + it("discards a custom ID when an invitation is cancelled", async () => { + const user = userEvent.setup(); + renderForm(false); + await user.click(screen.getByRole("button", { name: "+ Invite User" })); + await user.click(screen.getByRole("checkbox", { name: "Supply custom User ID" })); + fireEvent.change(screen.getByRole("textbox", { name: "User ID", exact: true }), { target: { value: "cancelled" } }); + await user.click( + within(screen.getByRole("dialog", { name: "Invite User" })).getByRole("button", { name: "Close" }), + ); + await user.click(screen.getByRole("button", { name: "+ Invite User" })); + expect(screen.getByRole("checkbox", { name: "Supply custom User ID" })).not.toBeChecked(); + await user.click(screen.getByRole("checkbox", { name: "Supply custom User ID" })); + expect(screen.getByRole("textbox", { name: "User ID", exact: true })).toHaveValue(""); + }); +}); diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx index 94b3726ede0..d04272cecd3 100644 --- a/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx +++ b/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx @@ -308,7 +308,7 @@ describe("CreateUserButton", () => { const dialog = screen.getByRole("dialog", { name: /invite user/i }); - expectControlBesideLabel(within(dialog).getByRole("checkbox")); + expectControlBesideLabel(within(dialog).getByRole("checkbox", { name: "Send invitation email" })); }); describe("organizations", () => { diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.tsx index 6fbe8f2d2bd..bf188d36716 100644 --- a/ui/litellm-dashboard/src/components/CreateUserButton.tsx +++ b/ui/litellm-dashboard/src/components/CreateUserButton.tsx @@ -14,7 +14,7 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { ChevronRight, CircleHelp, Info, UserPlus } from "lucide-react"; import React, { useEffect, useState } from "react"; -import { useForm } from "react-hook-form"; +import { useForm, useWatch } from "react-hook-form"; import { useTranslation } from "react-i18next"; import TeamDropdown from "./common_components/team_dropdown"; import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key"; @@ -58,6 +58,8 @@ interface UISettings { } interface CreateUserFormValues { + supply_user_id: boolean; + user_id?: string; user_email?: string; user_role: string; team_id?: string | null; @@ -68,6 +70,8 @@ interface CreateUserFormValues { } const EMBEDDED_DEFAULTS: CreateUserFormValues = { + supply_user_id: false, + user_id: "", user_email: undefined, user_role: "internal_user_viewer", team_id: undefined, @@ -76,6 +80,8 @@ const EMBEDDED_DEFAULTS: CreateUserFormValues = { }; const STANDALONE_DEFAULTS: CreateUserFormValues = { + supply_user_id: false, + user_id: "", user_email: undefined, user_role: "internal_user_viewer", team_id: undefined, @@ -95,11 +101,13 @@ const withoutCollapsedModels = (values: CreateUserFormValues, isPersonalKeyOpen: return rest; }; -const buildCreatePayload = (values: CreateUserFormValues): CreateUserFormValues & { organizations?: string[] } => { +const buildCreatePayload = (values: CreateUserFormValues) => { + const { supply_user_id, user_id, ...fields } = values; + const apiValues = supply_user_id ? { ...fields, user_id: user_id?.trim() } : fields; const withModels = (!values.models || values.models.length === 0) && values.user_role !== "proxy_admin" - ? { ...values, models: ["no-default-models"] } - : values; + ? { ...apiValues, models: ["no-default-models"] } + : apiValues; if (!withModels.organization_ids) { return withModels; } @@ -163,6 +171,7 @@ export const CreateUserButton: React.FC = ({ const [uiSettings, setUISettings] = useState(null); const defaultValues = isEmbedded ? EMBEDDED_DEFAULTS : STANDALONE_DEFAULTS; const form = useForm({ defaultValues }); + const supplyUserId = useWatch({ control: form.control, name: "supply_user_id" }); const [isModalVisible, setIsModalVisible] = useState(false); const [apiuser, setApiuser] = useState(false); const [userModels, setUserModels] = useState([]); @@ -205,6 +214,14 @@ export const CreateUserButton: React.FC = ({ }; const handleCreate = async (formValues: CreateUserFormValues) => { + if (formValues.supply_user_id && !formValues.user_id?.trim()) { + form.setError("user_id", { message: "Enter a User ID" }, { shouldFocus: true }); + return; + } + if (formValues.supply_user_id && formValues.user_id?.includes("\u0000")) { + form.setError("user_id", { message: "User ID must not contain NUL characters" }, { shouldFocus: true }); + return; + } try { toast.info(t("users.createUser.toasts.makingApiCall")); if (!isEmbedded) { @@ -255,6 +272,34 @@ export const CreateUserButton: React.FC = ({ ); + const userIdField = ( + <> + + {({ id, value, onChange, onBlur }) => ( + { + onChange(checked); + form.resetField("user_id"); + }} + onBlur={onBlur} + /> + )} + + {supplyUserId && ( + + {({ ref, value, ...control }) => } + + )} + + ); + const teamField = ( = ({ {userEmailField} + {userIdField} {roleField(t("users.createUser.fields.userRole"))} {teamField} {metadataField} @@ -357,6 +403,7 @@ export const CreateUserButton: React.FC = ({
{userEmailField} + {userIdField} {roleField( labelWithHint(t("users.createUser.fields.globalRole"), t("users.createUser.fields.globalRoleHint")), )} @@ -436,7 +483,7 @@ export const CreateUserButton: React.FC = ({ -
+
{!isViewOnly && ( <> @@ -219,7 +238,7 @@ export default function MigrationDetail({ onClick={() => statusMutation.mutate(AlertStatus.Resolved)} > - Mark resolved + {t("alerts.migrationDetail.markResolved")} ) : ( )} )} {modelGroupHref(selected.model.model_name) && ( - Model routing + {t("alerts.migrationDetail.modelRouting")} )}
-

- Resolve across all teams once owners confirm migration. New traffic reopens the alert. -

+

{t("alerts.migrationDetail.resolveHint")}

{statusMutation.isError && (

- Could not save alert status. Please retry. + {t("alerts.migrationDetail.saveError")}

)}
-

{params.team ? "Keys in this team" : "Affected keys"}

+

+ {params.team + ? t("alerts.migrationDetail.summary.teamKeys") + : t("alerts.migrationDetail.summary.affectedKeys")} +

{keyQuery.data?.total_api_keys.toLocaleString() ?? "—"}

-

Teams using this model

+

{t("alerts.migrationDetail.summary.teams")}

{teamQuery.data?.length ?? "—"}

-

Deprecation

+

{t("alerts.migrationDetail.summary.deprecation")}

@@ -276,14 +297,12 @@ export default function MigrationDetail({

- Affected keys{" "} + {t("alerts.migrationDetail.keysTable.title")}{" "} {keyQuery.data?.total_api_keys.toLocaleString() ?? "…"}

-

- Keys and owners that used this model in the last 30 days. Tags show actual usage with this model. -

+

{t("alerts.migrationDetail.keysTable.description")}

({ value: language, label: nativeName(language) }))} + value={i18n.language} + onValueChange={choose} + > + + + + + {languages.map((language) => ( + + {nativeName(language)} + + + + {language !== DEFAULT_LANGUAGE && ( + {englishName(language)} + )} + + ))} + + + + + ), + }} + /> + + + ); +} diff --git a/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx b/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx index 5d419274bc7..6066c9404c6 100644 --- a/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx +++ b/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx @@ -6,6 +6,7 @@ import { useDisableLiteAdmin } from "@/app/(dashboard)/hooks/useDisableLiteAdmin import { ARIA_AGENT_LABEL } from "@/components/liteadmin/aria-brand"; import { useDisableShowNewBadge } from "@/app/(dashboard)/hooks/useDisableShowNewBadge"; import { useDisableShowPrompts } from "@/app/(dashboard)/hooks/useDisableShowPrompts"; +import LanguageField from "./LanguageField"; import { emitLocalStorageChange, removeLocalStorageItem, setLocalStorageItem } from "@/utils/localStorageUtils"; import { navAccountDisplayName } from "@/components/Navbar/navDisplayName"; import CopyButton from "@/components/shared/CopyButton"; @@ -267,6 +268,7 @@ const SidebarAccountMenu: React.FC = ({ onLogout, colla />
)} +
diff --git a/ui/litellm-dashboard/src/components/chat/runChatRequest.test.ts b/ui/litellm-dashboard/src/components/chat/runChatRequest.test.ts index 60ab8772112..aca9364c52b 100644 --- a/ui/litellm-dashboard/src/components/chat/runChatRequest.test.ts +++ b/ui/litellm-dashboard/src/components/chat/runChatRequest.test.ts @@ -1,5 +1,6 @@ import OpenAI from "openai"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import i18n from "@/i18n/i18n"; import { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api"; import { generateChatImages } from "./imageHelpers"; import { toResponsesMessages } from "./attachmentHelpers"; @@ -192,4 +193,67 @@ describe("runChatRequest", () => { expect(options.onUpdate).toHaveBeenCalledExactlyOnceWith({ content: " [stopped]" }); expect(options.onResponseId).toHaveBeenCalledWith(null); }); + + describe("when the UI language is not English", () => { + beforeEach(async () => { + i18n.addResourceBundle( + "de", + "translation", + { + chat: { + runChatRequest: { genericError: "Fehler", fileInputHint: " Hinweis", researchNeedsBackground: "Hinweis" }, + }, + }, + true, + true, + ); + await i18n.changeLanguage("de"); + }); + + afterEach(async () => { + await i18n.changeLanguage("en"); + i18n.removeResourceBundle("de", "translation"); + }); + + it("should keep the generic error in English because it is sent back to the model", async () => { + const options = requestOptions(); + vi.mocked(makeOpenAIResponsesRequest).mockRejectedValue("offline"); + await runChatRequest(options, i18n.t.bind(i18n)); + expect(options.onUpdate).toHaveBeenLastCalledWith({ content: "Something went wrong. Please try again." }); + }); + + it("should keep the file input hint in English because it is sent back to the model", async () => { + const options = requestOptions({ + messages: [ + { + role: "user", + content: "Read the report", + attachments: [ + { + id: "report", + kind: "file", + name: "report.pdf", + mimeType: "application/pdf", + url: "data:application/pdf;base64,JVBERg==", + }, + ], + }, + ], + }); + const error = new OpenAI.APIError(400, { message: "Provider detail" }, "Provider detail", new Headers()); + vi.mocked(makeOpenAIResponsesRequest).mockRejectedValue(error); + await runChatRequest(options, i18n.t.bind(i18n)); + expect(options.onUpdate).toHaveBeenLastCalledWith({ + content: error.message + " Try a model that supports file inputs, or paste the relevant text.", + }); + }); + + it("should keep the deep research notice in English because it is sent back to the model", async () => { + const options = requestOptions({ research: true }); + await runChatRequest(options, i18n.t.bind(i18n)); + expect(options.onUpdate).toHaveBeenCalledExactlyOnceWith({ + content: "Enable background chats before starting deep research.", + }); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/chat/runChatRequest.ts b/ui/litellm-dashboard/src/components/chat/runChatRequest.ts index 7758f33254d..dc30f7d68f6 100644 --- a/ui/litellm-dashboard/src/components/chat/runChatRequest.ts +++ b/ui/litellm-dashboard/src/components/chat/runChatRequest.ts @@ -31,8 +31,8 @@ function requestErrorMessage( error: unknown, messages: ChatRequestOptions["messages"], mode: ChatRequestOptions["mode"], - t: TFunction = i18n.t.bind(i18n), ): string { + const t = i18n.getFixedT("en"); const message = error instanceof Error ? error.message : t("chat.runChatRequest.genericError"); const hasFile = mode === "chat" && messages.some((item) => item.attachments?.some((attachment) => attachment.kind === "file")); @@ -67,7 +67,7 @@ export async function runChatRequest( try { signal.throwIfAborted(); if (research && !background) { - onUpdate({ content: t("chat.runChatRequest.researchNeedsBackground") }); + onUpdate({ content: i18n.getFixedT("en")("chat.runChatRequest.researchNeedsBackground") }); return; } if (mode === "image") { @@ -125,7 +125,7 @@ export async function runChatRequest( } catch (error: unknown) { onResponseId(null); const stopped = signal.aborted || (error instanceof Error && error.name === "AbortError"); - const errorMessage = requestErrorMessage(error, messages, mode, t); + const errorMessage = requestErrorMessage(error, messages, mode); onUpdate({ content: stopped ? content + " [stopped]" : content || errorMessage }); } finally { if (mcpEvents.length) onUpdate({ mcpEvents }); diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/SkillFilters.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/SkillFilters.tsx new file mode 100644 index 00000000000..49088ec1364 --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/SkillFilters.tsx @@ -0,0 +1,89 @@ +import { Search } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/components/ui/button"; +import { InputGroup, InputGroupAddon, InputGroupInput } from "@/components/ui/input-group"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import type { ScanFilter } from "./skillCatalog"; + +interface SkillFiltersProps { + search: string; + tag: string; + scan: ScanFilter; + tags: string[]; + active: boolean; + setSearch: (value: string) => void; + setTag: (value: string) => void; + setScan: (value: ScanFilter) => void; + clear: () => void; +} + +export function SkillFilters({ + search, + tag, + scan, + tags, + active, + setSearch, + setTag, + setScan, + clear, +}: SkillFiltersProps) { + const { t } = useTranslation(); + const tagItems = [ + { value: "", label: t("skills.filters.allTags") }, + ...[...new Set([...tags, ...(tag ? [tag] : [])])].map((value) => ({ value, label: value })), + ]; + const scanItems = [ + { value: "all", label: t("skills.filters.allScans") }, + { value: "yes", label: t("skills.filters.scanned") }, + { value: "no", label: t("skills.filters.notScanned") }, + ]; + return ( +
+ + + + + setSearch(event.target.value)} + /> + + + + {active && ( + + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/SkillInstallDialog.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/SkillInstallDialog.tsx new file mode 100644 index 00000000000..4af2bd9a1cb --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/SkillInstallDialog.tsx @@ -0,0 +1,164 @@ +import { useState } from "react"; +import { useQuery, type UseQueryOptions } from "@tanstack/react-query"; +import { Check, Copy } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import { getProxyBaseUrl } from "@/components/networking"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { Label } from "@/components/ui/label"; +import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { createApiClient } from "@/lib/http/client"; +import { getAuthHeaderName } from "@/lib/http/runtime"; +import { buildSkillInstallCommand, findPublicSkill, SKILL_AGENTS, type SkillAgent } from "./skillInstall"; + +interface SkillInstallDialogProps { + name: string; + archivePath: string; + open: boolean; + onOpenChange: (open: boolean) => void; +} + +export default function SkillInstallDialog({ name, archivePath, open, onOpenChange }: SkillInstallDialogProps) { + const { t } = useTranslation(); + const [agent, setAgent] = useState("codex"); + const [scope, setScope] = useState("project"); + const [mode, setMode] = useState("authenticated"); + const [copyState, setCopyState] = useState<{ + command: string; + status: "pending" | "copied" | "failed"; + } | null>(null); + const gatewayUrl = getProxyBaseUrl().replace(/\/+$/, ""); + const archiveUrl = `${gatewayUrl}${archivePath}`; + const indexOptions: UseQueryOptions = { + queryKey: ["public-skill-install", gatewayUrl, archivePath], + queryFn: ({ signal }) => + createApiClient({ getBaseUrl: () => gatewayUrl }).get("/.well-known/agent-skills/index.json", { + signal, + credentials: "omit", + redirect: "error", + }), + enabled: open, + retry: false, + staleTime: Infinity, + gcTime: 0, + refetchOnMount: "always", + }; + const index = useQuery(indexOptions); + const publicSkill = index.isSuccess && !index.isFetching ? findPublicSkill(index.data, archiveUrl) : undefined; + const publicInstall = mode === "public" && publicSkill !== undefined; + const commandOptions = { + gatewayUrl, + archiveUrl, + name, + agent, + global: scope === "global", + publicSkill: publicInstall ? publicSkill : undefined, + authHeaderName: getAuthHeaderName(), + }; + const command = buildSkillInstallCommand(commandOptions); + const copied = copyState?.command === command && copyState.status === "copied"; + const copyPending = copyState?.command === command && copyState.status === "pending"; + const copyFailed = copyState?.command === command && copyState.status === "failed"; + const copyCommand = async () => { + if (!command) return; + setCopyState({ command, status: "pending" }); + try { + await navigator.clipboard.writeText(command); + setCopyState({ command, status: "copied" }); + } catch { + setCopyState({ command, status: "failed" }); + } + }; + + return ( + + + + {t("skills.install.title", { name })} + {t("skills.install.description")} + +
+
+ + items={SKILL_AGENTS} value={agent} onValueChange={(value) => value && setAgent(value)}> + + + + + {SKILL_AGENTS.map((item) => ( + + {item.label} + + ))} + + +
+
+

{t("skills.install.scope")}

+ + + + +
+
+ {publicSkill && ( + + + + + )} +
+

+ {t(publicInstall ? "skills.install.publicHint" : "skills.install.authenticatedHint")} +

+ {command ? ( +
+              {command}
+            
+ ) : ( +

+ {t("skills.install.unavailable")} +

+ )} +

+ {t(publicInstall ? "skills.install.publicRequirements" : "skills.install.authenticatedRequirements")} +

+
+
+

{t("skills.install.shareHint")}

+ +
+ {copyFailed && ( +

+ {t("skills.install.copyFailed")} +

+ )} +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/SkillMetadataColumns.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/SkillMetadataColumns.tsx new file mode 100644 index 00000000000..649a1696a9e --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/SkillMetadataColumns.tsx @@ -0,0 +1,95 @@ +import { DataTableSortHeader } from "@/components/shared/DataTable/DataTableSortHeader"; +import type { ColumnDef } from "@tanstack/react-table"; +import { DateCell, StatusBadge } from "@/components/shared/table_cells"; +import { Badge } from "@/components/ui/badge"; +import i18n from "@/i18n/i18n"; +import type { SkillCatalogMetadata } from "./skillCatalog"; + +const IDENTITY_LABELS = { + created_by: "skills.table.columns.createdBy", + approved_by: "skills.table.columns.approvedBy", +} as const; + +export function getSkillMetadataColumns(): ColumnDef[] { + return [ + { + id: "download_count", + accessorKey: "download_count", + header: ({ column }) => , + size: 120, + sortUndefined: "last", + accessorFn: (skill) => skill.download_count ?? undefined, + cell: ({ row }) => ( + {row.original.download_count?.toLocaleString(i18n.language) ?? "—"} + ), + }, + { + id: "keywords", + header: i18n.t("skills.table.columns.tags"), + size: 180, + enableSorting: false, + cell: ({ row }) => ( +
+ {row.original.keywords?.length + ? row.original.keywords.map((tag) => ( + + {tag} + + )) + : "—"} +
+ ), + }, + ...(["created_by", "approved_by"] as const).map( + (field): ColumnDef => ({ + id: field, + accessorKey: field, + size: 190, + enableSorting: true, + header: ({ column }) => , + cell: ({ row }) => ( + + {row.original[field] || i18n.t("skills.table.notRecorded")} + + ), + }), + ), + ...(["created_at", "updated_at"] as const).map( + (field): ColumnDef => ({ + id: field, + accessorKey: field, + size: 140, + enableSorting: true, + accessorFn: (skill) => (skill[field] ? new Date(skill[field]!).getTime() : undefined), + sortUndefined: "last", + header: ({ column }) => ( + + ), + cell: ({ row }) => , + }), + ), + { + id: "security_scan_passed", + accessorKey: "security_scan_passed", + size: 130, + enableSorting: true, + header: ({ column }) => ( + {i18n.t("skills.table.columns.securityScan")}} + /> + ), + cell: ({ row }) => ( + + + + ), + }, + ]; +} diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skillCatalog.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/skillCatalog.ts new file mode 100644 index 00000000000..ec057a00984 --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skillCatalog.ts @@ -0,0 +1,99 @@ +import type { SkillListQuery } from "./types"; +import type { SortingState } from "@tanstack/react-table"; +import { parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs"; +import { useUrlTableState, type UrlTableStateOptions } from "@/components/shared/DataTable/useUrlTableState"; +import { matchesSearchTerm } from "@/utils/searchUtils"; + +const SCAN_FILTERS = ["all", "yes", "no"] as const; +export type ScanFilter = (typeof SCAN_FILTERS)[number]; + +export interface SkillCatalogMetadata { + created_by?: string | null; + approved_by?: string | null; + published_by?: string | null; + created_at?: string | null; + updated_at?: string | null; + security_scan_passed?: boolean; + download_count?: number | null; + keywords?: string[] | null; +} + +export const SKILL_SORT_FIELDS = [ + "name", + "download_count", + "created_at", + "updated_at", + "created_by", + "approved_by", + "security_scan_passed", +] as const satisfies readonly NonNullable[]; + +export function skillSortQuery(sorting: SortingState): Pick { + return { + sort_by: SKILL_SORT_FIELDS.find((field) => field === sorting[0]?.id) ?? "name", + sort_order: sorting[0]?.desc ? "desc" : "asc", + }; +} + +const CATALOG_STATE: UrlTableStateOptions = { + keyPrefix: "skills_", + defaultPageSize: 50, + maxPageSize: 100, + sortFields: SKILL_SORT_FIELDS, + defaultSort: { id: "name", desc: false }, + filterColumns: [], +}; +const FILTER_PARSERS = { + tag: parseAsString.withDefault(""), + scan: parseAsStringLiteral(SCAN_FILTERS).withDefault("all"), +}; +const FILTER_OPTIONS = { urlKeys: { tag: "skills_filter_tag", scan: "skills_filter_scan" } }; + +export function useSkillFilters(sortFields: readonly string[] = SKILL_SORT_FIELDS) { + const table = useUrlTableState({ ...CATALOG_STATE, sortFields }); + const [filters, setFilters] = useQueryStates(FILTER_PARSERS, FILTER_OPTIONS); + const update = (next: { tag?: string; scan?: ScanFilter } | null) => { + void setFilters(next); + table.onPaginationChange({ ...table.pagination, pageIndex: 0 }); + }; + return { + ...table, + ...filters, + setTag: (tag: string) => update({ tag }), + setScan: (scan: ScanFilter) => update({ scan }), + clear: () => { + update(null); + table.setSearch(""); + }, + active: Boolean(table.search.trim() || filters.tag || filters.scan !== "all"), + }; +} + +export function skillTags(skills: SkillCatalogMetadata[]): string[] { + return [...new Set(skills.flatMap((skill) => skill.keywords ?? []))].sort((a, b) => a.localeCompare(b)); +} + +export function matchesSkillFilters( + skill: SkillCatalogMetadata & { + name: string; + description?: string | null; + domain?: string | null; + namespace?: string | null; + }, + filters: { search: string; tag: string; scan: ScanFilter }, +): boolean { + const matchesQuery = matchesSearchTerm(filters.search, [ + skill.name, + skill.description, + skill.domain, + skill.namespace, + skill.created_by, + skill.approved_by, + skill.published_by, + ...(skill.keywords ?? []), + ]); + const matchesTag = + !filters.tag || (skill.keywords ?? []).some((tag) => tag.toLowerCase() === filters.tag.toLowerCase()); + const matchesScan = filters.scan === "all" || Boolean(skill.security_scan_passed) === (filters.scan === "yes"); + return matchesQuery && matchesTag && matchesScan; +} diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.fixture.py b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.fixture.py new file mode 100644 index 00000000000..d67718b9621 --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.fixture.py @@ -0,0 +1,109 @@ +import errno +import json +import os +import pty +import select +import shutil +import signal +import sys +import tempfile +import time +import zipfile +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Final, TypeAlias, cast + +Environment: TypeAlias = Mapping[str, str] + + +def external_command(tool: str) -> None: + root: Final = Path(os.environ["SKILL_FIXTURE_ROOT"]) + args: Final = tuple(sys.argv[1:]) + header: Final = sys.stdin.read() if tool == "curl" and args[args.index("--header") + 1] == "@-" else "" + (root / f"{tool}.json").write_text(json.dumps({"args": args, "env": dict(os.environ), "header": header})) + if tool == "curl": + output: Final = Path(args[args.index("--output") + 1]) + (output.parent / ".skill-install-fixture").write_text(str(root)) + if os.environ["SKILL_FIXTURE_FAIL"] == "1": + sys.exit(22) + with zipfile.ZipFile(output, "w") as archive: + archive.writestr("SKILL.md", "fixture skill") + return + source: Final = Path(args[args.index("add") + 1]) + if "--copy" in args: + shutil.copytree(source, root / "installed") + else: + (root / "installed").symlink_to(source, target_is_directory=True) + + +def run(command: str, fail: bool) -> None: + secret: Final = "private-fixture-key" + with tempfile.TemporaryDirectory() as directory: + root: Final = Path(directory) + (root / "bin").mkdir() + (root / "tmp").mkdir() + for tool in ("curl", "npx"): + executable: Path = root / "bin" / tool + executable.write_text(f"#!{sys.executable}\n" + Path(__file__).read_text()) + executable.chmod(0o700) + environment: Final[Environment] = { + "PATH": f"{root / 'bin'}{os.pathsep}{os.defpath}", + "TMPDIR": str(root / "tmp"), + "SKILL_FIXTURE_ROOT": str(root), + "SKILL_FIXTURE_FAIL": "1" if fail else "0", + "skill_key": "previously-exported-value", + } + child, terminal = pty.fork() + if child == 0: + os.execve("/bin/bash", ["bash", "-c", command], environment) + output: bytes = b"" # rebind-ok: accumulate partial PTY reads + sent: bool = False # rebind-ok: send the key once after its prompt arrives + ended: bool = False # rebind-ok: distinguish PTY EOF from timeout cleanup + deadline: Final = time.monotonic() + 5 + try: + while True: + ready, _, _ = select.select([terminal], [], [], max(0, deadline - time.monotonic())) + if not ready: + raise TimeoutError("Install command did not finish") + try: + chunk: bytes = os.read(terminal, 65536) + except OSError as error: + if error.errno != errno.EIO: + raise + chunk = b"" + if not chunk: + ended = True + break + output += chunk + if not sent and b"LiteLLM API key: " in output: + os.write(terminal, f"{secret}\n".encode()) + sent = True + finally: + if not ended: + os.killpg(child, signal.SIGKILL) + os.close(terminal) + _, status = os.waitpid(child, 0) + curl: Final = cast(Mapping[str, object], json.loads((root / "curl.json").read_text())) # cast-ok: own JSON + record: Final = root / "npx.json" + npx: Final[Mapping[str, object] | None] = json.loads(record.read_text()) if record.exists() else None + curl_args: Final = cast(Sequence[str], curl["args"]) # cast-ok: fixture records argv + installed: Final = root / "installed" / "SKILL.md" + temporary: Final = Path(curl_args[curl_args.index("--output") + 1]).parent + sys.stdout.write(json.dumps({ + "status": os.waitstatus_to_exitcode(status), + "header": curl["header"], + "installArgs": npx["args"] if npx else None, + "telemetry": cast(Environment, npx["env"]).get("DISABLE_TELEMETRY") if npx else None, # cast-ok: own env + "secretLeaked": secret in json.dumps((output.decode(), curl_args, curl["env"], npx)), + "retained": installed.read_text() if installed.exists() else None, + "temporaryRemoved": not temporary.exists(), + })) + marker: Final = temporary / ".skill-install-fixture" + if marker.exists() and marker.read_text() == str(root): + shutil.rmtree(temporary) + + +if Path(sys.argv[0]).name in ("curl", "npx"): + external_command(Path(sys.argv[0]).name) +else: + run(sys.stdin.read(), sys.argv[1] == "fail") diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.test.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.test.ts new file mode 100644 index 00000000000..183457af162 --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.test.ts @@ -0,0 +1,107 @@ +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { buildSkillInstallCommand, findPublicSkill } from "./skillInstall"; + +const options = { + gatewayUrl: "https://gateway.example.test/proxy", + archiveUrl: "https://gateway.example.test/proxy/skill-hub/skills/review/archive", + name: "review", + agent: "codex" as const, + global: false, + authHeaderName: "X-LiteLLM-Key", +}; + +describe("skill install commands", () => { + it("quotes the public index name without executing shell substitutions", () => { + const name = "review' $(printf injected) `printf injected`"; + const command = buildSkillInstallCommand({ ...options, global: true, publicSkill: { name } }); + const result = spawnSync("bash", ["-c", `npx() { printf '%s\\n' "$DISABLE_TELEMETRY" "$@"; }; ${command}`], { + encoding: "utf8", + }); + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "1", + "skills@1.7.6", + "add", + options.gatewayUrl, + "--skill", + name, + "-a", + "codex", + "-g", + ]); + }); + + it.each(["success", "fail"])("executes a private install with safe credentials and cleanup on %s", (outcome) => { + const command = buildSkillInstallCommand({ ...options, agent: "claude-code", global: true })!; + const result = spawnSync( + "python3", + [fileURLToPath(new URL("./skillInstall.fixture.py", import.meta.url)), outcome], + { + input: command, + encoding: "utf8", + timeout: 10_000, + }, + ); + expect(result.status, result.stderr).toBe(0); + const expected = { + status: outcome === "success" ? 0 : 22, + header: "X-LiteLLM-Key: Bearer private-fixture-key\n", + secretLeaked: false, + temporaryRemoved: true, + retained: outcome === "success" ? "fixture skill" : null, + telemetry: outcome === "success" ? "1" : null, + }; + const receipt = JSON.parse(result.stdout) as typeof expected & { installArgs: string[] | null }; + expect(receipt).toMatchObject(expected); + expect(receipt.installArgs).toEqual( + outcome === "success" ? expect.arrayContaining(["--skill", "review", "-a", "claude-code", "-g"]) : null, + ); + }); + + it.each([ + "localhost", + "skills-install.localhost", + "127.0.0.2", + "[::1]", + "gateway.example.test", + "127.attacker.test", + "localhost.attacker.test", + ])("requires HTTPS for authenticated installs outside loopback at %s", (host) => { + const local = ["localhost", "skills-install.localhost", "127.0.0.2", "[::1]"].includes(host); + const httpOptions = { + ...options, + gatewayUrl: `http://${host}`, + archiveUrl: `http://${host}/skill-hub/skills/review/archive`, + }; + expect(buildSkillInstallCommand(httpOptions) === null).toBe(!local); + expect(buildSkillInstallCommand({ ...httpOptions, publicSkill: { name: "review" } })).not.toBeNull(); + }); + + it.each([ + { gatewayUrl: "https://user:password@gateway.example.test/proxy" }, + { archiveUrl: "https://other.example.test/skill.zip" }, + { archiveUrl: "https://gateway.example.test/proxy/skill-hub/submissions/pending/archive" }, + { authHeaderName: "Authorization\r\nX-Other: value" }, + { name: "review\nLITELLM_SKILL\nexit" }, + { gatewayUrl: "file:///private/skill" }, + ])("rejects credentials and values outside the managed archive contract", (invalid) => { + expect(buildSkillInstallCommand({ ...options, ...invalid })).toBeNull(); + }); + + it("uses archive identity instead of a colliding public skill name", () => { + const entry = { name: "review-2", type: "archive" as const, description: "Review", digest: "sha256:abc" }; + const wrong = { ...entry, name: "review", url: "https://other.example.test/skill.zip" }; + const right = { ...entry, url: `${options.archiveUrl}?digest=sha256:abc` }; + expect(findPublicSkill({ $schema: "", skills: [wrong, right] }, options.archiveUrl)).toMatchObject({ + name: right.name, + url: right.url, + }); + expect(findPublicSkill({ $schema: "", skills: [wrong] }, options.archiveUrl)).toBeUndefined(); + expect( + findPublicSkill({ skills: [{ ...right, url: `${right.url}&token=credential` }] }, options.archiveUrl), + ).toBeUndefined(); + expect(findPublicSkill({ skills: null }, options.archiveUrl)).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.ts new file mode 100644 index 00000000000..343bc82961e --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.ts @@ -0,0 +1,82 @@ +import { z } from "zod"; + +export const SKILL_AGENTS = [ + { value: "codex", label: "Codex" }, + { value: "claude-code", label: "Claude Code" }, + { value: "cursor", label: "Cursor" }, + { value: "github-copilot", label: "GitHub Copilot" }, + { value: "gemini-cli", label: "Gemini CLI" }, +] as const; + +export type SkillAgent = (typeof SKILL_AGENTS)[number]["value"]; +const skillsPackage = "skills@1.7.6"; +const publicIndexSchema = z.object({ + skills: z.array(z.object({ name: z.string(), type: z.literal("archive"), url: z.string() })), +}); + +const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\"'\"'")}'`; + +const parseUrl = (value: string): URL | null => { + try { + const url = new URL(value); + return /^https?:$/.test(url.protocol) && !url.username && !url.password ? url : null; + } catch { + return null; + } +}; + +export const findPublicSkill = (index: unknown, archiveUrl: string) => { + const parsed = publicIndexSchema.safeParse(index); + return parsed.success + ? parsed.data.skills.find((skill) => { + const url = parseUrl(skill.url); + if (!url || url.hash) return false; + return ( + `${url.origin}${url.pathname}` === archiveUrl && [...url.searchParams.keys()].every((key) => key === "digest") + ); + }) + : undefined; +}; + +interface SkillInstallOptions { + gatewayUrl: string; + archiveUrl: string; + name: string; + agent: SkillAgent; + global: boolean; + publicSkill?: { name: string }; + authHeaderName: string; +} + +export function buildSkillInstallCommand(options: SkillInstallOptions): string | null { + const gateway = parseUrl(options.gatewayUrl); + const archive = parseUrl(options.archiveUrl); + if (!gateway || gateway.search || gateway.hash) return null; + if (!archive || archive.origin !== gateway.origin) return null; + if (!/^[a-z0-9-]+$/.test(options.name)) return null; + if (!/^[!#$%&'*+.^_`|~\da-z-]+$/i.test(options.authHeaderName)) return null; + const gatewayUrl = gateway.href.replace(/\/+$/, ""); + if (archive.href !== `${gatewayUrl}/skill-hub/skills/${options.name}/archive`) return null; + const scope = options.global ? " -g" : ""; + if (options.publicSkill) { + return `DISABLE_TELEMETRY=1 npx ${skillsPackage} add ${shellQuote(gatewayUrl)} --skill ${shellQuote(options.publicSkill.name)} -a ${options.agent}${scope}`; + } + const loopback = + /^(?:localhost|\[::1\]|127(?:\.\d{1,3}){3})$/.test(gateway.hostname) || gateway.hostname.endsWith(".localhost"); + if (gateway.protocol !== "https:" && !loopback) return null; + return `bash <<'LITELLM_SKILL' +set -euo pipefail +set +x +skill_tmp=$(mktemp -d) +trap 'rm -rf "$skill_tmp"' EXIT +IFS= read -r -s -p 'LiteLLM API key: ' skill_key { vi.clearAllMocks(); testQueryClient.clear(); + testQueryClient.setQueryDefaults(["public-skill-install"], { refetchOnWindowFocus: true }); skillRequestMock.mockReset().mockRejectedValue(new Error("Unexpected skill request")); fetchMock.mockReset().mockImplementation((input, init) => { const requestUrl = input instanceof Request ? input.url : String(input); @@ -59,6 +62,7 @@ beforeEach(() => { }); afterEach(() => { + focusManager.setFocused(undefined); setGlobalLitellmHeaderName(); vi.restoreAllMocks(); vi.unstubAllGlobals(); @@ -187,3 +191,74 @@ it.each([skill.source, { source: "archive", url: `${getProxyBaseUrl()}/skill-hub expect(await screen.findByText(/No recorded releases yet/)).toBeInTheDocument(); }, ); + +it("copies the selected private install without sharing a session key and reports clipboard results", async () => { + const user = userEvent.setup(); + const pendingCopy = Promise.withResolvers(); + const writeText = vi.spyOn(navigator.clipboard, "writeText").mockReturnValueOnce(pendingCopy.promise); + setGlobalLitellmHeaderName("X-LiteLLM-Key"); + skillRequestMock.mockResolvedValue(Response.json({ detail: "Not Found" }, { status: 404 })); + renderWithProviders( + , + ); + expect(skillRequestMock).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("button", { name: "Install / Share" })); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Coding agent" }), "Claude Code"); + fireEvent.click(screen.getByRole("radio", { name: "All projects" })); + const command = screen.getByLabelText("Install command"); + expect(command).toHaveTextContent("-a claude-code --copy --yes -g"); + expect(command).toHaveTextContent("X-LiteLLM-Key"); + expect(command).not.toHaveTextContent("dashboard-token"); + expect(screen.queryByRole("radio", { name: "Public command" })).not.toBeInTheDocument(); + expect(skillRequestMock).toHaveBeenCalledWith( + `${getProxyBaseUrl()}/.well-known/agent-skills/index.json`, + expect.objectContaining({ + credentials: "omit", + redirect: "error", + headers: { Accept: "application/json", "Content-Type": "application/json" }, + }), + ); + fireEvent.click(screen.getByRole("button", { name: "Copy install command" })); + expect(writeText).toHaveBeenCalledWith(command.textContent); + expect(screen.queryByRole("button", { name: "Copied" })).not.toBeInTheDocument(); + pendingCopy.resolve(); + const copiedButton = await screen.findByRole("button", { name: "Copied" }); + writeText.mockRejectedValueOnce(new Error("Clipboard denied")); + fireEvent.click(copiedButton); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not copy the command"); +}); + +it("offers the indexed public skill name and discards its availability when reopened", async () => { + const portablePath = "/skill-hub/skills/my-skill/archive"; + skillRequestMock.mockResolvedValueOnce( + Response.json({ + skills: [{ name: "my-skill-2", type: "archive", url: `${getProxyBaseUrl()}${portablePath}?digest=sha256:abc` }], + }), + ); + renderWithProviders( + , + ); + fireEvent.click(screen.getByRole("button", { name: "Install / Share" })); + fireEvent.click(await screen.findByRole("radio", { name: "Public command" })); + focusManager.setFocused(false); + await act(async () => focusManager.setFocused(true)); + expect(skillRequestMock).toHaveBeenCalledTimes(1); + expect(screen.getByLabelText("Install command")).toHaveTextContent( + `DISABLE_TELEMETRY=1 npx skills@1.7.6 add '${getProxyBaseUrl()}' --skill 'my-skill-2' -a codex`, + ); + fireEvent.click(screen.getByRole("button", { name: "Close" })); + const refreshedIndex = Promise.withResolvers(); + skillRequestMock.mockReturnValueOnce(refreshedIndex.promise); + fireEvent.click(screen.getByRole("button", { name: "Install / Share" })); + expect(screen.queryByRole("radio", { name: "Public command" })).not.toBeInTheDocument(); + refreshedIndex.resolve(Response.json({ skills: [] })); + expect(screen.getByLabelText("Install command")).toHaveTextContent("LiteLLM API key"); +}); diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx index efdf4757c11..c519816f65f 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx @@ -19,6 +19,7 @@ import { } from "./helpers"; import type { Plugin, SkillHubSkill } from "./types"; import SkillVersionHistory from "./SkillVersionHistory"; +import SkillInstallDialog from "./SkillInstallDialog"; type SkillDetailData = Pick< SkillHubSkill, @@ -128,6 +129,7 @@ const SkillDetail: React.FC = ({ skill, onBack, accessToken, o const { t } = useTranslation(); const [activeTab, setActiveTab] = useState("overview"); const [copiedKey, setCopiedKey] = useState(null); + const [installOpen, setInstallOpen] = useState(false); const marketplaceRequiresKey = useClaudeCodeMarketplaceRequiresKey(); const visibleTab = activeTab === "downloads" && !isAdmin ? "overview" : activeTab; const portableArchivePath = skill.downloadAvailable @@ -141,6 +143,7 @@ const SkillDetail: React.FC = ({ skill, onBack, accessToken, o getProxyBaseUrl(), typeof window !== "undefined" ? window.location.origin : undefined, ); + const installDialogPath = skill.portable && installOpen ? archivePath : null; const copyToClipboard = (text: string, key: string) => { navigator.clipboard.writeText(text); @@ -199,19 +202,37 @@ const SkillDetail: React.FC = ({ skill, onBack, accessToken, o {/* Header */}
-
+

{skill.name}

- {onEditClick && ( - - )} +
+ {onEditClick && ( + + )} + {skill.portable && archivePath && ( + + )} +
{skill.description && (

{skill.description}

)}
+ {installDialogPath && ( + + )} + {/* Tab bar */}
diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts index c0560c09951..7574500e693 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts @@ -3,7 +3,7 @@ * API request/response shapes are synced from the generated OpenAPI types in @/lib/http/schema. */ -import type { components } from "@/lib/http/schema"; +import type { components, operations } from "@/lib/http/schema"; export interface PluginSource { source: "github" | "url" | "git-subdir" | "archive"; @@ -25,6 +25,8 @@ export type SkillRegisterRequest = Omit; + export type SkillHubSkill = components["schemas"]["SkillHubSkill"]; export type SkillMetadata = components["schemas"]["SkillHubMetadata"]; export type SkillImport = Omit & diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index ef7c7511276..1ffe8f6eaa9 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -625,7 +625,7 @@ describe("Sidebar (leftnav)", () => { renderWithProviders(); - expect(screen.getByRole("link", { name: "Skills" })).toHaveAttribute("href", "/ui/skills"); + expect(screen.getByRole("link", { name: "Skills Hub" })).toHaveAttribute("href", "/ui/skills"); }); it.each([ diff --git a/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts b/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts index e9fae756427..80c21e81e64 100644 --- a/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts +++ b/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ChatCompletion, ChatCompletionCreateParamsNonStreaming } from "openai/resources/chat/completions"; import { createGatewayClient } from "@/components/llm_calls/gateway_client"; import { registerAuthHeaderNameGetter } from "@/lib/http/runtime"; +import i18n from "@/i18n/i18n"; import { getPreferredLiteAdminModel, MAX_INPUT_LENGTH, @@ -889,6 +890,42 @@ describe("SDK conversation and action lifecycle", () => { ); }); +describe("LiteAdmin replies sent back to the model when the UI language is not English", () => { + beforeEach(async () => { + i18n.addResourceBundle( + "de", + "translation", + { aria: { agent: { stepLimit: "Schrittlimit", noAnswer: "Keine Antwort" } } }, + true, + true, + ); + await i18n.changeLanguage("de"); + }); + + afterEach(async () => { + await i18n.changeLanguage("en"); + i18n.removeResourceBundle("de", "translation"); + }); + + it("keeps the step limit reply in English", async () => { + const input = options(); + const model = transport(Array.from({ length: 6 }, () => completion([call("teams_list", teamsArgs)]))); + await runLiteAdmin(input, model.client); + expect(input.onMessage).toHaveBeenCalledExactlyOnceWith( + "I reached the step limit. Any completed actions remain applied; check the relevant page before continuing.", + ); + }); + + it("keeps the no answer reply in English", async () => { + const input = options(); + const model = transport([completion([], null)]); + await runLiteAdmin(input, model.client); + expect(input.onMessage).toHaveBeenCalledExactlyOnceWith( + "The model returned no answer. Try another request or model.", + ); + }); +}); + describe("inference destination", () => { const destinations = [ ["", "/gateway", "https://dashboard.example/ui/", "https://dashboard.example/gateway", false], diff --git a/ui/litellm-dashboard/src/components/liteadmin/agent.ts b/ui/litellm-dashboard/src/components/liteadmin/agent.ts index b9b8420a6e5..9acce836323 100644 --- a/ui/litellm-dashboard/src/components/liteadmin/agent.ts +++ b/ui/litellm-dashboard/src/components/liteadmin/agent.ts @@ -141,8 +141,8 @@ export async function runLiteAdmin(options: LiteAdminOptions, client?: OpenAI): active(); const message = completion.choices[0]?.message; if (message?.tool_calls?.length) { - options.onMessage(i18n.t("aria.agent.stepLimit")); + options.onMessage(i18n.getFixedT("en")("aria.agent.stepLimit")); } else if (!message?.content && !message?.refusal) { - options.onMessage(i18n.t("aria.agent.noAnswer")); + options.onMessage(i18n.getFixedT("en")("aria.agent.noAnswer")); } } diff --git a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx index 3fedeead150..e966836fa3a 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx @@ -82,6 +82,7 @@ const MODELS = [ const SKILLS: Plugin[] = [ { + download_count: 5, id: "skill-research", name: "research-assistant", description: "Answers research questions", @@ -92,11 +93,15 @@ const SKILLS: Plugin[] = [ category: "Analysis", keywords: ["evidence"], author: { name: "Research maintainers" }, + created_by: "writer@example.test", + approved_by: "reviewer@example.test", + security_scan_passed: true, enabled: true, created_at: null, updated_at: null, }, { + download_count: 9, id: "skill-review", name: "review-changes", description: "Checks pull requests", @@ -111,6 +116,7 @@ const SKILLS: Plugin[] = [ updated_at: null, }, { + download_count: 2, id: "skill-release", name: "release-notes", description: "Summarizes shipped changes", @@ -522,18 +528,18 @@ describe("PublicAiHub", () => { expect(screen.getByRole("link", { name: /release-notes/ })).toBeInTheDocument(); expect(screen.getByText("2 skills")).toBeInTheDocument(); - fireEvent.change(screen.getByRole("textbox", { name: "Search skills" }), { target: { value: "changelog" } }); + fireEvent.change(screen.getByRole("textbox", { name: /Search skills/ }), { target: { value: "changelog" } }); expect(screen.getByRole("link", { name: /release-notes/ })).toBeInTheDocument(); expect(screen.queryByRole("link", { name: /review-changes/ })).not.toBeInTheDocument(); - fireEvent.change(screen.getByRole("textbox", { name: "Search skills" }), { target: { value: "evidence" } }); + fireEvent.change(screen.getByRole("textbox", { name: /Search skills/ }), { target: { value: "evidence" } }); expect(screen.getByText('No skills match "evidence"')).toBeInTheDocument(); await user.click(screen.getByRole("button", { name: /^All domains/ })); expect(screen.getByRole("link", { name: /research-assistant/ })).toBeInTheDocument(); expect(screen.queryByRole("link", { name: /release-notes/ })).not.toBeInTheDocument(); - await user.click(screen.getByRole("button", { name: "Clear all" })); + await user.click(screen.getByRole("button", { name: "Clear filters" })); expect(screen.getByText("3 skills")).toBeInTheDocument(); - expect(screen.getByRole("textbox", { name: "Search skills" })).toHaveValue(""); + expect(screen.getByRole("textbox", { name: /Search skills/ })).toHaveValue(""); expect(screen.getByRole("checkbox", { name: "Engineering" })).not.toBeChecked(); expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); }); @@ -541,7 +547,7 @@ describe("PublicAiHub", () => { it("finds skills by name, description, namespace, domain, and keywords", async () => { renderWithProviders(, { searchParams: { tab: "skills" } }); await screen.findByRole("link", { name: /research-assistant/ }); - const search = screen.getByRole("textbox", { name: "Search skills" }); + const search = screen.getByRole("textbox", { name: /Search skills/ }); for (const value of ["research-assistant", "Answers research questions", "knowledge", "Research", "evidence"]) { fireEvent.change(search, { target: { value } }); @@ -586,3 +592,34 @@ describe("PublicAiHub", () => { expect(screen.queryByRole("heading", { level: 2, name: "release-notes" })).not.toBeInTheDocument(); }); }); + +it("filters skills by tag and scan status, searches people, and scopes filters to Skills", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { searchParams: { tab: "skills" }, onUrlUpdate }); + await screen.findByRole("link", { name: /research-assistant/ }); + await user.click(screen.getByRole("button", { name: "Downloads" })); + expect(within(screen.getAllByRole("row")[1]).getByRole("link", { name: /review-changes/ })).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Downloads" })); + expect(within(screen.getAllByRole("row")[1]).getByRole("link", { name: /release-notes/ })).toBeInTheDocument(); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by tag" }), "evidence"); + expect(screen.queryByRole("link", { name: /review-changes/ })).not.toBeInTheDocument(); + expect(screen.getByText("writer@example.test")).toBeInTheDocument(); + expect(screen.getByText("reviewer@example.test")).toBeInTheDocument(); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Security scan" }), "Security scan: No"); + expect(screen.queryByRole("link", { name: /research-assistant/ })).not.toBeInTheDocument(); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Security scan" }), "Security scan: Yes"); + fireEvent.change(screen.getByRole("textbox", { name: /Search skills/ }), { + target: { value: "reviewer@example.test" }, + }); + expect(await screen.findByRole("link", { name: /research-assistant/ })).toBeInTheDocument(); + await waitFor(() => + expect(onUrlUpdate).toHaveBeenLastCalledWith( + expect.objectContaining({ + queryString: expect.stringContaining("skills_filter_tag=evidence"), + }), + ), + ); + await user.click(screen.getByRole("tab", { name: "Model Hub" })); + expect(screen.queryByRole("combobox", { name: "Filter by tag" })).not.toBeInTheDocument(); +}); diff --git a/ui/litellm-dashboard/src/components/publicModelHub/ResourcePanes.tsx b/ui/litellm-dashboard/src/components/publicModelHub/ResourcePanes.tsx index 4827a6fcc35..bcfade1849f 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/ResourcePanes.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/ResourcePanes.tsx @@ -5,17 +5,18 @@ import { Puzzle } from "lucide-react"; import * as React from "react"; import { useTranslation } from "react-i18next"; -import { getSourceDisplayText } from "@/components/claude_code_plugins/helpers"; import type { Plugin } from "@/components/claude_code_plugins/types"; +import { DataTableSortHeader } from "@/components/shared/DataTable/DataTableSortHeader"; import { DataTable } from "@/components/shared/DataTable"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; -import { matchesSearchTerm } from "@/utils/searchUtils"; +import { SkillFilters } from "@/components/claude_code_plugins/SkillFilters"; +import { getSkillMetadataColumns } from "@/components/claude_code_plugins/SkillMetadataColumns"; +import { useSkillFilters, matchesSkillFilters, skillTags } from "@/components/claude_code_plugins/skillCatalog"; import { CatalogDetails } from "./CatalogDetails"; import { CatalogFilters } from "./CatalogFilters"; import { LoadFailed, NoResults, NotListed } from "./HubEmpty"; -import { SearchInput } from "./HubParts"; import { SkillInspector } from "./SkillInspector"; import { useOpenHref } from "./useHubUrlState"; @@ -63,7 +64,8 @@ function SkillLink({ export function SkillsPane({ rows, loading, error, onRetry, selected, onSelect }: SkillsPaneProps) { const { t } = useTranslation(); - const [query, setQuery] = React.useState(""); + const filters = useSkillFilters(); + const query = filters.search; const [domains, setDomains] = React.useState([]); const skills = React.useMemo(() => [...(rows ?? [])].sort((a, b) => a.name.localeCompare(b.name)), [rows]); const options = React.useMemo( @@ -77,65 +79,34 @@ export function SkillsPane({ rows, loading, error, onRetry, selected, onSelect } const shown = React.useMemo( () => skills.filter((skill) => { - const matchesQuery = matchesSearchTerm(query, [ - skill.name, - skill.description, - skill.domain, - skill.namespace, - ...(skill.keywords ?? []), - ]); + const matchesQuery = matchesSkillFilters(skill, { + search: filters.search, + tag: filters.tag, + scan: filters.scan, + }); const matchesDomain = domains.length === 0 || domains.includes(skill.domain ?? ""); return matchesQuery && matchesDomain; }), - [skills, query, domains], + [skills, filters.search, filters.tag, filters.scan, domains], ); const opened = skills.find((skill) => skill.name === selected); const missingName = opened === undefined && !loading ? selected : null; const showRefreshError = error && !loading && shown.length > 0; - const filtered = query.trim() !== "" || domains.length > 0; + const filtered = filters.active || domains.length > 0; const clear = () => { - setQuery(""); + filters.clear(); setDomains([]); }; const columns: ColumnDef[] = [ { accessorKey: "name", - header: t("publicModelHub.catalog.skill", { defaultValue: "Skill" }), + header: ({ column }) => ( + + ), meta: { skeleton: "twoLine", className: "min-w-48 whitespace-normal" }, cell: ({ row }) => , }, - { - id: "source", - header: t("publicModelHub.catalog.source", { defaultValue: "Source" }), - meta: { - className: "hidden max-w-64 text-xs whitespace-normal wrap-anywhere text-muted-foreground lg:table-cell", - }, - cell: ({ row }) => getSourceDisplayText(row.original.source), - }, - { - accessorKey: "namespace", - header: t("publicModelHub.catalog.namespace", { defaultValue: "Namespace" }), - meta: { - className: - "hidden max-w-40 font-mono text-xs whitespace-normal wrap-anywhere text-muted-foreground xl:table-cell", - }, - cell: ({ row }) => row.original.namespace || "-", - }, - { - accessorKey: "category", - header: t("publicModelHub.catalog.category", { defaultValue: "Category" }), - meta: { className: "max-w-40 text-xs whitespace-normal wrap-anywhere text-muted-foreground" }, - cell: ({ row }) => row.original.category || "-", - }, - { - accessorKey: "version", - header: t("publicModelHub.catalog.version", { defaultValue: "Version" }), - meta: { - className: - "hidden max-w-32 font-mono text-xs whitespace-normal wrap-anywhere text-muted-foreground sm:table-cell", - }, - cell: ({ row }) => row.original.version || "-", - }, + ...getSkillMetadataColumns(), ]; const countOptions = { count: shown.length, @@ -183,16 +154,7 @@ export function SkillsPane({ rows, loading, error, onRetry, selected, onSelect }
- - {filtered && ( - - )} +
{loading @@ -203,6 +165,9 @@ export function SkillsPane({ rows, loading, error, onRetry, selected, onSelect }
skill.name} isLoading={loading} diff --git a/ui/litellm-dashboard/src/components/publicModelHub/SkillInspector.tsx b/ui/litellm-dashboard/src/components/publicModelHub/SkillInspector.tsx index 121d352565a..df94b2ea4f8 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/SkillInspector.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/SkillInspector.tsx @@ -1,5 +1,6 @@ "use client"; +import { DateCell } from "@/components/shared/table_cells"; import { ExternalLink } from "lucide-react"; import { useTranslation } from "react-i18next"; @@ -32,8 +33,6 @@ function sourceValue(source: Plugin["source"]) { export function SkillInspector({ skill }: { skill: Plugin }) { const { t } = useTranslation(); const keywords = skill.keywords ?? []; - const addedAt = skill.created_at ? new Date(skill.created_at) : null; - const added = addedAt && !Number.isNaN(addedAt.getTime()) ? addedAt.toLocaleDateString() : null; return (
}, + { label: t("skills.table.columns.updatedAt"), value: }, + { label: t("skills.table.columns.createdBy"), value: skill.created_by || t("skills.table.notRecorded") }, + { + label: t("skills.table.columns.publishedBy"), + value: skill.published_by || t("skills.table.notRecorded"), + }, + { label: t("skills.table.columns.approvedBy"), value: skill.approved_by || t("skills.table.notRecorded") }, + { + label: t("skills.table.columns.securityScan"), + value: ( + + {t(skill.security_scan_passed ? "skills.table.publicYes" : "skills.table.publicNo")} + + ), + }, ], [ { diff --git a/ui/litellm-dashboard/src/i18n/I18nProvider.test.tsx b/ui/litellm-dashboard/src/i18n/I18nProvider.test.tsx new file mode 100644 index 00000000000..1cf0c8971a0 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/I18nProvider.test.tsx @@ -0,0 +1,82 @@ +import { render, screen } from "@testing-library/react"; +import i18next, { type InitOptions } from "i18next"; +import { useState } from "react"; +import { I18nextProvider, initReactI18next, useTranslation } from "react-i18next"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { LanguageGate } from "./I18nProvider"; +import { Locales, Messages } from "./languages"; +import en from "./locales/en.json"; + +const JAPANESE_LOGOUT = "ログアウト"; +const japanese: Messages = { shell: { sidebarAccountMenu: { logout: JAPANESE_LOGOUT } } }; + +const options: InitOptions = { + lng: "en", + fallbackLng: "en", + interpolation: { escapeValue: false }, + resources: { en: { translation: en } }, + initAsync: false, +}; + +const createI18n = () => { + const instance = i18next.createInstance(); + void instance.use(initReactI18next).init(options); + return instance; +}; + +function AccountMenuLabels() { + const { t, i18n } = useTranslation(); + const [labelReadOnce] = useState(() => i18n.t("shell.sidebarAccountMenu.logout")); + return ( + <> +

{t("shell.sidebarAccountMenu.logout")}

+

{labelReadOnce}

+

{t("shell.sidebarAccountMenu.changePassword")}

+ + ); +} + +const renderGate = (stored: string | null, locales: Locales) => { + const instance = createI18n(); + render( + + stored, write: vi.fn() }}> + + + , + ); +}; + +afterEach(() => { + document.documentElement.lang = "en"; + vi.restoreAllMocks(); +}); + +describe("LanguageGate", () => { + it("shows the stored language, with English for strings it hasn't translated", async () => { + renderGate("ja", new Map([["ja", async () => japanese]])); + + expect(await screen.findByTestId("logout")).toHaveTextContent(JAPANESE_LOGOUT); + expect(screen.getByTestId("logout-read-once")).toHaveTextContent(JAPANESE_LOGOUT); + expect(screen.getByTestId("change-password")).toHaveTextContent(en.shell.sidebarAccountMenu.changePassword); + expect(document.documentElement.lang).toBe("ja"); + }); + + it.each([null, "en", "fr"])("shows English without loading anything when the stored value is %j", (stored) => { + const load = vi.fn(async () => japanese); + renderGate(stored, new Map([["ja", load]])); + + expect(screen.getByTestId("logout")).toHaveTextContent(en.shell.sidebarAccountMenu.logout); + expect(load).not.toHaveBeenCalled(); + expect(document.documentElement.lang).toBe("en"); + }); + + it("falls back to English when the translations fail to load", async () => { + vi.spyOn(console, "warn").mockImplementation(() => undefined); + renderGate("ja", new Map([["ja", () => Promise.reject(new Error("offline"))]])); + + expect(await screen.findByTestId("logout")).toHaveTextContent(en.shell.sidebarAccountMenu.logout); + expect(document.documentElement.lang).toBe("en"); + }); +}); diff --git a/ui/litellm-dashboard/src/i18n/I18nProvider.tsx b/ui/litellm-dashboard/src/i18n/I18nProvider.tsx index f0ecdfe2568..f96658fbbb3 100644 --- a/ui/litellm-dashboard/src/i18n/I18nProvider.tsx +++ b/ui/litellm-dashboard/src/i18n/I18nProvider.tsx @@ -1,9 +1,55 @@ "use client"; +import type { i18n as I18n } from "i18next"; +import { useEffect, useState, useSyncExternalStore } from "react"; import { I18nextProvider } from "react-i18next"; +import LoadingScreen from "@/components/common_components/LoadingScreen"; + import i18n from "./i18n"; +import { browserLanguageStore, DEFAULT_LANGUAGE, LanguageStore, Locales, LOCALES, resolveLanguage } from "./languages"; + +type LanguageGateProps = { + i18n: I18n; + locales: Locales; + store: LanguageStore; + children: React.ReactNode; +}; + +const neverChanges = () => () => undefined; + +export function LanguageGate({ i18n, locales, store, children }: LanguageGateProps) { + const language = useSyncExternalStore( + neverChanges, + () => resolveLanguage(store.read(), locales), + () => DEFAULT_LANGUAGE, + ); + const [settled, setSettled] = useState(DEFAULT_LANGUAGE); + + useEffect(() => { + const load = locales.get(language); + if (!load) return; + load() + .then((messages) => { + i18n.addResourceBundle(language, "translation", messages); + return i18n.changeLanguage(language); + }) + .then(() => { + document.documentElement.lang = language; + }) + .catch((error: unknown) => console.warn(`Couldn't load the "${language}" translations`, error)) + .finally(() => setSettled(language)); + }, [i18n, language, locales]); + + return language === settled ? children : ; +} export default function I18nProvider({ children }: { children: React.ReactNode }) { - return {children}; + return ( + + + {children} + + + ); } diff --git a/ui/litellm-dashboard/src/i18n/languages.test.ts b/ui/litellm-dashboard/src/i18n/languages.test.ts new file mode 100644 index 00000000000..5a462d7a52d --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/languages.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; + +import { englishName, Locales, nativeName, resolveLanguage, supportedLanguages } from "./languages"; + +const locales: Locales = new Map([["ja", async () => ({})]]); + +describe("resolveLanguage", () => { + it("uses the stored language when it has translations", () => { + expect(resolveLanguage("ja", locales)).toBe("ja"); + }); + + it.each([null, "", "en", "fr", "JA", "ja-JP", "__proto__"])("shows English when the stored value is %j", (stored) => { + expect(resolveLanguage(stored, locales)).toBe("en"); + }); + + it("shows English when no language besides English has translations", () => { + expect(resolveLanguage("ja", new Map())).toBe("en"); + }); +}); + +describe("supportedLanguages", () => { + it("puts English first, then every language with translations", () => { + expect(supportedLanguages(locales)).toEqual(["en", "ja"]); + expect(supportedLanguages(new Map())).toEqual(["en"]); + }); +}); + +describe("language names", () => { + it("names a language in itself, starting with a capital letter", () => { + const spanish = nativeName("es"); + + expect(spanish.toLocaleLowerCase("es")).toBe( + new Intl.DisplayNames(["es"], { type: "language" }).of("es")?.toLocaleLowerCase("es"), + ); + expect(spanish.charAt(0)).toBe(spanish.charAt(0).toLocaleUpperCase("es")); + }); + + it("names a language in English for the secondary label", () => { + expect(englishName("ja")).toBe(new Intl.DisplayNames(["en"], { type: "language" }).of("ja")); + expect(englishName("ja")).not.toBe(nativeName("ja")); + }); +}); diff --git a/ui/litellm-dashboard/src/i18n/languages.ts b/ui/litellm-dashboard/src/i18n/languages.ts new file mode 100644 index 00000000000..616523a7e54 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/languages.ts @@ -0,0 +1,30 @@ +import { getLocalStorageItem, setLocalStorageItem } from "@/utils/localStorageUtils"; + +export type Messages = { readonly [key: string]: string | Messages }; +export type Locales = ReadonlyMap Promise>; +export type LanguageStore = { readonly read: () => string | null; readonly write: (language: string) => void }; + +export const DEFAULT_LANGUAGE = "en"; +export const LANGUAGE_STORAGE_KEY = "litellm_language"; + +export const LOCALES: Locales = new Map(); + +export const browserLanguageStore: LanguageStore = { + read: () => getLocalStorageItem(LANGUAGE_STORAGE_KEY), + write: (language) => setLocalStorageItem(LANGUAGE_STORAGE_KEY, language), +}; + +export const resolveLanguage = (stored: string | null, locales: Locales): string => + stored !== null && locales.has(stored) ? stored : DEFAULT_LANGUAGE; + +export const supportedLanguages = (locales: Locales): readonly string[] => [DEFAULT_LANGUAGE, ...locales.keys()]; + +const displayName = (language: string, displayLanguage: string): string => + new Intl.DisplayNames([displayLanguage], { type: "language" }).of(language) ?? language; + +export const nativeName = (language: string): string => { + const name = displayName(language, language); + return name.charAt(0).toLocaleUpperCase(language) + name.slice(1); +}; + +export const englishName = (language: string): string => displayName(language, DEFAULT_LANGUAGE); diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index b237a3444e5..c72b9b91ca7 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -1670,10 +1670,32 @@ } }, "skills": { + "install": { + "action": "Install / Share", + "title": "Install {{name}}", + "description": "Choose your coding agent and copy the command to your terminal", + "agent": "Coding agent", + "scope": "Install location", + "project": "Current project", + "global": "All projects", + "method": "Install method", + "authenticated": "Authenticated", + "public": "Public command", + "authenticatedHint": "Enter your own LiteLLM key at the terminal prompt. The gateway checks your access before downloading the skill", + "publicHint": "This skill is listed in the gateway's public index and can be installed without a key", + "command": "Install command", + "authenticatedRequirements": "Requires macOS or Linux with Bash, curl, unzip, Node.js 22.20 or later, and npm. Run from your project directory for a project install", + "publicRequirements": "Requires macOS or Linux with Node.js 22.20 or later and npm. Run from your project directory for a project install", + "shareHint": "Share this command with teammates. It contains no key", + "copy": "Copy install command", + "copied": "Copied", + "copyFailed": "Could not copy the command. Select and copy it manually", + "unavailable": "An install command is unavailable for this gateway URL. Authenticated installs require HTTPS or a localhost gateway" + }, "panel": { - "title": "Skills", - "intro": "Upload or import portable skills. Review proposed changes before publishing them to the Skills Hub.", - "addButton": "+ Add Skill", + "title": "Skills Hub", + "intro": "Register and share reusable skills for Codex, Claude Code, and other agents that support the open Agent Skills format.", + "addButton": "+ Register your skill", "deleteDialog": { "title": "Delete Skill", "confirm": "Are you sure you want to delete skill: ?", @@ -1696,8 +1718,8 @@ }, "table": { "empty": { - "title": "No skills found", - "description": "Add one to get started." + "title": "No skills registered yet", + "description": "Register your first skill to get started." }, "loading": "Loading skills…", "uncategorized": "Uncategorized", @@ -1717,15 +1739,22 @@ "description": "Description", "category": "Category", "public": "Public", - "createdAt": "Created At", + "createdAt": "Created", "actions": "Actions", "domain": "Domain", - "updatedAt": "Updated at", - "downloads": "Downloads" - } + "updatedAt": "Last updated", + "downloads": "Downloads", + "tags": "Tags", + "createdBy": "Created by", + "publishedBy": "Published by", + "approvedBy": "Approved by", + "securityScan": "Security scan" + }, + "notRecorded": "Not recorded", + "scanHelp": "Yes means this published version has a completed scan with no findings and a risk score of 0. No means it has not passed a recorded scan." }, "addForm": { - "title": "Add New Skill", + "title": "Register your skill", "sourceOptions": { "url": "Source URL", "upload": "Upload ZIP" @@ -1742,22 +1771,22 @@ "namespace": "Namespace (Optional)", "description": "Description (Optional)", "category": "Category (Optional)", - "keywords": "Keywords (Optional)", + "keywords": "Tags (Optional)", "version": "Version (Optional)", "authorName": "Author Name (Optional)", "authorEmail": "Author Email (Optional)" }, "hints": { "archive": "Choose a ZIP from your computer, up to 10 MiB. Include SKILL.md at the root or inside one skill folder, along with any supporting files. The skill will use the same visibility settings as skills added by URL.", - "sourceUrl": "Paste an HTTPS git repository URL from GitHub, GitLab, Bitbucket, or a self-hosted host (e.g. github.com/org/repo or github.com/org/repo/tree/main/my-skill), or an HTTPS link to a .zip archive of the skill hosted on S3 or any static file server. For a private repository use its SSH clone URL (git@ghe.example.com:org/repo.git) so Claude Code clones it with your own SSH key.", - "subPath": "Path within the repository where the skill lives (e.g., plugins/my-skill). Leave empty if the skill is at the repo root.", - "sha256": "Hex digest of the zip file. Claude Code refuses to install the archive if its checksum does not match.", + "sourceUrl": "Link to a Git repository, a skill folder, or a ZIP archive containing SKILL.md and any supporting files. For private repositories, use an SSH clone URL with access configured in your agent.", + "subPath": "Path within the repository where the skill lives (e.g., skills/my-skill). Leave empty if the skill is at the repo root.", + "sha256": "Optional SHA-256 digest for verifying the ZIP archive’s integrity.", "name": "Unique identifier in kebab-case format (e.g., my-skill)", - "domain": "Top-level grouping in the Skill Hub (e.g., Productivity)", + "domain": "Top-level grouping in the Skills Hub (e.g., Productivity)", "namespace": "Sub-grouping within domain (e.g., workflows)", "description": "Brief description of what the skill does", "category": "Select a category or enter a custom one", - "keywords": "Comma-separated list of keywords for search", + "keywords": "Comma-separated list of tags for search", "version": "Semantic version (e.g., 1.0.0)", "authorName": "Name of the skill author or organization", "authorEmail": "Contact email for the skill author" @@ -1778,13 +1807,13 @@ "detected": "Detected: {{source}}", "noCategories": "No matching categories", "cancel": "Cancel", - "adding": "Adding...", - "submit": "Add Skill", + "adding": "Registering...", + "submit": "Register skill", "errors": { "zipRequired": "Please choose a .zip file", "zipTooLarge": "The ZIP file must be 10 MiB or smaller", "urlRequired": "Please enter a repository or zip archive URL", - "subPathInvalid": "Subfolder must be a relative path like plugins/my-skill (letters, numbers, dots, hyphens, underscores)", + "subPathInvalid": "Subfolder must be a relative path like skills/my-skill (letters, numbers, dots, hyphens, underscores)", "sha256Invalid": "SHA-256 must be a 64-character hex digest", "nameRequired": "Please enter skill name", "nameInvalid": "Name must be kebab-case (lowercase, numbers, hyphens only)", @@ -1822,7 +1851,7 @@ "path": "Path (optional)", "domain": "Domain (optional)", "namespace": "Namespace (optional)", - "keywords": "Keywords (optional)" + "keywords": "Tags (optional)" }, "uploadHint": "Choose a ZIP up to 10 MiB containing SKILL.md. Its frontmatter supplies the skill name and description.", "replacementHint": "The replacement SKILL.md supplies the description. Domain, namespace, and keywords are preserved.", @@ -1899,7 +1928,7 @@ "source": "Source", "author": "Author", "homepage": "Homepage", - "keywords": "Keywords", + "keywords": "Tags", "category": "Category", "domain": "Domain", "namespace": "Namespace", @@ -1912,6 +1941,15 @@ "detail": { "subtitle": "Published skill metadata and portable archive", "private": "Private" + }, + "filters": { + "allTags": "All tags", + "tag": "Filter by tag", + "allScans": "All scan statuses", + "scanned": "Security scan: Yes", + "notScanned": "Security scan: No", + "search": "Search skills, tags, or people…", + "clear": "Clear filters" } }, "claudeCodePlugins": { @@ -1940,7 +1978,7 @@ "newer": "Newer", "older": "Older" }, - "back": "Skills", + "back": "Skills Hub", "tabs": { "overview": "Overview", "usage": "How to Use", @@ -1994,14 +2032,14 @@ "copy": "Copy", "copied": "Copied", "usage": { - "title": "Using this skill", + "title": "Use with Claude Code", "intro": "Once your proxy is set as a marketplace, enable this skill in Claude Code with one command:", "staleCatalog": "If you see \"Plugin {{name}} not found in marketplace\", update the catalog first:", "notConfigured": "Don't have the marketplace configured yet?", "seeSetup": "See one-time setup →" }, "setup": { - "title": "One-time marketplace setup", + "title": "Claude Code marketplace setup", "removeFirst": "If you added this marketplace before without a key, remove it first in Claude Code. Removing it also uninstalls the plugins you installed from it, so install them again from /plugin after you add the snippet:", "requiresKey": "This proxy requires a LiteLLM key to read the marketplace. Add this to and replace with your key:", "runCommand": "Run this command in Claude Code to register the marketplace:", @@ -2009,7 +2047,7 @@ } }, "makePublic": { - "title": "Publish to Skill Hub", + "title": "Publish to Skills Hub", "steps": { "select": "Select Skills", "confirm": "Confirm" @@ -2017,15 +2055,15 @@ "step1": { "title": "Select Skills to Publish", "selectAll": "Select All ({{total}})", - "description": "Selected skills will be visible to all users in the Skill Hub. Deselected skills will be unpublished.", + "description": "Selected skills will be visible to all users in the Skills Hub. Deselected skills will be unpublished.", "empty": "No skills registered yet.", "public": "Public", "selectedCount_one": "{{count}} skill will be published", "selectedCount_other": "{{count}} skills will be published" }, "step2": { - "title": "Confirm Publish to Skill Hub", - "note": "Note: Published skills will be visible to all users in the Skill Hub tab. Skills not in the list below will be unpublished.", + "title": "Confirm Publish to Skills Hub", + "note": "Note: Published skills will be visible to all users in the Skills Hub tab. Skills not in the list below will be unpublished.", "listTitle": "Skills to be published:", "total_one": "Total: {{count}} skill will be published", "total_other": "Total: {{count}} skills will be published" @@ -2038,8 +2076,8 @@ }, "toasts": { "selectAtLeastOne": "Please select at least one skill", - "published_one": "Skill Hub updated — {{count}} skill(s) published", - "published_other": "Skill Hub updated — {{count}} skill(s) published", + "published_one": "Skills Hub updated — {{count}} skill(s) published", + "published_other": "Skills Hub updated — {{count}} skill(s) published", "publishFailed": "Failed to update skills. Please try again." } } @@ -5710,6 +5748,151 @@ "userBanner": { "dismiss": "Dismiss banner" }, + "alerts": { + "page": { + "tabs": { + "modelDeprecations": "Model deprecations" + } + }, + "panel": { + "title": "Model deprecations", + "adminRequired": "Gateway admin access is required to view affected keys and teams.", + "back": "← Model deprecations", + "loadError": "Could not load alerts. No alert status has been changed.", + "retry": "Retry", + "loadingAlert": "Loading alert…", + "missingAlert": "No matching model alert was found in the last 30 days." + }, + "status": { + "open": "open", + "resolved": "resolved", + "dismissed": "dismissed" + }, + "inbox": { + "columns": { + "model": "Model", + "deprecation": "Deprecation", + "requests": "Requests · 30d", + "lastSeen": "Last seen (UTC)" + }, + "empty": { + "open": "No open model deprecations in the last 30 days.", + "resolved": "No resolved model deprecations in the last 30 days.", + "dismissed": "No dismissed model deprecations in the last 30 days." + }, + "reviewImpact": "Review impact →", + "loading": "Loading model deprecations…", + "searchPlaceholder": "Search models…" + }, + "timing": { + "deprecatesIn_one": "Deprecates in {{count}} day", + "deprecatesIn_other": "Deprecates in {{count}} days", + "today": "Deprecates today", + "deprecatedAgo_one": "Deprecated {{count}} day ago", + "deprecatedAgo_other": "Deprecated {{count}} days ago" + }, + "recommendation": { + "sources": { + "modelCostMap": "LiteLLM model cost map", + "deploymentConfig": "Deployment configuration" + }, + "notProvided": "Not provided", + "recommendedModel": "Recommended new model:", + "source": "Source:", + "providerNotice": "Provider deprecation notice ↗" + }, + "alertItem": { + "deprecatedTitle": "{{model}} is deprecated", + "retiresTitle": "{{model}} retires in {{days}} days", + "usage": "{{requests}} requests in the last 30 days · Last seen {{date}} UTC", + "reviewMigration": "Review migration" + }, + "emailSetup": { + "steps": { + "connect": { + "title": "Connect email delivery", + "description": "Add your SMTP settings and public gateway URL in Email Settings. Send a test email to verify delivery." + }, + "enable": { + "title": "Enable model deprecation emails", + "description": "In Email events, turn on Model Deprecations and save. This is off by default." + }, + "reminders": { + "title": "Owners get two reminders", + "description": "Affected key owners and team admins receive a reminder 30 days before retirement, and one final reminder 1 day before. Recipients are based on actual usage in the last 30 days. Dates are checked in UTC." + } + }, + "summary": "Give affected owners time to migrate. Email reminders at 30 days and 1 day.", + "trigger": "Set up email reminders", + "title": "Set up model deprecation emails", + "description": "Help key owners and team admins act before a provider retires a model. Only their affected usage is included.", + "openSettings": "Open Email Settings" + }, + "migrationDetail": { + "headings": { + "resolved": "Migration marked resolved", + "dismissed": "Model deprecation dismissed", + "deprecated": "Migration needed: model is deprecated", + "due": "Migration due {{date}}" + }, + "columns": { + "key": "Virtual key", + "team": "Team", + "owner": "Owner", + "tags": "Usage tags", + "requests": "Requests · 30d", + "spend": "Spend · 30d" + }, + "summary": { + "teamKeys": "Keys in this team", + "affectedKeys": "Affected keys", + "teams": "Teams using this model", + "deprecation": "Deprecation" + }, + "keysTable": { + "title": "Affected keys", + "description": "Keys and owners that used this model in the last 30 days. Tags show actual usage with this model.", + "teamFilter": "Filter affected keys by team", + "allTeams": "All teams", + "teamOption": "{{team}} · {{keys}} keys", + "loadError": "Could not load affected keys.", + "retry": "Retry" + }, + "unnamedKey": "Unnamed key", + "noTeam": "No team", + "notRecorded": "Not recorded", + "followUpCopied": "Follow-up copied", + "lastSeen": "Last seen {{date}} UTC · {{requests}} requests in the last 30 days", + "copyFollowUp": "Copy follow-up", + "markResolved": "Mark resolved", + "dismiss": "Dismiss", + "reopen": "Reopen", + "modelRouting": "Model routing", + "resolveHint": "Resolve across all teams once owners confirm migration. New traffic reopens the alert.", + "saveError": "Could not save alert status. Please retry.", + "teamsError": "Could not load team filters. The keys table still includes all matching teams." + }, + "usageChart": { + "keyFallback": "Key · {{id}}", + "loadError": "Could not load daily requests.", + "retry": "Retry", + "showFewer": "Show fewer key labels", + "showAll": "Show all key labels", + "title": "Daily requests", + "subtitleTeam": "Requests to this model, stacked by key · Selected team", + "subtitleAll": "Requests to this model, stacked by key · All teams", + "footnote": "Last 30 days · UTC · Includes zero-usage days. Today is partial." + }, + "usageTags": { + "loading": "Loading tags…", + "failed": "Tags unavailable", + "none": "No recorded tags", + "showFewer": "Show fewer usage tags", + "showMore": "Show {{hidden}} more usage tags", + "showLess": "Show less", + "more": "+{{hidden}} more" + } + }, "caching": { "dashboard": { "tabs": { @@ -6935,7 +7118,7 @@ "models": "Model Hub", "agents": "Agent Hub", "mcp": "MCP Hub", - "skills": "Skill Hub" + "skills": "Skills Hub" }, "models": { "title": "Available Models", @@ -7098,7 +7281,7 @@ "models": "Model Hub", "agents": "Agent Hub", "mcp": "MCP Hub", - "skills": "Skill Hub" + "skills": "Skills Hub" }, "models": { "chatTitle": "Models", @@ -7521,7 +7704,7 @@ "agents": "Agents", "mcpShort": "MCP", "mcpServers": "MCP servers", - "skills": "Skills" + "skills": "Skills Hub" }, "header": { "title": "AI Hub", @@ -7787,7 +7970,7 @@ "workflowRuns": "Workflow Runs", "memory": "Memory", "mcpServers": "MCP Servers", - "skills": "Skills", + "skills": "Skills Hub", "guardrails": "Guardrails", "policies": "Policies", "tools": "Tools", @@ -7862,7 +8045,7 @@ "uiTheme": "Customize dashboard appearance", "tagManagement": "Organize resources with tags", "prompts": "Manage and version prompt templates", - "skills": "Browse and manage Claude Code skills", + "skills": "Register and share reusable agent skills in the Skills Hub", "routerSettings": "Configure routing and load balancing settings", "telemetry": "Choose which anonymous usage telemetry this proxy collects and see the last report", "loggingAndAlerts": "Set up logging and alert configurations", @@ -7902,6 +8085,10 @@ "copyUserId": "Copy user ID", "hideAria": "Hide ", "toggleHideAria": "Toggle hide {{name}}", + "language": { + "label": "Language", + "fallbackNote": "Missing translations show in English. Help translate" + }, "changePassword": "Change Password", "logout": "Logout" }, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d14e490f606..d319252506d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29442,7 +29442,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "acreate_realtime_client_secret" | "arealtime_calls" | "acreate_realtime_transcription_session" | "acreate_realtime_translation_client_secret" | "arealtime_translation_calls" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "get_responses" | "aget_responses" | "delete_responses" | "adelete_responses" | "cancel_responses" | "acancel_responses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "acreate_realtime_client_secret" | "arealtime_calls" | "acreate_realtime_transcription_session" | "acreate_realtime_translation_client_secret" | "arealtime_translation_calls" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "get_responses" | "aget_responses" | "delete_responses" | "adelete_responses" | "cancel_responses" | "acancel_responses" | "compact_responses" | "acompact_responses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ @@ -30791,6 +30791,16 @@ export interface components { * @description If True, lets keys address Responses API ids that this proxy did not issue (raw provider ids, or ids issued before response-id encryption was configured). Such an id carries no owner, so no ownership check can run on it; ids this proxy did issue keep full ownership enforcement. Off by default, in which case an unrecognized response id is rejected with 403 */ allow_unmanaged_response_ids?: boolean | null; + /** + * Allowed Client Side S3 Buckets + * @description S3 bucket names a client may send as `s3_bucket_name` or `s3_output_bucket_name` in a request body, e.g. on /v1/batches. Any other bucket is rejected unless client-side credentials are allowed + */ + allowed_client_side_s3_buckets?: string[] | null; + /** + * Allowed Client Side S3 Endpoint Urls + * @description Exact `s3_endpoint_url` values a client may send in a request body. Any other value is rejected unless client-side credentials are allowed + */ + allowed_client_side_s3_endpoint_urls?: string[] | null; /** * Allowed File Extensions * @description the only file extensions (e.g. ['.jsonl', '.pdf', '.txt']) accepted on /v1/files uploads, for any purpose, matched case-insensitively against the uploaded filename. Files with any other extension, or none, are rejected. An empty list rejects every upload. Unset means no allowlist is applied @@ -36286,6 +36296,8 @@ export interface components { * @default false */ allow_client_keepalive_override: boolean | null; + /** Allow Legacy Cloud File Ids */ + allow_legacy_cloud_file_ids?: boolean | null; /** Annotation Cost Per Page */ annotation_cost_per_page?: number | null; /** Annotation Cost Per Page Batches */ @@ -42114,15 +42126,21 @@ export interface components { * @description Plugin item in list responses. */ PluginListItem: { + /** Approved By */ + approved_by?: string | null; author?: components["schemas"]["PluginAuthor"] | null; /** Category */ category?: string | null; /** Created At */ created_at: string | null; + /** Created By */ + created_by?: string | null; /** Description */ description: string | null; /** Domain */ domain?: string | null; + /** Download Count */ + download_count?: number | null; /** Enabled */ enabled: boolean; /** Homepage */ @@ -42137,6 +42155,13 @@ export interface components { name: string; /** Namespace */ namespace?: string | null; + /** Published By */ + published_by?: string | null; + /** + * Security Scan Passed + * @default false + */ + security_scan_passed: boolean; /** Source */ source: { [key: string]: string; @@ -46920,6 +46945,11 @@ export interface components { next_offset: number | null; /** Skills */ skills: components["schemas"]["SkillHubSkill"][]; + /** + * Tags + * @default [] + */ + tags: string[]; }; /** SkillHubMetadata */ SkillHubMetadata: { @@ -47005,15 +47035,21 @@ export interface components { }; /** SkillHubSkill */ SkillHubSkill: { + /** Approved By */ + approved_by?: string | null; /** Content Url */ content_url?: string | null; + /** Created At */ + created_at?: string | null; + /** Created By */ + created_by?: string | null; /** Description */ description?: string | null; /** Domain */ domain?: string | null; /** * Download Count - * @description Gateway ZIP downloads, included in proxy-admin catalog listings + * @description Gateway ZIP downloads, including repeat downloads */ download_count?: number | null; /** Download Url */ @@ -47026,6 +47062,13 @@ export interface components { name: string; /** Namespace */ namespace?: string | null; + /** Published By */ + published_by?: string | null; + /** + * Security Scan Passed + * @default false + */ + security_scan_passed: boolean; /** Source */ source: { [key: string]: string; @@ -52350,6 +52393,8 @@ export interface components { * @default false */ allow_client_keepalive_override: boolean | null; + /** Allow Legacy Cloud File Ids */ + allow_legacy_cloud_file_ids?: boolean | null; /** Annotation Cost Per Page */ annotation_cost_per_page?: number | null; /** Annotation Cost Per Page Batches */ @@ -74391,6 +74436,10 @@ export interface operations { query?: string; offset?: number; limit?: number; + tag?: string; + security_scan_passed?: boolean | null; + sort_by?: "name" | "download_count" | "updated_at" | "created_at" | "created_by" | "published_by" | "approved_by" | "security_scan_passed"; + sort_order?: "asc" | "desc"; }; header?: never; path?: never; diff --git a/uv.lock b/uv.lock index 954b9f90a75..fb591264fc3 100644 --- a/uv.lock +++ b/uv.lock @@ -22,6 +22,7 @@ pypdf = "2026-10-28T17:01:00Z" members = [ "litellm", "litellm-enterprise", + "litellm-proxy", "litellm-proxy-extras", ] constraints = [ @@ -5526,6 +5527,11 @@ name = "litellm-enterprise" version = "0.1.81" source = { editable = "enterprise" } +[[package]] +name = "litellm-proxy" +version = "1.109.0" +source = { editable = "packaging/litellm-proxy" } + [[package]] name = "litellm-proxy-extras" version = "0.4.112" diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json index 80b868e528b..bea38102884 100644 --- a/vscode-extension/package-lock.json +++ b/vscode-extension/package-lock.json @@ -128,9 +128,9 @@ } }, "node_modules/@azure/identity": { - "version": "4.13.3", - "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.13.3.tgz", - "integrity": "sha512-zGQPtqvXPgSA8yfV2CkIQ1qirqk0p9AIVpC5uEkdXQYcKl07QHvyaGYRnZOk0AsQUmxNb4wfkcwY5di8Z5xa9A==", + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.14.1.tgz", + "integrity": "sha512-HHuuFrmN1uuF9D5A6aAo5eHzNFNX4Q+P4SksZqDKse5YaJFFy/zDLG3wXHEV2FVGgetZ3QM8jGURNBtnQYf1NQ==", "dev": true, "license": "MIT", "dependencies": { @@ -143,8 +143,8 @@ "@azure/core-util": "^1.11.0", "@azure/logger": "^1.0.0", "@azure/msal-browser": "^5.5.0", - "@azure/msal-node": "^6.0.0", - "open": "^10.1.0", + "@azure/msal-node": "^7.1.0", + "open": "^11.0.0", "tslib": "^2.2.0" }, "engines": { @@ -189,19 +189,29 @@ } }, "node_modules/@azure/msal-node": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-6.0.1.tgz", - "integrity": "sha512-ixSO1Y/kCVRthRs+hSx/5qkwaunX1/RAePhlMN0wIpIQ4WEZ6AREGGnGd1AP0qspHVsAwzWQKGubpjh50JyJIQ==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-7.2.0.tgz", + "integrity": "sha512-Y87VJsMWF4jiqLqq8XYLb1o+rGc/HacwS3Zydi3bDtaVm5Sa8FlB7YrKrBFKRIYj27RJXAaDkErImMAqtM10Pg==", "dev": true, "license": "MIT", "dependencies": { - "@azure/msal-common": "16.14.1", + "@azure/msal-common": "16.16.0", "jsonwebtoken": "^9.0.0" }, "engines": { "node": ">=20" } }, + "node_modules/@azure/msal-node/node_modules/@azure/msal-common": { + "version": "16.16.0", + "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-16.16.0.tgz", + "integrity": "sha512-EZgd8h5nczsdRAQAgXA0A0pC5L/zj43n1jePXvUMgdwXMbH2zhb7b7Hp+Oe7ZOODK/sch2x+soD6FOrtgUbx/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", @@ -2219,6 +2229,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-in-ssh": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-in-ssh/-/is-in-ssh-1.0.0.tgz", + "integrity": "sha512-jYa6Q9rH90kR1vKB6NM7qqd1mge3Fx4Dhw5TVlK1MUBqhEOuCagrEHMevNuCcbECmXZ0ThXkRm+Ymr51HwEPAw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-inside-container": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", @@ -2784,19 +2807,21 @@ } }, "node_modules/open": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", - "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==", + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/open/-/open-11.0.4.tgz", + "integrity": "sha512-++Zlftm0kVLPmzC06t6epuWmcRMDbI4z5P3NNX979WA/k23+NtSOynEGzsVfZwguKw2mi5umVgnBlJQMwRz4Pg==", "dev": true, "license": "MIT", "dependencies": { - "default-browser": "^5.2.1", + "default-browser": "^5.5.1", "define-lazy-prop": "^3.0.0", + "is-in-ssh": "^1.0.0", "is-inside-container": "^1.0.0", - "wsl-utils": "^0.1.0" + "powershell-utils": "^0.2.1", + "wsl-utils": "^1.0.0" }, "engines": { - "node": ">=18" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -2915,6 +2940,19 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/powershell-utils": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/powershell-utils/-/powershell-utils-0.2.1.tgz", + "integrity": "sha512-C+y9x90UElAddDZmV4qOx9W53B61PO7cIqWz2dQsWlwswuq4mr8NEwytdGKboYbQlGZ3awrkTeNvcZiZNHnQ8A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/proper-lockfile": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", @@ -3505,16 +3543,17 @@ } }, "node_modules/wsl-utils": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz", - "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-1.0.1.tgz", + "integrity": "sha512-c16PKLIRxBYpR3jSWsdS7SFp0COm5SOQdiFjY818dYFHJbMGuLBszpF07LBWmOS1uzrHa9VGdynsEgoeKy0oOQ==", "dev": true, "license": "MIT", "dependencies": { - "is-wsl": "^3.1.0" + "is-wsl": "^3.1.0", + "powershell-utils": "^0.2.0" }, "engines": { - "node": ">=18" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus"