diff --git a/.github/actions/helm-oci-chart-releaser/action.yml b/.github/actions/helm-oci-chart-releaser/action.yml index 059277ed882..1823e262832 100644 --- a/.github/actions/helm-oci-chart-releaser/action.yml +++ b/.github/actions/helm-oci-chart-releaser/action.yml @@ -40,38 +40,33 @@ outputs: runs: using: composite steps: + - name: Helm | Setup + uses: azure/setup-helm@v4 + with: + version: v3.20.0 + - name: Helm | Login shell: bash run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }} - env: - HELM_EXPERIMENTAL_OCI: '1' - + - name: Helm | Dependency if: inputs.update_dependencies == 'true' shell: bash run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} - env: - HELM_EXPERIMENTAL_OCI: '1' - name: Helm | Package shell: bash run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }} - env: - HELM_EXPERIMENTAL_OCI: '1' - name: Helm | Push shell: bash run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }} - env: - HELM_EXPERIMENTAL_OCI: '1' - name: Helm | Logout shell: bash run: helm registry logout ${{ inputs.registry }} - env: - HELM_EXPERIMENTAL_OCI: '1' - name: Helm | Output id: output shell: bash - run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT \ No newline at end of file + run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml new file mode 100644 index 00000000000..d83fedcb2ae --- /dev/null +++ b/.github/workflows/test-litellm-matrix.yml @@ -0,0 +1,95 @@ +name: LiteLLM Unit Tests (Matrix) + +on: + pull_request: + branches: [main] + +# Cancel in-progress runs for the same PR +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + test-group: + # tests/test_litellm split by subdirectory (~560 files total) + - name: "llms" + path: "tests/test_litellm/llms" + workers: 4 + # tests/test_litellm/proxy split by subdirectory (~180 files total) + - name: "proxy-guardrails" + path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers" + workers: 4 + - name: "proxy-core" + path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine" + workers: 4 + - name: "proxy-misc" + path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py" + workers: 4 + - name: "integrations" + path: "tests/test_litellm/integrations" + workers: 4 + - name: "core-utils" + path: "tests/test_litellm/litellm_core_utils" + workers: 2 + - name: "other" + path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types" + workers: 4 + - name: "root" + path: "tests/test_litellm/test_*.py" + workers: 4 + # tests/proxy_unit_tests split alphabetically (~48 files total) + - name: "proxy-unit-a" + path: "tests/proxy_unit_tests/test_[a-o]*.py" + workers: 2 + - name: "proxy-unit-b" + path: "tests/proxy_unit_tests/test_[p-z]*.py" + workers: 2 + + name: test (${{ matrix.test-group.name }}) + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Poetry + uses: snok/install-poetry@v1 + + - name: Cache Poetry dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install pytest-retry==1.6.3 pytest-xdist google-genai==1.22.0 \ + google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core + + - name: Setup litellm-enterprise + run: | + cd enterprise && poetry run pip install -e . && cd .. + + - name: Run tests - ${{ matrix.test-group.name }} + run: | + poetry run pytest ${{ matrix.test-group.path }} \ + --tb=short -vv \ + --maxfail=10 \ + -n ${{ matrix.test-group.workers }} \ + --durations=20 diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index d9cf2e74a11..dc9b48c28f6 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -1,8 +1,12 @@ name: LiteLLM Mock Tests (folder - tests/test_litellm) +# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs +# the same tests in parallel across 10 jobs for faster CI times. +# Kept for manual debugging only. on: - pull_request: - branches: [ main ] + workflow_dispatch: # Manual trigger only + # pull_request: + # branches: [ main ] jobs: test: diff --git a/.semgrep/rules/README.md b/.semgrep/rules/README.md index 6cffcc32963..0dbb77cdd48 100644 --- a/.semgrep/rules/README.md +++ b/.semgrep/rules/README.md @@ -1,52 +1,22 @@ -# Custom Semgrep Rules +# Custom Semgrep rules for LiteLLM -All `.yml` files under `.semgrep/rules/` run in CI (CircleCI `semgrep` job). +Add custom rule YAML files here. Semgrep loads all `.yml`/`.yaml` files under this directory. -## Add a Rule - -* Add a `.yml` file under `.semgrep/rules///` - - -[Rule syntax →](https://semgrep.dev/docs/writing-rules/rule-syntax/) - -## Organizing Rules - -### Structure: language → domain - -``` -.semgrep/rules///.yml -``` - -Examples: - -- `python/security/unsafe-yaml-load.yml` -- `python/reliability/missing-timeout-http.yml` -- `python/performance/blocking-io-in-async.yml` - -### Rule metadata - -Match tags to the folder for consistent filtering: - -```yaml -metadata: - tags: [python, security] -``` - -### Severity expectations - -All rules must fail CI on findings. No warn-only rules. - -- Use `severity: ERROR` in rule metadata -- If a rule is noisy → refine until low false positives before adding - -## Run Locally +**Run only custom rules (CI / fail on findings):** ```bash semgrep scan --config .semgrep/rules . --error ``` -With Semgrep registry: +**Run with registry + custom rules:** ```bash semgrep scan --config auto --config .semgrep/rules . ``` + +**Layout:** + +- `python/` – Python-specific rules (security, patterns) +- Add more subdirs as needed (e.g. `generic/` for language-agnostic rules) + +See [Semgrep rule syntax](https://semgrep.dev/docs/writing-rules/rule-syntax/). diff --git a/.semgrep/rules/python/unbounded-memory.yml b/.semgrep/rules/python/unbounded-memory.yml new file mode 100644 index 00000000000..811ef689344 --- /dev/null +++ b/.semgrep/rules/python/unbounded-memory.yml @@ -0,0 +1,14 @@ +# Unbounded memory growth – data structures without a clear max limit +# Can lead to OOM under load. + +rules: + - id: unbounded-asyncio-queue + message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues). + severity: ERROR + languages: [python] + pattern-either: + - pattern: asyncio.Queue() + - pattern: asyncio.Queue(maxsize=0) + metadata: + category: correctness + cwe: "CWE-400: Uncontrolled Resource Consumption" \ No newline at end of file diff --git a/Makefile b/Makefile index b867d7ea35e..74031f418d6 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ # LiteLLM Makefile # Simple Makefile for running tests and basic development tasks -.PHONY: help test test-unit test-integration test-unit-helm \ +.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \ + test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \ + test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ info lint lint-dev format \ install-dev install-proxy-dev install-test-deps \ install-helm-unittest check-circular-imports check-import-safety @@ -25,6 +27,16 @@ help: @echo " make check-import-safety - Check import safety" @echo " make test - Run all tests" @echo " make test-unit - Run unit tests (tests/test_litellm)" + @echo " make test-unit-llms - Run LLM provider tests (~225 files)" + @echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)" + @echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)" + @echo " make test-unit-proxy-misc - Run proxy misc tests (~77 files)" + @echo " make test-unit-integrations - Run integration tests (~60 files)" + @echo " make test-unit-core-utils - Run core utils tests (~32 files)" + @echo " make test-unit-other - Run other tests (caching, responses, etc., ~69 files)" + @echo " make test-unit-root - Run root-level tests (~34 files)" + @echo " make test-proxy-unit-a - Run proxy_unit_tests (a-o, ~20 files)" + @echo " make test-proxy-unit-b - Run proxy_unit_tests (p-z, ~28 files)" @echo " make test-integration - Run integration tests" @echo " make test-unit-helm - Run helm unit tests" @@ -129,6 +141,38 @@ test: test-unit: install-test-deps poetry run pytest tests/test_litellm -x -vv -n 4 +# Matrix test targets (matching CI workflow groups) +test-unit-llms: install-test-deps + poetry run pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-guardrails: install-test-deps + poetry run pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-core: install-test-deps + poetry run pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-misc: install-test-deps + poetry run pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 + +test-unit-integrations: install-test-deps + poetry run pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20 + +test-unit-core-utils: install-test-deps + poetry run pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20 + +test-unit-other: install-test-deps + poetry run pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20 + +test-unit-root: install-test-deps + poetry run pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20 + +# Proxy unit tests (tests/proxy_unit_tests split alphabetically) +test-proxy-unit-a: install-test-deps + poetry run pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20 + +test-proxy-unit-b: install-test-deps + poetry run pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20 + test-integration: poetry run pytest tests/ -k "not test_litellm" diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index 8a08f0b4e29..0f6db331e50 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -26,6 +26,10 @@ version: 1.1.0 # It is recommended to use it with quotes. appVersion: v1.80.12 +annotations: + org.opencontainers.image.source: "https://github.com/BerriAI/litellm" + org.opencontainers.image.url: "https://docs.litellm.ai/" + dependencies: - name: "postgresql" version: ">=13.3.0" diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 64126bb0292..004377e19b3 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -59,7 +59,8 @@ RUN mkdir -p /var/lib/litellm/ui && \ mkdir -p "$folder_name" && \ mv "$html_file" "$folder_name/index.html"; \ fi; \ - done ) && \ + done && \ + touch .litellm_ui_ready ) && \ cd /app/ui/litellm-dashboard && rm -rf ./out # Build litellm wheel and place it in wheels dir (replace any PyPI wheels) diff --git a/docker/README.md b/docker/README.md index 6d81276bb4b..7027a30fdd7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -70,9 +70,12 @@ docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d This setup: - Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image. -- Runs the proxy as a non-root user with a read-only rootfs and only two writable tmpfs mounts: +- Runs the proxy as a non-root user with a read-only rootfs and only writable tmpfs mounts: - `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`) - `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`) +- Pre-builds and serves the admin UI from read-only paths: + - `/var/lib/litellm/ui` (pre-restructured Next.js UI with `.litellm_ui_ready` marker) + - `/var/lib/litellm/assets` (UI logos and assets) - Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines. You should also verify offline Prisma behaviour with: diff --git a/docs/my-website/blog/claude_opus_4_6/index.md b/docs/my-website/blog/claude_opus_4_6/index.md index 3fd70661543..82320472e13 100644 --- a/docs/my-website/blog/claude_opus_4_6/index.md +++ b/docs/my-website/blog/claude_opus_4_6/index.md @@ -389,6 +389,10 @@ Compaction blocks are also supported in streaming mode. You'll receive: ### Adaptive Thinking +:::note +When using `reasoning_effort` with Claude Opus 4.6, all values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets with `type: "enabled"`, pass the native `thinking` parameter directly (see "Native thinking param" tab below). +::: + @@ -434,6 +438,21 @@ curl --location 'http://0.0.0.0:4000/v1/messages' \ }' ``` + + + +Use the `thinking` parameter directly for adaptive thinking via the SDK: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-opus-4-6", + messages=[{"role": "user", "content": "Solve this complex problem: What is the optimal strategy for..."}], + thinking={"type": "adaptive"}, +) +``` + diff --git a/docs/my-website/blog/minimax_m2_5/index.md b/docs/my-website/blog/minimax_m2_5/index.md new file mode 100644 index 00000000000..50084fcc1e5 --- /dev/null +++ b/docs/my-website/blog/minimax_m2_5/index.md @@ -0,0 +1,394 @@ +--- +slug: minimax_m2_5 +title: "Day 0 Support: MiniMax-M2.5" +date: 2026-02-12T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +description: "Day 0 support for MiniMax-M2.5 on LiteLLM" +tags: [minimax, M2.5, llm] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway. + +## Supported Models + +LiteLLM supports the following MiniMax models: + +| Model | Description | Input Cost | Output Cost | Context Window | +|-------|-------------|------------|-------------|----------------| +| **MiniMax-M2.5** | Advanced reasoning, Agentic capabilities | $0.3/M tokens | $1.2/M tokens | 1M tokens | +| **MiniMax-M2.5-lightning** | Faster and More Agile (~100 tps) | $0.3/M tokens | $2.4/M tokens | 1M tokens | + +## Features Supported + +- **Prompt Caching**: Reduce costs with cached prompts ($0.03/M tokens for cache read, $0.375/M tokens for cache write) +- **Function Calling**: Built-in tool calling support +- **Reasoning**: Advanced reasoning capabilities with thinking support +- **System Messages**: Full system message support +- **Cost Tracking**: Automatic cost calculation for all requests + +## Docker Image + +```bash +docker pull litellm/litellm:v1.81.3-stable +``` + +## Usage - OpenAI Compatible API (/v1/chat/completions) + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: minimax-m2-5 + litellm_params: + model: minimax/MiniMax-M2.5 + api_key: os.environ/MINIMAX_API_KEY + api_base: https://api.minimax.io/v1 +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e MINIMAX_API_KEY=$MINIMAX_API_KEY \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:v1.81.3-stable \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +### With Reasoning Split + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "messages": [ + { + "role": "user", + "content": "Solve: 2+2=?" + } + ], + "extra_body": { + "reasoning_split": true + } +}' +``` + +## Usage - Anthropic Compatible API (/v1/messages) + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: minimax-m2-5 + litellm_params: + model: minimax/MiniMax-M2.5 + api_key: os.environ/MINIMAX_API_KEY + api_base: https://api.minimax.io/anthropic/v1/messages +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e MINIMAX_API_KEY=$MINIMAX_API_KEY \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:v1.81.3-stable \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "max_tokens": 1000, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +### With Thinking + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "max_tokens": 1000, + "thinking": { + "type": "enabled", + "budget_tokens": 1000 + }, + "messages": [ + { + "role": "user", + "content": "Solve: 2+2=?" + } + ] +}' +``` + +## Usage - LiteLLM SDK + +### OpenAI-compatible API + +```python +import litellm + +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[ + {"role": "user", "content": "Hello, how are you?"} + ], + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +print(response.choices[0].message.content) +``` + +### Anthropic-compatible API + +```python +import litellm + +response = litellm.anthropic.messages.acreate( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Hello, how are you?"}], + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/anthropic/v1/messages", + max_tokens=1000 +) + +print(response.choices[0].message.content) +``` + +### With Thinking + +```python +response = litellm.anthropic.messages.acreate( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Solve: 2+2=?"}], + thinking={"type": "enabled", "budget_tokens": 1000}, + api_key="your-minimax-api-key" +) + +# Access thinking content +for block in response.choices[0].message.content: + if hasattr(block, 'type') and block.type == 'thinking': + print(f"Thinking: {block.thinking}") +``` + +### With Reasoning Split (OpenAI API) + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[ + {"role": "user", "content": "Solve: 2+2=?"} + ], + extra_body={"reasoning_split": True}, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +# Access thinking and response +if hasattr(response.choices[0].message, 'reasoning_details'): + print(f"Thinking: {response.choices[0].message.reasoning_details}") +print(f"Response: {response.choices[0].message.content}") +``` + +## Cost Tracking + +LiteLLM automatically tracks costs for MiniMax-M2.5 requests. The pricing is: + +- **Input**: $0.3 per 1M tokens +- **Output**: $1.2 per 1M tokens +- **Cache Read**: $0.03 per 1M tokens +- **Cache Write**: $0.375 per 1M tokens + +### Accessing Cost Information + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Hello!"}], + api_key="your-minimax-api-key" +) + +# Access cost information +print(f"Cost: ${response._hidden_params.get('response_cost', 0)}") +``` + +## Streaming Support + +### OpenAI API + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Tell me a story"}], + stream=True, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + +### Streaming with Reasoning Split + +```python +stream = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[ + {"role": "user", "content": "Tell me a story"}, + ], + extra_body={"reasoning_split": True}, + stream=True, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +reasoning_buffer = "" +text_buffer = "" + +for chunk in stream: + if hasattr(chunk.choices[0].delta, "reasoning_details") and chunk.choices[0].delta.reasoning_details: + for detail in chunk.choices[0].delta.reasoning_details: + if "text" in detail: + reasoning_text = detail["text"] + new_reasoning = reasoning_text[len(reasoning_buffer):] + if new_reasoning: + print(new_reasoning, end="", flush=True) + reasoning_buffer = reasoning_text + + if chunk.choices[0].delta.content: + content_text = chunk.choices[0].delta.content + new_text = content_text[len(text_buffer):] if text_buffer else content_text + if new_text: + print(new_text, end="", flush=True) + text_buffer = content_text +``` + +## Using with Native SDKs + +### Anthropic SDK via LiteLLM Proxy + +```python +import os +os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000" +os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM proxy key + +import anthropic + +client = anthropic.Anthropic() + +message = client.messages.create( + model="minimax-m2-5", + max_tokens=1000, + system="You are a helpful assistant.", + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hi, how are you?" + } + ] + } + ] +) + +for block in message.content: + if block.type == "thinking": + print(f"Thinking:\n{block.thinking}\n") + elif block.type == "text": + print(f"Text:\n{block.text}\n") +``` + +### OpenAI SDK via LiteLLM Proxy + +```python +import os +os.environ["OPENAI_BASE_URL"] = "http://localhost:4000" +os.environ["OPENAI_API_KEY"] = "sk-1234" # Your LiteLLM proxy key + +from openai import OpenAI + +client = OpenAI() + +response = client.chat.completions.create( + model="minimax-m2-5", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hi, how are you?"}, + ], + extra_body={"reasoning_split": True}, +) + +# Access thinking and response +if hasattr(response.choices[0].message, 'reasoning_details'): + print(f"Thinking:\n{response.choices[0].message.reasoning_details[0]['text']}\n") +print(f"Text:\n{response.choices[0].message.content}\n") +``` diff --git a/docs/my-website/docs/adding_provider/generic_guardrail_api.md b/docs/my-website/docs/adding_provider/generic_guardrail_api.md index 482dedaa8a9..0931c349e48 100644 --- a/docs/my-website/docs/adding_provider/generic_guardrail_api.md +++ b/docs/my-website/docs/adding_provider/generic_guardrail_api.md @@ -93,6 +93,12 @@ Implement `POST /beta/litellm_basic_guardrail_api` "user_api_key_end_user_id": "end user id associated with the litellm virtual key used", "user_api_key_org_id": "org id associated with the litellm virtual key used" }, + "request_headers": { // optional: inbound request headers (allowlist). Allowed headers show their value; all others show "[present]" to indicate the header existed. + "User-Agent": "OpenAI/Python 2.17.0", + "Content-Type": "application/json", + "X-Request-Id": "[present]" + }, + "litellm_version": "1.x.y", // optional: LiteLLM library version running this proxy "input_type": "request", // "request" or "response" "litellm_call_id": "unique_call_id", // the call id of the individual LLM call "litellm_trace_id": "trace_id", // the trace id of the LLM call - useful if there are multiple LLM calls for the same conversation diff --git a/docs/my-website/docs/completion/web_search.md b/docs/my-website/docs/completion/web_search.md index 9ba66c730f0..1f5ba2dee4e 100644 --- a/docs/my-website/docs/completion/web_search.md +++ b/docs/my-website/docs/completion/web_search.md @@ -18,7 +18,7 @@ Each provider uses their own search backend: | Provider | Search Engine | Notes | |----------|---------------|-------| -| **OpenAI** (`gpt-4o-search-preview`, `gpt-4o-mini-search-preview`, `gpt-5-search-api`) | OpenAI's internal search | Real-time web data | +| **OpenAI** (`gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview`) | OpenAI's internal search | Real-time web data | | **xAI** (`grok-3`) | xAI's search + X/Twitter | Real-time social media data | | **Google AI/Vertex** (`gemini-2.0-flash`) | **Google Search** | Uses actual Google search results | | **Anthropic** (`claude-3-5-sonnet`) | Anthropic's web search | Real-time web data | @@ -45,6 +45,19 @@ Use `web_search_options` when you need to: **Anthropic Web Search Models**: Claude models that support web search: `claude-3-5-sonnet-latest`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-latest`, `claude-3-5-haiku-20241022`, `claude-3-7-sonnet-20250219` ::: +## OpenAI Web Search: Two Approaches + +OpenAI offers two distinct ways to use web search depending on the endpoint and model: + +| Approach | Endpoint | Models | How to enable | +|----------|----------|--------|---------------| +| **Search Models** | `/chat/completions` | `gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview` | Pass `web_search_options` parameter | +| **Web Search Tool** | `/responses` | `gpt-5`, `gpt-4.1`, `gpt-4o`, and other regular models | Pass `web_search_preview` tool | + +:::tip Search models search automatically +Search models like `gpt-5-search-api` **automatically search the web** even without the `web_search_options` parameter. Use `web_search_options` to set `search_context_size` (`"low"`, `"medium"`, `"high"`) or specify `user_location` for localized results. +::: + ## `/chat/completions` (litellm.completion) ### Quick Start @@ -56,7 +69,7 @@ Use `web_search_options` when you need to: from litellm import completion response = completion( - model="openai/gpt-4o-search-preview", + model="openai/gpt-5-search-api", messages=[ { "role": "user", @@ -76,31 +89,36 @@ response = completion( ```yaml model_list: - # OpenAI + # OpenAI search models + - model_name: gpt-5-search-api + litellm_params: + model: openai/gpt-5-search-api + api_key: os.environ/OPENAI_API_KEY + - model_name: gpt-4o-search-preview litellm_params: model: openai/gpt-4o-search-preview api_key: os.environ/OPENAI_API_KEY - + # xAI - model_name: grok-3 litellm_params: model: xai/grok-3 api_key: os.environ/XAI_API_KEY - + # Anthropic - model_name: claude-3-5-sonnet-latest litellm_params: model: anthropic/claude-3-5-sonnet-latest api_key: os.environ/ANTHROPIC_API_KEY - + # VertexAI - model_name: gemini-2-flash litellm_params: model: gemini-2.0-flash vertex_project: your-project-id vertex_location: us-central1 - + # Google AI Studio - model_name: gemini-2-flash-studio litellm_params: @@ -108,13 +126,13 @@ model_list: api_key: os.environ/GOOGLE_API_KEY ``` -2. Start the proxy +2. Start the proxy ```bash litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```python showLineNumbers from openai import OpenAI @@ -126,13 +144,18 @@ client = OpenAI( ) response = client.chat.completions.create( - model="grok-3", # or any other web search enabled model + model="gpt-5-search-api", # or any other web search enabled model messages=[ { "role": "user", "content": "What was a positive news story from today?" } - ] + ], + extra_body={ + "web_search_options": { + "search_context_size": "medium" + } + } ) ``` @@ -149,7 +172,7 @@ from litellm import completion # Customize search context size response = completion( - model="openai/gpt-4o-search-preview", + model="openai/gpt-5-search-api", messages=[ { "role": "user", @@ -257,6 +280,12 @@ response = client.chat.completions.create( ## `/responses` (litellm.responses) +Use the `web_search_preview` tool with models like `gpt-5`, `gpt-4.1`, `gpt-4o`, etc. + +:::info +Search-dedicated models like `gpt-5-search-api` and `gpt-4o-search-preview` do **not** support the `/responses` endpoint. Use them with `/chat/completions` + `web_search_options` instead (see above). +::: + ### Quick Start @@ -266,18 +295,14 @@ response = client.chat.completions.create( from litellm import responses response = responses( - model="openai/gpt-4o", - input=[ - { - "role": "user", - "content": "What was a positive news story from today?" - } - ], + model="openai/gpt-5", + input="What is the capital of France?", tools=[{ "type": "web_search_preview" # enables web search with default medium context size }] ) ``` + @@ -285,19 +310,24 @@ response = responses( ```yaml model_list: - - model_name: gpt-4o + - model_name: gpt-5 litellm_params: - model: openai/gpt-4o + model: openai/gpt-5 + api_key: os.environ/OPENAI_API_KEY + + - model_name: gpt-4.1 + litellm_params: + model: openai/gpt-4.1 api_key: os.environ/OPENAI_API_KEY ``` -2. Start the proxy +2. Start the proxy ```bash litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```python showLineNumbers from openai import OpenAI @@ -309,11 +339,11 @@ client = OpenAI( ) response = client.responses.create( - model="gpt-4o", + model="gpt-5", tools=[{ "type": "web_search_preview" }], - input="What was a positive news story from today?", + input="What is the capital of France?", ) print(response.output_text) @@ -331,13 +361,8 @@ from litellm import responses # Customize search context size response = responses( - model="openai/gpt-4o", - input=[ - { - "role": "user", - "content": "What was a positive news story from today?" - } - ], + model="openai/gpt-5", + input="What is the capital of France?", tools=[{ "type": "web_search_preview", "search_context_size": "low" # Options: "low", "medium" (default), "high" @@ -358,12 +383,12 @@ client = OpenAI( # Customize search context size response = client.responses.create( - model="gpt-4o", + model="gpt-5", tools=[{ "type": "web_search_preview", "search_context_size": "low" # Options: "low", "medium" (default), "high" }], - input="What was a positive news story from today?", + input="What is the capital of France?", ) print(response.output_text) @@ -417,14 +442,14 @@ model_list: web_search_options: search_context_size: "high" # Options: "low", "medium", "high" - # Different context size for different models - - model_name: gpt-4o-search-preview + # OpenAI search model with custom context size + - model_name: gpt-5-search-api litellm_params: - model: openai/gpt-4o-search-preview + model: openai/gpt-5-search-api api_key: os.environ/OPENAI_API_KEY web_search_options: search_context_size: "low" - + # Gemini with medium context (default) - model_name: gemini-2-flash litellm_params: @@ -449,6 +474,7 @@ Use `litellm.supports_web_search(model="model_name")` -> returns `True` if model ```python showLineNumbers # Check OpenAI models +assert litellm.supports_web_search(model="openai/gpt-5-search-api") == True assert litellm.supports_web_search(model="openai/gpt-4o-search-preview") == True # Check xAI models @@ -472,13 +498,20 @@ assert litellm.supports_web_search(model="gemini/gemini-2.0-flash") == True ```yaml model_list: # OpenAI + - model_name: gpt-5-search-api + litellm_params: + model: openai/gpt-5-search-api + api_key: os.environ/OPENAI_API_KEY + model_info: + supports_web_search: True + - model_name: gpt-4o-search-preview litellm_params: model: openai/gpt-4o-search-preview api_key: os.environ/OPENAI_API_KEY model_info: supports_web_search: True - + # xAI - model_name: grok-3 litellm_params: @@ -533,6 +566,12 @@ Expected Response ```json showLineNumbers { "data": [ + { + "model_group": "gpt-5-search-api", + "providers": ["openai"], + "max_tokens": 128000, + "supports_web_search": true + }, { "model_group": "gpt-4o-search-preview", "providers": ["openai"], diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 446d663c5ac..de5a4dc610c 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -1473,6 +1473,20 @@ LiteLLM translates OpenAI's `reasoning_effort` to Anthropic's `thinking` paramet | "medium" | "budget_tokens": 2048 | | "high" | "budget_tokens": 4096 | +:::note +For Claude Opus 4.6, all `reasoning_effort` values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets, pass the native `thinking` parameter directly: + +```python +from litellm import completion + +resp = completion( + model="anthropic/claude-opus-4-6", + messages=[{"role": "user", "content": "What is the capital of France?"}], + thinking={"type": "enabled", "budget_tokens": 1024}, +) +``` +::: + @@ -1614,8 +1628,65 @@ curl http://0.0.0.0:4000/v1/chat/completions \ +#### Adaptive Thinking (Claude Opus 4.6) + + +```python +response = litellm.completion( + model="anthropic/claude-opus-4-6", + messages=[{"role": "user", "content": "What is the optimal strategy for solving this problem?"}], + thinking={"type": "adaptive"}, +) +``` + + + + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "anthropic/claude-opus-4-6", + "messages": [{"role": "user", "content": "What is the optimal strategy for solving this problem?"}], + "thinking": {"type": "adaptive"} + }' +``` + + + + +#### Enabled Thinking with Budget + + + + +```python +response = litellm.completion( + model="anthropic/claude-opus-4-6", + messages=[{"role": "user", "content": "What is the capital of France?"}], + thinking={"type": "enabled", "budget_tokens": 5000}, +) +``` + + + + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "anthropic/claude-opus-4-6", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "thinking": {"type": "enabled", "budget_tokens": 5000} + }' +``` + + + ## **Passing Extra Headers to Anthropic API** diff --git a/docs/my-website/docs/providers/dashscope.md b/docs/my-website/docs/providers/dashscope.md index 565776d6c4c..3df0fbab1ba 100644 --- a/docs/my-website/docs/providers/dashscope.md +++ b/docs/my-website/docs/providers/dashscope.md @@ -1,7 +1,7 @@ -# Dashscope (Qwen API) +# Dashscope API (Qwen models) https://dashscope.console.aliyun.com/ -**We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests** +**We support ALL Qwen models (from Alibaba Cloud), just set `dashscope/` as a prefix when sending completion requests** ## API Key ```python @@ -9,6 +9,26 @@ https://dashscope.console.aliyun.com/ os.environ['DASHSCOPE_API_KEY'] ``` +## API Base +You can optionally specify the API base URL depending on your region: + +| Region | API Base | +|--------|----------| +| **International** | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` | +| **China/Beijing** | `https://dashscope.aliyuncs.com/compatible-mode/v1` | + +```python +# Set via environment variable +os.environ['DASHSCOPE_API_BASE'] = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + +# Or pass directly in the completion call +response = completion( + model="dashscope/qwen-turbo", + messages=[{"role": "user", "content": "hello"}], + api_base="https://dashscope-intl.aliyuncs.com/compatible-mode/v1" +) +``` + ## Sample Usage ```python from litellm import completion @@ -43,9 +63,7 @@ for chunk in response: ``` -## Supported Models - ALL Qwen Models Supported! -We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests - +## All supported Models [DashScope Model List](https://help.aliyun.com/zh/model-studio/compatibility-of-openai-with-dashscope?spm=a2c4g.11186623.help-menu-2400256.d_2_8_0.1efd516e2tTXBn&scm=20140722.H_2833609._.OR_help-T_cn~zh-V_1#7f9c78ae99pwz) diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 80645a51ac5..23940e1c54e 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -230,7 +230,70 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL These also support the `OPENAI_BASE_URL` environment variable, which can be used to specify a custom API endpoint. -## OpenAI Vision Models +### OpenAI Web Search Models + +OpenAI has two ways to use web search, depending on the endpoint: + +| Approach | Endpoint | Models | How to enable | +|----------|----------|--------|---------------| +| **Search Models** | `/chat/completions` | `gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview` | Pass `web_search_options` parameter | +| **Web Search Tool** | `/responses` | `gpt-5`, `gpt-4.1`, `gpt-4o`, and other regular models | Pass `web_search_preview` tool | + + + + +```python showLineNumbers +from litellm import completion + +response = completion( + model="openai/gpt-5-search-api", + messages=[{"role": "user", "content": "What is the capital of France?"}], + web_search_options={ + "search_context_size": "medium" # Options: "low", "medium", "high" + } +) +``` + + + + +```python showLineNumbers +from litellm import responses + +response = responses( + model="openai/gpt-5", + input="What is the capital of France?", + tools=[{ + "type": "web_search_preview", + "search_context_size": "low" + }] +) +``` + + + + +```yaml +model_list: + # Search model for /chat/completions + - model_name: gpt-5-search-api + litellm_params: + model: openai/gpt-5-search-api + api_key: os.environ/OPENAI_API_KEY + + # Regular model for /responses with web_search_preview tool + - model_name: gpt-5 + litellm_params: + model: openai/gpt-5 + api_key: os.environ/OPENAI_API_KEY +``` + + + + +For full details, see the [Web Search guide](../completion/web_search.md). + +## OpenAI Vision Models | Model Name | Function Call | |-----------------------|-----------------------------------------------------------------| | gpt-4o | `response = completion(model="gpt-4o", messages=messages)` | diff --git a/docs/my-website/docs/providers/openai/responses_api.md b/docs/my-website/docs/providers/openai/responses_api.md index 75eab1afac5..7799c93ccf2 100644 --- a/docs/my-website/docs/providers/openai/responses_api.md +++ b/docs/my-website/docs/providers/openai/responses_api.md @@ -37,6 +37,24 @@ for event in response: print(event) ``` +#### Web Search +```python showLineNumbers title="OpenAI Responses with Web Search" +import litellm + +response = litellm.responses( + model="openai/gpt-5", + input="What is the capital of France?", + tools=[{ + "type": "web_search_preview", + "search_context_size": "medium" # Options: "low", "medium", "high" + }] +) + +print(response) +``` + +For full details, see the [Web Search guide](../../completion/web_search.md). + #### Image Generation with Streaming ```python showLineNumbers title="OpenAI Streaming Image Generation" import litellm diff --git a/docs/my-website/docs/providers/scaleway.md b/docs/my-website/docs/providers/scaleway.md new file mode 100644 index 00000000000..ea57c24db30 --- /dev/null +++ b/docs/my-website/docs/providers/scaleway.md @@ -0,0 +1,62 @@ + +# Scaleway +LiteLLM supports all [models available on Scaleway Generative APIs ↗](https://www.scaleway.com/en/docs/generative-apis/reference-content/supported-models/). + +## Usage with LiteLLM Python SDK + +```python +import os +from litellm import completion + +os.environ["SCW_SECRET_KEY"] = "your-scaleway-secret-key" + +messages = [{"role": "user", "content": "Write a short poem"}] +response = completion(model="scaleway/qwen3-235b-a22b-instruct-2507", messages=messages) +print(response) +``` + +## Usage with LiteLLM Proxy + +### 1. Set Scaleway models in config.yaml + +```yaml +model_list: + - model_name: scaleway-model + litellm_params: + model: scaleway/qwen3-235b-a22b-instruct-2507 + api_key: "os.environ/SCW_SECRET_KEY" # ensure you have `SCW_SECRET_KEY` in your .env +``` + +### 2. Start proxy + +```bash +litellm --config config.yaml +``` + +### 3. Query proxy + +Assuming the proxy is running on [http://localhost:4000](http://localhost:4000): +```bash +curl http://localhost:4000/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer YOUR_LITELLM_MASTER_KEY" \ + -d '{ + "model": "scaleway-model", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Write a short poem" + } + ] + }' +``` +`-H "Authorization: Bearer YOUR_LITELLM_MASTER_KEY" ` is only required if you have set a LiteLLM master key + + +## Supported features + +Scaleway provider supports all features in [Generative APIs reference documentation ↗](https://www.scaleway.com/en/developers/api/generative-apis/), such as streaming, structured outputs and tool calling. diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index f4122731ef4..847f8623e72 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -555,7 +555,7 @@ router_settings: | DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20 | DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10 | DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602 -| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available** +| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy when `NUM_WORKERS` is not set. Default is 1. **We strongly recommend setting NUM_WORKERS to the number of vCPUs available** (e.g. `NUM_WORKERS=8` or `--num_workers 8`). | DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7 | DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03 | DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0 @@ -746,6 +746,7 @@ router_settings: | LITERAL_API_URL | API URL for Literal service | LITERAL_BATCH_SIZE | Batch size for Literal operations | LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints +| LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker. | LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours | LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API | LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518 @@ -760,6 +761,7 @@ router_settings: | LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems. | LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM | LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset. +| LITELLM_UI_PATH | Path to directory for Admin UI files. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/ui` in Docker. | LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval. | LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false. | LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours). diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index a5674bf2bc5..56a8b9566db 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -469,6 +469,7 @@ credential_list: api_version: "2023-05-15" credential_info: description: "Production credentials for EU region" + custom_llm_provider: "azure" ``` #### Key Parameters diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index a42d91a7d5f..994788a3ad9 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -250,11 +250,133 @@ The migrate deploy command: ### Read-only File System -If you see a `Permission denied` error, it means the LiteLLM pod is running with a read-only file system. +Running LiteLLM with `readOnlyRootFilesystem: true` is a Kubernetes security best practice that prevents container processes from writing to the root filesystem. LiteLLM fully supports this configuration. -To fix this, just set `LITELLM_MIGRATION_DIR="/path/to/writeable/directory"` in your environment. +#### Quick Fix for Permission Errors -LiteLLM will use this directory to write migration files. +If you see a `Permission denied` error, it means the LiteLLM pod is running with a read-only file system. LiteLLM needs writable directories for: +- **Database migrations**: Set `LITELLM_MIGRATION_DIR="/path/to/writable/directory"` +- **Admin UI**: Set `LITELLM_UI_PATH="/path/to/writable/directory"` +- **UI assets/logos**: Set `LITELLM_ASSETS_PATH="/path/to/writable/directory"` + +#### Complete Read-Only Filesystem Setup (Kubernetes) + +For production deployments with enhanced security, use this configuration: + +**Option 1: Using EmptyDir Volumes with InitContainer (Recommended)** + +This approach copies the pre-built UI from the Docker image to writable emptyDir volumes at pod startup. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm-proxy +spec: + template: + spec: + initContainers: + - name: setup-ui + image: ghcr.io/berriai/litellm:main-stable + command: + - sh + - -c + - | + cp -r /var/lib/litellm/ui/* /app/var/litellm/ui/ && \ + cp -r /var/lib/litellm/assets/* /app/var/litellm/assets/ + volumeMounts: + - name: ui-volume + mountPath: /app/var/litellm/ui + - name: assets-volume + mountPath: /app/var/litellm/assets + + containers: + - name: litellm + image: ghcr.io/berriai/litellm:main-stable + env: + - name: LITELLM_NON_ROOT + value: "true" + - name: LITELLM_UI_PATH + value: "/app/var/litellm/ui" + - name: LITELLM_ASSETS_PATH + value: "/app/var/litellm/assets" + - name: LITELLM_MIGRATION_DIR + value: "/app/migrations" + - name: PRISMA_BINARY_CACHE_DIR + value: "/app/cache/prisma-python/binaries" + - name: XDG_CACHE_HOME + value: "/app/cache" + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 101 + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: /app/config.yaml + subPath: config.yaml + readOnly: true + - name: ui-volume + mountPath: /app/var/litellm/ui + - name: assets-volume + mountPath: /app/var/litellm/assets + - name: cache + mountPath: /app/cache + - name: migrations + mountPath: /app/migrations + + volumes: + - name: config + configMap: + name: litellm-config + - name: ui-volume + emptyDir: + sizeLimit: 100Mi + - name: assets-volume + emptyDir: + sizeLimit: 10Mi + - name: cache + emptyDir: + sizeLimit: 500Mi + - name: migrations + emptyDir: + sizeLimit: 64Mi +``` + +**Option 2: Without UI (API-only deployment)** + +If you don't need the admin UI, you can run with minimal configuration: + +```yaml +env: + - name: LITELLM_NON_ROOT + value: "true" + - name: LITELLM_MIGRATION_DIR + value: "/app/migrations" +securityContext: + readOnlyRootFilesystem: true +``` + +The proxy will log a warning about the UI but API endpoints will work normally. + +#### Environment Variables for Read-Only Filesystems + +| Variable | Purpose | Default | +|----------|---------|---------| +| `LITELLM_UI_PATH` | Admin UI directory | `/var/lib/litellm/ui` (Docker) | +| `LITELLM_ASSETS_PATH` | UI assets/logos | `/var/lib/litellm/assets` (Docker) | +| `LITELLM_MIGRATION_DIR` | Database migrations | Package directory | +| `PRISMA_BINARY_CACHE_DIR` | Prisma binary cache | System default | +| `XDG_CACHE_HOME` | General cache directory | System default | + +#### Important Notes + +1. **Migrations**: Always set `LITELLM_MIGRATION_DIR` to a writable emptyDir path +2. **Prisma Cache**: Set `PRISMA_BINARY_CACHE_DIR` and `XDG_CACHE_HOME` to writable paths +3. **Server Root Path**: If using a custom `server_root_path`, you must pre-process UI files in your Dockerfile as the proxy cannot modify files at runtime with read-only filesystem +4. **Automatic Detection**: The UI is automatically detected as pre-restructured if it contains a `.litellm_ui_ready` marker file (created by the official Docker images) ## 10. Use a Separate Health Check App :::info diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index 140dfd4faf8..dd2b77712c4 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -1023,6 +1023,134 @@ curl http://localhost:4000/v1/responses \ +## Server-side compaction + +For long-running conversations, you can enable **server-side compaction** so that when the rendered context size crosses a threshold, the server automatically runs compaction in-stream and emits a compaction item—no separate `POST /v1/responses/compact` call is required. + +Supported on the OpenAI Responses API when using the `openai` or `azure` provider. Pass `context_management` with a compaction entry and `compact_threshold` (token count; minimum 1000). When the context crosses the threshold, the server compacts in-stream and continues. Chain turns with `previous_response_id` or by appending output items to your next input array. See [OpenAI Compaction guide](https://developers.openai.com/api/docs/guides/compaction) for details. + +For explicit control over when compaction runs, use the standalone compact endpoint (`POST /v1/responses/compact`) instead. + +### Python SDK + +```python showLineNumbers title="Server-side compaction with LiteLLM Python SDK" +import litellm + +# Non-streaming: enable compaction when context exceeds 200k tokens +response = litellm.responses( + model="openai/gpt-4o", + input="Your conversation input...", + context_management=[{"type": "compaction", "compact_threshold": 200000}], + max_output_tokens=1024, +) +print(response) + +# Streaming: same context_management, compaction runs in-stream if threshold is crossed +stream = litellm.responses( + model="openai/gpt-4o", + input="Your conversation input...", + context_management=[{"type": "compaction", "compact_threshold": 200000}], + stream=True, +) +for event in stream: + print(event) +``` + +### LiteLLM Proxy (AI Gateway) + +Use the OpenAI SDK with your proxy as `base_url`, or call the proxy with curl. The proxy forwards `context_management` to the provider. + +**OpenAI Python SDK (proxy as base_url):** + +```python showLineNumbers title="Server-side compaction via LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", # LiteLLM Proxy (AI Gateway) + api_key="your-proxy-api-key", +) + +response = client.responses.create( + model="openai/gpt-4o", + input="Your conversation input...", + context_management=[{"type": "compaction", "compact_threshold": 200000}], + max_output_tokens=1024, +) +print(response) +``` + +**curl (proxy):** + +```bash title="Server-side compaction via curl to LiteLLM Proxy" +curl -X POST "http://localhost:4000/v1/responses" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "openai/gpt-4o", + "input": "Your conversation input...", + "context_management": [{"type": "compaction", "compact_threshold": 200000}], + "max_output_tokens": 1024 + }' +``` + +## Shell tool + +The **Shell tool** lets the model run commands in a hosted container or local runtime (OpenAI Responses API). You pass `tools=[{"type": "shell", "environment": {...}}]`; the `environment` object configures the runtime (e.g. `type: "container_auto"` for auto-provisioned containers). See [OpenAI Shell tool guide](https://developers.openai.com/api/docs/guides/tools-shell) for full options. + +Supported when using the `openai` or `azure` provider with a model that supports the Shell tool. + +### Python SDK + +```python showLineNumbers title="Shell tool with LiteLLM Python SDK" +import litellm + +response = litellm.responses( + model="openai/gpt-5.2", + input="List files in /mnt/data and run python --version.", + tools=[{"type": "shell", "environment": {"type": "container_auto"}}], + tool_choice="auto", + max_output_tokens=1024, +) +``` + +### LiteLLM Proxy (AI Gateway) + +Use the OpenAI SDK with your proxy as `base_url`, or call the proxy with curl. The proxy forwards `tools` (including `type: "shell"`) to the provider. + +**OpenAI Python SDK (proxy as base_url):** + +```python showLineNumbers title="Shell tool via LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-proxy-api-key", +) + +response = client.responses.create( + model="openai/gpt-5.2", + input="List files in /mnt/data.", + tools=[{"type": "shell", "environment": {"type": "container_auto"}}], + tool_choice="auto", + max_output_tokens=1024, +) +``` + +**curl:** + +```bash title="Shell tool via curl to LiteLLM Proxy" +curl -X POST "http://localhost:4000/v1/responses" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "openai/gpt-5.2", + "input": "List files in /mnt/data.", + "tools": [{"type": "shell", "environment": {"type": "container_auto"}}], + "tool_choice": "auto", + "max_output_tokens": 1024 + }' +``` + ## Session Management LiteLLM Proxy supports session management for all supported models. This allows you to store and fetch conversation history (state) in LiteLLM Proxy. diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 60c87ddc458..d861b493e7f 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -874,6 +874,7 @@ const sidebars = { }, "providers/sambanova", "providers/sap", + "providers/scaleway", "providers/stability", "providers/synthetic", "providers/snowflake", diff --git a/enterprise/enterprise_hooks/__init__.py b/enterprise/enterprise_hooks/__init__.py index 9eb1c8960a6..e93c8c9150a 100644 --- a/enterprise/enterprise_hooks/__init__.py +++ b/enterprise/enterprise_hooks/__init__.py @@ -1,11 +1,15 @@ from typing import Dict, Literal, Type, Union from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles +from litellm_enterprise.proxy.hooks.managed_vector_stores import ( + _PROXY_LiteLLMManagedVectorStores, +) from litellm.integrations.custom_logger import CustomLogger ENTERPRISE_PROXY_HOOKS: Dict[str, Type[CustomLogger]] = { "managed_files": _PROXY_LiteLLMManagedFiles, + "managed_vector_stores": _PROXY_LiteLLMManagedVectorStores, } @@ -13,6 +17,7 @@ def get_enterprise_proxy_hook( hook_name: Union[ Literal[ "managed_files", + "managed_vector_stores", "max_parallel_requests", ], str, diff --git a/enterprise/litellm_enterprise/proxy/auth/route_checks.py b/enterprise/litellm_enterprise/proxy/auth/route_checks.py index 6f7cf9143f4..fc57292a8d2 100644 --- a/enterprise/litellm_enterprise/proxy/auth/route_checks.py +++ b/enterprise/litellm_enterprise/proxy/auth/route_checks.py @@ -41,6 +41,10 @@ class EnterpriseRouteChecks: return get_secret_bool("DISABLE_ADMIN_ENDPOINTS") is True + # Routes that should remain accessible even when LLM API endpoints are disabled. + # These are read-only model listing routes needed by the Admin UI. + LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"] + @staticmethod def should_call_route(route: str): """ @@ -58,6 +62,7 @@ class EnterpriseRouteChecks: ) elif ( RouteChecks.is_llm_api_route(route=route) + and route not in EnterpriseRouteChecks.LLM_API_EXEMPT_ROUTES and EnterpriseRouteChecks.is_llm_api_route_disabled() ): raise HTTPException( diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py b/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py new file mode 100644 index 00000000000..254d816039c --- /dev/null +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py @@ -0,0 +1,464 @@ +# What is this? +## This hook is used to manage vector stores with target_model_names support +## It allows creating vector stores across multiple models and managing them with unified IDs + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast + +from fastapi import HTTPException + +import litellm +from litellm import Router, verbose_logger +from litellm._uuid import uuid +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.base_llm.managed_resources import BaseManagedResource +from litellm.llms.base_llm.managed_resources.utils import ( + generate_unified_id_string, + is_base64_encoded_unified_id, +) +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.vector_stores import ( + VectorStoreCreateOptionalRequestParams, + VectorStoreCreateResponse, +) + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + + from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache + from litellm.proxy.utils import PrismaClient as _PrismaClient + + Span = Union[_Span, Any] + InternalUsageCache = _InternalUsageCache + PrismaClient = _PrismaClient +else: + Span = Any + InternalUsageCache = Any + PrismaClient = Any + + +class _PROXY_LiteLLMManagedVectorStores( + CustomLogger, BaseManagedResource[VectorStoreCreateResponse] +): + """ + Managed vector stores with target_model_names support. + + This class provides functionality to: + - Create vector stores across multiple models + - Retrieve vector stores by unified ID + - Delete vector stores from all models + - List vector stores created by a user + """ + + def __init__( + self, internal_usage_cache: InternalUsageCache, prisma_client: PrismaClient + ): + CustomLogger.__init__(self) + BaseManagedResource.__init__(self, internal_usage_cache, prisma_client) + + # ============================================================================ + # ABSTRACT METHOD IMPLEMENTATIONS + # ============================================================================ + + @property + def resource_type(self) -> str: + """Return the resource type identifier.""" + return "vector_store" + + @property + def table_name(self) -> str: + """Return the database table name for vector stores.""" + # Prisma converts model name LiteLLM_ManagedVectorStoreTable to litellm_managedvectorstoretable + return "litellm_managedvectorstoretable" + + def get_unified_resource_id_format( + self, + resource_object: VectorStoreCreateResponse, + target_model_names_list: List[str], + ) -> str: + """ + Generate the format string for the unified vector store ID. + + Format: + litellm_proxy:vector_store;unified_id,;target_model_names,;resource_id,;model_id, + """ + # VectorStoreCreateResponse is a TypedDict, so resource_object is a dictionary + # Extract provider resource ID from the response + provider_resource_id = resource_object.get("id", "") + + # Model ID is stored in hidden params if the response object supports it + # For TypedDict responses, we need to check if _hidden_params was added + hidden_params: Dict[str, Any] = {} + if hasattr(resource_object, "_hidden_params"): + hidden_params = getattr(resource_object, "_hidden_params", {}) or {} + model_id = hidden_params.get("model_id", "") + + return generate_unified_id_string( + resource_type=self.resource_type, + unified_uuid=str(uuid.uuid4()), + target_model_names=target_model_names_list, + provider_resource_id=provider_resource_id, + model_id=model_id, + ) + + async def create_resource_for_model( + self, + llm_router: Router, + model: str, + request_data: Dict[str, Any], + litellm_parent_otel_span: Span, + ) -> VectorStoreCreateResponse: + """ + Create a vector store for a specific model. + + Args: + llm_router: LiteLLM router instance + model: Model name to create vector store for + request_data: Request data for vector store creation + litellm_parent_otel_span: OpenTelemetry span for tracing + + Returns: + VectorStoreCreateResponse from the provider + """ + # Use the router to create the vector store + response = await llm_router.avector_store_create( + model=model, **request_data + ) + return response + + # ============================================================================ + # VECTOR STORE CRUD OPERATIONS + # ============================================================================ + + async def acreate_vector_store( + self, + create_request: VectorStoreCreateOptionalRequestParams, + llm_router: Router, + target_model_names_list: List[str], + litellm_parent_otel_span: Span, + user_api_key_dict: UserAPIKeyAuth, + ) -> VectorStoreCreateResponse: + """ + Create a vector store across multiple models. + + Args: + create_request: Vector store creation request parameters + llm_router: LiteLLM router instance + target_model_names_list: List of target model names + litellm_parent_otel_span: OpenTelemetry span for tracing + user_api_key_dict: User API key authentication details + + Returns: + VectorStoreCreateResponse with unified ID + """ + verbose_logger.info( + f"Creating managed vector store for models: {target_model_names_list}" + ) + + # Create vector store for each model + # Convert TypedDict to Dict[str, Any] for base class compatibility + request_data_dict: Dict[str, Any] = dict(create_request) + responses = await self.create_resource_for_each_model( + llm_router=llm_router, + request_data=request_data_dict, + target_model_names_list=target_model_names_list, + litellm_parent_otel_span=litellm_parent_otel_span, + ) + + # Generate unified ID + unified_id = self.generate_unified_resource_id( + resource_objects=responses, + target_model_names_list=target_model_names_list, + ) + + # Extract model mappings from responses + model_mappings: Dict[str, str] = {} + for response in responses: + hidden_params = getattr(response, "_hidden_params", {}) or {} + model_id = hidden_params.get("model_id") + if model_id: + # VectorStoreCreateResponse is a TypedDict, use dict access + model_mappings[model_id] = response["id"] + + verbose_logger.debug( + f"Created vector stores with model mappings: {model_mappings}" + ) + + # Store in database + await self.store_unified_resource_id( + unified_resource_id=unified_id, + resource_object=responses[0], # Store first response as template + litellm_parent_otel_span=litellm_parent_otel_span, + model_mappings=model_mappings, + user_api_key_dict=user_api_key_dict, + ) + + # Return response with unified ID + # VectorStoreCreateResponse is a TypedDict, so we need to create a new dict with the unified ID + response = responses[0].copy() + response["id"] = unified_id + + verbose_logger.info( + f"Successfully created managed vector store with unified ID: {unified_id}" + ) + + return response + + async def alist_vector_stores( + self, + user_api_key_dict: UserAPIKeyAuth, + limit: Optional[int] = None, + after: Optional[str] = None, + order: Optional[str] = None, + ) -> Dict[str, Any]: + """ + List vector stores created by a user. + + Args: + user_api_key_dict: User API key authentication details + limit: Maximum number of vector stores to return + after: Cursor for pagination + order: Sort order ('asc' or 'desc') + + Returns: + Dictionary with list of vector stores and pagination info + """ + # Use the base class method + return await self.list_user_resources( + user_api_key_dict=user_api_key_dict, + limit=limit, + after=after, + ) + + # ============================================================================ + # ACCESS CONTROL + # ============================================================================ + + async def check_vector_store_access( + self, vector_store_id: str, user_api_key_dict: UserAPIKeyAuth + ) -> bool: + """ + Check if user has access to a vector store. + + Args: + vector_store_id: The unified vector store ID + user_api_key_dict: User API key authentication details + + Returns: + True if user has access, False otherwise + """ + is_unified_id = is_base64_encoded_unified_id(vector_store_id) + + if is_unified_id: + # Check access for managed vector store + return await self.can_user_access_unified_resource_id( + vector_store_id, + user_api_key_dict, + ) + + # Not a managed vector store, allow access + return True + + async def check_managed_vector_store_access( + self, data: Dict, user_api_key_dict: UserAPIKeyAuth + ) -> bool: + """ + Check if user has access to a managed vector store in request data. + + Args: + data: Request data containing vector_store_id + user_api_key_dict: User API key authentication details + + Returns: + True if this is a managed vector store and user has access + + Raises: + HTTPException: If user doesn't have access + """ + vector_store_id = cast(Optional[str], data.get("vector_store_id")) + is_unified_id = ( + is_base64_encoded_unified_id(vector_store_id) + if vector_store_id + else False + ) + + if is_unified_id and vector_store_id: + if await self.can_user_access_unified_resource_id( + vector_store_id, user_api_key_dict + ): + return True + else: + raise HTTPException( + status_code=403, + detail=f"User {user_api_key_dict.user_id} does not have access to vector store {vector_store_id}", + ) + + return False + + # ============================================================================ + # PRE-CALL HOOK (For Router Integration) + # ============================================================================ + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: Any, + data: Dict, + call_type: str, + ) -> Union[Exception, str, Dict, None]: + """ + Pre-call hook to handle vector store operations. + + This hook intercepts vector store requests and: + - Validates access for managed vector stores + - Transforms unified IDs to provider-specific IDs + - Adds model routing information + + Args: + user_api_key_dict: User API key authentication details + cache: Cache instance + data: Request data + call_type: Type of call being made + + Returns: + Modified request data or None + """ + from litellm.llms.base_llm.managed_resources.utils import ( + is_base64_encoded_unified_id, + parse_unified_id, + ) + + # Handle vector store search operations + if call_type == "avector_store_search": + vector_store_id = data.get("vector_store_id") + + if vector_store_id: + # Check if it's a managed vector store ID + decoded_id = is_base64_encoded_unified_id(vector_store_id) + + if decoded_id: + verbose_logger.debug( + f"Processing managed vector store search: {vector_store_id}" + ) + + # Check access + has_access = await self.can_user_access_unified_resource_id( + vector_store_id, user_api_key_dict + ) + + if not has_access: + raise HTTPException( + status_code=403, + detail=f"User {user_api_key_dict.user_id} does not have access to vector store {vector_store_id}", + ) + + # Parse the unified ID to extract components + parsed_id = parse_unified_id(vector_store_id) + + if parsed_id: + # Extract the model ID and provider resource ID + model_id = parsed_id.get("model_id") + provider_resource_id = parsed_id.get("provider_resource_id") + target_model_names = parsed_id.get("target_model_names", []) + + verbose_logger.debug( + f"Decoded vector store - model_id: {model_id}, provider_resource_id: {provider_resource_id}, target_model_names: {target_model_names}" + ) + + # Determine which model to use for routing + # Priority: model_id (deployment ID) > first target_model_name + routing_model = None + if model_id: + routing_model = model_id + elif target_model_names and len(target_model_names) > 0: + routing_model = target_model_names[0] + + # Set the model for routing + if routing_model: + data["model"] = routing_model + verbose_logger.info( + f"Routing vector store search to model: {routing_model}" + ) + + # Replace the unified ID with the provider-specific ID + if provider_resource_id: + data["vector_store_id"] = provider_resource_id + verbose_logger.debug( + f"Replaced unified ID with provider resource ID: {provider_resource_id}" + ) + + # Handle vector store retrieve/delete operations + elif call_type in ("avector_store_retrieve", "avector_store_delete"): + await self.check_managed_vector_store_access(data, user_api_key_dict) + + # If it's a managed vector store, we'll handle it in the endpoint + # No need to transform here as the endpoint will route to the hook + + return data + + # ============================================================================ + # POST-CALL HOOK (For Response Transformation) + # ============================================================================ + + async def async_post_call_success_hook( + self, + data: Dict, + user_api_key_dict: UserAPIKeyAuth, + response: Any, + ) -> Any: + """ + Post-call hook to transform responses. + + This hook can be used to transform responses if needed. + For now, it just passes through the response. + + Args: + data: Request data + user_api_key_dict: User API key authentication details + response: Response from the provider + + Returns: + Potentially modified response + """ + # Currently no transformation needed + return response + + # ============================================================================ + # DEPLOYMENT FILTERING + # ============================================================================ + + async def async_filter_deployments( # type: ignore[override] + self, + model: str, + healthy_deployments: List, + messages: Optional[List] = None, + request_kwargs: Optional[Dict] = None, + parent_otel_span: Optional[Span] = None, + ) -> List[Dict]: + """ + Filter deployments based on vector store availability. + + This is used by the router to select only deployments that have + the vector store available. + + Note: This method signature is a compromise between CustomLogger and BaseManagedResource + parent classes which have incompatible signatures. The type: ignore[override] is necessary + due to this multiple inheritance conflict. + + Args: + model: Model name + healthy_deployments: List of healthy deployments + messages: Messages (unused for vector stores, required by CustomLogger interface) + request_kwargs: Request kwargs containing vector_store_id and mappings + parent_otel_span: OpenTelemetry span for tracing + + Returns: + Filtered list of deployments + """ + return await BaseManagedResource.async_filter_deployments( + self, + model=model, + healthy_deployments=healthy_deployments, + request_kwargs=request_kwargs, + parent_otel_span=parent_otel_span, + resource_id_key="vector_store_id", + ) diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35-py3-none-any.whl new file mode 100644 index 00000000000..8a443f38ef5 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35.tar.gz new file mode 100644 index 00000000000..4dde13b32e2 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35.tar.gz differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl new file mode 100644 index 00000000000..c98d9cfcfac Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz new file mode 100644 index 00000000000..c8c33404620 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz differ diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql new file mode 100644 index 00000000000..f3a0821d37f --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "LiteLLM_PolicyAttachmentTable" ADD COLUMN "tags" TEXT[] DEFAULT ARRAY[]::TEXT[]; + diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql new file mode 100644 index 00000000000..67e75e84c4a --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql @@ -0,0 +1,33 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- AlterTable +ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- AlterTable +ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- AlterTable +ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- CreateTable +CREATE TABLE "LiteLLM_AccessGroupTable" ( + "access_group_id" TEXT NOT NULL, + "access_group_name" TEXT NOT NULL, + "description" TEXT, + "access_model_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "access_mcp_server_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "access_agent_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "assigned_team_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "assigned_key_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_by" TEXT, + + CONSTRAINT "LiteLLM_AccessGroupTable_pkey" PRIMARY KEY ("access_group_id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_AccessGroupTable_access_group_name_key" ON "LiteLLM_AccessGroupTable"("access_group_name"); + diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260213105436_add_managed_vector_store_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260213105436_add_managed_vector_store_table/migration.sql new file mode 100644 index 00000000000..0835875220f --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260213105436_add_managed_vector_store_table/migration.sql @@ -0,0 +1,22 @@ +-- CreateTable +CREATE TABLE "LiteLLM_ManagedVectorStoreTable" ( + "id" TEXT NOT NULL, + "unified_resource_id" TEXT NOT NULL, + "resource_object" JSONB, + "model_mappings" JSONB NOT NULL, + "flat_model_resource_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "storage_backend" TEXT, + "storage_url" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT, + "updated_at" TIMESTAMP(3) NOT NULL, + "updated_by" TEXT, + + CONSTRAINT "LiteLLM_ManagedVectorStoreTable_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_ManagedVectorStoreTable_unified_resource_id_key" ON "LiteLLM_ManagedVectorStoreTable"("unified_resource_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_ManagedVectorStoreTable_unified_resource_id_idx" ON "LiteLLM_ManagedVectorStoreTable"("unified_resource_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 558dfcc9517..02dddc74e3e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -128,6 +128,7 @@ model LiteLLM_TeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team @@ -161,6 +162,7 @@ model LiteLLM_DeletedTeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) @@ -293,6 +295,7 @@ model LiteLLM_VerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") budget_id String? @@ -348,6 +351,7 @@ model LiteLLM_DeletedVerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") router_settings Json? @default("{}") @@ -766,6 +770,22 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t @@index([model_object_id]) } +model LiteLLM_ManagedVectorStoreTable { + id String @id @default(uuid()) + unified_resource_id String @unique // The base64 encoded unified vector store ID + resource_object Json? // Stores the VectorStoreCreateResponse + model_mappings Json // Maps model_id -> provider_vector_store_id + flat_model_resource_ids String[] @default([]) // Flat list of provider vector store IDs for faster querying + storage_backend String? // Storage backend name (if applicable) + storage_url String? // Storage URL (if applicable) + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @updatedAt + updated_by String? + + @@index([unified_resource_id]) +} + model LiteLLM_ManagedVectorStoresTable { vector_store_id String @id custom_llm_provider String @@ -920,3 +940,23 @@ model LiteLLM_PolicyAttachmentTable { updated_at DateTime @default(now()) @updatedAt updated_by String? } + +//Unified Access Groups table for storing unified access groups +model LiteLLM_AccessGroupTable { + access_group_id String @id @default(uuid()) + access_group_name String @unique + description String? + + // Resource memberships - explicit arrays per type + access_model_ids String[] @default([]) + access_mcp_server_ids String[] @default([]) + access_agent_ids String[] @default([]) + + assigned_team_ids String[] @default([]) + assigned_key_ids String[] @default([]) + + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? +} \ No newline at end of file diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index e0a769a5edf..eda49bfb9fa 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.34" +version = "0.4.36" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.34" +version = "0.4.36" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index 538ee727612..0fdbac63feb 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -175,6 +175,7 @@ _async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # Custo pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False +standard_logging_payload_excluded_fields: Optional[List[str]] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index ebe9af9d85c..2af6ed8f09e 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -275,7 +275,6 @@ LLM_CONFIG_NAMES = ( "LmStudioEmbeddingConfig", "NscaleConfig", "PerplexityChatConfig", - "PerplexityResponsesConfig", "AzureOpenAIO1Config", "IBMWatsonXAIConfig", "IBMWatsonXChatConfig", diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 5edb8067a08..a066a5e95bc 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -19,6 +19,7 @@ "mcp-client-2025-11-20": "mcp-client-2025-11-20", "mcp-client-2025-04-04": "mcp-client-2025-04-04", "mcp-servers-2025-12-04": "mcp-servers-2025-12-04", + "oauth-2025-04-20": "oauth-2025-04-20", "output-128k-2025-02-19": "output-128k-2025-02-19", "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05", "skills-2025-10-02": "skills-2025-10-02", diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index f80eae20f3b..16a467e00cb 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -39,11 +39,19 @@ async def _handle_completed_batch( batch: Batch, custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"], model_name: Optional[str] = None, + litellm_params: Optional[dict] = None, ) -> Tuple[float, Usage, List[str]]: - """Helper function to process a completed batch and handle logging""" + """Helper function to process a completed batch and handle logging + + Args: + batch: The batch object + custom_llm_provider: The LLM provider + model_name: Optional model name + litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.) + """ # Get batch results file_content_dictionary = await _get_batch_output_file_content_as_dictionary( - batch, custom_llm_provider + batch, custom_llm_provider, litellm_params=litellm_params ) # Calculate costs and usage @@ -187,9 +195,16 @@ def calculate_vertex_ai_batch_cost_and_usage( async def _get_batch_output_file_content_as_dictionary( batch: Batch, custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai", + litellm_params: Optional[dict] = None, ) -> List[dict]: """ Get the batch output file content as a list of dictionaries + + Args: + batch: The batch object + custom_llm_provider: The LLM provider + litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.) + Required for Azure and other providers that need authentication """ from litellm.files.main import afile_content from litellm.proxy.openai_files_endpoints.common_utils import ( @@ -211,13 +226,50 @@ async def _get_batch_output_file_content_as_dictionary( except (IndexError, AttributeError) as e: verbose_logger.error(f"Failed to extract LLM output file ID from unified file ID: {batch.output_file_id}, error: {e}") - _file_content = await afile_content( - file_id=file_id, - custom_llm_provider=custom_llm_provider, - ) + # Build kwargs for afile_content with credentials from litellm_params + file_content_kwargs = { + "file_id": file_id, + "custom_llm_provider": custom_llm_provider, + } + + # Extract and add credentials for file access + credentials = _extract_file_access_credentials(litellm_params) + file_content_kwargs.update(credentials) + + _file_content = await afile_content(**file_content_kwargs) return _get_file_content_as_dictionary(_file_content.content) +def _extract_file_access_credentials(litellm_params: Optional[dict]) -> dict: + """ + Extract credentials from litellm_params for file access operations. + + This method extracts relevant authentication and configuration parameters + needed for accessing files across different providers (Azure, Vertex AI, etc.). + + Args: + litellm_params: Dictionary containing litellm parameters with credentials + + Returns: + Dictionary containing only the credentials needed for file access + """ + credentials = {} + + if litellm_params: + # List of credential keys that should be passed to file operations + credential_keys = [ + "api_key", "api_base", "api_version", "organization", + "azure_ad_token", "azure_ad_token_provider", + "vertex_project", "vertex_location", "vertex_credentials", + "timeout", "max_retries" + ] + for key in credential_keys: + if key in litellm_params: + credentials[key] = litellm_params[key] + + return credentials + + def _get_file_content_as_dictionary(file_content: bytes) -> List[dict]: """ Get the file content as a list of dictionaries from JSON Lines format diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 3edc3f42820..6df570c72b9 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -12,7 +12,8 @@ import asyncio import time import traceback from concurrent.futures import ThreadPoolExecutor -from typing import TYPE_CHECKING, Any, List, Optional, Union +from threading import Lock +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union if TYPE_CHECKING: from litellm.types.caching import RedisPipelineIncrementOperation @@ -71,6 +72,7 @@ class DualCache(BaseCache): self.last_redis_batch_access_time = LimitedSizeOrderedDict( max_size=default_max_redis_batch_cache_size ) + self._last_redis_batch_access_time_lock = Lock() self.redis_batch_cache_expiry = ( default_redis_batch_cache_expiry or litellm.default_redis_batch_cache_expiry @@ -236,22 +238,46 @@ class DualCache(BaseCache): except Exception: verbose_logger.error(traceback.format_exc()) - def get_redis_batch_keys( + def _reserve_redis_batch_keys( self, current_time: float, keys: List[str], result: List[Any], - ) -> List[str]: - sublist_keys = [] - for key, value in zip(keys, result): - if value is None: + ) -> Tuple[List[str], Dict[str, Optional[float]]]: + """ + Atomically choose keys to fetch from Redis and reserve their access time. + This prevents check-then-act races under concurrent async callers. + """ + sublist_keys: List[str] = [] + previous_access_times: Dict[str, Optional[float]] = {} + + with self._last_redis_batch_access_time_lock: + for key, value in zip(keys, result): + if value is not None: + continue + if ( key not in self.last_redis_batch_access_time or current_time - self.last_redis_batch_access_time[key] >= self.redis_batch_cache_expiry ): sublist_keys.append(key) - return sublist_keys + previous_access_times[key] = self.last_redis_batch_access_time.get( + key + ) + self.last_redis_batch_access_time[key] = current_time + + return sublist_keys, previous_access_times + + def _rollback_redis_batch_key_reservations( + self, previous_access_times: Dict[str, Optional[float]] + ) -> None: + with self._last_redis_batch_access_time_lock: + for key, previous_time in previous_access_times.items(): + if previous_time is None: + self.last_redis_batch_access_time.pop(key, None) + else: + self.last_redis_batch_access_time[key] = previous_time async def async_batch_get_cache( self, @@ -276,19 +302,23 @@ class DualCache(BaseCache): - check the redis cache """ current_time = time.time() - sublist_keys = self.get_redis_batch_keys(current_time, keys, result) + sublist_keys, previous_access_times = self._reserve_redis_batch_keys( + current_time, keys, result + ) - # Only hit Redis if the last access time was more than 5 seconds ago + # Only hit Redis if enough time has passed since last access. if len(sublist_keys) > 0: - # If not found in in-memory cache, try fetching from Redis - redis_result = await self.redis_cache.async_batch_get_cache( - sublist_keys, parent_otel_span=parent_otel_span - ) - - # Update the last access time for ALL queried keys - # This includes keys with None values to throttle repeated Redis queries - for key in sublist_keys: - self.last_redis_batch_access_time[key] = current_time + try: + # If not found in in-memory cache, try fetching from Redis + redis_result = await self.redis_cache.async_batch_get_cache( + sublist_keys, parent_otel_span=parent_otel_span + ) + except Exception: + # Do not throttle subsequent callers if the Redis read fails. + self._rollback_redis_batch_key_reservations( + previous_access_times + ) + raise # Short-circuit if redis_result is None or contains only None values if redis_result is None or all(v is None for v in redis_result.values()): diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 753a94295b3..e546a0dbb02 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -227,6 +227,84 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return input_items, instructions + def _map_optional_params_to_responses_api_request( + self, + optional_params: dict, + responses_api_request: "ResponsesAPIOptionalRequestParams", + ) -> None: + """Map optional_params into responses_api_request (mutates in place).""" + for key, value in optional_params.items(): + if value is None: + continue + if key in ("max_tokens", "max_completion_tokens"): + responses_api_request["max_output_tokens"] = value + elif key == "tools" and value is not None: + responses_api_request["tools"] = ( + self._convert_tools_to_responses_format( + cast(List[Dict[str, Any]], value) + ) + ) + elif key == "response_format": + text_format = self._transform_response_format_to_text_format(value) + if text_format: + responses_api_request["text"] = text_format # type: ignore + elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): + responses_api_request[key] = value # type: ignore + elif key == "previous_response_id": + responses_api_request["previous_response_id"] = value + elif key == "reasoning_effort": + responses_api_request["reasoning"] = self._map_reasoning_effort(value) + elif key == "web_search_options": + self._add_web_search_tool(responses_api_request, value) + + def _build_sanitized_litellm_params( + self, litellm_params: dict + ) -> Dict[str, Any]: + """Build sanitized litellm_params with merged metadata.""" + responses_optional_param_keys = set( + ResponsesAPIOptionalRequestParams.__annotations__.keys() + ) + sanitized: Dict[str, Any] = { + key: value + for key, value in litellm_params.items() + if key not in responses_optional_param_keys + } + legacy_metadata = litellm_params.get("metadata") + existing_litellm_metadata = litellm_params.get("litellm_metadata") + merged_litellm_metadata: Dict[str, Any] = {} + if isinstance(legacy_metadata, dict): + merged_litellm_metadata.update(legacy_metadata) + if isinstance(existing_litellm_metadata, dict): + merged_litellm_metadata.update(existing_litellm_metadata) + if merged_litellm_metadata: + sanitized["litellm_metadata"] = merged_litellm_metadata + else: + sanitized.pop("litellm_metadata", None) + return sanitized + + def _merge_responses_api_request_into_request_data( + self, + request_data: Dict[str, Any], + responses_api_request: "ResponsesAPIOptionalRequestParams", + instructions: Optional[str], + ) -> None: + """Add non-None values from responses_api_request into request_data.""" + for key, value in responses_api_request.items(): + if value is None: + continue + if key == "instructions" and instructions: + request_data["instructions"] = instructions + elif key == "stream_options" and isinstance(value, dict): + request_data["stream_options"] = value.get("include_obfuscation") + elif key == "user" and isinstance(value, str): + # OpenAI API requires user param to be max 64 chars - truncate if longer + if len(value) <= 64: + request_data["user"] = value + else: + request_data["user"] = value[:64] + else: + request_data[key] = value + def transform_request( self, model: str, @@ -251,36 +329,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if instructions: responses_api_request["instructions"] = instructions - # Map optional parameters - for key, value in optional_params.items(): - if value is None: - continue - if key in ("max_tokens", "max_completion_tokens"): - responses_api_request["max_output_tokens"] = value - elif key == "tools" and value is not None: - # Convert chat completion tools to responses API tools format - responses_api_request["tools"] = ( - self._convert_tools_to_responses_format( - cast(List[Dict[str, Any]], value) - ) - ) - elif key == "response_format": - # Convert response_format to text.format - text_format = self._transform_response_format_to_text_format(value) - if text_format: - responses_api_request["text"] = text_format # type: ignore - elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): - responses_api_request[key] = value # type: ignore - elif key == "metadata": - responses_api_request["metadata"] = value - elif key == "previous_response_id": - responses_api_request["previous_response_id"] = value - elif key == "reasoning_effort": - responses_api_request["reasoning"] = self._map_reasoning_effort(value) - elif key == "web_search_options": - self._add_web_search_tool(responses_api_request, value) + self._map_optional_params_to_responses_api_request( + optional_params, responses_api_request + ) - # Get stream parameter from litellm_params if not in optional_params stream = optional_params.get("stream") or litellm_params.get("stream", False) verbose_logger.debug(f"Chat provider: Stream parameter: {stream}") @@ -304,11 +356,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): setattr(litellm_logging_obj, "call_type", CallTypes.responses.value) + sanitized_litellm_params = self._build_sanitized_litellm_params( + litellm_params + ) + request_data = { "model": api_model, "input": input_items, "litellm_logging_obj": litellm_logging_obj, - **litellm_params, + **sanitized_litellm_params, "client": client, } @@ -316,18 +372,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): f"Chat provider: Final request model={api_model}, input_items={len(input_items)}" ) - # Add non-None values from responses_api_request - for key, value in responses_api_request.items(): - if value is not None: - if key == "instructions" and instructions: - request_data["instructions"] = instructions - elif key == "stream_options" and isinstance(value, dict): - request_data["stream_options"] = value.get("include_obfuscation") - elif key == "user": # string can't be longer than 64 characters - if isinstance(value, str) and len(value) <= 64: - request_data["user"] = value - else: - request_data[key] = value + self._merge_responses_api_request_into_request_data( + request_data, responses_api_request, instructions + ) if headers: request_data["extra_headers"] = headers diff --git a/litellm/constants.py b/litellm/constants.py index 88c57d3ce4c..addd659be73 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -101,6 +101,11 @@ MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE = int( MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL = int( os.getenv("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", "3600") ) + +# Default npm cache directory for STDIO MCP servers. +# npm/npx needs a writable cache dir; in containers the default (~/.npm) +# may not exist or be read-only. /tmp is always writable. +MCP_NPM_CACHE_DIR = os.getenv("MCP_NPM_CACHE_DIR", "/tmp/.npm_mcp_cache") MCP_OAUTH2_TOKEN_CACHE_MIN_TTL = int( os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10") ) @@ -1011,10 +1016,12 @@ BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[ BEDROCK_CONVERSE_MODELS = [ "qwen.qwen3-coder-480b-a35b-v1:0", + "qwen.qwen3-coder-next", "qwen.qwen3-235b-a22b-2507-v1:0", "qwen.qwen3-coder-30b-a3b-v1:0", "qwen.qwen3-32b-v1:0", "deepseek.v3-v1:0", + "deepseek.v3.2", "openai.gpt-oss-20b-1:0", "openai.gpt-oss-120b-1:0", "anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1057,6 +1064,8 @@ BEDROCK_CONVERSE_MODELS = [ "amazon.nova-pro-v1:0", "writer.palmyra-x4-v1:0", "writer.palmyra-x5-v1:0", + "minimax.minimax-m2.1", + "moonshotai.kimi-k2.5", ] diff --git a/litellm/integrations/cloudzero/transform.py b/litellm/integrations/cloudzero/transform.py index c36833a6dbf..b40a71da1c6 100644 --- a/litellm/integrations/cloudzero/transform.py +++ b/litellm/integrations/cloudzero/transform.py @@ -141,7 +141,7 @@ class CBFTransformer: # Required CBF fields 'time/usage_start': usage_date.isoformat() if usage_date else None, # Required: ISO-formatted UTC datetime 'cost/cost': float(row.get('spend', 0.0)), # Required: billed cost - 'resource/id': model, # Send model name + 'resource/id': resource_id, # CZRN (CloudZero Resource Name) # Usage metrics for token consumption 'usage/amount': total_tokens, # Numeric value of tokens consumed diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 407bc581f71..af6d651803e 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -29,6 +29,7 @@ from litellm.types.utils import ( LLMResponseTypes, StandardLoggingGuardrailInformation, ) +from fastapi.exceptions import HTTPException if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -624,7 +625,9 @@ class CustomGuardrail(CustomLogger): This gets logged on downsteam Langfuse, DataDog, etc. """ # Convert None to empty dict to satisfy type requirements - guardrail_response = {} if response is None else response + guardrail_response: Union[Dict[str, Any], str] = ( + {} if response is None else response + ) # For apply_guardrail functions in custom_code_guardrail scenario, # simplify the logged response to "allow", "deny", or "mask" @@ -648,6 +651,23 @@ class CustomGuardrail(CustomLogger): ) return response + @staticmethod + def _is_guardrail_intervention(e: Exception) -> bool: + """ + Returns True if the exception represents an intentional guardrail block + (this was logged previously as an API failure - guardrail_failed_to_respond). + + Guardrails signal intentional blocks by raising: + - HTTPException with status 400 (content policy violation) + - ModifyResponseException (passthrough mode violation) + """ + + if isinstance(e, ModifyResponseException): + return True + if isinstance(e, HTTPException) and e.status_code == 400: + return True + return False + def _process_error( self, e: Exception, @@ -662,6 +682,11 @@ class CustomGuardrail(CustomLogger): This gets logged on downsteam Langfuse, DataDog, etc. """ + guardrail_status: GuardrailStatus = ( + "guardrail_intervened" + if self._is_guardrail_intervention(e) + else "guardrail_failed_to_respond" + ) # For custom_code_guardrail scenario, log as "deny" instead of full exception # Check if this is from custom_code_guardrail by checking the class name guardrail_response: Union[Exception, str] = e @@ -671,7 +696,7 @@ class CustomGuardrail(CustomLogger): self.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response=guardrail_response, request_data=request_data, - guardrail_status="guardrail_failed_to_respond", + guardrail_status=guardrail_status, duration=duration, start_time=start_time, end_time=end_time, diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 4a341863d4b..c244363e389 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -774,15 +774,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac self, model_call_details: Dict ) -> Dict: """ - Only redacts messages and responses when self.turn_off_message_logging is True + Redacts or excludes fields from StandardLoggingPayload before callbacks receive it. + This method handles two features: + 1. turn_off_message_logging: When True, redacts messages and responses + 2. standard_logging_payload_excluded_fields: Removes specified fields entirely - By default, self.turn_off_message_logging is False and this does nothing. - - Return a redacted deepcopy of the provided logging payload. + Return a modified copy of the provided logging payload. This is useful for logging payloads that contain sensitive information. """ + import litellm from copy import copy from litellm import Choices, Message, ModelResponse @@ -790,14 +792,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac turn_off_message_logging: bool = getattr( self, "turn_off_message_logging", False ) + excluded_fields: Optional[List[str]] = getattr( + litellm, "standard_logging_payload_excluded_fields", None + ) - if turn_off_message_logging is False: + # Early return if no processing needed + if turn_off_message_logging is False and not excluded_fields: return model_call_details # Only make a shallow copy of the top-level dict to avoid deepcopy issues # with complex objects like AuthenticationError that may be present model_call_details_copy = copy(model_call_details) - redacted_str = "redacted-by-litellm" standard_logging_object = model_call_details.get("standard_logging_object") if standard_logging_object is None: return model_call_details_copy @@ -805,39 +810,58 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac # Make a copy of just the standard_logging_object to avoid modifying the original standard_logging_object_copy = copy(standard_logging_object) - if standard_logging_object_copy.get("messages") is not None: - standard_logging_object_copy["messages"] = [ - Message(content=redacted_str).model_dump() - ] + # Handle excluded fields - remove them entirely from the payload + if excluded_fields: + for field in excluded_fields: + if field in standard_logging_object_copy: + del standard_logging_object_copy[field] - if standard_logging_object_copy.get("response") is not None: - response = standard_logging_object_copy["response"] - # Check if this is a ResponsesAPIResponse (has "output" field) - if isinstance(response, dict) and "output" in response: - # Make a copy to avoid modifying the original - from copy import deepcopy + # Handle turn_off_message_logging - redact messages and responses (if not already excluded) + if turn_off_message_logging: + redacted_str = "redacted-by-litellm" - response_copy = deepcopy(response) - # Redact content in output array - if isinstance(response_copy.get("output"), list): - for output_item in response_copy["output"]: - if isinstance(output_item, dict) and "content" in output_item: - if isinstance(output_item["content"], list): - # Redact text in content items - for content_item in output_item["content"]: - if ( - isinstance(content_item, dict) - and "text" in content_item - ): - content_item["text"] = redacted_str - standard_logging_object_copy["response"] = response_copy - else: - # Standard ModelResponse format - model_response = ModelResponse( - choices=[Choices(message=Message(content=redacted_str))] - ) - model_response_dict = model_response.model_dump() - standard_logging_object_copy["response"] = model_response_dict + if ( + "messages" not in (excluded_fields or []) + and standard_logging_object_copy.get("messages") is not None + ): + standard_logging_object_copy["messages"] = [ + Message(content=redacted_str).model_dump() + ] + + if ( + "response" not in (excluded_fields or []) + and standard_logging_object_copy.get("response") is not None + ): + response = standard_logging_object_copy["response"] + # Check if this is a ResponsesAPIResponse (has "output" field) + if isinstance(response, dict) and "output" in response: + # Make a copy to avoid modifying the original + from copy import deepcopy + + response_copy = deepcopy(response) + # Redact content in output array + if isinstance(response_copy.get("output"), list): + for output_item in response_copy["output"]: + if ( + isinstance(output_item, dict) + and "content" in output_item + ): + if isinstance(output_item["content"], list): + # Redact text in content items + for content_item in output_item["content"]: + if ( + isinstance(content_item, dict) + and "text" in content_item + ): + content_item["text"] = redacted_str + standard_logging_object_copy["response"] = response_copy + else: + # Standard ModelResponse format + model_response = ModelResponse( + choices=[Choices(message=Message(content=redacted_str))] + ) + model_response_dict = model_response.model_dump() + standard_logging_object_copy["response"] = model_response_dict model_call_details_copy["standard_logging_object"] = ( standard_logging_object_copy diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 3ddcae69315..dde44cced36 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -70,6 +70,11 @@ class ExceptionCheckers: Check if an error string indicates a context window exceeded error. """ _error_str_lowercase = error_str.lower() + # Exclude param validation errors (e.g. OpenAI "user" param max 64 chars) + if "string_above_max_length" in _error_str_lowercase: + return False + if "invalid 'user'" in _error_str_lowercase and "string too long" in _error_str_lowercase: + return False known_exception_substrings = [ "exceed context limit", "this model's maximum context length is", @@ -98,16 +103,18 @@ class ExceptionCheckers: """ Check if an error string indicates a content policy violation error. """ + _lower = error_str.lower() known_exception_substrings = [ - "invalid_request_error", "content_policy_violation", + "responsibleaipolicyviolation", "the response was filtered due to the prompt triggering azure openai's content management", "your task failed as a result of our safety system", "the model produced invalid content", "content_filter_policy", + "your request was rejected as a result of our safety system", ] for substring in known_exception_substrings: - if substring in error_str.lower(): + if substring in _lower: return True return False @@ -2060,6 +2067,19 @@ def exception_type( # type: ignore # noqa: PLR0915 if isinstance(body_dict, dict): if isinstance(body_dict.get("error"), dict): azure_error_code = body_dict["error"].get("code") # type: ignore[index] + # Also check inner_error for + # ResponsibleAIPolicyViolation which indicates a + # content policy violation even when the top-level + # code is generic (e.g. "invalid_request_error"). + if azure_error_code != "content_policy_violation": + _inner = ( + body_dict["error"].get("inner_error") # type: ignore[index] + or body_dict["error"].get("innererror") # type: ignore[index] + ) + if isinstance(_inner, dict) and _inner.get( + "code" + ) == "ResponsibleAIPolicyViolation": + azure_error_code = "content_policy_violation" else: azure_error_code = body_dict.get("code") except Exception: diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 718773a1b16..8ab4ec15b07 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -51,7 +51,7 @@ def handle_cohere_chat_model_custom_llm_provider( if custom_llm_provider == "cohere" and model in litellm.cohere_chat_models: return model, "cohere_chat" - if "/" in model: + if model and "/" in model: _custom_llm_provider, _model = model.split("/", 1) if ( _custom_llm_provider @@ -84,7 +84,7 @@ def handle_anthropic_text_model_custom_llm_provider( ): return model, "anthropic_text" - if "/" in model: + if model and "/" in model: _custom_llm_provider, _model = model.split("/", 1) if ( _custom_llm_provider @@ -113,6 +113,12 @@ def get_llm_provider( # noqa: PLR0915 Return model, custom_llm_provider, dynamic_api_key, api_base """ try: + # Early validation - model is required + if model is None: + raise ValueError( + "model parameter is required but was None. Please provide a valid model name." + ) + if litellm.LiteLLMProxyChatConfig._should_use_litellm_proxy_by_default( litellm_params=litellm_params ): diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 82a7af64f97..bdbbc7579b7 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2331,7 +2331,7 @@ class Logging(LiteLLMLoggingBaseClass): result, LiteLLMBatch ): litellm_params = self.litellm_params or {} - litellm_metadata = litellm_params.get("litellm_metadata", {}) + litellm_metadata = litellm_params.get("litellm_metadata") or {} if ( litellm_metadata.get("batch_ignore_default_logging", False) is True ): # polling job will query these frequently, don't spam db logs @@ -2369,6 +2369,7 @@ class Logging(LiteLLMLoggingBaseClass): ) = await _handle_completed_batch( batch=result, custom_llm_provider=self.custom_llm_provider, + litellm_params=self.litellm_params, ) result._hidden_params["response_cost"] = response_cost @@ -3127,7 +3128,7 @@ class Logging(LiteLLMLoggingBaseClass): self, dynamic_success_callbacks: Optional[List], global_callbacks: List ) -> List: if dynamic_success_callbacks is None: - return global_callbacks + return list(global_callbacks) return list(set(dynamic_success_callbacks + global_callbacks)) def _remove_internal_litellm_callbacks(self, callbacks: List) -> List: diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index acd08f0a569..9938cd7979b 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -664,35 +664,34 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): reasoning_effort: Optional[Union[REASONING_EFFORT, str]], model: str, ) -> Optional[AnthropicThinkingParam]: + if reasoning_effort is None or reasoning_effort == "none": + return None if AnthropicConfig._is_claude_opus_4_6(model): return AnthropicThinkingParam( type="adaptive", ) + elif reasoning_effort == "low": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, + ) + elif reasoning_effort == "medium": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + ) + elif reasoning_effort == "high": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, + ) + elif reasoning_effort == "minimal": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, + ) else: - if reasoning_effort is None: - return None - elif reasoning_effort == "low": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, - ) - elif reasoning_effort == "medium": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - ) - elif reasoning_effort == "high": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, - ) - elif reasoning_effort == "minimal": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, - ) - else: - raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}") + raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}") def _extract_json_schema_from_response_format( self, value: Optional[dict] diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index cb23d21fbc9..c665e084261 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -38,9 +38,18 @@ def optionally_handle_anthropic_oauth( Returns: Tuple of (updated headers, api_key) """ + # Check Authorization header (passthrough / forwarded requests) auth_header = headers.get("authorization", "") if auth_header and auth_header.startswith(f"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}"): api_key = auth_header.replace("Bearer ", "") + headers.pop("x-api-key", None) + headers["anthropic-beta"] = ANTHROPIC_OAUTH_BETA_HEADER + headers["anthropic-dangerous-direct-browser-access"] = "true" + return headers, api_key + # Check api_key directly (standard chat/completion flow) + if api_key and api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX): + headers.pop("x-api-key", None) + headers["authorization"] = f"Bearer {api_key}" headers["anthropic-beta"] = ANTHROPIC_OAUTH_BETA_HEADER headers["anthropic-dangerous-direct-browser-access"] = "true" return headers, api_key @@ -108,7 +117,9 @@ class AnthropicModelInfo(BaseLLMModelInfo): if tools is None: return False for tool in tools: - if "type" in tool and tool["type"].startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value): + if "type" in tool and tool["type"].startswith( + ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value + ): return True return False @@ -134,111 +145,126 @@ class AnthropicModelInfo(BaseLLMModelInfo): """ if not tools: return False - + for tool in tools: tool_type = tool.get("type", "") - if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]: + if tool_type in [ + "tool_search_tool_regex_20251119", + "tool_search_tool_bm25_20251119", + ]: return True return False - + def is_programmatic_tool_calling_used(self, tools: Optional[List]) -> bool: """ Check if programmatic tool calling is being used (tools with allowed_callers field). - + Returns True if any tool has allowed_callers containing 'code_execution_20250825'. """ if not tools: return False - + for tool in tools: # Check top-level allowed_callers allowed_callers = tool.get("allowed_callers", None) if allowed_callers and isinstance(allowed_callers, list): if "code_execution_20250825" in allowed_callers: return True - + # Check function.allowed_callers for OpenAI format tools function = tool.get("function", {}) if isinstance(function, dict): function_allowed_callers = function.get("allowed_callers", None) - if function_allowed_callers and isinstance(function_allowed_callers, list): + if function_allowed_callers and isinstance( + function_allowed_callers, list + ): if "code_execution_20250825" in function_allowed_callers: return True - + return False - + def is_input_examples_used(self, tools: Optional[List]) -> bool: """ Check if input_examples is being used in any tools. - + Returns True if any tool has input_examples field. """ if not tools: return False - + for tool in tools: # Check top-level input_examples input_examples = tool.get("input_examples", None) - if input_examples and isinstance(input_examples, list) and len(input_examples) > 0: + if ( + input_examples + and isinstance(input_examples, list) + and len(input_examples) > 0 + ): return True - + # Check function.input_examples for OpenAI format tools function = tool.get("function", {}) if isinstance(function, dict): function_input_examples = function.get("input_examples", None) - if function_input_examples and isinstance(function_input_examples, list) and len(function_input_examples) > 0: + if ( + function_input_examples + and isinstance(function_input_examples, list) + and len(function_input_examples) > 0 + ): return True - + return False - - def is_effort_used(self, optional_params: Optional[dict], model: Optional[str] = None) -> bool: + + def is_effort_used( + self, optional_params: Optional[dict], model: Optional[str] = None + ) -> bool: """ Check if effort parameter is being used. - + Returns True if effort-related parameters are present. """ if not optional_params: return False - + # Check if reasoning_effort is provided for Claude Opus 4.5 if model and ("opus-4-5" in model.lower() or "opus_4_5" in model.lower()): reasoning_effort = optional_params.get("reasoning_effort") if reasoning_effort and isinstance(reasoning_effort, str): return True - + # Check if output_config is directly provided output_config = optional_params.get("output_config") if output_config and isinstance(output_config, dict): effort = output_config.get("effort") if effort and isinstance(effort, str): return True - + return False def is_code_execution_tool_used(self, tools: Optional[List]) -> bool: """ Check if code execution tool is being used. - + Returns True if any tool has type "code_execution_20250825". """ if not tools: return False - + for tool in tools: tool_type = tool.get("type", "") if tool_type == "code_execution_20250825": return True return False - + def is_container_with_skills_used(self, optional_params: Optional[dict]) -> bool: """ Check if container with skills is being used. - + Returns True if optional_params contains container with skills. """ if not optional_params: return False - + container = optional_params.get("container") if container and isinstance(container, dict): skills = container.get("skills") @@ -256,10 +282,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): def get_computer_tool_beta_header(self, computer_tool_version: str) -> str: """ Get the appropriate beta header for a given computer tool version. - + Args: computer_tool_version: The computer tool version (e.g., 'computer_20250124', 'computer_20241022') - + Returns: The corresponding beta header string """ @@ -282,37 +308,37 @@ class AnthropicModelInfo(BaseLLMModelInfo): ) -> List[str]: """ Get list of common beta headers based on the features that are active. - + Returns: List of beta header strings """ from litellm.types.llms.anthropic import ( ANTHROPIC_EFFORT_BETA_HEADER, ) - + betas = [] - + # Detect features effort_used = self.is_effort_used(optional_params, model) - + if effort_used: betas.append(ANTHROPIC_EFFORT_BETA_HEADER) # effort-2025-11-24 - + if computer_tool_used: beta_header = self.get_computer_tool_beta_header(computer_tool_used) betas.append(beta_header) - + # Anthropic no longer requires the prompt-caching beta header # Prompt caching now works automatically when cache_control is used in messages # Reference: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching - + if file_id_used: betas.append("files-api-2025-04-14") betas.append("code-execution-2025-05-22") - + if mcp_server_used: betas.append("mcp-client-2025-04-04") - + return list(set(betas)) def get_anthropic_headers( @@ -351,27 +377,35 @@ class AnthropicModelInfo(BaseLLMModelInfo): # Tool search, programmatic tool calling, and input_examples all use the same beta header if tool_search_used or programmatic_tool_calling_used or input_examples_used: from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER + betas.add(ANTHROPIC_TOOL_SEARCH_BETA_HEADER) - + # Effort parameter uses a separate beta header if effort_used: from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER + betas.add(ANTHROPIC_EFFORT_BETA_HEADER) - + # Code execution tool uses a separate beta header if code_execution_tool_used: betas.add("code-execution-2025-08-25") - + # Container with skills uses a separate beta header if container_with_skills_used: betas.add("skills-2025-10-02") + _is_oauth = api_key and api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX) headers = { "anthropic-version": anthropic_version or "2023-06-01", - "x-api-key": api_key, "accept": "application/json", "content-type": "application/json", } + if _is_oauth: + headers["authorization"] = f"Bearer {api_key}" + headers["anthropic-dangerous-direct-browser-access"] = "true" + betas.add(ANTHROPIC_OAUTH_BETA_HEADER) + else: + headers["x-api-key"] = api_key if user_anthropic_beta_headers is not None: betas.update(user_anthropic_beta_headers) @@ -381,7 +415,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): # Vertex AI requires web search beta header for web search to work if web_search_tool_used: from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES - headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value + + headers[ + "anthropic-beta" + ] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value elif len(betas) > 0: headers["anthropic-beta"] = ",".join(betas) @@ -398,7 +435,9 @@ class AnthropicModelInfo(BaseLLMModelInfo): api_base: Optional[str] = None, ) -> Dict: # Check for Anthropic OAuth token in headers - headers, api_key = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key) + headers, api_key = optionally_handle_anthropic_oauth( + headers=headers, api_key=api_key + ) if api_key is None: raise litellm.AuthenticationError( message="Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params. Please set `ANTHROPIC_API_KEY` in your environment vars", @@ -416,11 +455,15 @@ class AnthropicModelInfo(BaseLLMModelInfo): file_id_used = self.is_file_id_used(messages=messages) web_search_tool_used = self.is_web_search_tool_used(tools=tools) tool_search_used = self.is_tool_search_used(tools=tools) - programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools) + programmatic_tool_calling_used = self.is_programmatic_tool_calling_used( + tools=tools + ) input_examples_used = self.is_input_examples_used(tools=tools) effort_used = self.is_effort_used(optional_params=optional_params, model=model) code_execution_tool_used = self.is_code_execution_tool_used(tools=tools) - container_with_skills_used = self.is_container_with_skills_used(optional_params=optional_params) + container_with_skills_used = self.is_container_with_skills_used( + optional_params=optional_params + ) user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( anthropic_beta_header=headers.get("anthropic-beta") ) @@ -499,7 +542,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): def get_token_counter(self) -> Optional[BaseTokenCounter]: """ Factory method to create an Anthropic token counter. - + Returns: AnthropicTokenCounter instance for this provider. """ diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 8f2f3bf3545..8275ba2b3e1 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -49,15 +49,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): # TODO: Add Anthropic `metadata` support # "metadata", ] - + @staticmethod def _filter_billing_headers_from_system(system_param): """ Filter out x-anthropic-billing-header metadata from system parameter. - + Args: system_param: Can be a string or a list of system message content blocks - + Returns: Filtered system parameter (string or list), or None if all content was filtered """ @@ -74,7 +74,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): text = content_block.get("text", "") content_type = content_block.get("type", "") # Skip text blocks that start with billing header - if content_type == "text" and text.startswith("x-anthropic-billing-header:"): + if content_type == "text" and text.startswith( + "x-anthropic-billing-header:" + ): continue filtered_list.append(content_block) else: @@ -111,11 +113,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): import os # Check for Anthropic OAuth token in Authorization header - headers, api_key = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key) + headers, api_key = optionally_handle_anthropic_oauth( + headers=headers, api_key=api_key + ) if api_key is None: api_key = os.getenv("ANTHROPIC_API_KEY") - if "x-api-key" not in headers and api_key: + if "x-api-key" not in headers and "authorization" not in headers and api_key: headers["x-api-key"] = api_key if "anthropic-version" not in headers: headers["anthropic-version"] = DEFAULT_ANTHROPIC_API_VERSION @@ -149,7 +153,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): message="max_tokens is required for Anthropic /v1/messages API", status_code=400, ) - + # Filter out x-anthropic-billing-header from system messages system_param = anthropic_messages_optional_request_params.get("system") if system_param is not None: @@ -159,7 +163,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): else: # Remove system parameter if all content was filtered out anthropic_messages_optional_request_params.pop("system", None) - + ####### get required params for all anthropic messages requests ###### verbose_logger.debug(f"TRANSFORMATION DEBUG - Messages: {messages}") anthropic_messages_request: AnthropicMessagesRequest = AnthropicMessagesRequest( @@ -244,25 +248,29 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): edits = context_management_param.get("edits", []) has_compact = False has_other = False - + for edit in edits: edit_type = edit.get("type", "") if edit_type == "compact_20260112": has_compact = True else: has_other = True - + # Add compact header if any compact edits exist if has_compact: beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - + # Add context management header if any other edits exist if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.add( + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value + ) # Check for structured outputs if optional_params.get("output_format") is not None: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value) + beta_values.add( + ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value + ) # Check for fast mode if optional_params.get("speed") == "fast": diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 76fa713ca8c..44ee51d14ab 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -901,7 +901,20 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): if response.json()["status"] == "failed": error_data = response.json() - raise AzureOpenAIError(status_code=400, message=json.dumps(error_data)) + # Preserve Azure error details (e.g. content_policy_violation, + # inner_error, content_filter_results) as structured body so + # exception_type() can route them correctly. + _error_body = error_data.get("error", error_data) + _error_msg = ( + _error_body.get("message", "Image generation failed") + if isinstance(_error_body, dict) + else json.dumps(error_data) + ) + raise AzureOpenAIError( + status_code=400, + message=_error_msg, + body=error_data, + ) result = response.json()["result"] return httpx.Response( @@ -999,7 +1012,20 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): if response.json()["status"] == "failed": error_data = response.json() - raise AzureOpenAIError(status_code=400, message=json.dumps(error_data)) + # Preserve Azure error details (e.g. content_policy_violation, + # inner_error, content_filter_results) as structured body so + # exception_type() can route them correctly. + _error_body = error_data.get("error", error_data) + _error_msg = ( + _error_body.get("message", "Image generation failed") + if isinstance(_error_body, dict) + else json.dumps(error_data) + ) + raise AzureOpenAIError( + status_code=400, + message=_error_msg, + body=error_data, + ) result = response.json()["result"] return httpx.Response( diff --git a/litellm/llms/base_llm/managed_resources/__init__.py b/litellm/llms/base_llm/managed_resources/__init__.py new file mode 100644 index 00000000000..5eb9b46f89f --- /dev/null +++ b/litellm/llms/base_llm/managed_resources/__init__.py @@ -0,0 +1,41 @@ +""" +Managed Resources Module + +This module provides base classes and utilities for managing resources +(files, vector stores, etc.) with target_model_names support. + +The BaseManagedResource class provides common functionality for: +- Storing unified resource IDs with model mappings +- Retrieving resources by unified ID +- Deleting resources across multiple models +- Creating resources for multiple models +- Filtering deployments based on model mappings +""" + +from .base_managed_resource import BaseManagedResource +from .utils import ( + decode_unified_id, + encode_unified_id, + extract_model_id_from_unified_id, + extract_provider_resource_id_from_unified_id, + extract_resource_type_from_unified_id, + extract_target_model_names_from_unified_id, + extract_unified_uuid_from_unified_id, + generate_unified_id_string, + is_base64_encoded_unified_id, + parse_unified_id, +) + +__all__ = [ + "BaseManagedResource", + "is_base64_encoded_unified_id", + "extract_target_model_names_from_unified_id", + "extract_resource_type_from_unified_id", + "extract_unified_uuid_from_unified_id", + "extract_model_id_from_unified_id", + "extract_provider_resource_id_from_unified_id", + "generate_unified_id_string", + "encode_unified_id", + "decode_unified_id", + "parse_unified_id", +] diff --git a/litellm/llms/base_llm/managed_resources/base_managed_resource.py b/litellm/llms/base_llm/managed_resources/base_managed_resource.py new file mode 100644 index 00000000000..3c8ce748ade --- /dev/null +++ b/litellm/llms/base_llm/managed_resources/base_managed_resource.py @@ -0,0 +1,605 @@ +# What is this? +## Base class for managing resources (files, vector stores, etc.) with target_model_names support +## This provides common functionality for creating, retrieving, and managing resources across multiple models + +import base64 +import json +from abc import ABC, abstractmethod +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generic, + List, + Optional, + TypeVar, + Union, + cast, +) + +from litellm import verbose_logger +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.utils import SpecialEnums + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + + from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache + from litellm.proxy.utils import PrismaClient as _PrismaClient + from litellm.router import Router as _Router + + Span = Union[_Span, Any] + InternalUsageCache = _InternalUsageCache + PrismaClient = _PrismaClient + Router = _Router +else: + Span = Any + InternalUsageCache = Any + PrismaClient = Any + Router = Any + +# Generic type for resource objects +ResourceObjectType = TypeVar('ResourceObjectType') + + +class BaseManagedResource(ABC, Generic[ResourceObjectType]): + """ + Base class for managing resources with target_model_names support. + + This class provides common functionality for: + - Storing unified resource IDs with model mappings + - Retrieving resources by unified ID + - Deleting resources across multiple models + - Creating resources for multiple models + - Filtering deployments based on model mappings + + Subclasses should implement: + - resource_type: str property + - table_name: str property + - create_resource_for_model: method to create resource on a specific model + - get_unified_resource_id_format: method to generate unified ID format + """ + + def __init__( + self, + internal_usage_cache: InternalUsageCache, + prisma_client: PrismaClient, + ): + self.internal_usage_cache = internal_usage_cache + self.prisma_client = prisma_client + + # ============================================================================ + # ABSTRACT METHODS + # ============================================================================ + + @property + @abstractmethod + def resource_type(self) -> str: + """ + Return the resource type identifier (e.g., 'file', 'vector_store', 'vector_store_file'). + Used for logging and unified ID generation. + """ + pass + + @property + @abstractmethod + def table_name(self) -> str: + """ + Return the database table name for this resource type. + Example: 'litellm_managedfiletable', 'litellm_managedvectorstoretable' + """ + pass + + @abstractmethod + def get_unified_resource_id_format( + self, + resource_object: ResourceObjectType, + target_model_names_list: List[str], + ) -> str: + """ + Generate the format string for the unified resource ID. + + This should return a string that will be base64 encoded. + Example for files: + "litellm_proxy:application/json;unified_id,{uuid};target_model_names,{models};..." + + Args: + resource_object: The resource object returned from the provider + target_model_names_list: List of target model names + + Returns: + Format string to be base64 encoded + """ + pass + + @abstractmethod + async def create_resource_for_model( + self, + llm_router: Router, + model: str, + request_data: Dict[str, Any], + litellm_parent_otel_span: Span, + ) -> ResourceObjectType: + """ + Create a resource for a specific model. + + Args: + llm_router: LiteLLM router instance + model: Model name to create resource for + request_data: Request data for resource creation + litellm_parent_otel_span: OpenTelemetry span for tracing + + Returns: + Resource object from the provider + """ + pass + + # ============================================================================ + # COMMON STORAGE OPERATIONS + # ============================================================================ + + async def store_unified_resource_id( + self, + unified_resource_id: str, + resource_object: Optional[ResourceObjectType], + litellm_parent_otel_span: Optional[Span], + model_mappings: Dict[str, str], + user_api_key_dict: UserAPIKeyAuth, + additional_db_fields: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Store unified resource ID with model mappings in cache and database. + + Args: + unified_resource_id: The unified resource ID (base64 encoded) + resource_object: The resource object to store (can be None) + litellm_parent_otel_span: OpenTelemetry span for tracing + model_mappings: Dictionary mapping model_id -> provider_resource_id + user_api_key_dict: User API key authentication details + additional_db_fields: Additional fields to store in database + """ + verbose_logger.info( + f"Storing LiteLLM Managed {self.resource_type} with id={unified_resource_id} in cache" + ) + + # Prepare cache data + cache_data = { + "unified_resource_id": unified_resource_id, + "resource_object": resource_object, + "model_mappings": model_mappings, + "flat_model_resource_ids": list(model_mappings.values()), + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, + } + + # Add additional fields if provided + if additional_db_fields: + cache_data.update(additional_db_fields) + + # Store in cache + if resource_object is not None: + await self.internal_usage_cache.async_set_cache( + key=unified_resource_id, + value=cache_data, + litellm_parent_otel_span=litellm_parent_otel_span, + ) + + # Prepare database data + db_data = { + "unified_resource_id": unified_resource_id, + "model_mappings": json.dumps(model_mappings), + "flat_model_resource_ids": list(model_mappings.values()), + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, + } + + # Add resource object if available + if resource_object is not None: + # Handle both dict and Pydantic models + if hasattr(resource_object, "model_dump_json"): + db_data["resource_object"] = resource_object.model_dump_json() # type: ignore + elif isinstance(resource_object, dict): + db_data["resource_object"] = json.dumps(resource_object) + + # Extract storage metadata from hidden params if present + hidden_params = getattr(resource_object, "_hidden_params", {}) or {} + if "storage_backend" in hidden_params: + db_data["storage_backend"] = hidden_params["storage_backend"] + if "storage_url" in hidden_params: + db_data["storage_url"] = hidden_params["storage_url"] + + # Add additional fields to database + if additional_db_fields: + db_data.update(additional_db_fields) + + # Store in database + table = getattr(self.prisma_client.db, self.table_name) + result = await table.create(data=db_data) + + verbose_logger.debug( + f"LiteLLM Managed {self.resource_type} with id={unified_resource_id} stored in db: {result}" + ) + + async def get_unified_resource_id( + self, + unified_resource_id: str, + litellm_parent_otel_span: Optional[Span] = None, + ) -> Optional[Dict[str, Any]]: + """ + Retrieve unified resource by ID from cache or database. + + Args: + unified_resource_id: The unified resource ID to retrieve + litellm_parent_otel_span: OpenTelemetry span for tracing + + Returns: + Dictionary containing resource data or None if not found + """ + # Check cache first + result = cast( + Optional[dict], + await self.internal_usage_cache.async_get_cache( + key=unified_resource_id, + litellm_parent_otel_span=litellm_parent_otel_span, + ), + ) + + if result: + return result + + # Check database + table = getattr(self.prisma_client.db, self.table_name) + db_object = await table.find_first( + where={"unified_resource_id": unified_resource_id} + ) + + if db_object: + return db_object.model_dump() + + return None + + async def delete_unified_resource_id( + self, + unified_resource_id: str, + litellm_parent_otel_span: Optional[Span] = None, + ) -> Optional[ResourceObjectType]: + """ + Delete unified resource from cache and database. + + Args: + unified_resource_id: The unified resource ID to delete + litellm_parent_otel_span: OpenTelemetry span for tracing + + Returns: + The deleted resource object or None if not found + """ + # Get old value from database + table = getattr(self.prisma_client.db, self.table_name) + initial_value = await table.find_first( + where={"unified_resource_id": unified_resource_id} + ) + + if initial_value is None: + raise Exception( + f"LiteLLM Managed {self.resource_type} with id={unified_resource_id} not found" + ) + + # Delete from cache + await self.internal_usage_cache.async_set_cache( + key=unified_resource_id, + value=None, + litellm_parent_otel_span=litellm_parent_otel_span, + ) + + # Delete from database + await table.delete(where={"unified_resource_id": unified_resource_id}) + + return initial_value.resource_object + + async def can_user_access_unified_resource_id( + self, + unified_resource_id: str, + user_api_key_dict: UserAPIKeyAuth, + litellm_parent_otel_span: Optional[Span] = None, + ) -> bool: + """ + Check if user has access to the unified resource ID. + + Uses get_unified_resource_id() which checks cache first before hitting the database, + avoiding direct DB queries in the critical request path. + + Args: + unified_resource_id: The unified resource ID to check + user_api_key_dict: User API key authentication details + litellm_parent_otel_span: OpenTelemetry span for tracing + + Returns: + True if user has access, False otherwise + """ + user_id = user_api_key_dict.user_id + + # Use cached method instead of direct DB query + resource = await self.get_unified_resource_id( + unified_resource_id, litellm_parent_otel_span + ) + + if resource: + return resource.get("created_by") == user_id + + return False + + # ============================================================================ + # MODEL MAPPING OPERATIONS + # ============================================================================ + + async def get_model_resource_id_mapping( + self, + resource_ids: List[str], + litellm_parent_otel_span: Span, + ) -> Dict[str, Dict[str, str]]: + """ + Get model-specific resource IDs for a list of unified resource IDs. + + Args: + resource_ids: List of unified resource IDs + litellm_parent_otel_span: OpenTelemetry span for tracing + + Returns: + Dictionary mapping unified_resource_id -> model_id -> provider_resource_id + + Example: + { + "unified_resource_id_1": { + "model_id_1": "provider_resource_id_1", + "model_id_2": "provider_resource_id_2" + } + } + """ + resource_id_mapping: Dict[str, Dict[str, str]] = {} + + for resource_id in resource_ids: + # Get unified resource from cache/db + unified_resource_object = await self.get_unified_resource_id( + resource_id, litellm_parent_otel_span + ) + + if unified_resource_object: + model_mappings = unified_resource_object.get("model_mappings", {}) + + # Handle both JSON string and dict + if isinstance(model_mappings, str): + model_mappings = json.loads(model_mappings) + + resource_id_mapping[resource_id] = model_mappings + + return resource_id_mapping + + # ============================================================================ + # RESOURCE CREATION OPERATIONS + # ============================================================================ + + async def create_resource_for_each_model( + self, + llm_router: Router, + request_data: Dict[str, Any], + target_model_names_list: List[str], + litellm_parent_otel_span: Span, + ) -> List[ResourceObjectType]: + """ + Create a resource for each model in the target list. + + Args: + llm_router: LiteLLM router instance + request_data: Request data for resource creation + target_model_names_list: List of target model names + litellm_parent_otel_span: OpenTelemetry span for tracing + + Returns: + List of resource objects created for each model + """ + if llm_router is None: + raise Exception("LLM Router not initialized. Ensure models added to proxy.") + + responses = [] + for model in target_model_names_list: + individual_response = await self.create_resource_for_model( + llm_router=llm_router, + model=model, + request_data=request_data, + litellm_parent_otel_span=litellm_parent_otel_span, + ) + responses.append(individual_response) + return responses + + def generate_unified_resource_id( + self, + resource_objects: List[ResourceObjectType], + target_model_names_list: List[str], + ) -> str: + """ + Generate a unified resource ID from multiple resource objects. + + Args: + resource_objects: List of resource objects from different models + target_model_names_list: List of target model names + + Returns: + Base64 encoded unified resource ID + """ + # Use the first resource object to generate the format + unified_id_format = self.get_unified_resource_id_format( + resource_object=resource_objects[0], + target_model_names_list=target_model_names_list, + ) + + # Convert to URL-safe base64 and strip padding + base64_unified_id = ( + base64.urlsafe_b64encode(unified_id_format.encode()).decode().rstrip("=") + ) + + return base64_unified_id + + def extract_model_mappings_from_responses( + self, + resource_objects: List[ResourceObjectType], + ) -> Dict[str, str]: + """ + Extract model mappings from resource objects. + + Args: + resource_objects: List of resource objects from different models + + Returns: + Dictionary mapping model_id -> provider_resource_id + """ + model_mappings: Dict[str, str] = {} + + for resource_object in resource_objects: + # Get hidden params if available + hidden_params = getattr(resource_object, "_hidden_params", {}) or {} + model_resource_id_mapping = hidden_params.get("model_resource_id_mapping") + + if model_resource_id_mapping and isinstance(model_resource_id_mapping, dict): + model_mappings.update(model_resource_id_mapping) + + return model_mappings + + # ============================================================================ + # DEPLOYMENT FILTERING + # ============================================================================ + + async def async_filter_deployments( + self, + model: str, + healthy_deployments: List, + request_kwargs: Optional[Dict] = None, + parent_otel_span: Optional[Span] = None, + resource_id_key: str = "resource_id", + ) -> List[Dict]: + """ + Filter deployments based on model mappings for a resource. + + This is used by the router to select only deployments that have + the resource available. + + Args: + model: Model name + healthy_deployments: List of healthy deployments + request_kwargs: Request kwargs containing resource_id and mappings + parent_otel_span: OpenTelemetry span for tracing + resource_id_key: Key to use for resource ID in request_kwargs + + Returns: + Filtered list of deployments + """ + if request_kwargs is None: + return healthy_deployments + + resource_id = cast(Optional[str], request_kwargs.get(resource_id_key)) + model_resource_id_mapping = cast( + Optional[Dict[str, Dict[str, str]]], + request_kwargs.get("model_resource_id_mapping"), + ) + + allowed_model_ids = [] + if resource_id and model_resource_id_mapping: + model_id_dict = model_resource_id_mapping.get(resource_id, {}) + allowed_model_ids = list(model_id_dict.keys()) + + if len(allowed_model_ids) == 0: + return healthy_deployments + + return [ + deployment + for deployment in healthy_deployments + if deployment.get("model_info", {}).get("id") in allowed_model_ids + ] + + # ============================================================================ + # UTILITY METHODS + # ============================================================================ + + def get_unified_id_prefix(self) -> str: + """ + Get the prefix for unified IDs for this resource type. + + Returns: + Prefix string (e.g., "litellm_proxy:") + """ + return SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value + + async def list_user_resources( + self, + user_api_key_dict: UserAPIKeyAuth, + limit: Optional[int] = None, + after: Optional[str] = None, + additional_filters: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """ + List resources created by a user. + + Args: + user_api_key_dict: User API key authentication details + limit: Maximum number of resources to return + after: Cursor for pagination + additional_filters: Additional filters to apply + + Returns: + Dictionary with list of resources and pagination info + """ + where_clause: Dict[str, Any] = {} + + # Filter by user who created the resource + if user_api_key_dict.user_id: + where_clause["created_by"] = user_api_key_dict.user_id + + if after: + where_clause["id"] = {"gt": after} + + # Add additional filters + if additional_filters: + where_clause.update(additional_filters) + + # Fetch resources + fetch_limit = limit or 20 + table = getattr(self.prisma_client.db, self.table_name) + resources = await table.find_many( + where=where_clause, + take=fetch_limit, + order={"created_at": "desc"}, + ) + + resource_objects: List[Any] = [] + for resource in resources: + try: + # Stop once we have enough + if len(resource_objects) >= (limit or 20): + break + + # Parse resource object + resource_data = resource.resource_object + if isinstance(resource_data, str): + resource_data = json.loads(resource_data) + + # Set unified ID + if hasattr(resource_data, "id"): + resource_data.id = resource.unified_resource_id + elif isinstance(resource_data, dict): + resource_data["id"] = resource.unified_resource_id + + resource_objects.append(resource_data) + + except Exception as e: + verbose_logger.warning( + f"Failed to parse {self.resource_type} object " + f"{resource.unified_resource_id}: {e}" + ) + continue + + return { + "object": "list", + "data": resource_objects, + "first_id": resource_objects[0].id if resource_objects else None, + "last_id": resource_objects[-1].id if resource_objects else None, + "has_more": len(resource_objects) == (limit or 20), + } diff --git a/litellm/llms/base_llm/managed_resources/utils.py b/litellm/llms/base_llm/managed_resources/utils.py new file mode 100644 index 00000000000..0d843b6d128 --- /dev/null +++ b/litellm/llms/base_llm/managed_resources/utils.py @@ -0,0 +1,364 @@ +""" +Utility functions for managed resources. + +This module provides common utility functions that can be used across +different managed resource types (files, vector stores, etc.). +""" + +import base64 +import re +from typing import List, Optional, Union, Literal + + +def is_base64_encoded_unified_id( + resource_id: str, + prefix: str = "litellm_proxy:", +) -> Union[str, Literal[False]]: + """ + Check if a resource ID is a base64 encoded unified ID. + + Args: + resource_id: The resource ID to check + prefix: The expected prefix for unified IDs + + Returns: + Decoded string if valid unified ID, False otherwise + """ + # Ensure resource_id is a string + if not isinstance(resource_id, str): + return False + + # Add padding back if needed + padded = resource_id + "=" * (-len(resource_id) % 4) + + # Decode from base64 + try: + decoded = base64.urlsafe_b64decode(padded).decode() + if decoded.startswith(prefix): + return decoded + else: + return False + except Exception: + return False + + +def extract_target_model_names_from_unified_id( + unified_id: str, +) -> List[str]: + """ + Extract target model names from a unified resource ID. + + Args: + unified_id: The unified resource ID (decoded or encoded) + + Returns: + List of target model names + + Example: + unified_id = "litellm_proxy:vector_store;unified_id,uuid;target_model_names,gpt-4,gemini-2.0" + returns: ["gpt-4", "gemini-2.0"] + """ + try: + # Ensure unified_id is a string + if not isinstance(unified_id, str): + return [] + + # Decode if it's base64 encoded + decoded_id = is_base64_encoded_unified_id(unified_id) + if decoded_id: + unified_id = decoded_id + + # Extract model names using regex + match = re.search(r"target_model_names,([^;]+)", unified_id) + if match: + # Split on comma and strip whitespace from each model name + return [model.strip() for model in match.group(1).split(",")] + + return [] + except Exception: + return [] + + +def extract_resource_type_from_unified_id( + unified_id: str, +) -> Optional[str]: + """ + Extract resource type from a unified resource ID. + + Args: + unified_id: The unified resource ID (decoded or encoded) + + Returns: + Resource type string or None + + Example: + unified_id = "litellm_proxy:vector_store;unified_id,uuid;..." + returns: "vector_store" + """ + try: + # Ensure unified_id is a string + if not isinstance(unified_id, str): + return None + + # Decode if it's base64 encoded + decoded_id = is_base64_encoded_unified_id(unified_id) + if decoded_id: + unified_id = decoded_id + + # Extract resource type (comes after prefix and before first semicolon) + match = re.search(r"litellm_proxy:([^;]+)", unified_id) + if match: + return match.group(1).strip() + + return None + except Exception: + return None + + +def extract_unified_uuid_from_unified_id( + unified_id: str, +) -> Optional[str]: + """ + Extract the UUID from a unified resource ID. + + Args: + unified_id: The unified resource ID (decoded or encoded) + + Returns: + UUID string or None + + Example: + unified_id = "litellm_proxy:vector_store;unified_id,abc-123;..." + returns: "abc-123" + """ + try: + # Ensure unified_id is a string + if not isinstance(unified_id, str): + return None + + # Decode if it's base64 encoded + decoded_id = is_base64_encoded_unified_id(unified_id) + if decoded_id: + unified_id = decoded_id + + # Extract UUID + match = re.search(r"unified_id,([^;]+)", unified_id) + if match: + return match.group(1).strip() + + return None + except Exception: + return None + + +def extract_model_id_from_unified_id( + unified_id: str, +) -> Optional[str]: + """ + Extract model ID from a unified resource ID. + + Args: + unified_id: The unified resource ID (decoded or encoded) + + Returns: + Model ID string or None + + Example: + unified_id = "litellm_proxy:vector_store;...;model_id,gpt-4-model-id;..." + returns: "gpt-4-model-id" + """ + try: + # Ensure unified_id is a string + if not isinstance(unified_id, str): + return None + + # Decode if it's base64 encoded + decoded_id = is_base64_encoded_unified_id(unified_id) + if decoded_id: + unified_id = decoded_id + + # Extract model ID + match = re.search(r"model_id,([^;]+)", unified_id) + if match: + return match.group(1).strip() + + return None + except Exception: + return None + + +def extract_provider_resource_id_from_unified_id( + unified_id: str, +) -> Optional[str]: + """ + Extract provider resource ID from a unified resource ID. + + Args: + unified_id: The unified resource ID (decoded or encoded) + + Returns: + Provider resource ID string or None + + Example: + unified_id = "litellm_proxy:vector_store;...;resource_id,vs_abc123;..." + returns: "vs_abc123" + """ + try: + # Ensure unified_id is a string + if not isinstance(unified_id, str): + return None + + # Decode if it's base64 encoded + decoded_id = is_base64_encoded_unified_id(unified_id) + if decoded_id: + unified_id = decoded_id + + # Extract resource ID (try multiple patterns for different resource types) + patterns = [ + r"resource_id,([^;]+)", + r"vector_store_id,([^;]+)", + r"file_id,([^;]+)", + ] + + for pattern in patterns: + match = re.search(pattern, unified_id) + if match: + return match.group(1).strip() + + return None + except Exception: + return None + + +def generate_unified_id_string( + resource_type: str, + unified_uuid: str, + target_model_names: List[str], + provider_resource_id: str, + model_id: str, + additional_fields: Optional[dict] = None, +) -> str: + """ + Generate a unified ID string (before base64 encoding). + + Args: + resource_type: Type of resource (e.g., "vector_store", "file") + unified_uuid: UUID for this unified resource + target_model_names: List of target model names + provider_resource_id: Resource ID from the provider + model_id: Model ID from the router + additional_fields: Additional fields to include in the ID + + Returns: + Unified ID string (not yet base64 encoded) + + Example: + generate_unified_id_string( + resource_type="vector_store", + unified_uuid="abc-123", + target_model_names=["gpt-4", "gemini"], + provider_resource_id="vs_xyz", + model_id="model-id-123", + ) + returns: "litellm_proxy:vector_store;unified_id,abc-123;target_model_names,gpt-4,gemini;resource_id,vs_xyz;model_id,model-id-123" + """ + # Build the unified ID string + parts = [ + f"litellm_proxy:{resource_type}", + f"unified_id,{unified_uuid}", + f"target_model_names,{','.join(target_model_names)}", + f"resource_id,{provider_resource_id}", + f"model_id,{model_id}", + ] + + # Add additional fields if provided + if additional_fields: + for key, value in additional_fields.items(): + parts.append(f"{key},{value}") + + return ";".join(parts) + + +def encode_unified_id(unified_id_string: str) -> str: + """ + Encode a unified ID string to base64. + + Args: + unified_id_string: The unified ID string to encode + + Returns: + Base64 encoded unified ID (URL-safe, padding stripped) + """ + return ( + base64.urlsafe_b64encode(unified_id_string.encode()) + .decode() + .rstrip("=") + ) + + +def decode_unified_id(encoded_unified_id: str) -> Optional[str]: + """ + Decode a base64 encoded unified ID. + + Args: + encoded_unified_id: The base64 encoded unified ID + + Returns: + Decoded unified ID string or None if invalid + """ + try: + # Add padding back if needed + padded = encoded_unified_id + "=" * (-len(encoded_unified_id) % 4) + + # Decode from base64 + decoded = base64.urlsafe_b64decode(padded).decode() + + # Verify it starts with the expected prefix + if decoded.startswith("litellm_proxy:"): + return decoded + + return None + except Exception: + return None + + +def parse_unified_id( + unified_id: str, +) -> Optional[dict]: + """ + Parse a unified ID into its components. + + Args: + unified_id: The unified ID (encoded or decoded) + + Returns: + Dictionary with parsed components or None if invalid + + Example: + { + "resource_type": "vector_store", + "unified_uuid": "abc-123", + "target_model_names": ["gpt-4", "gemini"], + "provider_resource_id": "vs_xyz", + "model_id": "model-id-123" + } + """ + try: + # Decode if needed + decoded_id = decode_unified_id(unified_id) + if not decoded_id: + # Maybe it's already decoded + if unified_id.startswith("litellm_proxy:"): + decoded_id = unified_id + else: + return None + + return { + "resource_type": extract_resource_type_from_unified_id(decoded_id), + "unified_uuid": extract_unified_uuid_from_unified_id(decoded_id), + "target_model_names": extract_target_model_names_from_unified_id(decoded_id), + "provider_resource_id": extract_provider_resource_id_from_unified_id(decoded_id), + "model_id": extract_model_id_from_unified_id(decoded_id), + } + except Exception: + return None diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 2c041adba7a..477fa3316d1 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -246,6 +246,11 @@ class AmazonAnthropicClaudeMessagesConfig( "sonnet_4.5", "sonnet-4-5", "sonnet_4_5", + # Opus 4.6 + "opus-4.6", + "opus_4.6", + "opus-4-6", + "opus_4_6", ] return any(pattern in model_lower for pattern in supported_patterns) diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index 1b03ec47643..fb98006c7e4 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -4,7 +4,7 @@ import os import ssl import typing import urllib.request -from typing import Callable, Dict, Optional, Union +from typing import Any, Callable, Dict, Optional, Union import aiohttp import aiohttp.client_exceptions @@ -248,26 +248,25 @@ class LiteLLMAiohttpTransport(AiohttpTransport): # Only pass ssl kwarg when explicitly configured, to avoid # overriding the session/connector defaults with None (which is # not a valid value for aiohttp's ssl parameter). - ssl_kwargs: Dict[str, Union[bool, ssl.SSLContext]] = {} - if ssl_verify is not None: - ssl_kwargs["ssl"] = ssl_verify - - response = await client_session.request( - method=request.method, - url=YarlURL(str(request.url), encoded=True), - headers=request.headers, - data=data, - allow_redirects=False, - auto_decompress=False, - timeout=ClientTimeout( + request_kwargs: Dict[str, Any] = { + "method": request.method, + "url": YarlURL(str(request.url), encoded=True), + "headers": request.headers, + "data": data, + "allow_redirects": False, + "auto_decompress": False, + "timeout": ClientTimeout( sock_connect=timeout.get("connect"), sock_read=timeout.get("read"), connect=timeout.get("pool"), ), - proxy=proxy, - server_hostname=sni_hostname, - **ssl_kwargs, - ).__aenter__() + "proxy": proxy, + "server_hostname": sni_hostname, + } + if ssl_verify is not None: + request_kwargs["ssl"] = ssl_verify + + response = await client_session.request(**request_kwargs).__aenter__() return response diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 95f411c397c..5cf6efe5ba2 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1206,7 +1206,28 @@ def get_async_httpx_client( If not present, creates a new client Caches the new client and returns it. + + Note: When shared_session is provided, the cache is bypassed to ensure + the user's session (with its trace_configs, connector settings, etc.) + is used for the request. """ + # When shared_session is provided, bypass cache and create a new handler + # that uses the user's session directly. This preserves the user's + # session configuration including trace_configs for aiohttp tracing. + if shared_session is not None: + verbose_logger.debug( + f"shared_session provided (ID: {id(shared_session)}), bypassing client cache" + ) + if params is not None: + handler_params = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"} + handler_params["shared_session"] = shared_session + return AsyncHTTPHandler(**handler_params) + else: + return AsyncHTTPHandler( + timeout=httpx.Timeout(timeout=600.0, connect=5.0), + shared_session=shared_session, + ) + _params_key_name = "" if params is not None: for key, value in params.items(): @@ -1233,12 +1254,10 @@ def get_async_httpx_client( if params is not None: # Filter out params that are only used for cache key, not for AsyncHTTPHandler.__init__ handler_params = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"} - handler_params["shared_session"] = shared_session _new_client = AsyncHTTPHandler(**handler_params) else: _new_client = AsyncHTTPHandler( timeout=httpx.Timeout(timeout=600.0, connect=5.0), - shared_session=shared_session, ) cache.set_cache( diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 16368907070..5b9840d95b0 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -772,12 +772,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator): def chunk_parser(self, chunk: dict) -> ModelResponseStream: try: - return ModelResponseStream( - id=chunk["id"], - object="chat.completion.chunk", - created=chunk.get("created"), - model=chunk.get("model"), - choices=chunk.get("choices", []), - ) + kwargs = { + "id": chunk["id"], + "object": "chat.completion.chunk", + "created": chunk.get("created"), + "model": chunk.get("model"), + "choices": chunk.get("choices", []), + } + if "usage" in chunk and chunk["usage"] is not None: + kwargs["usage"] = chunk["usage"] + return ModelResponseStream(**kwargs) except Exception as e: raise e diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index cc2439b431a..3e089682097 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast, get_type_hin import httpx from openai.types.responses import ResponseReasoningItem -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError import litellm from litellm._logging import verbose_logger @@ -240,25 +240,26 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): event_pydantic_model = OpenAIResponsesAPIConfig.get_event_model_class( event_type=event_type ) - # Defensive: Some OpenAI-compatible providers may send `error.code: null`. - # Pydantic will raise a ValidationError when it expects a string but gets None. - # Coalesce a None `error.code` to a stable default string so streaming - # iteration does not crash (see issue report). This keeps behavior similar - # to previous fixes (coalesce before validation) and lets higher-level - # handlers still receive an `ErrorEvent` object. + # Some OpenAI-compatible providers send error.code: null; coalesce so validation succeeds. try: error_obj = parsed_chunk.get("error") if isinstance(error_obj, dict) and error_obj.get("code") is None: - # Preserve other fields, but ensure `code` is a non-null string parsed_chunk = dict(parsed_chunk) parsed_chunk["error"] = dict(error_obj) parsed_chunk["error"]["code"] = "unknown_error" except Exception: - # If anything unexpected happens here, fall back to attempting - # instantiation and let higher-level handlers manage errors. verbose_logger.debug("Failed to coalesce error.code in parsed_chunk") - return event_pydantic_model(**parsed_chunk) + try: + return event_pydantic_model(**parsed_chunk) + except ValidationError: + verbose_logger.debug( + "Pydantic validation failed for %s with chunk %s, " + "falling back to model_construct", + event_pydantic_model.__name__, + parsed_chunk, + ) + return event_pydantic_model.model_construct(**parsed_chunk) @staticmethod def get_event_model_class(event_type: str) -> Any: @@ -307,6 +308,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ResponsesAPIStreamEvents.MCP_CALL_FAILED: MCPCallFailedEvent, ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE: ImageGenerationPartialImageEvent, ResponsesAPIStreamEvents.ERROR: ErrorEvent, + # Shell tool events: passthrough as GenericEvent so payload is preserved + ResponsesAPIStreamEvents.SHELL_CALL_IN_PROGRESS: GenericEvent, + ResponsesAPIStreamEvents.SHELL_CALL_COMPLETED: GenericEvent, + ResponsesAPIStreamEvents.SHELL_CALL_OUTPUT: GenericEvent, } model_class = event_models.get(cast(ResponsesAPIStreamEvents, event_type)) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index b4f9cbe42de..1b1b1c2f8cc 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -26,6 +26,10 @@ "max_completion_tokens": "max_tokens" } }, + "scaleway": { + "base_url": "https://api.scaleway.ai/v1", + "api_key_env": "SCW_SECRET_KEY" + }, "synthetic": { "base_url": "https://api.synthetic.new/openai/v1", "api_key_env": "SYNTHETIC_API_KEY", diff --git a/litellm/llms/sagemaker/embedding/transformation.py b/litellm/llms/sagemaker/embedding/transformation.py index bd8abc5e01a..04b201380fc 100644 --- a/litellm/llms/sagemaker/embedding/transformation.py +++ b/litellm/llms/sagemaker/embedding/transformation.py @@ -102,11 +102,18 @@ class SagemakerEmbeddingConfig(BaseEmbeddingConfig): status_code=raw_response.status_code ) - if "embedding" not in response_data: + # Handle both raw array format (TEI) and wrapped format (standard HF) + if isinstance(response_data, list): + # TEI and some HF models return raw embedding arrays directly + embeddings = response_data + elif isinstance(response_data, dict) and "embedding" in response_data: + # Standard HF format with "embedding" key + embeddings = response_data["embedding"] + else: raise SagemakerError( - status_code=500, message="HF response missing 'embedding' field" + status_code=500, + message=f"Unexpected response format. Expected list or dict with 'embedding' key, got: {type(response_data).__name__}", ) - embeddings = response_data["embedding"] if not isinstance(embeddings, list): raise SagemakerError( diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 00b461dcda0..6a5ac92816b 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -529,6 +529,17 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 raise e +def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None: + """Pop extra_body from optional_params and shallow-merge into data, deep-merging dict values.""" + extra_body: Optional[dict] = optional_params.pop("extra_body", None) + if extra_body is not None: + for k, v in extra_body.items(): + if k in data and isinstance(data[k], dict) and isinstance(v, dict): + data[k].update(v) + else: + data[k] = v + + def _transform_request_body( messages: List[AllMessageValues], model: str, @@ -619,6 +630,7 @@ def _transform_request_body( # Only add labels for Vertex AI endpoints (not Google GenAI/AI Studio) and only if non-empty if labels and custom_llm_provider != LlmProviders.GEMINI: data["labels"] = labels + _pop_and_merge_extra_body(data, optional_params) except Exception as e: raise e diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 04ae4b6beb8..bef83b6d35e 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -480,7 +480,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): tool = {VertexToolName.COMPUTER_USE.value: computer_use_config} # Handle OpenAI-style web_search and web_search_preview tools # Transform them to Gemini's googleSearch tool - elif "type" in tool and tool["type"] in ("web_search", "web_search_preview"): + elif "type" in tool and tool["type"] in ( + "web_search", + "web_search_preview", + ): verbose_logger.info( f"Gemini: Transforming OpenAI-style '{tool['type']}' tool to googleSearch" ) @@ -1196,6 +1199,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "PROHIBITED_CONTENT": "The token generation was stopped as the response was flagged for the prohibited contents.", "SPII": "The token generation was stopped as the response was flagged for Sensitive Personally Identifiable Information (SPII) contents.", "IMAGE_SAFETY": "The token generation was stopped as the response was flagged for image safety reasons.", + "IMAGE_PROHIBITED_CONTENT": "The token generation was stopped as the response was flagged for prohibited image content.", } @staticmethod @@ -1218,6 +1222,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "SPII": "content_filter", "MALFORMED_FUNCTION_CALL": "malformed_function_call", # openai doesn't have a way of representing this "IMAGE_SAFETY": "content_filter", + "IMAGE_PROHIBITED_CONTENT": "content_filter", } def translate_exception_str(self, exception_string: str): @@ -1630,7 +1635,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): completion_image_tokens = response_tokens_details.image_tokens or 0 completion_audio_tokens = response_tokens_details.audio_tokens or 0 calculated_text_tokens = ( - candidates_token_count - completion_image_tokens - completion_audio_tokens + candidates_token_count + - completion_image_tokens + - completion_audio_tokens ) response_tokens_details.text_tokens = calculated_text_tokens ######################################################### @@ -2248,6 +2255,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): citation_metadata # older approach - maintaining to prevent regressions ) + ## ADD TRAFFIC TYPE ## + traffic_type = completion_response.get("usageMetadata", {}).get( + "trafficType" + ) + if traffic_type: + model_response._hidden_params.setdefault("provider_specific_fields", {})["traffic_type"] = traffic_type + except Exception as e: raise VertexAIError( message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format( @@ -2906,6 +2920,12 @@ class ModelResponseIterator: PromptTokensDetailsWrapper, usage.prompt_tokens_details ).web_search_requests = web_search_requests + traffic_type = processed_chunk.get("usageMetadata", {}).get( + "trafficType" + ) + if traffic_type: + model_response._hidden_params.setdefault("provider_specific_fields", {})["traffic_type"] = traffic_type + setattr(model_response, "usage", usage) # type: ignore model_response._hidden_params["is_finished"] = False diff --git a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py index 08b93145e50..1be9cd820a3 100644 --- a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py @@ -115,8 +115,13 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase): vertex_project = self.get_vertex_ai_project(litellm_params) vertex_location = self.get_vertex_ai_location(litellm_params) - # Construct full rag corpus path - full_rag_corpus = f"projects/{vertex_project}/locations/{vertex_location}/ragCorpora/{vector_store_id}" + # Handle both full corpus path and just corpus ID + if vector_store_id.startswith("projects/"): + # Already a full path + full_rag_corpus = vector_store_id + else: + # Just the corpus ID, construct full path + full_rag_corpus = f"projects/{vertex_project}/locations/{vertex_location}/ragCorpora/{vector_store_id}" # Build the request body for Vertex AI RAG API request_body: Dict[str, Any] = { diff --git a/litellm/main.py b/litellm/main.py index bca023e65ec..80a2f74c571 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7383,6 +7383,16 @@ def stream_chunk_builder( # noqa: PLR0915 setattr(response, "usage", usage) + # Propagate provider_specific_fields from the last chunk (contains provider + # metadata like traffic_type set during streaming) + for chunk in reversed(chunks): + hidden = getattr(chunk, "_hidden_params", None) + if hidden and "provider_specific_fields" in hidden: + response._hidden_params.setdefault( + "provider_specific_fields", {} + ).update(hidden["provider_specific_fields"]) + break + # Add cost to usage object if include_cost_in_streaming_usage is True if litellm.include_cost_in_streaming_usage and logging_obj is not None: setattr( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f6edcf7efd0..e6b7cf17297 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5859,6 +5859,7 @@ "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, "azure_ai/ministral-3b": { @@ -6104,6 +6105,32 @@ "output_cost_per_token": 2.4e-05, "supports_tool_choice": true }, + "bedrock/ap-northeast-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-northeast-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6115,6 +6142,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6127,15 +6181,18 @@ "supports_reasoning": true }, "bedrock/moonshotai.kimi-k2.5": { - "input_cost_per_token": 7.3e-07, + "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.03e-06, + "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_video_input": true, + "supports_vision": true }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.18e-06, @@ -6155,6 +6212,32 @@ "mode": "chat", "output_cost_per_token": 7.2e-07 }, + "bedrock/ap-south-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-south-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", @@ -6166,6 +6249,86 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/ap-south-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-south-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.05e-06, "litellm_provider": "bedrock", @@ -6184,6 +6347,46 @@ "mode": "chat", "output_cost_per_token": 6.9e-07 }, + "bedrock/eu-north-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-north-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-north-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "input_cost_per_second": 0.01635, "litellm_provider": "bedrock", @@ -6271,6 +6474,32 @@ "output_cost_per_token": 2.4e-05, "supports_tool_choice": true }, + "bedrock/eu-central-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-central-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.86e-06, "litellm_provider": "bedrock", @@ -6289,6 +6518,32 @@ "mode": "chat", "output_cost_per_token": 6.5e-07 }, + "bedrock/eu-west-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-west-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.45e-06, "litellm_provider": "bedrock", @@ -6307,6 +6562,32 @@ "mode": "chat", "output_cost_per_token": 7.8e-07 }, + "bedrock/eu-west-2/minimax.minimax-m2.1": { + "input_cost_per_token": 4.7e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.86e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-west-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 7.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.86e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", @@ -6337,6 +6618,32 @@ "output_cost_per_token": 9.1e-07, "supports_tool_choice": true }, + "bedrock/eu-south-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-south-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, "litellm_provider": "bedrock", @@ -6371,6 +6678,32 @@ "mode": "chat", "output_cost_per_token": 1.01e-06 }, + "bedrock/sa-east-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/sa-east-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6382,6 +6715,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/sa-east-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/sa-east-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "input_cost_per_second": 0.011, "litellm_provider": "bedrock", @@ -6518,6 +6878,32 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-east-1/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6529,6 +6915,59 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-east-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6540,6 +6979,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-east-2/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "input_cost_per_token": 9.6e-07, "litellm_provider": "bedrock", @@ -6946,6 +7412,32 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-west-2/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-west-2/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6957,6 +7449,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-west-2/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-west-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, "cache_read_input_token_cost": 8e-08, @@ -10870,6 +11389,19 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "dolphin": { "input_cost_per_token": 5e-07, "litellm_provider": "nlp_cloud", @@ -15784,44 +16316,14 @@ "tpm": 250000 }, "gemini/gemini-2.5-flash-preview-tts": { - "cache_read_input_token_cost": 3.75e-08, - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 1.5e-07, + "input_cost_per_token": 3e-07, "litellm_provider": "gemini", - "max_audio_length_hours": 8.4, - "max_audio_per_prompt": 1, - "max_images_per_prompt": 3000, - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_pdf_size_mb": 30, - "max_tokens": 65535, - "max_video_length": 1, - "max_videos_per_prompt": 10, - "mode": "chat", - "output_cost_per_reasoning_token": 3.5e-06, - "output_cost_per_token": 6e-07, - "rpm": 10, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "mode": "audio_speech", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions" - ], - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "audio" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000 + "/v1/audio/speech" + ] }, "gemini/gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -21370,6 +21872,19 @@ "output_cost_per_token": 1.2e-06, "supports_system_messages": true }, + "minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, "litellm_provider": "minimax", @@ -21432,6 +21947,36 @@ "max_input_tokens": 1000000, "max_output_tokens": 8192 }, + "minimax/MiniMax-M2.5": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, + "minimax/MiniMax-M2.5-lightning": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, "minimax/MiniMax-M2": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, @@ -22096,6 +22641,20 @@ "supports_reasoning": true, "supports_system_messages": true }, + "moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "moonshot/kimi-k2-0711-preview": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 6e-07, @@ -22147,9 +22706,10 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat", + "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, "moonshot/kimi-latest": { @@ -22611,6 +23171,19 @@ "output_cost_per_token": 2.3e-07, "supports_system_messages": true }, + "nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 6e-08, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.4e-07, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "o1": { "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, @@ -23616,36 +24189,6 @@ "output_cost_per_token": 2e-07, "supports_system_messages": true }, - "openrouter/anthropic/claude-2": { - "input_cost_per_token": 1.102e-05, - "litellm_provider": "openrouter", - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 3.268e-05, - "supports_tool_choice": true - }, - "openrouter/anthropic/claude-3-5-haiku": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_tokens": 200000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "supports_function_calling": true, - "supports_tool_choice": true - }, - "openrouter/anthropic/claude-3-5-haiku-20241022": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "tool_use_system_prompt_tokens": 264 - }, "openrouter/anthropic/claude-3-haiku": { "input_cost_per_image": 0.0004, "input_cost_per_token": 2.5e-07, @@ -23657,43 +24200,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/anthropic/claude-3-haiku-20240307": { - "input_cost_per_token": 2.5e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 264 - }, - "openrouter/anthropic/claude-3-opus": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 395 - }, - "openrouter/anthropic/claude-3-sonnet": { - "input_cost_per_image": 0.0048, - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_tokens": 200000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/anthropic/claude-3.5-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "openrouter", @@ -23709,20 +24215,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-3.5-sonnet:beta": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 - }, "openrouter/anthropic/claude-3.7-sonnet": { "input_cost_per_image": 0.0048, "input_cost_per_token": 3e-06, @@ -23740,31 +24232,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-3.7-sonnet:beta": { - "input_cost_per_image": 0.0048, - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 - }, - "openrouter/anthropic/claude-instant-v1": { - "input_cost_per_token": 1.63e-06, - "litellm_provider": "openrouter", - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 5.51e-06, - "supports_tool_choice": true - }, "openrouter/anthropic/claude-opus-4": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 1.875e-05, @@ -23903,30 +24370,6 @@ "source": "https://openrouter.ai/api/v1/models/bytedance/ui-tars-1.5-7b", "supports_tool_choice": true }, - "openrouter/cognitivecomputations/dolphin-mixtral-8x7b": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 32769, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/cohere/command-r-plus": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_tool_choice": true - }, - "openrouter/databricks/dbrx-instruct": { - "input_cost_per_token": 6e-07, - "litellm_provider": "openrouter", - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_tool_choice": true - }, "openrouter/deepseek/deepseek-chat": { "input_cost_per_token": 1.4e-07, "litellm_provider": "openrouter", @@ -23994,17 +24437,6 @@ "supports_reasoning": false, "supports_tool_choice": true }, - "openrouter/deepseek/deepseek-coder": { - "input_cost_per_token": 1.4e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 66000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2.8e-07, - "supports_prompt_caching": true, - "supports_tool_choice": true - }, "openrouter/deepseek/deepseek-r1": { "input_cost_per_token": 5.5e-07, "input_cost_per_token_cache_hit": 1.4e-07, @@ -24035,14 +24467,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "openrouter/fireworks/firellava-13b": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, "openrouter/google/gemini-2.0-flash-001": { "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 7e-07, @@ -24204,46 +24628,6 @@ "supports_web_search": true, "tpm": 800000 }, - "openrouter/google/gemini-pro-1.5": { - "input_cost_per_image": 0.00265, - "input_cost_per_token": 2.5e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.5e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "openrouter/google/gemini-pro-vision": { - "input_cost_per_image": 0.0025, - "input_cost_per_token": 1.25e-07, - "litellm_provider": "openrouter", - "max_tokens": 45875, - "mode": "chat", - "output_cost_per_token": 3.75e-07, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "openrouter/google/palm-2-chat-bison": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 25804, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/google/palm-2-codechat-bison": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 20070, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, "openrouter/gryphe/mythomax-l2-13b": { "input_cost_per_token": 1.875e-06, "litellm_provider": "openrouter", @@ -24252,14 +24636,6 @@ "output_cost_per_token": 1.875e-06, "supports_tool_choice": true }, - "openrouter/jondurbin/airoboros-l2-70b-2.1": { - "input_cost_per_token": 1.3875e-05, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.3875e-05, - "supports_tool_choice": true - }, "openrouter/mancer/weaver": { "input_cost_per_token": 5.625e-06, "litellm_provider": "openrouter", @@ -24268,30 +24644,6 @@ "output_cost_per_token": 5.625e-06, "supports_tool_choice": true }, - "openrouter/meta-llama/codellama-34b-instruct": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-2-13b-chat": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-2-70b-chat": { - "input_cost_per_token": 1.5e-06, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_tool_choice": true - }, "openrouter/meta-llama/llama-3-70b-instruct": { "input_cost_per_token": 5.9e-07, "litellm_provider": "openrouter", @@ -24300,38 +24652,6 @@ "output_cost_per_token": 7.9e-07, "supports_tool_choice": true }, - "openrouter/meta-llama/llama-3-70b-instruct:nitro": { - "input_cost_per_token": 9e-07, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-3-8b-instruct:extended": { - "input_cost_per_token": 2.25e-07, - "litellm_provider": "openrouter", - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2.25e-06, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-3-8b-instruct:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 0.0, - "supports_tool_choice": true - }, - "openrouter/microsoft/wizardlm-2-8x22b:nitro": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1e-06, - "supports_tool_choice": true - }, "openrouter/minimax/minimax-m2": { "input_cost_per_token": 2.55e-07, "litellm_provider": "openrouter", @@ -24345,20 +24665,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "openrouter/mistralai/devstral-2512:free": { - "input_cost_per_image": 0, - "input_cost_per_token": 0, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 0, - "supports_function_calling": true, - "supports_prompt_caching": false, - "supports_tool_choice": true, - "supports_vision": false - }, "openrouter/mistralai/devstral-2512": { "input_cost_per_image": 0, "input_cost_per_token": 1.5e-07, @@ -24437,14 +24743,6 @@ "output_cost_per_token": 1.3e-07, "supports_tool_choice": true }, - "openrouter/mistralai/mistral-7b-instruct:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 0.0, - "supports_tool_choice": true - }, "openrouter/mistralai/mistral-large": { "input_cost_per_token": 8e-06, "litellm_provider": "openrouter", @@ -24489,16 +24787,9 @@ "source": "https://openrouter.ai/moonshotai/kimi-k2.5", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, - "openrouter/nousresearch/nous-hermes-llama2-13b": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, "openrouter/openai/gpt-3.5-turbo": { "input_cost_per_token": 1.5e-06, "litellm_provider": "openrouter", @@ -24523,17 +24814,6 @@ "output_cost_per_token": 6e-05, "supports_tool_choice": true }, - "openrouter/openai/gpt-4-vision-preview": { - "input_cost_per_image": 0.01445, - "input_cost_per_token": 1e-05, - "litellm_provider": "openrouter", - "max_tokens": 130000, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1": { "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, @@ -24551,23 +24831,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-2025-04-14": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 8e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1-mini": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, @@ -24585,23 +24848,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-mini-2025-04-14": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 4e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1.6e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1-nano": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, @@ -24619,23 +24865,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-nano-2025-04-14": { - "cache_read_input_token_cost": 2.5e-08, - "input_cost_per_token": 1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 4e-07, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4o": { "input_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", @@ -24707,11 +24936,8 @@ "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "output_cost_per_token": 1.4e-05, - "supported_endpoints": [ - "/v1/responses" - ], "supported_modalities": [ "text", "image" @@ -24872,58 +25098,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/o1-mini": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-mini-2024-09-12": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-preview": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-preview-2024-09-12": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, "openrouter/openai/o3-mini": { "input_cost_per_token": 1.1e-06, "litellm_provider": "openrouter", @@ -24952,14 +25126,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "openrouter/pygmalionai/mythalion-13b": { - "input_cost_per_token": 1.875e-06, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.875e-06, - "supports_tool_choice": true - }, "openrouter/qwen/qwen-2.5-coder-32b-instruct": { "input_cost_per_token": 1.8e-07, "litellm_provider": "openrouter", @@ -25051,20 +25217,6 @@ "supports_tool_choice": true, "supports_web_search": true }, - "openrouter/x-ai/grok-4-fast:free": { - "input_cost_per_token": 0, - "litellm_provider": "openrouter", - "max_input_tokens": 2000000, - "max_output_tokens": 30000, - "max_tokens": 30000, - "mode": "chat", - "output_cost_per_token": 0, - "source": "https://openrouter.ai/x-ai/grok-4-fast:free", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": false - }, "openrouter/z-ai/glm-4.6": { "input_cost_per_token": 4e-07, "litellm_provider": "openrouter", @@ -25937,6 +26089,19 @@ "supports_system_messages": true, "supports_vision": true }, + "qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "recraft/recraftv2": { "litellm_provider": "recraft", "mode": "image_generation", @@ -28373,6 +28538,30 @@ "supports_reasoning": true, "supports_tool_choice": false }, + "us.deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "eu.deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "us.meta.llama3-1-405b-instruct-v1:0": { "input_cost_per_token": 5.32e-06, "litellm_provider": "bedrock", @@ -30799,6 +30988,21 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/zai-org/glm-5-maas": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "vertex_ai/mistral-medium-3": { "input_cost_per_token": 4e-07, "litellm_provider": "vertex_ai-mistral_models", @@ -32302,6 +32506,20 @@ "supports_vision": true, "supports_web_search": true }, + "zai.glm-4.7": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "zai/glm-4.7": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 1.1e-07, @@ -32454,6 +32672,23 @@ "1280x720" ] }, + "openai/sora-2-pro-high-res": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.5, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "1024x1792", + "1792x1024" + ] + }, "azure/sora-2": { "litellm_provider": "azure", "mode": "video_generation", @@ -35975,5 +36210,715 @@ "mode": "chat", "output_cost_per_token": 0, "supports_reasoning": true + }, + "tts-1-1106": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "openai", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "tts-1-hd-1106": { + "input_cost_per_character": 3e-05, + "litellm_provider": "openai", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "gpt-4o-mini-tts-2025-03-20": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "mode": "audio_speech", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_second": 0.00025, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "audio" + ] + }, + "gpt-4o-mini-tts-2025-12-15": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "mode": "audio_speech", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_second": 0.00025, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "audio" + ] + }, + "gpt-4o-mini-transcribe-2025-03-20": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 16000, + "max_output_tokens": 2000, + "mode": "audio_transcription", + "output_cost_per_token": 5e-06, + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "gpt-4o-mini-transcribe-2025-12-15": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 16000, + "max_output_tokens": 2000, + "mode": "audio_transcription", + "output_cost_per_token": 5e-06, + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "gpt-5-search-api": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "gpt-5-search-api-2025-10-14": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "gpt-realtime-mini-2025-10-06": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "gpt-realtime-mini-2025-12-15": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "sora-2": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.1, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "720x1280", + "1280x720" + ] + }, + "sora-2-pro": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.3, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "720x1280", + "1280x720" + ] + }, + "sora-2-pro-high-res": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.5, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "1024x1792", + "1792x1024" + ] + }, + "chatgpt-image-latest": { + "cache_read_input_image_token_cost": 2.5e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 1e-05, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 4e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "gemini-2.0-flash-exp-image-generation": { + "input_cost_per_token": 0.0, + "litellm_provider": "gemini", + "max_images_per_prompt": 3000, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.039, + "output_cost_per_token": 0.0, + "source": "https://ai.google.dev/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "gemini/gemini-2.0-flash-exp-image-generation": { + "input_cost_per_token": 0.0, + "litellm_provider": "gemini", + "max_images_per_prompt": 3000, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.039, + "output_cost_per_token": 0.0, + "source": "https://ai.google.dev/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "gemini/gemini-2.0-flash-lite-001": { + "cache_read_input_token_cost": 1.875e-08, + "deprecation_date": "2026-03-31", + "input_cost_per_audio_token": 7.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_pdf_size_mb": 50, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 3e-07, + "rpm": 4000, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 4000000 + }, + "gemini-2.5-flash-native-audio-latest": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-native-audio-preview-09-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-native-audio-preview-12-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-latest": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-preview-09-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-preview-12-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-preview-tts": { + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "mode": "audio_speech", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "gemini-flash-latest": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "rpm": 100000, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 8000000 + }, + "gemini-flash-lite-latest": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_audio_token": 3e-07, + "input_cost_per_token": 1e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 4e-07, + "output_cost_per_token": 4e-07, + "rpm": 15, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-lite", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 250000 + }, + "gemini-pro-latest": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "rpm": 2000, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, + "gemini/gemini-pro-latest": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "rpm": 2000, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, + "gemini-exp-1206": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "rpm": 100000, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 8000000 } } diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 548e3bc3dbf..ed4fb133478 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -536,10 +536,25 @@ class MCPRequestHandler: user_api_key_auth: Optional[UserAPIKeyAuth] = None, ) -> List[str]: try: - # Get key object permission (already loaded in main auth flow) + # Get key object permission (already loaded in main auth flow, or fetch from DB) key_object_permission = MCPRequestHandler._get_key_object_permission( user_api_key_auth ) + if key_object_permission is None and user_api_key_auth and user_api_key_auth.object_permission_id: + from litellm.proxy.auth.auth_checks import get_object_permission + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + if prisma_client is not None: + key_object_permission = await get_object_permission( + object_permission_id=user_api_key_auth.object_permission_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) if key_object_permission is None: return [] diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index fd251488db4..e003841ac09 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -14,6 +14,7 @@ import re from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Union, cast from urllib.parse import urlparse +import anyio from fastapi import HTTPException from httpx import HTTPStatusError from mcp import ReadResourceResult, Resource @@ -887,8 +888,15 @@ class MCPServerManager: # Handle stdio transport if transport == MCPTransport.stdio: - # For stdio, we need to get the stdio config from the server - resolved_env = stdio_env if stdio_env is not None else server.env or {} + resolved_env = stdio_env if stdio_env is not None else dict(server.env or {}) + + # Ensure npm-based STDIO MCP servers have a writable cache dir. + # In containers the default (~/.npm or /app/.npm) may not exist + # or be read-only, causing npx to fail with ENOENT. + if "NPM_CONFIG_CACHE" not in resolved_env: + from litellm.constants import MCP_NPM_CACHE_DIR + + resolved_env["NPM_CONFIG_CACHE"] = MCP_NPM_CACHE_DIR stdio_config: Optional[MCPStdioConfig] = None if server.command and server.args is not None: stdio_config = MCPStdioConfig( @@ -1437,6 +1445,9 @@ class MCPServerManager: """ Fetch tools from MCP client with timeout and error handling. + Uses anyio.fail_after() instead of asyncio.wait_for() to avoid conflicts + with the MCP SDK's anyio TaskGroup. See GitHub issue #20715 for details. + Args: client: MCP client instance server_name: Name of the server for logging @@ -1444,24 +1455,12 @@ class MCPServerManager: Returns: List of tools from the server """ - - async def _list_tools_task(): - try: + try: + with anyio.fail_after(30.0): tools = await client.list_tools() verbose_logger.debug(f"Tools from {server_name}: {tools}") return tools - except asyncio.CancelledError: - verbose_logger.warning(f"Client operation cancelled for {server_name}") - return [] - except Exception as e: - verbose_logger.warning( - f"Client operation failed for {server_name}: {str(e)}" - ) - return [] - - try: - return await asyncio.wait_for(_list_tools_task(), timeout=30.0) - except asyncio.TimeoutError: + except TimeoutError: verbose_logger.warning(f"Timeout while listing tools from {server_name}") return [] except asyncio.CancelledError: @@ -2481,6 +2480,9 @@ class MCPServerManager: except asyncio.TimeoutError: health_check_error = "Health check timed out after 10 seconds" status = "unhealthy" + except asyncio.CancelledError: + health_check_error = "Health check was cancelled" + status = "unknown" except Exception as e: health_check_error = str(e) status = "unhealthy" diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 58cd8c99e7b..da0df653106 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -24,6 +24,7 @@ from fastapi import FastAPI, HTTPException from pydantic import AnyUrl, ConfigDict from starlette.requests import Request as StarletteRequest from starlette.types import Receive, Scope, Send +from starlette.responses import JSONResponse from litellm._logging import verbose_logger from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG @@ -41,6 +42,9 @@ from litellm.proxy._experimental.mcp_server.utils import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, +) from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall @@ -842,6 +846,7 @@ if MCP_AVAILABLE: raw_headers: Optional[Dict[str, str]] = None, log_list_tools_to_spendlogs: bool = False, list_tools_log_source: Optional[str] = None, + litellm_trace_id: Optional[str] = None, ) -> List[MCPTool]: """ Helper method to fetch tools from MCP servers based on server filtering criteria. @@ -879,6 +884,7 @@ if MCP_AVAILABLE: "model": "MCP: list_tools", "call_type": CallTypes.list_mcp_tools.value, "litellm_call_id": list_tools_call_id, + "litellm_trace_id": litellm_trace_id, "metadata": { "spend_logs_metadata": spend_logs_metadata, }, @@ -894,13 +900,14 @@ if MCP_AVAILABLE: ], } - # Attach user identifiers when available (matches call_mcp_tool style) + # Attach user identifiers using the standard helper if user_api_key_auth is not None: - user_api_key = getattr(user_api_key_auth, "api_key", None) - if user_api_key: - cast(dict, list_tools_request_data["metadata"])[ - "user_api_key" - ] = user_api_key + + LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( + data=list_tools_request_data, + user_api_key_dict=user_api_key_auth, + _metadata_variable_name="metadata", + ) user_identifier = getattr( user_api_key_auth, "end_user_id", None @@ -1907,18 +1914,27 @@ if MCP_AVAILABLE: raw_headers, ) - def _strip_stale_mcp_session_header( + async def _handle_stale_mcp_session( scope: Scope, + receive: Receive, + send: Send, mgr: "StreamableHTTPSessionManager", - ) -> None: + ) -> bool: """ - Strip stale ``mcp-session-id`` headers so the session manager - creates a fresh session instead of returning 404 "Session not found". + Handle stale MCP session IDs to prevent "Session not found" errors. - When clients like VSCode reconnect after a reload they may resend a - session id that has already been cleaned up. Rather than letting the - SDK return a 404 error loop, we detect the stale id and remove the - header so a brand-new session is created transparently. + When clients reconnect after a server restart or session cleanup, they may + send a session ID that no longer exists. This function handles two scenarios: + + 1. Non-DELETE requests: Strip the stale session ID header so the session + manager creates a fresh session transparently. + + 2. DELETE requests: Return success (200) immediately for idempotent behavior, + since the desired state (session doesn't exist) is already achieved. + + Returns: + True if the request was handled (DELETE on non-existent session) + False if the request should continue to the session manager Fixes https://github.com/BerriAI/litellm/issues/20292 """ @@ -1930,10 +1946,30 @@ if MCP_AVAILABLE: break if _session_id is None: - return + return False known_sessions = getattr(mgr, "_server_instances", None) - if known_sessions is not None and _session_id not in known_sessions: + if known_sessions is None or _session_id in known_sessions: + # Session exists or we can't check - let the session manager handle it + return False + + # Session doesn't exist - handle based on request method + method = scope.get("method", "").upper() + + if method == "DELETE": + # Idempotent DELETE: session doesn't exist, return success + verbose_logger.info( + f"DELETE request for non-existent MCP session '{_session_id}'. " + "Returning success (idempotent DELETE)." + ) + success_response = JSONResponse( + status_code=200, + content={"message": "Session terminated successfully"} + ) + await success_response(scope, receive, send) + return True + else: + # Non-DELETE: strip stale session ID to allow new session creation verbose_logger.warning( "MCP session ID '%s' not found in active sessions. " "Stripping stale header to force new session creation.", @@ -1943,6 +1979,7 @@ if MCP_AVAILABLE: (k, v) for k, v in scope["headers"] if k != _mcp_session_header ] + return False async def handle_streamable_http_mcp( scope: Scope, receive: Receive, send: Send @@ -2005,7 +2042,12 @@ if MCP_AVAILABLE: # Give it a moment to start up await asyncio.sleep(0.1) - _strip_stale_mcp_session_header(scope, session_manager) + # Handle stale session IDs - either strip them for reconnection + # or return success for idempotent DELETE operations + handled = await _handle_stale_mcp_session(scope, receive, send, session_manager) + if handled: + # Request was fully handled (e.g., DELETE on non-existent session) + return await session_manager.handle_request(scope, receive, send) except HTTPException: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 87ff4a66e08..45476900a26 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -514,6 +514,8 @@ class LiteLLMRoutes(enum.Enum): "/user/delete", "/user/info", "/user/list", + "/user/daily/activity", + "/user/daily/activity/aggregated", # team "/team/new", "/team/update", @@ -526,6 +528,7 @@ class LiteLLMRoutes(enum.Enum): "/team/available", "/team/permissions_list", "/team/permissions_update", + "/team/daily/activity", # model "/model/new", "/model/update", @@ -893,6 +896,7 @@ class KeyRequestBase(GenerateRequestBase): Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] ] = None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm router_settings: Optional[UpdateRouterConfig] = None + access_group_ids: Optional[List[str]] = None class LiteLLMKeyType(str, enum.Enum): @@ -1502,6 +1506,7 @@ class TeamBase(LiteLLMPydanticObjectBase): models: list = [] blocked: bool = False router_settings: Optional[dict] = None + access_group_ids: Optional[List[str]] = None class NewTeamRequest(TeamBase): @@ -1589,6 +1594,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): model_tpm_limit: Optional[Dict[str, int]] = None allowed_vector_store_indexes: Optional[List[AllowedVectorStoreIndexItem]] = None router_settings: Optional[dict] = None + access_group_ids: Optional[List[str]] = None class ResetTeamBudgetRequest(LiteLLMPydanticObjectBase): @@ -2177,6 +2183,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase): updated_by: Optional[str] = None object_permission_id: Optional[str] = None object_permission: Optional[LiteLLM_ObjectPermissionTable] = None + access_group_ids: Optional[List[str]] = None rotation_count: Optional[int] = 0 # Number of times key has been rotated auto_rotate: Optional[bool] = False # Whether this key should be auto-rotated rotation_interval: Optional[str] = None # How often to rotate (e.g., "30d", "90d") @@ -3945,6 +3952,18 @@ class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase): file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob, ResponsesAPIResponse] +class LiteLLM_ManagedVectorStoreTable(LiteLLMPydanticObjectBase): + """Table for managing vector stores with target_model_names support.""" + unified_resource_id: str + resource_object: Optional[Any] = None # VectorStoreCreateResponse + model_mappings: Dict[str, str] + flat_model_resource_ids: List[str] + created_by: Optional[str] + updated_by: Optional[str] + storage_backend: Optional[str] = None + storage_url: Optional[str] = None + + class EnterpriseLicenseData(TypedDict, total=False): expiration_date: str user_id: str diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 42f10ff8598..ba4e3b42c37 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -925,10 +925,11 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 if isinstance( api_key, str ): # if generated token, make sure it starts with sk-. + _masked_key = "{}****{}".format(api_key[:4], api_key[-4:]) if len(api_key) > 8 else "****" assert api_key.startswith( "sk-" ), "LiteLLM Virtual Key expected. Received={}, expected to start with 'sk-'.".format( - api_key + _masked_key ) # prevent token hashes from being used else: verbose_logger.warning( diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f33b2412260..a02bc7f9e51 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -282,7 +282,7 @@ def _override_openai_response_model( if isinstance(response_obj, dict): downstream_model = response_obj.get("model") if downstream_model != requested_model: - verbose_proxy_logger.warning( + verbose_proxy_logger.debug( "%s: response model mismatch - requested=%r downstream=%r. Overriding response['model'] to requested model.", log_context, requested_model, @@ -301,7 +301,7 @@ def _override_openai_response_model( downstream_model = getattr(response_obj, "model", None) if downstream_model != requested_model: - verbose_proxy_logger.warning( + verbose_proxy_logger.debug( "%s: response model mismatch - requested=%r downstream=%r. Overriding response.model to requested model.", log_context, requested_model, diff --git a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py index c96564252d0..b41ff121622 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py @@ -112,7 +112,8 @@ class SpendUpdateQueue(BaseUpdateQueue): for update in updates: _key = f"{update.get('entity_type')}:{update.get('entity_id')}" if _key not in _in_memory_map: - _in_memory_map[_key] = update + # avoid mutating caller-owned dicts while aggregating queue entries + _in_memory_map[_key] = update.copy() else: current_cost = _in_memory_map[_key].get("response_cost", 0) or 0 update_cost = update.get("response_cost", 0) or 0 diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index 9018675d7a5..9cded6f0ac2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -5,10 +5,12 @@ # +-------------------------------------------------------------+ # Thank you users! We ❤️ you! - Krrish & Ishaan +import fnmatch import os from typing import TYPE_CHECKING, Any, Dict, Literal, Optional from litellm._logging import verbose_proxy_logger +from litellm._version import version as litellm_version from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_guardrail import ( CustomGuardrail, @@ -31,6 +33,110 @@ if TYPE_CHECKING: GUARDRAIL_NAME = "generic_guardrail_api" +# Headers whose values are forwarded as-is (case-insensitive). Glob patterns supported (e.g. x-stainless-*, x-litellm*). +_HEADER_VALUE_ALLOWLIST = frozenset({ + "host", + "accept-encoding", + "connection", + "accept", + "content-type", + "user-agent", + "x-stainless-*", + "x-litellm-*", + "content-length", +}) + +# Placeholder for headers that exist but are not on the allowlist (we don't expose their value). +_HEADER_PRESENT_PLACEHOLDER = "[present]" + + +def _header_value_allowed(header_name: str) -> bool: + """Return True if this header's value may be forwarded (allowlist, including globs).""" + lower = header_name.lower() + if lower in _HEADER_VALUE_ALLOWLIST: + return True + for pattern in _HEADER_VALUE_ALLOWLIST: + if "*" in pattern and fnmatch.fnmatch(lower, pattern): + return True + return False + + +def _sanitize_inbound_headers(headers: Any) -> Optional[Dict[str, str]]: + """ + Sanitize inbound headers before passing them to a 3rd party guardrail service. + + - Allowlist: only headers in the allowlist have their values forwarded (exact + glob: x-stainless-*, x-litellm-*). + - All other headers are included with value "[present]" so the guardrail knows the header existed. + - Coerces values to str (for JSON serialization). + """ + if not headers or not isinstance(headers, dict): + return None + + sanitized: Dict[str, str] = {} + for k, v in headers.items(): + if k is None: + continue + key = str(k) + if _header_value_allowed(key): + try: + sanitized[key] = str(v) + except Exception: + continue + else: + sanitized[key] = _HEADER_PRESENT_PLACEHOLDER + + return sanitized or None + + +def _extract_inbound_headers( + request_data: dict, logging_obj: Optional["LiteLLMLoggingObj"] +) -> Optional[Dict[str, str]]: + """ + Extract inbound headers from available request context. + + We try multiple locations to support different call paths: + - proxy endpoints: request_data["proxy_server_request"]["headers"] + - if the guardrail is passed the proxy_server_request object directly + - metadata headers captured in litellm_pre_call_utils + - response hooks: fallback to logging_obj.model_call_details + """ + # 1) Most common path (proxy): full request context in proxy_server_request + headers = request_data.get("proxy_server_request", {}).get("headers") + if headers: + return _sanitize_inbound_headers(headers) + + # 2) Some guardrails pass proxy_server_request as request_data itself + headers = request_data.get("headers") + if headers: + return _sanitize_inbound_headers(headers) + + # 3) Pre-call: headers stored in request metadata + metadata_headers = (request_data.get("metadata") or {}).get("headers") + if metadata_headers: + return _sanitize_inbound_headers(metadata_headers) + + litellm_metadata_headers = (request_data.get("litellm_metadata") or {}).get( + "headers" + ) + if litellm_metadata_headers: + return _sanitize_inbound_headers(litellm_metadata_headers) + + # 4) Post-call: headers not present on response; fallback to logging object + if logging_obj and getattr(logging_obj, "model_call_details", None): + try: + details = logging_obj.model_call_details or {} + headers = ( + details.get("litellm_params", {}) + .get("metadata", {}) + .get("headers", None) + ) + if headers: + return _sanitize_inbound_headers(headers) + except Exception: + pass + + return None + class GenericGuardrailAPI(CustomGuardrail): """ @@ -207,6 +313,7 @@ class GenericGuardrailAPI(CustomGuardrail): # Extract user API key metadata user_metadata = self._extract_user_api_key_metadata(request_data) + inbound_headers = _extract_inbound_headers(request_data=request_data, logging_obj=logging_obj) # Create request payload guardrail_request = GenericGuardrailAPIRequest( @@ -214,6 +321,8 @@ class GenericGuardrailAPI(CustomGuardrail): litellm_trace_id=logging_obj.litellm_trace_id if logging_obj else None, texts=texts, request_data=user_metadata, + request_headers=inbound_headers, + litellm_version=litellm_version, images=images, tools=tools, structured_messages=structured_messages, diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index 9d64aea8910..9b2b43c6fcb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -29,10 +29,7 @@ from fastapi import HTTPException from litellm import Router from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import ( - CustomGuardrail, - log_guardrail_information, -) +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ModelResponseStream @@ -1056,7 +1053,6 @@ class ContentFilterGuardrail(CustomGuardrail): masked_entity_count=masked_entity_count, ) - @log_guardrail_information async def apply_guardrail( self, inputs: "GenericGuardrailAPIInputs", diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 38462094b11..2d3f048f81b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -330,6 +330,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): end_time: Optional[float] = None, duration: Optional[float] = None, event_type: Optional[GuardrailEventHooks] = None, + original_inputs: Optional[dict] = None, ): """ Override to store only the Model Armor API response, not the entire data dict. diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index a196937ef6c..6160fb41439 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -5,14 +5,9 @@ OpenAI Moderation Guardrail Integration for LiteLLM from typing import ( TYPE_CHECKING, - Any, - AsyncGenerator, - Dict, - List, Literal, Optional, Type, - Union, ) from fastapi import HTTPException @@ -20,7 +15,7 @@ from fastapi import HTTPException from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, - log_guardrail_information, + log_guardrail_information ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.custom_httpx.http_handler import ( @@ -32,10 +27,8 @@ from litellm.types.utils import GenericGuardrailAPIInputs from .base import OpenAIGuardrailBase if TYPE_CHECKING: - from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import OpenAIModerationResponse from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel - from litellm.types.utils import ModelResponse, ModelResponseStream class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): @@ -236,108 +229,6 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): # Moderation doesn't modify content, just blocks - return inputs unchanged return inputs - @log_guardrail_information - async def async_post_call_streaming_iterator_hook( - self, - user_api_key_dict: "UserAPIKeyAuth", - response: Any, - request_data: Dict[str, Any], - ) -> AsyncGenerator["ModelResponseStream", None]: - """ - Process streaming response chunks for OpenAI moderation. - - Collects all chunks from the stream, assembles them into a complete response, - and applies moderation check. If content violates moderation policy, raises HTTPException. - """ - # Import here to avoid circular imports - from litellm.llms.base_llm.base_model_iterator import MockResponseIterator - from litellm.main import stream_chunk_builder - from litellm.types.utils import TextCompletionResponse - - verbose_proxy_logger.debug("OpenAI Moderation: Running streaming response scan") - - # Collect all chunks to process them together - all_chunks: List["ModelResponseStream"] = [] - async for chunk in response: - all_chunks.append(chunk) - - # Assemble the complete response from chunks - assembled_model_response: Optional[ - Union["ModelResponse", TextCompletionResponse] - ] = stream_chunk_builder( - chunks=all_chunks, - ) - - if isinstance(assembled_model_response, (type(None), TextCompletionResponse)): - # If we can't assemble a ModelResponse or it's a text completion, - # just yield the original chunks without moderation - verbose_proxy_logger.warning( - "OpenAI Moderation: Could not assemble ModelResponse from chunks, skipping moderation" - ) - for chunk in all_chunks: - yield chunk - return - - # Extract response text for moderation - response_text = self._extract_response_text(assembled_model_response) - if response_text: - verbose_proxy_logger.debug( - f"OpenAI Moderation: Streaming response text: {response_text[:100]}..." # Log first 100 chars - ) - - # Make moderation request - this will raise HTTPException if content is flagged - moderation_response = await self.async_make_request( - input_text=response_text, - ) - - # Check if content is flagged and raise exception if needed - self._check_moderation_result(moderation_response) - - # If we reach here, content passed moderation - yield the original chunks - mock_response = MockResponseIterator(model_response=assembled_model_response) - - # Return the reconstructed stream - async for chunk in mock_response: - yield chunk - - def _extract_response_text(self, response: "ModelResponse") -> Optional[str]: - """ - Extract text content from the model response for moderation. - """ - if not hasattr(response, "choices") or not response.choices: - return None - - response_texts = [] - for choice in response.choices: - try: - # Try to get content from message (chat completion) - message = getattr(choice, "message", None) - if message: - content = getattr(message, "content", None) - if content and isinstance(content, str): - response_texts.append(content) - continue - - # Try to get text (text completion) - text = getattr(choice, "text", None) - if text and isinstance(text, str): - response_texts.append(text) - continue - - # Try to get content from delta (streaming) - delta = getattr(choice, "delta", None) - if delta: - content = getattr(delta, "content", None) - if content and isinstance(content, str): - response_texts.append(content) - continue - - except (AttributeError, TypeError): - # Skip choices that don't have expected attributes - continue - - return "\n".join(response_texts) if response_texts else None - @staticmethod def get_config_model() -> Optional[Type["GuardrailConfigModel"]]: """ diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 3984384aae4..b3cc9236b1d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -386,7 +386,11 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): continue return final_results except Exception as e: - raise e + # Sanitize exception to avoid leaking the original text (which may + # contain API keys or other secrets) in error responses. + raise Exception( + f"Presidio PII analysis failed: {type(e).__name__}" + ) from e async def anonymize_text( self, @@ -443,9 +447,15 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) return redacted_text["text"] else: - raise Exception(f"Invalid anonymizer response: {redacted_text}") + raise Exception("Invalid anonymizer response: received None") except Exception as e: - raise e + # Sanitize exception to avoid leaking the original text (which may + # contain API keys or other secrets) in error responses. + if "Invalid anonymizer response" in str(e): + raise + raise Exception( + f"Presidio PII anonymization failed: {type(e).__name__}" + ) from e def filter_analyze_results_by_score( self, analyze_results: Union[List[PresidioAnalyzeResponseItem], Dict] diff --git a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py index ff00cd73ca5..b0b18166d2d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py @@ -21,6 +21,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel GUARDRAIL_TIMEOUT = 5 @@ -334,3 +335,11 @@ class ZscalerAIGuard(CustomGuardrail): user_facing_error = self._create_user_facing_error(f"{str(e)})") # This exception will be caught by the proxy and returned to the user raise HTTPException(status_code=500, detail=user_facing_error) + + @staticmethod + def get_config_model() -> Optional[type["GuardrailConfigModel"]]: + from litellm.types.proxy.guardrails.guardrail_hooks.zscaler_ai_guard import ( + ZscalerAIGuardConfigModel, + ) + + return ZscalerAIGuardConfigModel diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index c3da6892209..c0903a35b6d 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -12,6 +12,7 @@ from litellm._uuid import uuid from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy.utils import PrismaClient +from litellm.proxy.types_utils.utils import get_instance_fn from litellm.secret_managers.main import get_secret from litellm.types.guardrails import ( Guardrail, @@ -489,7 +490,7 @@ class InMemoryGuardrailHandler: config_file_path: Optional[str] = None, ) -> Optional[CustomGuardrail]: """ - Initialize a Custom Guardrail from a python file + Initialize a Custom Guardrail from a python file or module path This initializes it by adding it to the litellm callback manager """ @@ -498,26 +499,12 @@ class InMemoryGuardrailHandler: "GuardrailsAIException - Please pass the config_file_path to initialize_guardrails_v2" ) - _file_name, _class_name = guardrail_type.split(".") verbose_proxy_logger.debug( - "Initializing custom guardrail: %s, file_name: %s, class_name: %s", + "Initializing custom guardrail: %s", guardrail_type, - _file_name, - _class_name, ) - directory = os.path.dirname(config_file_path) - module_file_path = os.path.join(directory, _file_name) + ".py" - - spec = importlib.util.spec_from_file_location(_class_name, module_file_path) # type: ignore - if not spec: - raise ImportError( - f"Could not find a module specification for {module_file_path}" - ) - - module = importlib.util.module_from_spec(spec) # type: ignore - spec.loader.exec_module(module) # type: ignore - _guardrail_class = getattr(module, _class_name) + _guardrail_class = get_instance_fn(guardrail_type, config_file_path=config_file_path) mode = litellm_params.mode if mode is None: diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index 78a371ad66e..45b1bd8653f 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -255,12 +255,24 @@ class _PROXY_BatchRateLimiter(CustomLogger): BatchFileUsage with total_tokens and request_count """ try: - # Read file content - file_content = await litellm.afile_content( - file_id=file_id, - custom_llm_provider=custom_llm_provider, - user_api_key_dict=user_api_key_dict, + # Check if this is a managed file (base64 encoded unified file ID) + from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, ) + is_managed_file = _is_base64_encoded_unified_file_id(file_id) + if is_managed_file and user_api_key_dict is not None: + # For managed files, use the managed files hook directly + file_content = await self._fetch_managed_file_content( + file_id=file_id, + user_api_key_dict=user_api_key_dict, + ) + else: + # For non-managed files, use the standard litellm.afile_content + file_content = await litellm.afile_content( + file_id=file_id, + custom_llm_provider=custom_llm_provider, + user_api_key_dict=user_api_key_dict, + ) file_content_as_dict = _get_file_content_as_dictionary( file_content.content @@ -282,6 +294,67 @@ class _PROXY_BatchRateLimiter(CustomLogger): ) raise + async def _fetch_managed_file_content( + self, + file_id: str, + user_api_key_dict: UserAPIKeyAuth, + ) -> Any: + """ + Fetch file content from managed files hook. + + This is needed for managed files because they require proper user context + to verify file ownership and access permissions. + + Args: + file_id: The managed file ID (base64 encoded) + user_api_key_dict: User authentication information + + Returns: + HttpxBinaryResponseContent with the file content + """ + from litellm.llms.base_llm.files.transformation import BaseFileEndpoints + + # Import proxy_server dependencies at runtime to avoid circular imports + try: + from litellm.proxy.proxy_server import llm_router, proxy_logging_obj + except ImportError as e: + raise ValueError( + f"Cannot import proxy_server dependencies: {str(e)}. " + "Managed files require proxy_server to be initialized." + ) + + # Get the managed files hook + if proxy_logging_obj is None: + raise ValueError( + "proxy_logging_obj not available. Cannot access managed files hook." + ) + + managed_files_obj = proxy_logging_obj.get_proxy_hook("managed_files") + if managed_files_obj is None: + raise ValueError( + "Managed files hook not found. Cannot access managed file." + ) + + if not isinstance(managed_files_obj, BaseFileEndpoints): + raise ValueError( + "Managed files hook is not a BaseFileEndpoints instance." + ) + + if llm_router is None: + raise ValueError( + "llm_router not available. Cannot access managed files." + ) + + # Use the managed files hook to get file content + # This properly handles user permissions and file ownership + file_content = await managed_files_obj.afile_content( + file_id=file_id, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + llm_router=llm_router, + ) + + return file_content + async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index a8325d34612..c07f30f8646 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -150,15 +150,26 @@ class KeyManagementEventHooks: existing_key_row.key_alias or f"virtual-key-{existing_key_row.token}" ) + new_secret_name = ( + response.key_alias + or data.key_alias + or f"virtual-key-{response.token_id}" + ) + verbose_proxy_logger.info( + "Updating secret in secret manager: secret_name=%s", + new_secret_name, + ) team_id = getattr(existing_key_row, "team_id", None) await KeyManagementEventHooks._rotate_virtual_key_in_secret_manager( current_secret_name=initial_secret_name, - new_secret_name=response.key_alias - or data.key_alias - or f"virtual-key-{response.token_id}", + new_secret_name=new_secret_name, new_secret_value=response.key, team_id=team_id, ) + verbose_proxy_logger.info( + "Secret updated in secret manager: secret_name=%s", + new_secret_name, + ) except Exception as e: verbose_proxy_logger.warning( f"Failed to rotate virtual key in secret manager: {e}" diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index dab5fb1bfd5..37b79e6d065 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -202,8 +202,8 @@ class _ProxyDBLogger(CustomLogger): max_budget=end_user_max_budget, ) else: - if kwargs["stream"] is not True or ( - kwargs["stream"] is True and "complete_streaming_response" in kwargs + if kwargs.get("stream") is not True or ( + kwargs.get("stream") is True and "complete_streaming_response" in kwargs ): if sl_object is not None: cost_tracking_failure_debug_info: Union[dict, str] = ( diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py new file mode 100644 index 00000000000..33b81e85654 --- /dev/null +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -0,0 +1,264 @@ +from typing import List + +from fastapi import APIRouter, Depends, HTTPException, status + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.utils import get_prisma_client_or_throw +from litellm.types.access_group import ( + AccessGroupCreateRequest, + AccessGroupResponse, + AccessGroupUpdateRequest, +) + +router = APIRouter( + tags=["access group management"], +) + + +def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={"error": CommonProxyErrors.not_allowed_access.value}, + ) + + +def _record_to_response(record) -> AccessGroupResponse: + return AccessGroupResponse( + access_group_id=record.access_group_id, + access_group_name=record.access_group_name, + description=record.description, + access_model_ids=record.access_model_ids, + access_mcp_server_ids=record.access_mcp_server_ids, + access_agent_ids=record.access_agent_ids, + assigned_team_ids=record.assigned_team_ids, + assigned_key_ids=record.assigned_key_ids, + created_at=record.created_at, + created_by=record.created_by, + updated_at=record.updated_at, + updated_by=record.updated_by, + ) + + +@router.post( + "/v1/access_group", + response_model=AccessGroupResponse, + status_code=status.HTTP_201_CREATED, +) +async def create_access_group( + data: AccessGroupCreateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> AccessGroupResponse: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + existing = await prisma_client.db.litellm_accessgrouptable.find_unique( + where={"access_group_name": data.access_group_name} + ) + if existing is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Access group '{data.access_group_name}' already exists", + ) + + try: + record = await prisma_client.db.litellm_accessgrouptable.create( + data={ + "access_group_name": data.access_group_name, + "description": data.description, + "access_model_ids": data.access_model_ids or [], + "access_mcp_server_ids": data.access_mcp_server_ids or [], + "access_agent_ids": data.access_agent_ids or [], + "assigned_team_ids": data.assigned_team_ids or [], + "assigned_key_ids": data.assigned_key_ids or [], + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, + } + ) + except Exception as e: + # Race condition: another request created the same name between find_unique and create. + # Prisma raises UniqueViolationError (P2002) or similar for unique constraint. + if "unique constraint" in str(e).lower() or "P2002" in str(e): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Access group '{data.access_group_name}' already exists", + ) + raise + return _record_to_response(record) + + +@router.get( + "/v1/access_group", + response_model=List[AccessGroupResponse], +) +async def list_access_groups( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> List[AccessGroupResponse]: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + records = await prisma_client.db.litellm_accessgrouptable.find_many( + order={"created_at": "desc"} + ) + return [_record_to_response(r) for r in records] + + +@router.get( + "/v1/access_group/{access_group_id}", + response_model=AccessGroupResponse, +) +async def get_access_group( + access_group_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> AccessGroupResponse: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + record = await prisma_client.db.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if record is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + return _record_to_response(record) + + +@router.put( + "/v1/access_group/{access_group_id}", + response_model=AccessGroupResponse, +) +async def update_access_group( + access_group_id: str, + data: AccessGroupUpdateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> AccessGroupResponse: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + existing = await prisma_client.db.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if existing is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + + update_data: dict = {"updated_by": user_api_key_dict.user_id} + for field, value in data.model_dump(exclude_unset=True).items(): + update_data[field] = value + + record = await prisma_client.db.litellm_accessgrouptable.update( + where={"access_group_id": access_group_id}, + data=update_data, + ) + return _record_to_response(record) + + +@router.delete( + "/v1/access_group/{access_group_id}", + status_code=status.HTTP_204_NO_CONTENT, +) +async def delete_access_group( + access_group_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> None: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + try: + async with prisma_client.db.tx() as tx: + existing = await tx.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if existing is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + + # Remove access_group_id from teams and keys that reference it + teams_with_group = await tx.litellm_teamtable.find_many( + where={"access_group_ids": {"hasSome": [access_group_id]}} + ) + for team in teams_with_group: + updated_ids = [tid for tid in (team.access_group_ids or []) if tid != access_group_id] + await tx.litellm_teamtable.update( + where={"team_id": team.team_id}, + data={"access_group_ids": updated_ids}, + ) + + keys_with_group = await tx.litellm_verificationtoken.find_many( + where={"access_group_ids": {"hasSome": [access_group_id]}} + ) + for key in keys_with_group: + updated_ids = [kid for kid in (key.access_group_ids or []) if kid != access_group_id] + await tx.litellm_verificationtoken.update( + where={"token": key.token}, + data={"access_group_ids": updated_ids}, + ) + + await tx.litellm_accessgrouptable.delete( + where={"access_group_id": access_group_id} + ) + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception( + "delete_access_group failed: access_group_id=%s error=%s", + access_group_id, + e, + ) + if PrismaDBExceptionHandler.is_database_connection_error(e): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=CommonProxyErrors.db_not_connected_error.value, + ) + if "P2025" in str(e) or ("record" in str(e).lower() and "not found" in str(e).lower()): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to delete access group. Please try again.", + ) + + +# Alias routes for /v1/unified_access_group +router.add_api_route( + "/v1/unified_access_group", + create_access_group, + methods=["POST"], + response_model=AccessGroupResponse, + status_code=status.HTTP_201_CREATED, +) +router.add_api_route( + "/v1/unified_access_group", + list_access_groups, + methods=["GET"], + response_model=List[AccessGroupResponse], +) +router.add_api_route( + "/v1/unified_access_group/{access_group_id}", + get_access_group, + methods=["GET"], + response_model=AccessGroupResponse, +) +router.add_api_route( + "/v1/unified_access_group/{access_group_id}", + update_access_group, + methods=["PUT"], + response_model=AccessGroupResponse, +) +router.add_api_route( + "/v1/unified_access_group/{access_group_id}", + delete_access_group, + methods=["DELETE"], + status_code=status.HTTP_204_NO_CONTENT, +) diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 942758e3bab..d6f6d1337a3 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -383,6 +383,17 @@ def _update_metadata_field(updated_kv: dict, field_name: str) -> None: updated_kv["metadata"] = {field_name: _value} +def _has_non_empty_value(value: Any) -> bool: + """Check if a value has real content (not None, not empty list, not blank string).""" + if value is None: + return False + if isinstance(value, list) and len(value) == 0: + return False + if isinstance(value, str) and value.strip() == "": + return False + return True + + def _update_metadata_fields(updated_kv: dict) -> None: """ Helper function to update all metadata fields (both premium and standard). @@ -391,7 +402,7 @@ def _update_metadata_fields(updated_kv: dict) -> None: updated_kv: The key-value dict being used for the update """ for field in LiteLLM_ManagementEndpoint_MetadataFields_Premium: - if field in updated_kv and updated_kv[field] is not None: + if field in updated_kv and _has_non_empty_value(updated_kv[field]): _update_metadata_field(updated_kv=updated_kv, field_name=field) for field in LiteLLM_ManagementEndpoint_MetadataFields: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 2e71759072d..d15c51afe7b 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -628,10 +628,11 @@ async def _common_key_generation_helper( # noqa: PLR0915 # Validate user-provided key format if data.key is not None and not data.key.startswith("sk-"): + _masked = "{}****{}".format(data.key[:4], data.key[-4:]) if len(data.key) > 8 else "****" raise HTTPException( status_code=400, detail={ - "error": f"Invalid key format. LiteLLM Virtual Key must start with 'sk-'. Received: {data.key}" + "error": f"Invalid key format. LiteLLM Virtual Key must start with 'sk-'. Received: {_masked}" }, ) @@ -2770,6 +2771,7 @@ async def can_modify_verification_token( Rules: - Proxy admin can modify any key + - Internal jobs service account can modify any key (for auto-rotation) - For team keys: only team admin or key owner can modify - For personal keys: only key owner can modify @@ -2782,13 +2784,19 @@ async def can_modify_verification_token( Returns: True if user can modify the key, False otherwise """ + from litellm.constants import LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME + is_team_key = _is_team_key(data=key_info) # 1. Proxy admin can modify any key if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value: return True - # 2. For team keys: only team admin or key owner can modify + # 2. Internal jobs service account can modify any key (for auto-rotation) + if user_api_key_dict.api_key == LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME: + return True + + # 3. For team keys: only team admin or key owner can modify if is_team_key and key_info.team_id is not None: # Get team object to check if user is team admin team_table = await get_team_object( @@ -2818,7 +2826,7 @@ async def can_modify_verification_token( # Not team admin and doesn't own the key return False - # 3. For personal keys: only key owner can modify + # 4. For personal keys: only key owner can modify if key_info.user_id is not None and key_info.user_id == user_api_key_dict.user_id: return True @@ -3179,7 +3187,7 @@ def get_new_token(data: Optional[RegenerateKeyRequest]) -> str: dependencies=[Depends(user_api_key_auth)], ) @management_endpoint_wrapper -async def regenerate_key_fn( +async def regenerate_key_fn( # noqa: PLR0915 key: Optional[str] = None, data: Optional[RegenerateKeyRequest] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -3330,6 +3338,10 @@ async def regenerate_key_fn( detail={"error": "You are not authorized to regenerate this key"}, ) + verbose_proxy_logger.info( + "Key regeneration requested: key_alias=%s", + getattr(_key_in_db, "key_alias", None), + ) verbose_proxy_logger.debug("key_in_db: %s", _key_in_db) new_token = get_new_token(data=data) @@ -3380,6 +3392,10 @@ async def regenerate_key_fn( **updated_token_dict, ) + verbose_proxy_logger.info( + "Key regeneration completed: key_alias=%s", + getattr(_key_in_db, "key_alias", None), + ) asyncio.create_task( KeyManagementEventHooks.async_key_rotated_hook( data=data, diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 597521ae773..8c4d4e7937e 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -10,10 +10,13 @@ Endpoints here: - DELETE `/v1/mcp/server/{server_id}` - Deletes the mcp server given `server_id`. - GET `/v1/mcp/tools - lists all the tools available for a key - GET `/v1/mcp/access_groups` - lists all available MCP access groups +- GET `/v1/mcp/discover` - Returns curated list of well-known MCP servers for discovery UI """ import importlib +import json +import os from dataclasses import dataclass from datetime import datetime, timedelta from typing import Any, Dict, Iterable, List, Literal, Optional @@ -1176,3 +1179,88 @@ if MCP_AVAILABLE: except Exception as e: verbose_proxy_logger.exception(f"Error making agent public: {e}") raise HTTPException(status_code=500, detail=str(e)) + + # --- MCP Discovery --- + + _MCP_REGISTRY_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "mcp_registry.json", + ) + + _mcp_registry_cache: Optional[Dict[str, Any]] = None + + def _load_mcp_registry() -> Dict[str, Any]: + """Load the curated MCP registry from disk. Cached after first read.""" + global _mcp_registry_cache + if _mcp_registry_cache is not None: + return _mcp_registry_cache + try: + with open(_MCP_REGISTRY_PATH, "r") as f: + data: Dict[str, Any] = json.load(f) + except Exception as e: + verbose_proxy_logger.warning( + f"Failed to load MCP registry from {_MCP_REGISTRY_PATH}: {e}" + ) + data = {"servers": []} + _mcp_registry_cache = data + return data + + @router.get( + "/discover", + description="Returns a curated list of well-known MCP servers for discovery UI", + dependencies=[Depends(user_api_key_auth)], + ) + async def discover_mcp_servers( + query: Optional[str] = Query( + None, description="Search filter for server names and descriptions" + ), + category: Optional[str] = Query( + None, description="Filter by category" + ), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + """ + Returns a curated list of well-known MCP servers that can be added to the proxy. + + Used by the UI to show a discovery grid when adding new MCP servers. + """ + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail={ + "error": "Only proxy admins can access MCP discovery. Your role={}".format( + user_api_key_dict.user_role + ) + }, + ) + + registry = _load_mcp_registry() + servers = registry.get("servers", []) + + # Apply query filter + if query: + query_lower = query.lower() + servers = [ + s + for s in servers + if query_lower in s.get("name", "").lower() + or query_lower in s.get("title", "").lower() + or query_lower in s.get("description", "").lower() + ] + + # Apply category filter + if category: + servers = [ + s for s in servers if s.get("category", "") == category + ] + + # Extract unique categories from the full list (before filtering) + all_servers = registry.get("servers", []) + categories = sorted( + set(s.get("category", "Other") for s in all_servers) + ) + + return { + "servers": servers, + "categories": categories, + } diff --git a/litellm/proxy/mcp_registry.json b/litellm/proxy/mcp_registry.json new file mode 100644 index 00000000000..2e1e8f64eae --- /dev/null +++ b/litellm/proxy/mcp_registry.json @@ -0,0 +1,426 @@ +{ + "servers": [ + { + "name": "github", + "title": "GitHub", + "description": "Manage repos, issues, PRs, and workflows through natural language", + "icon_url": "https://cdn.simpleicons.org/github", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.github%2Fgithub-mcp-server", + "transport": "http", + "url": "https://api.githubcopilot.com/mcp/", + "env_vars": [ + {"name": "GITHUB_PERSONAL_ACCESS_TOKEN", "description": "GitHub Personal Access Token", "secret": true} + ] + }, + { + "name": "gitlab", + "title": "GitLab", + "description": "Official GitLab MCP Server for project and repository management", + "icon_url": "https://cdn.simpleicons.org/gitlab", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.gitlab%2Fmcp", + "transport": "http", + "url": "https://gitlab.com/api/v4/mcp", + "env_vars": [ + {"name": "GITLAB_PERSONAL_ACCESS_TOKEN", "description": "GitLab Personal Access Token", "secret": true} + ] + }, + { + "name": "atlassian", + "title": "Atlassian (Jira & Confluence)", + "description": "Jira issues, Confluence pages, and Atlassian product integration", + "icon_url": "https://cdn.simpleicons.org/atlassian", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.atlassian%2Fatlassian-mcp-server", + "transport": "sse", + "url": "https://mcp.atlassian.com/v1/sse", + "env_vars": [] + }, + { + "name": "linear", + "title": "Linear", + "description": "Issue tracking, project management, and team workflow automation", + "icon_url": "https://cdn.simpleicons.org/linear", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/app.linear%2Flinear", + "transport": "sse", + "url": "https://mcp.linear.app/sse", + "env_vars": [] + }, + { + "name": "sentry", + "title": "Sentry", + "description": "Error monitoring, issue tracking, and debugging for AI assistants", + "icon_url": "https://cdn.simpleicons.org/sentry", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.getsentry%2Fsentry-mcp", + "transport": "stdio", + "command": "npx", + "args": ["-y", "@sentry/mcp-server"], + "env_vars": [ + {"name": "SENTRY_ACCESS_TOKEN", "description": "Sentry Access Token", "secret": true} + ] + }, + { + "name": "slack", + "title": "Slack", + "description": "Channel management, messaging, and Slack workspace integration", + "icon_url": "https://cdn.simpleicons.org/slack", + "category": "Communication", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-slack"], + "env_vars": [ + {"name": "SLACK_BOT_TOKEN", "description": "Slack Bot User OAuth Token", "secret": true}, + {"name": "SLACK_TEAM_ID", "description": "Slack Team/Workspace ID", "secret": false} + ] + }, + { + "name": "discord", + "title": "Discord", + "description": "Discord server management, messaging, and bot integration", + "icon_url": "https://cdn.simpleicons.org/discord", + "category": "Communication", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-discord"], + "env_vars": [ + {"name": "DISCORD_BOT_TOKEN", "description": "Discord Bot Token", "secret": true} + ] + }, + { + "name": "postgresql", + "title": "PostgreSQL", + "description": "Query and manage PostgreSQL databases with read-only access", + "icon_url": "https://cdn.simpleicons.org/postgresql", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-postgres"], + "env_vars": [ + {"name": "POSTGRES_CONNECTION_STRING", "description": "PostgreSQL connection string (e.g., postgresql://user:pass@host:5432/db)", "secret": true} + ] + }, + { + "name": "sqlite", + "title": "SQLite", + "description": "Query and manage SQLite databases", + "icon_url": "https://cdn.simpleicons.org/sqlite", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-sqlite"], + "env_vars": [ + {"name": "SQLITE_DB_PATH", "description": "Path to SQLite database file", "secret": false} + ] + }, + { + "name": "mysql", + "title": "MySQL", + "description": "Query and manage MySQL databases", + "icon_url": "https://cdn.simpleicons.org/mysql", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-mysql"], + "env_vars": [ + {"name": "MYSQL_HOST", "description": "MySQL host", "secret": false}, + {"name": "MYSQL_USER", "description": "MySQL username", "secret": false}, + {"name": "MYSQL_PASSWORD", "description": "MySQL password", "secret": true}, + {"name": "MYSQL_DATABASE", "description": "MySQL database name", "secret": false} + ] + }, + { + "name": "mongodb", + "title": "MongoDB", + "description": "Query and manage MongoDB databases and collections", + "icon_url": "https://cdn.simpleicons.org/mongodb", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-mongodb"], + "env_vars": [ + {"name": "MONGODB_CONNECTION_STRING", "description": "MongoDB connection string", "secret": true} + ] + }, + { + "name": "redis", + "title": "Redis", + "description": "Interact with Redis key-value stores", + "icon_url": "https://cdn.simpleicons.org/redis", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-redis"], + "env_vars": [ + {"name": "REDIS_URL", "description": "Redis connection URL (e.g., redis://localhost:6379)", "secret": true} + ] + }, + { + "name": "snowflake", + "title": "Snowflake", + "description": "MCP Server for Snowflake from Snowflake Labs", + "icon_url": "https://cdn.simpleicons.org/snowflake", + "category": "Databases", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.Snowflake-Labs%2Fmcp", + "transport": "stdio", + "command": "uvx", + "args": ["snowflake-labs-mcp"], + "env_vars": [ + {"name": "SNOWFLAKE_ACCOUNT", "description": "Snowflake account identifier (e.g., xy12345.us-east-1)", "secret": false}, + {"name": "SNOWFLAKE_USER", "description": "Snowflake username", "secret": false}, + {"name": "SNOWFLAKE_PASSWORD", "description": "Snowflake password", "secret": true} + ] + }, + { + "name": "notion", + "title": "Notion", + "description": "Official Notion MCP server for pages and databases", + "icon_url": "https://cdn.simpleicons.org/notion", + "category": "Productivity", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.notion%2Fmcp", + "transport": "sse", + "url": "https://mcp.notion.com/sse", + "env_vars": [] + }, + { + "name": "google_drive", + "title": "Google Drive", + "description": "Search and access files in Google Drive", + "icon_url": "https://cdn.simpleicons.org/googledrive", + "category": "Productivity", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-gdrive"], + "env_vars": [ + {"name": "GOOGLE_CLIENT_ID", "description": "Google OAuth Client ID", "secret": false}, + {"name": "GOOGLE_CLIENT_SECRET", "description": "Google OAuth Client Secret", "secret": true} + ] + }, + { + "name": "google_calendar", + "title": "Google Calendar", + "description": "Manage events and calendars in Google Calendar", + "icon_url": "https://cdn.simpleicons.org/googlecalendar", + "category": "Productivity", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-google-calendar"], + "env_vars": [ + {"name": "GOOGLE_CLIENT_ID", "description": "Google OAuth Client ID", "secret": false}, + {"name": "GOOGLE_CLIENT_SECRET", "description": "Google OAuth Client Secret", "secret": true} + ] + }, + { + "name": "obsidian", + "title": "Obsidian", + "description": "Read, search, and manage Obsidian vault notes and files", + "icon_url": "https://cdn.simpleicons.org/obsidian", + "category": "Productivity", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-obsidian"], + "env_vars": [ + {"name": "OBSIDIAN_VAULT_PATH", "description": "Path to Obsidian vault directory", "secret": false} + ] + }, + { + "name": "brave_search", + "title": "Brave Search", + "description": "Web results, images, videos, and AI summaries via Brave Search API", + "icon_url": "https://cdn.simpleicons.org/brave", + "category": "Search", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.brave%2Fbrave-search-mcp-server", + "transport": "stdio", + "command": "npx", + "args": ["-y", "@brave/brave-search-mcp-server"], + "env_vars": [ + {"name": "BRAVE_API_KEY", "description": "Brave Search API Key", "secret": true} + ] + }, + { + "name": "exa", + "title": "Exa", + "description": "Fast, intelligent web search and web crawling", + "icon_url": "https://cdn.simpleicons.org/exa", + "category": "Search", + "registry_url": "https://registry.modelcontextprotocol.io/servers/ai.exa%2Fexa", + "transport": "http", + "url": "https://mcp.exa.ai/mcp", + "env_vars": [ + {"name": "EXA_API_KEY", "description": "Exa API Key", "secret": true} + ] + }, + { + "name": "tavily", + "title": "Tavily", + "description": "AI-optimized search engine for research and retrieval", + "icon_url": "https://cdn.simpleicons.org/tavily", + "category": "Search", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-tavily"], + "env_vars": [ + {"name": "TAVILY_API_KEY", "description": "Tavily API Key", "secret": true} + ] + }, + { + "name": "puppeteer", + "title": "Puppeteer", + "description": "Browser automation, web scraping, and screenshot capture", + "icon_url": "https://cdn.simpleicons.org/puppeteer", + "category": "Web & Browser", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-puppeteer"], + "env_vars": [] + }, + { + "name": "playwright", + "title": "Playwright", + "description": "Browser automation and testing with Playwright", + "icon_url": "https://cdn.simpleicons.org/playwright", + "category": "Web & Browser", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-playwright"], + "env_vars": [] + }, + { + "name": "browserbase", + "title": "Browserbase", + "description": "Cloud browser automation and session management", + "icon_url": "https://cdn.simpleicons.org/browserbase", + "category": "Web & Browser", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-browserbase"], + "env_vars": [ + {"name": "BROWSERBASE_API_KEY", "description": "Browserbase API Key", "secret": true}, + {"name": "BROWSERBASE_PROJECT_ID", "description": "Browserbase Project ID", "secret": false} + ] + }, + { + "name": "aws", + "title": "AWS", + "description": "Interact with Amazon Web Services resources and APIs", + "icon_url": "https://cdn.simpleicons.org/amazonaws", + "category": "Cloud", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-aws"], + "env_vars": [ + {"name": "AWS_ACCESS_KEY_ID", "description": "AWS Access Key ID", "secret": true}, + {"name": "AWS_SECRET_ACCESS_KEY", "description": "AWS Secret Access Key", "secret": true}, + {"name": "AWS_REGION", "description": "AWS Region (e.g., us-east-1)", "secret": false} + ] + }, + { + "name": "cloudflare", + "title": "Cloudflare", + "description": "Manage Cloudflare Workers, KV, R2, D1, and more", + "icon_url": "https://cdn.simpleicons.org/cloudflare", + "category": "Cloud", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.cloudflare.mcp%2Fmcp", + "transport": "sse", + "url": "https://bindings.mcp.cloudflare.com/sse", + "env_vars": [] + }, + { + "name": "filesystem", + "title": "Filesystem", + "description": "Read, write, and manage files and directories on disk", + "icon_url": "https://cdn.simpleicons.org/files", + "category": "System", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem"], + "env_vars": [] + }, + { + "name": "docker", + "title": "Docker", + "description": "Manage Docker containers, images, and networks", + "icon_url": "https://cdn.simpleicons.org/docker", + "category": "System", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-docker"], + "env_vars": [] + }, + { + "name": "stripe", + "title": "Stripe", + "description": "Manage payments, customers, and subscriptions via Stripe", + "icon_url": "https://cdn.simpleicons.org/stripe", + "category": "Finance", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.stripe%2Fmcp", + "transport": "http", + "url": "https://mcp.stripe.com", + "env_vars": [] + }, + { + "name": "shopify", + "title": "Shopify", + "description": "Manage Shopify stores, products, orders, and customers", + "icon_url": "https://cdn.simpleicons.org/shopify", + "category": "E-Commerce", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-shopify"], + "env_vars": [ + {"name": "SHOPIFY_ACCESS_TOKEN", "description": "Shopify Admin API Access Token", "secret": true}, + {"name": "SHOPIFY_STORE_URL", "description": "Shopify Store URL (e.g., mystore.myshopify.com)", "secret": false} + ] + }, + { + "name": "twilio", + "title": "Twilio", + "description": "Send SMS, make calls, and manage communication via Twilio", + "icon_url": "https://cdn.simpleicons.org/twilio", + "category": "Communication", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-twilio"], + "env_vars": [ + {"name": "TWILIO_ACCOUNT_SID", "description": "Twilio Account SID", "secret": false}, + {"name": "TWILIO_AUTH_TOKEN", "description": "Twilio Auth Token", "secret": true} + ] + }, + { + "name": "supabase", + "title": "Supabase", + "description": "Manage Supabase projects, databases, and storage", + "icon_url": "https://cdn.simpleicons.org/supabase", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-supabase"], + "env_vars": [ + {"name": "SUPABASE_URL", "description": "Supabase Project URL", "secret": false}, + {"name": "SUPABASE_SERVICE_ROLE_KEY", "description": "Supabase Service Role Key", "secret": true} + ] + } + ] +} diff --git a/litellm/proxy/policy_engine/policy_resolve_endpoints.py b/litellm/proxy/policy_engine/policy_resolve_endpoints.py index eb4d3fc5845..318e990ff12 100644 --- a/litellm/proxy/policy_engine/policy_resolve_endpoints.py +++ b/litellm/proxy/policy_engine/policy_resolve_endpoints.py @@ -12,6 +12,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query from litellm._logging import verbose_proxy_logger from litellm.constants import MAX_POLICY_ESTIMATE_IMPACT_ROWS from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry @@ -85,7 +86,6 @@ def _filter_keys_by_tags(keys: list, tag_patterns: list) -> tuple: Returns (named_aliases, unnamed_count). """ - from litellm.proxy.auth.route_checks import RouteChecks affected: list = [] unnamed_count = 0 @@ -111,7 +111,6 @@ def _filter_teams_by_tags(teams: list, tag_patterns: list) -> tuple: Returns (named_aliases, unnamed_count). """ - from litellm.proxy.auth.route_checks import RouteChecks affected: list = [] unnamed_count = 0 @@ -141,7 +140,6 @@ async def _find_affected_by_team_patterns( Returns (new_teams, new_keys, unnamed_keys_count). """ - from litellm.proxy.auth.route_checks import RouteChecks new_teams: list = [] matched_team_ids: list = [] @@ -178,7 +176,6 @@ async def _find_affected_keys_by_alias( prisma_client: object, key_patterns: list, existing_keys: list ) -> list: """Find keys whose alias matches the given patterns.""" - from litellm.proxy.auth.route_checks import RouteChecks affected: list = [] diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 2bc1c8f8e98..2509a80b140 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -318,7 +318,7 @@ class ProxyInitializationHelpers: @click.option( "--num_workers", default=DEFAULT_NUM_WORKERS_LITELLM_PROXY, - help="Number of uvicorn / gunicorn workers to spin up. By default, it equals the number of logical CPUs in the system, or 4 workers if that cannot be determined.", + help="Number of uvicorn / gunicorn workers to spin up. Default is 1 (from DEFAULT_NUM_WORKERS_LITELLM_PROXY)", envvar="NUM_WORKERS", ) @click.option("--api_base", default=None, help="API base URL.") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6286d6dd1ca..63109916ab1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -393,6 +393,9 @@ from litellm.proxy.management_endpoints.tag_management_endpoints import ( from litellm.proxy.management_endpoints.team_callback_endpoints import ( router as team_callback_router, ) +from litellm.proxy.management_endpoints.access_group_endpoints import ( + router as access_group_router, +) from litellm.proxy.management_endpoints.team_endpoints import router as team_router from litellm.proxy.management_endpoints.team_endpoints import ( update_team, @@ -1051,98 +1054,236 @@ try: except FileNotFoundError: return False + def _validate_ui_directory(ui_path: str) -> bool: + """ + Verify UI directory has minimum required structure. + + Checks for: + - Directory exists + - Has index.html (main entry point) + - Has _next directory (Next.js assets) + + Returns True if UI directory appears valid and servable. + """ + if not os.path.isdir(ui_path): + return False + + # Must have main index.html + if not os.path.exists(os.path.join(ui_path, "index.html")): + return False + + # Must have _next directory with Next.js assets + next_dir = os.path.join(ui_path, "_next") + if not os.path.isdir(next_dir): + return False + + return True + + def _is_ui_pre_restructured(ui_dir: str) -> bool: + """ + Detect if UI directory is already pre-restructured and ready to serve. + + Returns True if: + 1. Marker file .litellm_ui_ready exists (created by Dockerfile), OR + 2. Restructuring pattern detected (subdirectories with index.html inside) + + This allows skipping copy/restructure operations on read-only filesystems. + """ + if not os.path.isdir(ui_dir): + return False + + # Primary signal: marker file created by Dockerfile + marker_file = os.path.join(ui_dir, ".litellm_ui_ready") + if os.path.exists(marker_file): + verbose_proxy_logger.debug(f"Found UI ready marker: {marker_file}") + return True + + # Fallback signal: Detect restructuring pattern + # After restructuring, routes exist as directories with index.html inside + # (e.g., login/index.html instead of login.html) + # Check for main index.html first (basic UI structure requirement) + if not os.path.exists(os.path.join(ui_dir, "index.html")): + return False + + # Look for ANY subdirectory with index.html (proves restructuring happened) + # Ignore directories starting with _ (Next.js internals like _next) + try: + for entry in os.scandir(ui_dir): + if entry.is_dir() and not entry.name.startswith("_"): + index_path = os.path.join(entry.path, "index.html") + if os.path.exists(index_path): + # Found at least one restructured route - this proves the pattern + verbose_proxy_logger.debug( + f"Detected restructured UI via pattern: found {entry.name}/index.html" + ) + return True + except (PermissionError, OSError) as e: + verbose_proxy_logger.debug( + f"Could not scan {ui_dir} for restructuring detection: {e}" + ) + return False + + # No restructured routes found + return False + + def _try_populate_ui_directory( + source_path: str, target_path: str + ) -> tuple[bool, str]: + """ + Attempt to populate target UI directory from source. + + Returns: (success: bool, error_message: str) + """ + try: + os.makedirs(target_path, exist_ok=True) + if not _dir_has_content(target_path) and _dir_has_content(source_path): + shutil.copytree( + source_path, + target_path, + dirs_exist_ok=True, + ) + verbose_proxy_logger.info(f"Successfully populated UI at {target_path}") + return True, "" + else: + return False, "Source or target directory state invalid" + except (PermissionError, OSError) as e: + return False, str(e) + # Use a writable runtime UI directory whenever possible. # This prevents mutating the packaged UI directory (e.g. site-packages or the repo checkout) # and ensures extensionless routes like /ui/login work via /index.html. is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - # Only use runtime UI path in Docker/non-root environments - # In local development, use the packaged UI directly + # Determine runtime UI path + # Priority: LITELLM_UI_PATH env var > default path based on is_non_root if is_non_root: - # Use /var/lib/litellm/ui for Docker (more secure than /tmp) - runtime_ui_path = "/var/lib/litellm/ui" + default_runtime_ui_path = "/var/lib/litellm/ui" + else: + default_runtime_ui_path = packaged_ui_path - if _dir_has_content(runtime_ui_path): + runtime_ui_path = os.getenv("LITELLM_UI_PATH", default_runtime_ui_path) + + # Validate packaged UI before proceeding + if not _validate_ui_directory(packaged_ui_path): + verbose_proxy_logger.error( + f"Packaged UI at {packaged_ui_path} is invalid or incomplete. " + f"UI may not function correctly." + ) + + # Decision tree for UI path selection: + # 1. If runtime path == packaged path: use packaged UI directly + # 2. If runtime UI exists and is pre-restructured: use it + # 3. If runtime UI exists but not restructured: use it (will restructure later) + # 4. If runtime UI missing: try to populate from packaged UI + # 4a. If population succeeds: use runtime UI + # 4b. If population fails: fall back to packaged UI + + should_use_runtime_path = runtime_ui_path != packaged_ui_path + + if should_use_runtime_path: + is_pre_restructured = _is_ui_pre_restructured(runtime_ui_path) + has_content = _dir_has_content(runtime_ui_path) + + # Case 2: Runtime UI exists and is ready + if has_content and is_pre_restructured: verbose_proxy_logger.info( - f"Using pre-built UI for non-root Docker: {runtime_ui_path}" + f"Using pre-restructured UI at {runtime_ui_path}" ) ui_path = runtime_ui_path + + # Case 3: Runtime UI exists but needs restructuring + elif has_content and not is_pre_restructured: + verbose_proxy_logger.warning( + f"UI at {runtime_ui_path} has content but is not properly restructured. " + f"Will attempt to restructure in place." + ) + ui_path = runtime_ui_path + + # Case 4: Runtime UI missing - try to populate else: - verbose_proxy_logger.error( - f"UI not found at {runtime_ui_path}. Attempting to populate it from packaged UI." - ) - verbose_proxy_logger.error( - f"Path exists: {os.path.exists(runtime_ui_path)}, Has content: {_dir_has_content(runtime_ui_path)}" + verbose_proxy_logger.info( + f"UI not found at {runtime_ui_path}. Attempting to populate from packaged UI." ) - try: - os.makedirs(runtime_ui_path, exist_ok=True) - if not _dir_has_content(runtime_ui_path) and _dir_has_content( - packaged_ui_path - ): - shutil.copytree( - packaged_ui_path, - runtime_ui_path, - dirs_exist_ok=True, - ) - except Exception as e: - verbose_proxy_logger.exception( - f"Failed to populate runtime UI directory {runtime_ui_path} from {packaged_ui_path}: {e}" - ) + success, error = _try_populate_ui_directory( + packaged_ui_path, runtime_ui_path + ) + + if success: + # Case 4a: Population succeeded + ui_path = runtime_ui_path else: - if _dir_has_content(runtime_ui_path): - verbose_proxy_logger.info( - f"Using populated UI for non-root Docker: {runtime_ui_path}" - ) - ui_path = runtime_ui_path + # Case 4b: Population failed - fall back to packaged UI + verbose_proxy_logger.warning( + f"Failed to populate UI at {runtime_ui_path}: {error}. " + f"Falling back to packaged UI at {packaged_ui_path}. " + f"For read-only deployments, pre-build UI in Dockerfile " + f"or set LITELLM_UI_PATH to a writable emptyDir volume." + ) + ui_path = packaged_ui_path else: - # Local development: use packaged UI directly, no runtime copy needed - verbose_proxy_logger.info( - f"Using packaged UI directory for local development: {packaged_ui_path}" - ) + # Case 1: Using packaged UI directly (local development) + verbose_proxy_logger.info(f"Using packaged UI directory: {packaged_ui_path}") ui_path = packaged_ui_path - # Only modify files if a custom server root path is set + + # Validate final UI path + if not _validate_ui_directory(ui_path): + verbose_proxy_logger.error( + f"Selected UI path {ui_path} is invalid or incomplete. UI may not work correctly." + ) + + # Only modify files if a custom server root path is set AND filesystem is writable if server_root_path and server_root_path != "/": - # Iterate through files in the UI directory - for root, dirs, files in os.walk(ui_path): - for filename in files: - file_path = os.path.join(root, filename) - # Skip binary files and files that don't need path replacement - if filename.endswith( - ( - ".png", - ".jpg", - ".jpeg", - ".gif", - ".ico", - ".woff", - ".woff2", - ".ttf", - ".eot", - ) - ): - continue - try: - with open(file_path, "r", encoding="utf-8") as f: - content = f.read() + # Check if UI path is writable + is_writable = os.access(ui_path, os.W_OK) - # Replace the asset prefix with the server root path - modified_content = content.replace( - f"{litellm_asset_prefix}", - f"{server_root_path}", - ) + if not is_writable: + verbose_proxy_logger.warning( + f"Cannot apply server_root_path replacements to UI at {ui_path}: " + f"path is not writable. Ensure server_root_path is '/' or pre-process " + f"UI files in Dockerfile with custom server_root_path." + ) + else: + # Iterate through files in the UI directory + for root, dirs, files in os.walk(ui_path): + for filename in files: + file_path = os.path.join(root, filename) + # Skip binary files and files that don't need path replacement + if filename.endswith( + ( + ".png", + ".jpg", + ".jpeg", + ".gif", + ".ico", + ".woff", + ".woff2", + ".ttf", + ".eot", + ) + ): + continue + try: + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() - # Replace the /.well-known/litellm-ui-config with the server root path - modified_content = modified_content.replace( - "/litellm/.well-known/litellm-ui-config", - f"{server_root_path}/.well-known/litellm-ui-config", - ) + # Replace the asset prefix with the server root path + modified_content = content.replace( + f"{litellm_asset_prefix}", + f"{server_root_path}", + ) - with open(file_path, "w", encoding="utf-8") as f: - f.write(modified_content) - except UnicodeDecodeError: - # Skip binary files that can't be decoded - continue + # Replace the /.well-known/litellm-ui-config with the server root path + modified_content = modified_content.replace( + "/litellm/.well-known/litellm-ui-config", + f"{server_root_path}/.well-known/litellm-ui-config", + ) + + with open(file_path, "w", encoding="utf-8") as f: + f.write(modified_content) + except (UnicodeDecodeError, PermissionError, OSError): + # Skip binary files or files we can't write to + continue # # Mount the _next directory at the root level app.mount( @@ -1186,14 +1327,22 @@ try: continue # Handle HTML file restructuring - # Always restructure the directory we actually serve. - # This is critical for extensionless routes like /ui/login (expects login/index.html). - # In development, we restructure directly in _experimental/out. - # In non-root Docker, we restructure in /var/lib/litellm/ui. + # Only restructure if: + # 1. UI is not already pre-restructured + # 2. Filesystem is writable try: - if is_non_root and ui_path == "/var/lib/litellm/ui": + is_pre_restructured = _is_ui_pre_restructured(ui_path) + is_writable = os.access(ui_path, os.W_OK) + + if is_pre_restructured: verbose_proxy_logger.info( - f"Skipping runtime UI restructuring for non-root Docker. UI at {ui_path} is pre-restructured." + f"Skipping UI restructuring: {ui_path} is already pre-restructured" + ) + elif not is_writable: + verbose_proxy_logger.warning( + f"Cannot restructure UI at {ui_path}: path is not writable. " + f"UI may not work correctly for extensionless routes. " + f"Pre-build and restructure UI in Dockerfile for read-only deployments." ) else: _restructure_ui_html_files(ui_path) @@ -4716,8 +4865,10 @@ async def async_assistants_data_generator( if isinstance(e, HTTPException): raise e else: - error_traceback = traceback.format_exc() - error_msg = f"{str(e)}\n\n{error_traceback}" + # Only include the error message, not the traceback. + # The traceback is already logged above via verbose_proxy_logger.exception(). + # Including it in the SSE response leaks internal details to clients. + error_msg = str(e) proxy_exception = ProxyException( message=getattr(e, "message", error_msg), @@ -4764,7 +4915,7 @@ def _restamp_streaming_chunk_model( chunk.get("model") if isinstance(chunk, dict) else getattr(chunk, "model", None) ) if not model_mismatch_logged and downstream_model != requested_model_from_client: - verbose_proxy_logger.warning( + verbose_proxy_logger.debug( "litellm_call_id=%s: streaming chunk model mismatch - requested=%r downstream=%r. Overriding model to requested.", request_data.get("litellm_call_id"), requested_model_from_client, @@ -4867,8 +5018,10 @@ async def async_data_generator( elif isinstance(e, StreamingCallbackError): error_msg = str(e) else: - error_traceback = traceback.format_exc() - error_msg = f"{str(e)}\n\n{error_traceback}" + # Only include the error message, not the traceback. + # The traceback is already logged above via verbose_proxy_logger.exception(). + # Including it in the SSE response leaks internal details to clients. + error_msg = str(e) proxy_exception = ProxyException( message=getattr(e, "message", error_msg), @@ -10294,18 +10447,34 @@ async def get_image(): default_site_logo = os.path.join(current_dir, "logo.jpg") is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - assets_dir = "/var/lib/litellm/assets" if is_non_root else current_dir - if is_non_root: - os.makedirs(assets_dir, exist_ok=True) + # Determine assets directory + # Priority: LITELLM_ASSETS_PATH env var > default based on is_non_root + default_assets_dir = "/var/lib/litellm/assets" if is_non_root else current_dir + assets_dir = os.getenv("LITELLM_ASSETS_PATH", default_assets_dir) + # Try to create assets_dir if it doesn't exist (simple try/except approach) + if not os.path.exists(assets_dir): + try: + os.makedirs(assets_dir, exist_ok=True) + verbose_proxy_logger.debug(f"Created assets directory at {assets_dir}") + except (PermissionError, OSError) as e: + verbose_proxy_logger.warning( + f"Cannot create assets directory at {assets_dir}: {e}. " + f"Logo caching may not work. Using current directory for assets." + ) + assets_dir = current_dir + + # Determine default logo path default_logo = ( - os.path.join(assets_dir, "logo.jpg") if is_non_root else default_site_logo + os.path.join(assets_dir, "logo.jpg") + if assets_dir != current_dir + else default_site_logo ) - if is_non_root and not os.path.exists(default_logo): + if assets_dir != current_dir and not os.path.exists(default_logo): default_logo = default_site_logo - cache_dir = assets_dir if is_non_root else current_dir + cache_dir = assets_dir if os.access(assets_dir, os.W_OK) else current_dir cache_path = os.path.join(cache_dir, "cached_logo.jpg") # [OPTIMIZATION] Check if the cached image exists first @@ -11832,6 +12001,7 @@ app.include_router(enterprise_router) app.include_router(ui_discovery_endpoints_router) app.include_router(agent_endpoints_router) app.include_router(a2a_router) +app.include_router(access_group_router) ######################################################## # MCP Server ######################################################## diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 37ed0182663..390b0415d15 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -128,6 +128,7 @@ model LiteLLM_TeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) @@ -160,6 +161,7 @@ model LiteLLM_DeletedTeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases @@ -291,6 +293,7 @@ model LiteLLM_VerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") budget_id String? @@ -346,6 +349,7 @@ model LiteLLM_DeletedVerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") router_settings Json? @default("{}") @@ -764,6 +768,22 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t @@index([model_object_id]) } +model LiteLLM_ManagedVectorStoreTable { + id String @id @default(uuid()) + unified_resource_id String @unique // The base64 encoded unified vector store ID + resource_object Json? // Stores the VectorStoreCreateResponse + model_mappings Json // Maps model_id -> provider_vector_store_id + flat_model_resource_ids String[] @default([]) // Flat list of provider vector store IDs for faster querying + storage_backend String? // Storage backend name (if applicable) + storage_url String? // Storage URL (if applicable) + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @updatedAt + updated_by String? + + @@index([unified_resource_id]) +} + model LiteLLM_ManagedVectorStoresTable { vector_store_id String @id custom_llm_provider String @@ -917,3 +937,23 @@ model LiteLLM_PolicyAttachmentTable { updated_at DateTime @default(now()) @updatedAt updated_by String? } + +//Unified Access Groups table for storing unified access groups +model LiteLLM_AccessGroupTable { + access_group_id String @id @default(uuid()) + access_group_name String @unique + description String? + + // Resource memberships - explicit arrays per type + access_model_ids String[] @default([]) + access_mcp_server_ids String[] @default([]) + access_agent_ids String[] @default([]) + + assigned_team_ids String[] @default([]) + assigned_key_ids String[] @default([]) + + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? +} \ No newline at end of file diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index afbc57360e2..9c0190bdfec 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1878,13 +1878,15 @@ async def ui_view_spend_logs( # noqa: PLR0915 verbose_proxy_logger.debug("data= %s", json.dumps(data, indent=4, default=str)) - return { - "data": data, - "total": total_records, - "page": page, - "page_size": page_size, - "total_pages": total_pages, - } + return await _build_ui_spend_logs_response( + prisma_client, + data, + total_records, + page, + page_size, + total_pages, + enrich_session_counts=not is_v2, + ) except Exception as e: verbose_proxy_logger.exception(f"Error in ui_view_spend_logs: {e}") raise handle_exception_on_proxy(e) @@ -3129,6 +3131,91 @@ async def ui_view_session_spend_logs( ) +async def _build_ui_spend_logs_response( + prisma_client: "PrismaClient", + data: list, + total_records: int, + page: int, + page_size: int, + total_pages: int, + enrich_session_counts: bool = True, +) -> dict: + """ + Build the paginated response for the UI spend-logs endpoint. + + When ``enrich_session_counts`` is ``True`` (the default for the v1/UI + endpoint), each row is enriched with ``session_total_count`` so the + frontend knows which sessions are expandable (multi-call sessions). + For every row that carries a ``session_id``, a single ``GROUP BY`` query + fetches the total number of logs in each referenced session. Rows without + a ``session_id`` default to ``1``. + + When ``enrich_session_counts`` is ``False`` (v2 endpoint), rows are + serialised without the extra query. + + Args: + prisma_client: The connected Prisma client instance. + data: A list of Prisma model instances (must support ``.model_dump()`` + and have a ``session_id`` attribute). + total_records: Total number of matching records (for pagination). + page: Current page number. + page_size: Number of items per page. + total_pages: Total number of pages. + enrich_session_counts: Whether to add ``session_total_count`` to each + row. Defaults to ``True``. + + Returns: + A dict with ``data`` (enriched rows), ``total``, ``page``, + ``page_size``, and ``total_pages``. + """ + count_map: dict[str, int] = {} + if enrich_session_counts: + session_ids = list( + {row.session_id for row in data if getattr(row, "session_id", None)} + ) + if session_ids: + # NOTE: This GROUP BY runs on every v1/UI page load. The IN clause + # is bounded by page_size (typically 25-50 distinct session IDs). + # If performance degrades at scale, consider short-lived caching or + # folding the count into the main query via a window function. + counts = await prisma_client.db.litellm_spendlogs.group_by( + by=["session_id"], + where={"session_id": {"in": session_ids}}, + count={"session_id": True}, + ) + count_map = { + r["session_id"]: r["_count"]["session_id"] + for r in counts + if r.get("session_id") + } + + if enrich_session_counts: + enriched: List[dict] = [] + for row in data: + row_dict = ( + dict(row) + if isinstance(row, dict) + else row.model_dump() + ) + sid = row_dict.get("session_id") + row_dict["session_total_count"] = count_map.get(sid, 1) if sid else 1 + enriched.append(row_dict) + response_data: list = enriched + else: + # v2 path: return raw Prisma model instances so FastAPI applies its + # own Pydantic-aware serialisation (preserves alias handling, custom + # serializers, etc.). + response_data = data # type: ignore[assignment] + + return { + "data": response_data, + "total": total_records, + "page": page, + "page_size": page_size, + "total_pages": total_pages, + } + + def _build_status_filter_condition(status_filter: Optional[str]) -> Dict[str, Any]: """ Helper function to build the status filter condition for database queries. diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0aace65ff6b..d977751004c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -740,7 +740,7 @@ class ProxyLogging: self, dynamic_success_callbacks: Optional[List], global_callbacks: List ) -> List: if dynamic_success_callbacks is None: - return global_callbacks + return list(global_callbacks) return list(set(dynamic_success_callbacks + global_callbacks)) def _parse_pre_mcp_call_hook_response( diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 9ba12537bc8..0775e05f4fa 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -136,10 +136,17 @@ async def vector_store_search( if "vector_store_id" not in data: data["vector_store_id"] = vector_store_id + # Check for legacy vector store registry (non-managed vector stores) data = _update_request_data_with_litellm_managed_vector_store_registry( data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict ) + # The managed_vector_stores pre-call hook will handle: + # 1. Decoding managed vector store IDs + # 2. Extracting model and provider resource ID + # 3. Setting up proper routing + # 4. Authentication checks + processor = ProxyBaseLLMRequestProcessing(data=data) try: return await processor.base_process_llm_request( @@ -181,6 +188,14 @@ async def vector_store_create( API Reference: https://platform.openai.com/docs/api-reference/vector-stores/create + + Supports target_model_names parameter for creating vector stores across multiple models: + ```json + { + "name": "my-vector-store", + "target_model_names": "gpt-4,gemini-2.0" + } + ``` """ from litellm.proxy.proxy_server import ( _read_request_body, @@ -198,6 +213,47 @@ async def vector_store_create( ) data = await _read_request_body(request=request) + + # Check for target_model_names parameter + target_model_names = data.pop("target_model_names", None) + + if target_model_names: + # Use managed vector stores for multi-model support + if isinstance(target_model_names, str): + target_model_names_list = [m.strip() for m in target_model_names.split(",")] + elif isinstance(target_model_names, list): + target_model_names_list = target_model_names + else: + raise HTTPException( + status_code=400, + detail="target_model_names must be a comma-separated string or list of model names", + ) + + # Get managed vector stores hook + managed_vector_stores = proxy_logging_obj.get_proxy_hook("managed_vector_stores") + if managed_vector_stores is None: + raise HTTPException( + status_code=500, + detail="Managed vector stores not configured. Please ensure the proxy is initialized with database support.", + ) + + if llm_router is None: + raise HTTPException( + status_code=500, + detail="LLM Router not initialized. Ensure models are added to proxy.", + ) + + # Create vector store across multiple models + response = await managed_vector_stores.acreate_vector_store( + create_request=data, + llm_router=llm_router, + target_model_names_list=target_model_names_list, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + user_api_key_dict=user_api_key_dict, + ) + + return response + processor = ProxyBaseLLMRequestProcessing(data=data) try: return await processor.base_process_llm_request( diff --git a/litellm/proxy/vector_store_files_endpoints/endpoints.py b/litellm/proxy/vector_store_files_endpoints/endpoints.py index 6ac5cd57724..e3e022c9cfe 100644 --- a/litellm/proxy/vector_store_files_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_files_endpoints/endpoints.py @@ -1,9 +1,9 @@ -from typing import Dict, Optional +from typing import TYPE_CHECKING, Dict, Optional -import litellm from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse +import litellm from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -12,18 +12,252 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( get_custom_llm_provider_from_request_headers, get_custom_llm_provider_from_request_query, ) +from litellm.proxy.openai_files_endpoints.common_utils import ( + handle_model_based_routing, + prepare_data_with_credentials, +) from litellm.proxy.vector_store_endpoints.utils import ( is_allowed_to_call_vector_store_files_endpoint, ) from litellm.types.utils import LlmProviders +if TYPE_CHECKING: + from litellm.router import Router + router = APIRouter() +def _update_request_data_with_managed_file_id( + data: Dict, + file_id: str, + request: Request, + llm_router: Optional["Router"] = None, +) -> tuple[Dict, Optional[str]]: + """ + Update request data with model routing information from managed file ID. + + This function handles two types of file IDs: + 1. Simple encoded file IDs (format: litellm:{file_id};model,{model}) + 2. Unified managed file IDs (format: litellm_proxy:{mime};unified_id,{uuid};...;llm_output_file_id,{file_id};...) + + For unified managed file IDs, it: + - Decodes the unified ID to extract the actual provider file ID (llm_output_file_id) + - Extracts the model routing information (target_model_names) + - Updates data with credentials for the correct deployment + + Args: + data: Request data to update + file_id: File ID (can be managed/encoded or regular) + request: FastAPI request object + llm_router: LiteLLM router for credential lookup (required for managed files) + + Returns: + Tuple of (updated request data, original_managed_file_id) + - original_managed_file_id is the original file_id if it was managed/encoded, None otherwise + """ + import re + + from litellm import verbose_logger + from litellm.llms.base_llm.managed_resources.utils import ( + is_base64_encoded_unified_id, + parse_unified_id, + ) + + # First, check if this is a unified managed file ID (base64 encoded) + decoded_id = is_base64_encoded_unified_id(file_id) + + if decoded_id: + # This is a unified managed file ID + verbose_logger.debug( + f"Processing unified managed file ID: {file_id}" + ) + + # Parse the unified ID to extract components + parsed_id = parse_unified_id(file_id) + + if parsed_id: + target_model_names = parsed_id.get("target_model_names", []) + + # Extract the actual provider file ID from llm_output_file_id field + # Format: litellm_proxy:...;llm_output_file_id,{actual_file_id};... + llm_output_file_id = None + try: + match = re.search(r"llm_output_file_id,([^;]+)", decoded_id) + if match: + llm_output_file_id = match.group(1).strip() + except Exception: + pass + + verbose_logger.debug( + f"Decoded unified file ID - target_model_names: {target_model_names}, llm_output_file_id: {llm_output_file_id}" + ) + + # Set the model for routing + if target_model_names and len(target_model_names) > 0: + routing_model = target_model_names[0] + data["model"] = routing_model + + # Get credentials for the model + if llm_router: + credentials = llm_router.get_deployment_credentials_with_provider( + model_id=routing_model + ) + if credentials: + prepare_data_with_credentials( + data=data, + credentials=credentials, + file_id=llm_output_file_id, # Use the actual provider file ID + ) + verbose_logger.info( + f"Routing vector store file operation to model: {routing_model}, file_id: {file_id} -> {llm_output_file_id}" + ) + return data, file_id # Return original managed file ID + + # If we extracted the provider file ID but no routing, still use it + if llm_output_file_id: + data["file_id"] = llm_output_file_id + verbose_logger.debug( + f"Replaced unified file ID with provider file ID: {llm_output_file_id}" + ) + return data, file_id # Return original managed file ID + + return data, file_id if decoded_id else None + + # Fall back to simple encoded file ID handling (format: litellm:{file_id};model,{model}) + should_route, model_used, original_file_id, credentials = handle_model_based_routing( + file_id=file_id, + request=request, + llm_router=llm_router, + data=data, + check_file_id_encoding=True, + ) + + if should_route: + # Use model-based routing with credentials from config + prepare_data_with_credentials( + data=data, + credentials=credentials, # type: ignore + file_id=original_file_id, # Use decoded file ID if from encoded ID + ) + + verbose_logger.debug( + f"Routing vector store file operation using model: {model_used}" + + (f", file_id: {file_id} -> {original_file_id}" if original_file_id else "") + ) + return data, file_id # Return original file ID for response replacement + + return data, None + + +def _replace_file_id_in_response(response, original_file_id: str): + """ + Replace the provider file ID in the response with the original managed file ID. + + This ensures that when a user sends a managed file ID, they get back the same + managed file ID in the response, not the decoded provider file ID. + + Args: + response: The response object from the provider + original_file_id: The original managed file ID to restore + + Returns: + Modified response with original file ID + """ + if response is None: + return response + + # Handle different response types + if isinstance(response, dict): + # For dict responses (e.g., VectorStoreFileDeleteResponse) + if "id" in response: + response["id"] = original_file_id + if "file_id" in response: + response["file_id"] = original_file_id + elif hasattr(response, "id"): + # For object responses (e.g., VectorStoreFileObject) + response.id = original_file_id + elif hasattr(response, "file_id"): + response.file_id = original_file_id + + return response + + def _update_request_data_with_litellm_managed_vector_store_registry( data: Dict, vector_store_id: str, + llm_router: Optional["Router"] = None, ) -> Dict: + """ + Update request data with model routing information from managed vector store. + + This function handles two types of vector stores: + 1. Legacy vector stores from registry (non-managed) + 2. Managed vector stores with unified IDs (requires decoding) + + For managed vector stores, this function: + - Decodes the unified vector store ID + - Extracts the model_id and provider resource ID + - Sets data["model"] so the router can use the correct deployment credentials + - Replaces the unified ID with the provider-specific ID + + Args: + data: Request data to update + vector_store_id: Vector store ID (can be unified or legacy) + llm_router: LiteLLM router for credential lookup (required for managed vector stores) + + Returns: + Updated request data with model routing information + """ + from litellm import verbose_logger + from litellm.llms.base_llm.managed_resources.utils import ( + is_base64_encoded_unified_id, + parse_unified_id, + ) + + # Check if this is a managed vector store ID (base64 encoded unified ID) + decoded_id = is_base64_encoded_unified_id(vector_store_id) + + if decoded_id: + # This is a managed vector store - decode and extract routing information + verbose_logger.debug( + f"Processing managed vector store ID: {vector_store_id}" + ) + + parsed_id = parse_unified_id(vector_store_id) + + if parsed_id: + model_id = parsed_id.get("model_id") + provider_resource_id = parsed_id.get("provider_resource_id") + target_model_names = parsed_id.get("target_model_names", []) + + verbose_logger.debug( + f"Decoded vector store - model_id: {model_id}, provider_resource_id: {provider_resource_id}, target_model_names: {target_model_names}" + ) + + # Set the model for routing - this tells the router which deployment to use + # The router will automatically get the credentials from the deployment + routing_model = None + if model_id: + routing_model = model_id + elif target_model_names and len(target_model_names) > 0: + routing_model = target_model_names[0] + + if routing_model: + data["model"] = routing_model + verbose_logger.info( + f"Routing vector store files operation to model: {routing_model}" + ) + + # Replace unified vector store ID with provider resource ID + if provider_resource_id: + data["vector_store_id"] = provider_resource_id + verbose_logger.debug( + f"Replaced unified vector store ID with provider resource ID: {provider_resource_id}" + ) + + return data + + # Legacy path: Check vector store registry for non-managed vector stores if litellm.vector_store_registry is not None: vector_store_to_run = ( litellm.vector_store_registry.get_litellm_managed_vector_store_from_registry( @@ -42,6 +276,7 @@ def _update_request_data_with_litellm_managed_vector_store_registry( if "litellm_params" in vector_store_to_run: litellm_params = vector_store_to_run.get("litellm_params", {}) or {} data.update(litellm_params) + return data @@ -128,8 +363,16 @@ async def vector_store_file_create( if "vector_store_id" not in data: data["vector_store_id"] = vector_store_id + # Handle managed file IDs if present in request body + original_managed_file_id = None + if "file_id" in data: + data, original_managed_file_id = _update_request_data_with_managed_file_id( + data=data, file_id=data["file_id"], request=request, llm_router=llm_router + ) + + # Then handle managed vector store IDs data = _update_request_data_with_litellm_managed_vector_store_registry( - data=data, vector_store_id=vector_store_id + data=data, vector_store_id=vector_store_id, llm_router=llm_router ) provider_enum = await _resolve_provider(data=data, request=request) @@ -145,7 +388,7 @@ async def vector_store_file_create( processor = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + response = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -163,6 +406,12 @@ async def vector_store_file_create( user_api_base=user_api_base, version=version, ) + + # Replace provider file ID with original managed file ID in response + if original_managed_file_id: + response = _replace_file_id_in_response(response, original_managed_file_id) + + return response except Exception as e: # noqa: BLE001 raise await processor._handle_llm_api_exception( e=e, @@ -209,7 +458,7 @@ async def vector_store_file_list( data.update(query_params) data = _update_request_data_with_litellm_managed_vector_store_registry( - data=data, vector_store_id=vector_store_id + data=data, vector_store_id=vector_store_id, llm_router=llm_router ) provider_enum = await _resolve_provider(data=data, request=request) @@ -290,8 +539,14 @@ async def vector_store_file_retrieve( "file_id": file_id, } + # Handle managed file IDs first + data, original_managed_file_id = _update_request_data_with_managed_file_id( + data=data, file_id=file_id, request=request, llm_router=llm_router + ) + + # Then handle managed vector store IDs data = _update_request_data_with_litellm_managed_vector_store_registry( - data=data, vector_store_id=vector_store_id + data=data, vector_store_id=vector_store_id, llm_router=llm_router ) provider_enum = await _resolve_provider(data=data, request=request) @@ -307,7 +562,7 @@ async def vector_store_file_retrieve( processor = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + response = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -325,6 +580,12 @@ async def vector_store_file_retrieve( user_api_base=user_api_base, version=version, ) + + # Replace provider file ID with original managed file ID in response + if original_managed_file_id: + response = _replace_file_id_in_response(response, original_managed_file_id) + + return response except Exception as e: # noqa: BLE001 raise await processor._handle_llm_api_exception( e=e, @@ -372,8 +633,14 @@ async def vector_store_file_content( "file_id": file_id, } + # Handle managed file IDs first + data, original_managed_file_id = _update_request_data_with_managed_file_id( + data=data, file_id=file_id, request=request, llm_router=llm_router + ) + + # Then handle managed vector store IDs data = _update_request_data_with_litellm_managed_vector_store_registry( - data=data, vector_store_id=vector_store_id + data=data, vector_store_id=vector_store_id, llm_router=llm_router ) provider_enum = await _resolve_provider(data=data, request=request) @@ -389,7 +656,7 @@ async def vector_store_file_content( processor = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + response = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -407,6 +674,12 @@ async def vector_store_file_content( user_api_base=user_api_base, version=version, ) + + # Replace provider file ID with original managed file ID in response + if original_managed_file_id: + response = _replace_file_id_in_response(response, original_managed_file_id) + + return response except Exception as e: # noqa: BLE001 raise await processor._handle_llm_api_exception( e=e, @@ -454,8 +727,14 @@ async def vector_store_file_update( data["vector_store_id"] = vector_store_id data["file_id"] = file_id + # Handle managed file IDs first + data, original_managed_file_id = _update_request_data_with_managed_file_id( + data=data, file_id=file_id, request=request, llm_router=llm_router + ) + + # Then handle managed vector store IDs data = _update_request_data_with_litellm_managed_vector_store_registry( - data=data, vector_store_id=vector_store_id + data=data, vector_store_id=vector_store_id, llm_router=llm_router ) provider_enum = await _resolve_provider(data=data, request=request) @@ -471,7 +750,7 @@ async def vector_store_file_update( processor = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + response = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -489,6 +768,12 @@ async def vector_store_file_update( user_api_base=user_api_base, version=version, ) + + # Replace provider file ID with original managed file ID in response + if original_managed_file_id: + response = _replace_file_id_in_response(response, original_managed_file_id) + + return response except Exception as e: # noqa: BLE001 raise await processor._handle_llm_api_exception( e=e, @@ -536,8 +821,14 @@ async def vector_store_file_delete( "file_id": file_id, } + # Handle managed file IDs first + data, original_managed_file_id = _update_request_data_with_managed_file_id( + data=data, file_id=file_id, request=request, llm_router=llm_router + ) + + # Then handle managed vector store IDs data = _update_request_data_with_litellm_managed_vector_store_registry( - data=data, vector_store_id=vector_store_id + data=data, vector_store_id=vector_store_id, llm_router=llm_router ) provider_enum = await _resolve_provider(data=data, request=request) @@ -553,7 +844,7 @@ async def vector_store_file_delete( processor = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + response = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -571,6 +862,12 @@ async def vector_store_file_delete( user_api_base=user_api_base, version=version, ) + + # Replace provider file ID with original managed file ID in response + if original_managed_file_id: + response = _replace_file_id_in_response(response, original_managed_file_id) + + return response except Exception as e: # noqa: BLE001 raise await processor._handle_llm_api_exception( e=e, diff --git a/litellm/rag/ingestion/__init__.py b/litellm/rag/ingestion/__init__.py index e34a715085c..264bd6b5e41 100644 --- a/litellm/rag/ingestion/__init__.py +++ b/litellm/rag/ingestion/__init__.py @@ -4,11 +4,17 @@ RAG Ingestion classes for different providers. from litellm.rag.ingestion.base_ingestion import BaseRAGIngestion from litellm.rag.ingestion.bedrock_ingestion import BedrockRAGIngestion +from litellm.rag.ingestion.gemini_ingestion import GeminiRAGIngestion from litellm.rag.ingestion.openai_ingestion import OpenAIRAGIngestion +from litellm.rag.ingestion.s3_vectors_ingestion import S3VectorsRAGIngestion +from litellm.rag.ingestion.vertex_ai_ingestion import VertexAIRAGIngestion __all__ = [ "BaseRAGIngestion", "BedrockRAGIngestion", + "GeminiRAGIngestion", "OpenAIRAGIngestion", + "S3VectorsRAGIngestion", + "VertexAIRAGIngestion", ] diff --git a/litellm/rag/ingestion/vertex_ai_ingestion.py b/litellm/rag/ingestion/vertex_ai_ingestion.py new file mode 100644 index 00000000000..47a94185d1d --- /dev/null +++ b/litellm/rag/ingestion/vertex_ai_ingestion.py @@ -0,0 +1,478 @@ +""" +Vertex AI-specific RAG Ingestion implementation. + +Vertex AI RAG Engine handles embedding and chunking internally when files are uploaded, +so this implementation skips the embedding step and directly uploads files to RAG corpora. + +Based on: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/rag-api-v1 +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast + +from litellm._logging import verbose_logger +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url +from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +from litellm.rag.ingestion.base_ingestion import BaseRAGIngestion + +if TYPE_CHECKING: + from litellm import Router + from litellm.types.rag import RAGIngestOptions + + +class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase): + """ + Vertex AI RAG Engine ingestion implementation. + + Key differences from base: + - Embedding is handled by Vertex AI RAG Engine when files are uploaded + - Files are uploaded using the RAG API (import or upload) + - Chunking is done by Vertex AI RAG Engine (supports custom chunking config) + - Supports Google Cloud Storage (GCS) and Google Drive sources + - Supports custom parsing configurations (layout parser, LLM parser) + """ + + def __init__( + self, + ingest_options: "RAGIngestOptions", + router: Optional["Router"] = None, + ): + BaseRAGIngestion.__init__(self, ingest_options=ingest_options, router=router) + VertexBase.__init__(self) + + # Extract Vertex AI specific configs from vector_store_config + litellm_params = dict(self.vector_store_config) + + # Get project, location, and credentials using VertexBase methods + self.project_id = self.safe_get_vertex_ai_project(litellm_params) + self.location = self.get_vertex_ai_location(litellm_params) or "us-central1" + self.vertex_credentials = self.safe_get_vertex_ai_credentials(litellm_params) + + async def embed( + self, + chunks: List[str], + ) -> Optional[List[List[float]]]: + """ + Vertex AI RAG Engine handles embedding internally - skip this step. + + Returns: + None (Vertex AI embeds when files are uploaded to RAG corpus) + """ + # Vertex AI RAG Engine handles embedding when files are uploaded + return None + + async def store( + self, + file_content: Optional[bytes], + filename: Optional[str], + content_type: Optional[str], + chunks: List[str], + embeddings: Optional[List[List[float]]], + ) -> Tuple[Optional[str], Optional[str]]: + """ + Store content in Vertex AI RAG corpus. + + Vertex AI workflow: + 1. Create RAG corpus (if not provided) + 2. Upload file using RAG API (Vertex AI handles chunking/embedding) + + Args: + file_content: Raw file bytes + filename: Name of the file + content_type: MIME type + chunks: Ignored - Vertex AI handles chunking + embeddings: Ignored - Vertex AI handles embedding + + Returns: + Tuple of (rag_corpus_id, file_id) + """ + if not self.project_id: + raise ValueError( + "vertex_project is required for Vertex AI RAG ingestion. " + "Set it in vector_store config." + ) + + # Get or create RAG corpus + rag_corpus_id = self.vector_store_config.get("vector_store_id") + if not rag_corpus_id: + rag_corpus_id = await self._create_rag_corpus( + display_name=self.ingest_name or "litellm-rag-corpus", + description=self.vector_store_config.get("description"), + ) + + # Upload file to RAG corpus + result_file_id = None + if file_content and filename and rag_corpus_id: + result_file_id = await self._upload_file_to_corpus( + rag_corpus_id=rag_corpus_id, + filename=filename, + file_content=file_content, + content_type=content_type, + ) + + return rag_corpus_id, result_file_id + + async def _create_rag_corpus( + self, + display_name: str, + description: Optional[str] = None, + ) -> str: + """ + Create a Vertex AI RAG corpus. + + Args: + display_name: Display name for the corpus + description: Optional description + + Returns: + RAG corpus ID (format: projects/{project}/locations/{location}/ragCorpora/{corpus_id}) + """ + # Get access token using VertexBase method + access_token, project_id = self._ensure_access_token( + credentials=self.vertex_credentials, + project_id=self.project_id, + custom_llm_provider="vertex_ai", + ) + + # Use the project_id from token if not set + if not self.project_id: + self.project_id = project_id + + # Construct URL using vertex base URL helper + base_url = get_vertex_base_url(self.location) + url = ( + f"{base_url}/v1beta1/" + f"projects/{self.project_id}/locations/{self.location}/ragCorpora" + ) + + # Build request body with camelCase keys (Vertex AI API format) + request_body: Dict[str, Any] = { + "displayName": display_name, + } + + if description: + request_body["description"] = description + + # Add vector database config if specified + vector_db_config = self.vector_store_config.get("vector_db_config") + if vector_db_config: + request_body["vectorDbConfig"] = vector_db_config + + # Add embedding model config if specified + embedding_model = self.vector_store_config.get("embedding_model") + if embedding_model: + if "vectorDbConfig" not in request_body: + request_body["vectorDbConfig"] = {} + request_body["vectorDbConfig"]["ragEmbeddingModelConfig"] = { + "vertexPredictionEndpoint": { + "endpoint": embedding_model + } + } + + verbose_logger.debug(f"Creating RAG corpus: {url}") + verbose_logger.debug(f"Request body: {json.dumps(request_body, indent=2)}") + + client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.RAG, + params={"timeout": 60.0}, + ) + + response = await client.post( + url, + json=request_body, + headers={ + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + }, + ) + if response.status_code not in [200, 201]: + error_msg = f"Failed to create RAG corpus: {response.text}" + verbose_logger.error(error_msg) + raise Exception(error_msg) + + response_data = response.json() + verbose_logger.debug(f"Create corpus response: {json.dumps(response_data, indent=2)}") + + # The response is a long-running operation + # Check if it's already done or if we need to poll + if response_data.get("done"): + # Operation completed immediately + corpus_name = response_data.get("response", {}).get("name", "") + else: + # Need to poll the operation + operation_name = response_data.get("name", "") + verbose_logger.debug(f"Polling operation: {operation_name}") + corpus_name = await self._poll_operation( + operation_name=operation_name, + access_token=access_token, + ) + + verbose_logger.debug(f"Created RAG corpus: {corpus_name}") + return corpus_name + + async def _poll_operation( + self, + operation_name: str, + access_token: str, + max_retries: int = 30, + retry_delay: float = 2.0, + ) -> str: + """ + Poll a long-running operation until it completes. + + Args: + operation_name: The operation name (e.g., "operations/123456") + access_token: Access token for authentication + max_retries: Maximum number of polling attempts + retry_delay: Delay between polling attempts in seconds + + Returns: + The corpus name from the completed operation + + Raises: + Exception: If operation fails or times out + """ + import asyncio + + base_url = get_vertex_base_url(self.location) + # Operation name is like: projects/{project}/locations/{location}/operations/{operation_id} + # We need to construct the full URL + url = f"{base_url}/v1beta1/{operation_name}" + + client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.RAG, + params={"timeout": 60.0}, + ) + + for attempt in range(max_retries): + response = await client.get( + url, + headers={ + "Authorization": f"Bearer {access_token}", + }, + ) + + if response.status_code != 200: + error_msg = f"Failed to poll operation: {response.text}" + verbose_logger.error(error_msg) + raise Exception(error_msg) + + operation_data = response.json() + + if operation_data.get("done"): + # Check for errors + if "error" in operation_data: + error = operation_data["error"] + raise Exception(f"Operation failed: {error}") + + # Extract corpus name from response + corpus_name = operation_data.get("response", {}).get("name", "") + if corpus_name: + return corpus_name + else: + raise Exception(f"No corpus name in operation response: {operation_data}") + + verbose_logger.debug(f"Operation not done yet, attempt {attempt + 1}/{max_retries}") + await asyncio.sleep(retry_delay) + + raise Exception(f"Operation timed out after {max_retries} attempts") + + async def _upload_file_to_corpus( + self, + rag_corpus_id: str, + filename: str, + file_content: bytes, + content_type: Optional[str], + ) -> str: + """ + Upload a file to Vertex AI RAG corpus using multipart upload. + + Args: + rag_corpus_id: RAG corpus resource name + filename: Name of the file + file_content: File content bytes + content_type: MIME type + + Returns: + File ID or resource name + """ + # Get access token using VertexBase method + access_token, _ = self._ensure_access_token( + credentials=self.vertex_credentials, + project_id=self.project_id, + custom_llm_provider="vertex_ai", + ) + + # Construct upload URL using vertex base URL helper + base_url = get_vertex_base_url(self.location) + url = ( + f"{base_url}/upload/v1beta1/" + f"{rag_corpus_id}/ragFiles:upload" + ) + + # Build metadata for the file with snake_case keys (as per upload API docs) + metadata: Dict[str, Any] = { + "rag_file": { + "display_name": filename, + } + } + + # Add description if provided + description = self.vector_store_config.get("file_description") + if description: + metadata["rag_file"]["description"] = description + + # Add chunking configuration if provided + chunking_strategy = self.chunking_strategy + if chunking_strategy and isinstance(chunking_strategy, dict): + chunk_size = chunking_strategy.get("chunk_size") + chunk_overlap = chunking_strategy.get("chunk_overlap") + + if chunk_size or chunk_overlap: + if "upload_rag_file_config" not in metadata: + metadata["upload_rag_file_config"] = {} + + metadata["upload_rag_file_config"]["rag_file_transformation_config"] = { + "rag_file_chunking_config": { + "fixed_length_chunking": {} + } + } + + chunking_config = metadata["upload_rag_file_config"][ + "rag_file_transformation_config" + ]["rag_file_chunking_config"]["fixed_length_chunking"] + + if chunk_size: + chunking_config["chunk_size"] = chunk_size + if chunk_overlap: + chunking_config["chunk_overlap"] = chunk_overlap + + verbose_logger.debug(f"Uploading file to RAG corpus: {url}") + verbose_logger.debug(f"Metadata: {json.dumps(metadata, indent=2)}") + + # Prepare multipart form data + files = { + "metadata": (None, json.dumps(metadata), "application/json"), + "file": (filename, file_content, content_type or "application/octet-stream"), + } + client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.RAG, + params={"timeout": 300.0}, # Longer timeout for large files + ) + + response = await client.post( + url, + files=files, + headers={ + "Authorization": f"Bearer {access_token}", + "X-Goog-Upload-Protocol": "multipart", + }, + ) + + if response.status_code not in [200, 201]: + error_msg = f"Failed to upload file: {response.text}" + verbose_logger.error(error_msg) + raise Exception(error_msg) + + # Parse response to get file ID + try: + response_data = response.json() + # The response should contain the rag_file resource name + file_id = response_data.get("ragFile", {}).get("name", "") + if not file_id: + file_id = response_data.get("name", "") + + verbose_logger.debug(f"Upload complete. File ID: {file_id}") + return file_id + except Exception as e: + verbose_logger.warning(f"Could not parse upload response: {e}") + return "uploaded" + + async def _import_files_from_gcs( + self, + rag_corpus_id: str, + gcs_uris: List[str], + ) -> str: + """ + Import files from Google Cloud Storage into RAG corpus. + + Args: + rag_corpus_id: RAG corpus resource name + gcs_uris: List of GCS URIs (e.g., ["gs://bucket/file.pdf"]) + + Returns: + Operation name for tracking import progress + """ + # Get access token using VertexBase method + access_token, _ = self._ensure_access_token( + credentials=self.vertex_credentials, + project_id=self.project_id, + custom_llm_provider="vertex_ai", + ) + + # Construct import URL using vertex base URL helper + base_url = get_vertex_base_url(self.location) + url = ( + f"{base_url}/v1beta1/" + f"{rag_corpus_id}/ragFiles:import" + ) + + # Build request body with camelCase keys (Vertex AI API format) + request_body: Dict[str, Any] = { + "importRagFilesConfig": { + "gcsSource": { + "uris": gcs_uris + } + } + } + + # Add chunking configuration if provided + chunking_strategy = self.chunking_strategy + if chunking_strategy and isinstance(chunking_strategy, dict): + chunk_size = chunking_strategy.get("chunk_size") + chunk_overlap = chunking_strategy.get("chunk_overlap") + + if chunk_size or chunk_overlap: + request_body["importRagFilesConfig"]["ragFileChunkingConfig"] = { + "chunkSize": chunk_size or 1024, + "chunkOverlap": chunk_overlap or 200, + } + + # Add max embedding requests per minute if specified + max_embedding_qpm = self.vector_store_config.get("max_embedding_requests_per_min") + if max_embedding_qpm: + request_body["importRagFilesConfig"]["maxEmbeddingRequestsPerMin"] = max_embedding_qpm + + verbose_logger.debug(f"Importing files from GCS: {url}") + verbose_logger.debug(f"Request body: {json.dumps(request_body, indent=2)}") + + client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.RAG, + params={"timeout": 60.0}, + ) + + response = await client.post( + url, + json=request_body, + headers={ + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + }, + ) + + if response.status_code not in [200, 201]: + error_msg = f"Failed to import files: {response.text}" + verbose_logger.error(error_msg) + raise Exception(error_msg) + + response_data = response.json() + operation_name = response_data.get("name", "") + + verbose_logger.debug(f"Import operation started: {operation_name}") + return operation_name diff --git a/litellm/rag/main.py b/litellm/rag/main.py index 571f78f2f70..6091d300256 100644 --- a/litellm/rag/main.py +++ b/litellm/rag/main.py @@ -32,6 +32,7 @@ from litellm.rag.ingestion.bedrock_ingestion import BedrockRAGIngestion from litellm.rag.ingestion.gemini_ingestion import GeminiRAGIngestion from litellm.rag.ingestion.openai_ingestion import OpenAIRAGIngestion from litellm.rag.ingestion.s3_vectors_ingestion import S3VectorsRAGIngestion +from litellm.rag.ingestion.vertex_ai_ingestion import VertexAIRAGIngestion from litellm.rag.rag_query import RAGQuery from litellm.types.rag import ( RAGIngestOptions, @@ -50,6 +51,7 @@ INGESTION_REGISTRY: Dict[str, Type[BaseRAGIngestion]] = { "bedrock": BedrockRAGIngestion, "gemini": GeminiRAGIngestion, "s3_vectors": S3VectorsRAGIngestion, + "vertex_ai": VertexAIRAGIngestion, } diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 2bff4e23c78..08e31c59662 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1512,6 +1512,12 @@ class LiteLLMCompletionResponsesConfig: user=getattr(chat_completion_response, "user", None), ) responses_api_response._hidden_params = getattr(chat_completion_response, "_hidden_params", {}) + + # Surface provider-specific fields (generic passthrough from any provider) + provider_fields = responses_api_response._hidden_params.get("provider_specific_fields") + if provider_fields: + responses_api_response.provider_specific_fields = provider_fields + return responses_api_response @staticmethod diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 8f524690be1..e943789a1cd 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -181,6 +181,7 @@ async def aresponses_api_with_mcp( ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + litellm_trace_id=kwargs.get("litellm_trace_id"), ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( original_mcp_tools @@ -681,6 +682,7 @@ def responses( _is_async=_is_async, stream=stream, extra_headers=extra_headers, + extra_body=extra_body, **kwargs, ) diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index 4a640a61064..377ce396457 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -127,6 +127,7 @@ async def acompletion_with_mcp( # noqa: PLR0915 ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + litellm_trace_id=kwargs.get("litellm_trace_id"), ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( @@ -235,7 +236,10 @@ async def acompletion_with_mcp( # noqa: PLR0915 def _add_mcp_list_tools_to_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: """Add mcp_list_tools to the first chunk.""" - from litellm.types.utils import StreamingChoices, add_provider_specific_fields + from litellm.types.utils import ( + StreamingChoices, + add_provider_specific_fields, + ) if not self.openai_tools: return chunk @@ -258,7 +262,10 @@ async def acompletion_with_mcp( # noqa: PLR0915 def _add_mcp_tool_metadata_to_final_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: """Add mcp_tool_calls and mcp_call_results to the final chunk.""" - from litellm.types.utils import StreamingChoices, add_provider_specific_fields + from litellm.types.utils import ( + StreamingChoices, + add_provider_specific_fields, + ) if hasattr(chunk, "choices") and chunk.choices: for choice in chunk.choices: diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 297ccf4355e..805a1958552 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -6,10 +6,10 @@ from typing import ( Dict, Iterable, List, + Literal, Optional, Tuple, Union, - Literal, ) from litellm._logging import verbose_logger @@ -29,6 +29,7 @@ from litellm.utils import Rules, function_setup if TYPE_CHECKING: from mcp.types import Tool as MCPTool + from litellm.proxy.utils import ProxyLogging else: MCPTool = Any @@ -97,6 +98,7 @@ class LiteLLM_Proxy_MCP_Handler: async def _get_mcp_tools_from_manager( user_api_key_auth: Any, mcp_tools_with_litellm_proxy: Optional[Iterable[ToolParam]], + litellm_trace_id: Optional[str] = None, ) -> tuple[List[MCPTool], List[str]]: """ Get available tools from the MCP server manager. @@ -109,13 +111,13 @@ class LiteLLM_Proxy_MCP_Handler: List of MCP tools List names of allowed MCP servers """ - from litellm.proxy._experimental.mcp_server.server import ( - _get_tools_from_mcp_servers, - _get_allowed_mcp_servers_from_mcp_server_names, - ) from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) + from litellm.proxy._experimental.mcp_server.server import ( + _get_allowed_mcp_servers_from_mcp_server_names, + _get_tools_from_mcp_servers, + ) mcp_servers: List[str] = [] if mcp_tools_with_litellm_proxy: @@ -136,6 +138,7 @@ class LiteLLM_Proxy_MCP_Handler: mcp_server_auth_headers=None, log_list_tools_to_spendlogs=True, list_tools_log_source="responses", + litellm_trace_id=litellm_trace_id, ) allowed_mcp_server_ids = ( await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) @@ -239,7 +242,9 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _process_mcp_tools_to_openai_format( - user_api_key_auth: Any, mcp_tools_with_litellm_proxy: List[ToolParam] + user_api_key_auth: Any, + mcp_tools_with_litellm_proxy: List[ToolParam], + litellm_trace_id: Optional[str] = None, ) -> tuple[List[Any], dict[str, str]]: """ Centralized method to process MCP tools through the complete pipeline. @@ -247,6 +252,7 @@ class LiteLLM_Proxy_MCP_Handler: Args: user_api_key_auth: User authentication info for access control mcp_tools_with_litellm_proxy: ToolParam objects with server_url starting with "litellm_proxy" + litellm_trace_id: Optional trace ID for linking list_mcp_tools spend logs to parent request Returns: List of tools in OpenAI format ready to be sent to the LLM @@ -258,6 +264,7 @@ class LiteLLM_Proxy_MCP_Handler: ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( user_api_key_auth, mcp_tools_with_litellm_proxy, + litellm_trace_id=litellm_trace_id, ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( @@ -268,7 +275,9 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _process_mcp_tools_without_openai_transform( - user_api_key_auth: Any, mcp_tools_with_litellm_proxy: List[ToolParam] + user_api_key_auth: Any, + mcp_tools_with_litellm_proxy: List[ToolParam], + litellm_trace_id: Optional[str] = None, ) -> tuple[List[Any], dict[str, str]]: """ Process MCP tools through filtering and deduplication pipeline without OpenAI transformation. @@ -291,6 +300,7 @@ class LiteLLM_Proxy_MCP_Handler: ) = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager( user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + litellm_trace_id=litellm_trace_id, ) # Step 2: Filter tools based on allowed_tools parameter @@ -495,14 +505,13 @@ class LiteLLM_Proxy_MCP_Handler: """Execute tool calls and return results.""" from fastapi import HTTPException + from litellm._uuid import uuid from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.proxy.proxy_server import proxy_logging_obj - from litellm._uuid import uuid - tool_results = [] tool_call_id: Optional[str] = None rules_obj = Rules() @@ -1025,7 +1034,6 @@ class LiteLLM_Proxy_MCP_Handler: List of MCP tool execution events for streaming """ from litellm._uuid import uuid - from litellm.responses.mcp.mcp_streaming_iterator import create_mcp_call_events tool_execution_events: List[Any] = [] @@ -1108,8 +1116,8 @@ class LiteLLM_Proxy_MCP_Handler: """Add custom output elements to the final response for MCP tool execution.""" # Import the required classes for creating output items import json - from litellm._uuid import uuid + from litellm._uuid import uuid from litellm.types.responses.main import GenericResponseOutputItem, OutputText # Create output element for initial MCP tools diff --git a/litellm/router.py b/litellm/router.py index 37fa3926b4d..888c97ca0b1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -110,15 +110,15 @@ from litellm.router_utils.handle_error import ( async_raise_no_deployment_exception, send_llm_exception_alert, ) +from litellm.router_utils.pre_call_checks.model_rate_limit_check import ( + ModelRateLimitingCheck, +) from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import ( PromptCachingDeploymentCheck, ) from litellm.router_utils.pre_call_checks.responses_api_deployment_check import ( ResponsesApiDeploymentCheck, ) -from litellm.router_utils.pre_call_checks.model_rate_limit_check import ( - ModelRateLimitingCheck, -) from litellm.router_utils.router_callbacks.track_deployment_metrics import ( increment_deployment_failures_for_current_minute, increment_deployment_successes_for_current_minute, @@ -896,14 +896,11 @@ class Router: def _initialize_vector_store_endpoints(self): """Initialize vector store endpoints.""" - from litellm.vector_stores.main import acreate, asearch, create, search + from litellm.vector_stores.main import asearch, create, search self.avector_store_search = self.factory_function( asearch, call_type="avector_store_search" ) - self.avector_store_create = self.factory_function( - acreate, call_type="avector_store_create" - ) self.vector_store_search = self.factory_function( search, call_type="vector_store_search" ) @@ -1159,6 +1156,8 @@ class Router: self._initialize_vector_store_file_endpoints() self._initialize_google_genai_endpoints() self._initialize_ocr_search_endpoints() + # Override vector store methods with router-aware implementations + self._override_vector_store_methods_for_router() self._initialize_video_endpoints() self._initialize_container_endpoints() self._initialize_skills_endpoints() @@ -2283,7 +2282,7 @@ class Router: item = FlowItem( priority=priority, # 👈 SET PRIORITY FOR REQUEST request_id=_request_id, # 👈 SET REQUEST ID - model_name="gpt-3.5-turbo", # 👈 SAME as 'Router' + model_name=model, # 👈 SAME as 'Router' ) ### [fin] ### @@ -2325,6 +2324,10 @@ class Router: setattr(e, "priority", priority) raise e else: + # Clean up the request from the scheduler queue also before raising the timeout exception + await self.scheduler.remove_request( + request_id=item.request_id, model_name=item.model_name + ) raise litellm.Timeout( message="Request timed out while polling queue", model=model, @@ -2386,6 +2389,10 @@ class Router: setattr(e, "priority", priority) raise e else: + # Clean up the request from the scheduler queue also before raising the timeout exception + await self.scheduler.remove_request( + request_id=item.request_id, model_name=item.model_name + ) raise litellm.Timeout( message="Request timed out while polling queue", model=model, @@ -3813,6 +3820,112 @@ class Router: self.fail_calls[model] += 1 raise e + #### VECTOR STORES API #### + async def avector_store_create( + self, + model: Union[str, None], + **kwargs, + ): + """ + Create a vector store for a specific model. + + Args: + model: Model name from router config + **kwargs: Vector store creation parameters + + Returns: + VectorStoreCreateResponse + """ + try: + # If model is None, use the factory function approach (direct SDK call) + if model is None: + from litellm.vector_stores.main import acreate + + # Use the factory function to handle the call + factory_fn = self.factory_function( + acreate, call_type="avector_store_create" + ) + return await factory_fn(**kwargs) + + from litellm.vector_stores import acreate as avector_store_create_sdk + parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs) + deployment = await self.async_get_available_deployment( + model=model, + messages=[{"role": "user", "content": "vector-store-api-fake-text"}], + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) + data = deployment["litellm_params"].copy() + model_name = data["model"] + self._update_kwargs_with_deployment( + deployment=deployment, kwargs=kwargs, function_name="avector_store_create" + ) + + model_client = self._get_async_openai_model_client( + deployment=deployment, + kwargs=kwargs, + ) + self.total_calls[model_name] += 1 + + # Get custom provider + _, custom_llm_provider, _, _ = get_llm_provider(model=data["model"]) + + response = avector_store_create_sdk( + **{ + **data, + "custom_llm_provider": custom_llm_provider, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } + ) + + rpm_semaphore = self._get_client( + deployment=deployment, + kwargs=kwargs, + client_type="max_parallel_requests", + ) + + if rpm_semaphore is not None and isinstance( + rpm_semaphore, asyncio.Semaphore + ): + async with rpm_semaphore: + await self.async_routing_strategy_pre_call_checks( + deployment=deployment, parent_otel_span=parent_otel_span + ) + response = await response + else: + await self.async_routing_strategy_pre_call_checks( + deployment=deployment, parent_otel_span=parent_otel_span + ) + response = await response + + self.success_calls[model_name] += 1 + verbose_router_logger.info( + f"litellm.avector_store_create(model={model_name})\033[32m 200 OK\033[0m" + ) + + return response + except Exception as e: + verbose_router_logger.exception( + f"litellm.avector_store_create(model={model})\033[31m Exception {str(e)}\033[0m" + ) + if model is not None: + self.fail_calls[model] += 1 + raise e + + + def _override_vector_store_methods_for_router(self): + """ + Override factory-generated vector store methods with router-aware implementations. + This is called after _initialize_vector_store_endpoints() to ensure our custom + methods that handle deployment selection and credential injection are used instead + of the generic factory-generated ones. + """ + # Store references to the custom methods defined above + # These methods handle proper routing through deployments + pass # The methods are already defined as instance methods above + async def acreate_batch( self, model: str, @@ -4519,9 +4632,21 @@ class Router: ): """ Initialize the Vector Store API endpoints on the router. + + If a model is provided in kwargs, use model-based routing to get + the deployment credentials. Otherwise, call the original function directly. """ if custom_llm_provider and "custom_llm_provider" not in kwargs: kwargs["custom_llm_provider"] = custom_llm_provider + + # If model is provided, use generic API call with fallbacks for proper routing + if kwargs.get("model"): + return await self._ageneric_api_call_with_fallbacks( + original_function=original_function, + **kwargs, + ) + + # Otherwise, call the original function directly return await original_function(**kwargs) async def _init_containers_api_endpoints( @@ -5039,7 +5164,7 @@ class Router: else: _healthy_deployments = [] _timeout = self._time_to_sleep_before_retry( - e=original_exception, + e=e, remaining_retries=remaining_retries, num_retries=num_retries, healthy_deployments=_healthy_deployments, diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 9e4001b67b9..64dc5fe4741 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -41,6 +41,53 @@ from litellm.types.utils import GenericBudgetConfigType, StandardLoggingPayload DEFAULT_REDIS_SYNC_INTERVAL = 1 +class _LiteLLMParamsDictView: + """ + Lightweight attribute view over `litellm_params` dict. + + This avoids pydantic construction in request hot-path while preserving + attribute-style access used by `litellm.get_llm_provider(...)`. + """ + + __slots__ = ("_params",) + + def __init__(self, params: Dict[str, Any]): + self._params = params + + def __getattr__(self, key: str) -> Any: + return self._params.get(key) + + def __getitem__(self, key: str) -> Any: + return self._params.get(key) + + def __contains__(self, key: str) -> bool: + return key in self._params + + def get(self, key: str, default: Any = None) -> Any: + return self._params.get(key, default) + + def keys(self): + return self._params.keys() + + def values(self): + return self._params.values() + + def items(self): + return self._params.items() + + def __iter__(self): + return iter(self._params) + + def __len__(self) -> int: + return len(self._params) + + def dict(self) -> Dict[str, Any]: + return dict(self._params) + + def model_dump(self) -> Dict[str, Any]: + return dict(self._params) + + class RouterBudgetLimiting(CustomLogger): def __init__( self, @@ -98,6 +145,7 @@ class RouterBudgetLimiting(CustomLogger): cache_keys, provider_configs, deployment_configs, + deployment_providers, ) = await self._async_get_cache_keys_for_router_budget_limiting( healthy_deployments=healthy_deployments, request_kwargs=request_kwargs, @@ -123,6 +171,7 @@ class RouterBudgetLimiting(CustomLogger): healthy_deployments=healthy_deployments, provider_configs=provider_configs, deployment_configs=deployment_configs, + deployment_providers=deployment_providers, spend_map=spend_map, potential_deployments=potential_deployments, request_tags=_get_tags_from_request_kwargs( @@ -145,6 +194,7 @@ class RouterBudgetLimiting(CustomLogger): healthy_deployments: List[Dict[str, Any]], provider_configs: Dict[str, GenericBudgetInfo], deployment_configs: Dict[str, GenericBudgetInfo], + deployment_providers: List[Optional[str]], spend_map: Dict[str, float], request_tags: List[str], ) -> Tuple[List[Dict[str, Any]], str]: @@ -161,12 +211,15 @@ class RouterBudgetLimiting(CustomLogger): """ # Filter deployments based on both provider and deployment budgets deployment_above_budget_info: str = "" - for deployment in healthy_deployments: + for idx, deployment in enumerate(healthy_deployments): is_within_budget = True # Check provider budget if self.provider_budget_config: - provider = self._get_llm_provider_for_deployment(deployment) + if idx < len(deployment_providers): + provider = deployment_providers[idx] + else: + provider = self._get_llm_provider_for_deployment(deployment) if provider in provider_configs: config = provider_configs[provider] if config.max_budget is None: @@ -230,24 +283,32 @@ class RouterBudgetLimiting(CustomLogger): self, healthy_deployments: List[Dict[str, Any]], request_kwargs: Optional[Dict] = None, - ) -> Tuple[List[str], Dict[str, GenericBudgetInfo], Dict[str, GenericBudgetInfo]]: + ) -> Tuple[ + List[str], + Dict[str, GenericBudgetInfo], + Dict[str, GenericBudgetInfo], + List[Optional[str]], + ]: """ Returns list of cache keys to fetch from router cache for budget limiting and provider and deployment configs Returns: - Tuple[List[str], Dict[str, GenericBudgetInfo], Dict[str, GenericBudgetInfo]]: + Tuple[List[str], Dict[str, GenericBudgetInfo], Dict[str, GenericBudgetInfo], List[Optional[str]]]: - List of cache keys to fetch from router cache for budget limiting - Dict of provider budget configs `provider_configs` - Dict of deployment budget configs `deployment_configs` + - List of resolved providers aligned by deployment index `deployment_providers` """ cache_keys: List[str] = [] provider_configs: Dict[str, GenericBudgetInfo] = {} deployment_configs: Dict[str, GenericBudgetInfo] = {} + deployment_providers: List[Optional[str]] = [] for deployment in healthy_deployments: # Check provider budgets if self.provider_budget_config: provider = self._get_llm_provider_for_deployment(deployment) + deployment_providers.append(provider) if provider is not None: budget_config = self._get_budget_config_for_provider(provider) if ( @@ -280,7 +341,12 @@ class RouterBudgetLimiting(CustomLogger): cache_keys.append( f"tag_spend:{_tag}:{_tag_budget_config.budget_duration}" ) - return cache_keys, provider_configs, deployment_configs + return ( + cache_keys, + provider_configs, + deployment_configs, + deployment_providers, + ) async def _get_or_set_budget_start_time( self, start_time_key: str, current_time: float, ttl_seconds: int @@ -597,12 +663,23 @@ class RouterBudgetLimiting(CustomLogger): def _get_llm_provider_for_deployment(self, deployment: Dict) -> Optional[str]: try: - _litellm_params: LiteLLM_Params = LiteLLM_Params( - **deployment.get("litellm_params", {"model": ""}) - ) + deployment_litellm_params = deployment.get("litellm_params") or {} + + if isinstance(deployment_litellm_params, LiteLLM_Params): + model = deployment_litellm_params.model or "" + provider_resolution_params: Any = deployment_litellm_params + elif isinstance(deployment_litellm_params, dict): + model = deployment_litellm_params.get("model") or "" + provider_resolution_params = _LiteLLMParamsDictView( + deployment_litellm_params + ) + else: + model = "" + provider_resolution_params = _LiteLLMParamsDictView({}) + _, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=_litellm_params.model, - litellm_params=_litellm_params, + model=str(model), + litellm_params=provider_resolution_params, ) except Exception: verbose_router_logger.error( diff --git a/litellm/scheduler.py b/litellm/scheduler.py index 5f3dd4cbf61..0221e249848 100644 --- a/litellm/scheduler.py +++ b/litellm/scheduler.py @@ -92,6 +92,17 @@ class Scheduler: return True + async def remove_request(self, request_id: str, model_name: str) -> None: + """ + Remove a specific request from the priority queue for a model. + Used when a request times out while waiting in the queue. + """ + queue = await self.get_queue(model_name=model_name) + filtered_queue = [item for item in queue if item[1] != request_id] + heapq.heapify(filtered_queue) # restore heap invariant after filtering + await self.save_queue(queue=filtered_queue, model_name=model_name) + print_verbose(f"Removed request_id: {request_id} from queue for model: {model_name}") + async def peek(self, id: str, model_name: str, health_deployments: list) -> bool: """Return if the id is at the top of the queue. Don't pop the value from heap.""" queue = await self.get_queue(model_name=model_name) diff --git a/litellm/secret_managers/aws_secret_manager_v2.py b/litellm/secret_managers/aws_secret_manager_v2.py index 8edfc48336b..c1b4d019dcf 100644 --- a/litellm/secret_managers/aws_secret_manager_v2.py +++ b/litellm/secret_managers/aws_secret_manager_v2.py @@ -3,7 +3,8 @@ This is a file for the AWS Secret Manager Integration Handles Async Operations for: - Read Secret -- Write Secret +- Write Secret (CreateSecret) +- Update Secret (PutSecretValue) - for in-place rotation when alias is preserved - Delete Secret Relevant issue: https://github.com/BerriAI/litellm/issues/1883 @@ -42,11 +43,11 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): aws_profile_name: Optional[str] = None, aws_web_identity_token: Optional[str] = None, aws_sts_endpoint: Optional[str] = None, - **kwargs + **kwargs, ): BaseSecretManager.__init__(self, **kwargs) BaseAWSLLM.__init__(self, **kwargs) - + # Store AWS authentication settings self.aws_region_name = aws_region_name self.aws_role_name = aws_role_name @@ -61,7 +62,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): # AWS_REGION_NAME is only strictly required if not using a profile or role # When using IAM roles, the region can come from multiple sources if ( - "AWS_REGION_NAME" not in os.environ + "AWS_REGION_NAME" not in os.environ and "AWS_REGION" not in os.environ and "AWS_DEFAULT_REGION" not in os.environ ): @@ -83,22 +84,36 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): return try: cls.validate_environment() - + # Extract AWS settings from key_management_settings if provided aws_kwargs = {} if key_management_settings is not None: aws_kwargs = { - "aws_region_name": getattr(key_management_settings, "aws_region_name", None), - "aws_role_name": getattr(key_management_settings, "aws_role_name", None), - "aws_session_name": getattr(key_management_settings, "aws_session_name", None), - "aws_external_id": getattr(key_management_settings, "aws_external_id", None), - "aws_profile_name": getattr(key_management_settings, "aws_profile_name", None), - "aws_web_identity_token": getattr(key_management_settings, "aws_web_identity_token", None), - "aws_sts_endpoint": getattr(key_management_settings, "aws_sts_endpoint", None), + "aws_region_name": getattr( + key_management_settings, "aws_region_name", None + ), + "aws_role_name": getattr( + key_management_settings, "aws_role_name", None + ), + "aws_session_name": getattr( + key_management_settings, "aws_session_name", None + ), + "aws_external_id": getattr( + key_management_settings, "aws_external_id", None + ), + "aws_profile_name": getattr( + key_management_settings, "aws_profile_name", None + ), + "aws_web_identity_token": getattr( + key_management_settings, "aws_web_identity_token", None + ), + "aws_sts_endpoint": getattr( + key_management_settings, "aws_sts_endpoint", None + ), } # Remove None values aws_kwargs = {k: v for k, v in aws_kwargs.items() if v is not None} - + litellm.secret_manager_client = cls(**aws_kwargs) litellm._key_management_system = KeyManagementSystem.AWS_SECRET_MANAGER @@ -246,13 +261,13 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): return primary_secret_kv_pairs.get(secret_name) async def async_write_secret( - self, - secret_name: str, - secret_value: str, - description: Optional[str] = None, - optional_params: Optional[dict] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, - tags: Optional[Union[dict, list]] = None + self, + secret_name: str, + secret_value: str, + description: Optional[str] = None, + optional_params: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + tags: Optional[Union[dict, list]] = None, ) -> dict: """ Async function to write a secret to AWS Secrets Manager @@ -312,6 +327,94 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): except httpx.TimeoutException: raise ValueError("Timeout error occurred") + async def async_put_secret_value( + self, + secret_name: str, + secret_value: str, + optional_params: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + ) -> dict: + """ + Async function to update an existing secret's value in AWS Secrets Manager. + + Uses PutSecretValue to update in place. Use this when rotating a secret + that keeps the same name (current_secret_name == new_secret_name). + + Args: + secret_name: Name of the existing secret to update + secret_value: New value to store + optional_params: Additional AWS parameters + timeout: Request timeout + + Returns: + dict: Response from AWS Secrets Manager containing update details + """ + from litellm._uuid import uuid + + data: Dict[str, Any] = { + "SecretId": secret_name, + "SecretString": secret_value, + "ClientRequestToken": str(uuid.uuid4()), + } + + endpoint_url, headers, body = self._prepare_request( + action="PutSecretValue", + secret_name=secret_name, + secret_value=secret_value, + optional_params=optional_params, + request_data=data, + ) + + async_client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.SecretManager, + params={"timeout": timeout}, + ) + + try: + response = await async_client.post( + url=endpoint_url, headers=headers, data=body.decode("utf-8") + ) + response.raise_for_status() + return response.json() + except httpx.HTTPStatusError as err: + raise ValueError(f"HTTP error occurred: {err.response.text}") + except httpx.TimeoutException: + raise ValueError("Timeout error occurred") + + async def async_rotate_secret( + self, + current_secret_name: str, + new_secret_name: str, + new_secret_value: str, + optional_params: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + ) -> dict: + """ + Rotate a secret. When current_secret_name == new_secret_name (in-place + update), uses PutSecretValue instead of create+delete to avoid + ResourceExistsException. + """ + if current_secret_name == new_secret_name: + # Same alias: update in place via PutSecretValue + verbose_logger.info( + "Secret rotated in-place (PutSecretValue): secret_name=%s", + current_secret_name, + ) + return await self.async_put_secret_value( + secret_name=current_secret_name, + secret_value=new_secret_value, + optional_params=optional_params, + timeout=timeout, + ) + # Different names: create new, delete old (base class logic) + return await super().async_rotate_secret( + current_secret_name=current_secret_name, + new_secret_name=new_secret_name, + new_secret_value=new_secret_value, + optional_params=optional_params, + timeout=timeout, + ) + async def async_delete_secret( self, secret_name: str, @@ -375,7 +478,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): except ImportError: raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") optional_params = optional_params or {} - + # Build optional_params from instance settings if not provided # This allows the IAM role settings to be used for Secret Manager calls if not optional_params.get("aws_role_name") and self.aws_role_name: @@ -388,11 +491,14 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): optional_params["aws_external_id"] = self.aws_external_id if not optional_params.get("aws_profile_name") and self.aws_profile_name: optional_params["aws_profile_name"] = self.aws_profile_name - if not optional_params.get("aws_web_identity_token") and self.aws_web_identity_token: + if ( + not optional_params.get("aws_web_identity_token") + and self.aws_web_identity_token + ): optional_params["aws_web_identity_token"] = self.aws_web_identity_token if not optional_params.get("aws_sts_endpoint") and self.aws_sts_endpoint: optional_params["aws_sts_endpoint"] = self.aws_sts_endpoint - + boto3_credentials_info = self._get_boto_credentials_from_optional_params( optional_params ) @@ -431,12 +537,3 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): prepped = request.prepare() return endpoint_url, prepped.headers, body - - -# if __name__ == "__main__": -# print("loading aws secret manager v2") -# aws_secret_manager_v2 = AWSSecretsManagerV2() -# import asyncio -# print("writing secret to aws secret manager v2") -# asyncio.run(aws_secret_manager_v2.async_write_secret(secret_name="test_secret_3", secret_value="test_value_2")) -# print("reading secret from aws secret manager v2") diff --git a/litellm/types/access_group.py b/litellm/types/access_group.py new file mode 100644 index 00000000000..3a6b75768ef --- /dev/null +++ b/litellm/types/access_group.py @@ -0,0 +1,38 @@ +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel + + +class AccessGroupCreateRequest(BaseModel): + access_group_name: str + description: Optional[str] = None + access_model_ids: Optional[List[str]] = None + access_mcp_server_ids: Optional[List[str]] = None + access_agent_ids: Optional[List[str]] = None + assigned_team_ids: Optional[List[str]] = None + assigned_key_ids: Optional[List[str]] = None + + +class AccessGroupUpdateRequest(BaseModel): + description: Optional[str] = None + access_model_ids: Optional[List[str]] = None + access_mcp_server_ids: Optional[List[str]] = None + access_agent_ids: Optional[List[str]] = None + assigned_team_ids: Optional[List[str]] = None + assigned_key_ids: Optional[List[str]] = None + + +class AccessGroupResponse(BaseModel): + access_group_id: str + access_group_name: str + description: Optional[str] = None + access_model_ids: List[str] + access_mcp_server_ids: List[str] + access_agent_ids: List[str] + assigned_team_ids: List[str] + assigned_key_ids: List[str] + created_at: datetime + created_by: Optional[str] = None + updated_at: datetime + updated_by: Optional[str] = None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 299b47199ed..4ab81f8fd57 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1058,7 +1058,20 @@ class ComputerToolParam(TypedDict, total=False): type: Required[Union[Literal["computer_use_preview"], str]] -ALL_RESPONSES_API_TOOL_PARAMS = Union[ToolParam, ComputerToolParam] +class ShellToolParam(TypedDict, total=False): + """ + Shell tool for Responses API: run commands in hosted containers or local runtime. + See https://developers.openai.com/api/docs/guides/tools-shell. + """ + + type: Required[Union[Literal["shell"], str]] + """The type of tool. Use ``\"shell\"``.""" + + environment: Required[Dict[str, Any]] + """Environment config: ``type`` (e.g. ``\"container_auto\"``, ``\"container_reference\"``, ``\"local\"``), optional ``container_id``, ``network_policy``, ``domain_secrets``, ``skills``.""" + + +ALL_RESPONSES_API_TOOL_PARAMS = Union[ToolParam, ComputerToolParam, ShellToolParam] class PromptObject(TypedDict, total=False): @@ -1074,6 +1087,19 @@ class PromptObject(TypedDict, total=False): """Optional version of the prompt template.""" +class ContextManagementEntry(TypedDict, total=False): + """ + Context management configuration entry for a request. + See https://developers.openai.com/api/docs/guides/compaction. + """ + + type: str + """The context management entry type. Currently only ``'compaction'`` is supported.""" + + compact_threshold: int + """Token threshold at which compaction is triggered for this entry. Minimum 1000.""" + + class ResponsesAPIOptionalRequestParams(TypedDict, total=False): """TypedDict for Optional parameters supported by the responses API.""" @@ -1104,6 +1130,8 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): partial_images: Optional[ int ] # Number of partial images to generate (1-3) for streaming image generation + context_management: Optional[List[ContextManagementEntry]] + """Context management configuration. E.g. [{\"type\": \"compaction\", \"compact_threshold\": 200000}] for server-side compaction (minimum 1000).""" class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): @@ -1189,7 +1217,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): top_p: Optional[float] = None max_output_tokens: Optional[int] = None previous_response_id: Optional[str] = None - reasoning: Optional[Reasoning] = None + reasoning: Optional[Dict[str, Any]] = None status: Optional[str] = None text: Optional[Union["ResponseText", Dict[str, Any]]] = None truncation: Optional[Literal["auto", "disabled"]] = None @@ -1199,6 +1227,18 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): # Define private attributes using PrivateAttr _hidden_params: dict = PrivateAttr(default_factory=dict) + @field_validator("reasoning", mode="before") + @classmethod + def validate_reasoning_to_dict(cls, value: Any) -> Optional[Dict[str, Any]]: + """Accept API reasoning dict (including effort 'none'/'xhigh'); always store as dict.""" + if value is None: + return None + if isinstance(value, dict): + return value + if hasattr(value, "model_dump"): + return value.model_dump() + return value + @field_validator("usage", mode="before") @classmethod def validate_usage(cls, value): @@ -1307,6 +1347,11 @@ class ResponsesAPIStreamEvents(str, Enum): # Image generation events IMAGE_GENERATION_PARTIAL_IMAGE = "image_generation.partial_image" + # Shell tool events (Responses API; passthrough via GenericEvent) + SHELL_CALL_IN_PROGRESS = "response.shell_call.in_progress" + SHELL_CALL_COMPLETED = "response.shell_call.completed" + SHELL_CALL_OUTPUT = "response.shell_call_output.done" + # Error event ERROR = "error" @@ -1593,12 +1638,12 @@ class ImageGenerationPartialImageEvent(BaseLiteLLMOpenAIResponseObject): class ErrorEventError(BaseLiteLLMOpenAIResponseObject): - """Nested error object within ErrorEvent""" + """Nested error object within ErrorEvent.""" type: str # e.g., 'invalid_request_error' code: str # e.g., 'context_length_exceeded' message: str - param: Optional[str] + param: Optional[str] = None class ErrorEvent(BaseLiteLLMOpenAIResponseObject): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py index 96d78cf8827..21f6f5b3b4e 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py @@ -60,6 +60,14 @@ class GenericGuardrailAPIRequest(BaseModel): tools: Optional[List[ChatCompletionToolParam]] = None texts: Optional[List[str]] = None request_data: GenericGuardrailAPIMetadata + request_headers: Optional[Dict[str, str]] = Field( + default=None, + description="Sanitized inbound request headers from the original proxy request.", + ) + litellm_version: Optional[str] = Field( + default=None, + description="LiteLLM library version running this proxy.", + ) additional_provider_specific_params: Optional[Dict[str, Any]] = None tool_calls: Optional[ Union[List[ChatCompletionToolCallChunk], List[ChatCompletionMessageToolCall]] diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py b/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py new file mode 100644 index 00000000000..7cbdf751e1b --- /dev/null +++ b/litellm/types/proxy/guardrails/guardrail_hooks/zscaler_ai_guard.py @@ -0,0 +1,132 @@ +from typing import Optional + +from pydantic import Field, model_validator + +from litellm._logging import verbose_proxy_logger +from litellm.types.guardrails import GuardrailParamUITypes + +from .base import GuardrailConfigModel + + +class ZscalerAIGuardConfigModel(GuardrailConfigModel): + api_key: Optional[str] = Field( + default=None, + description=( + "API key for Zscaler AI Guard authentication. " + "If not provided, falls back to ZSCALER_AI_GUARD_API_KEY environment variable." + ), + ) + + api_base: Optional[str] = Field( + default=None, + description=( + "Zscaler AI Guard API endpoint. Determines policy resolution behavior:\n" + "• /execute-policy (default) - Requires explicit policy_id in configuration\n" + "• /resolve-and-execute-policy - Infers policy from user-api-key-alias header\n" + "Default: https://api.us1.zseclipse.net/v1/detection/execute-policy\n" + "Falls back to ZSCALER_AI_GUARD_URL environment variable." + ), + json_schema_extra={ + "examples": [ + "https://api.us1.zseclipse.net/v1/detection/execute-policy", + "https://api.us1.zseclipse.net/v1/detection/resolve-and-execute-policy", + ] + }, + ) + + policy_id: Optional[int] = Field( + default=None, + description=( + "Global policy ID for Zscaler AI Guard. Required when using /execute-policy endpoint.\n\n" + "Set to 0 or leave empty when using /resolve-and-execute-policy with dynamic policy resolution.\n" + "Falls back to ZSCALER_AI_GUARD_POLICY_ID environment variable." + ), + json_schema_extra={ + "ui_hint": "conditional_required", + "condition": "Required when api_base ends with /execute-policy", + }, + ) + + send_user_api_key_alias: Optional[bool] = Field( + default=False, + description=( + "Send user API key alias in request headers as 'user-api-key-alias'. " + "CRITICAL when using /resolve-and-execute-policy endpoint - the policy is inferred from this value. " + "Also useful for tracking/auditing with /execute-policy endpoint." + ), + json_schema_extra={ + "ui_type": GuardrailParamUITypes.BOOL, + "ui_hint": "recommended_when", + "condition": "Recommended when api_base ends with /resolve-and-execute-policy", + }, + ) + + send_user_api_key_user_id: Optional[bool] = Field( + default=False, + description=( + "Send user API key user_id in request headers as 'user-api-key-user-id'. " + "Enables user-level tracking and analytics in Zscaler AI Guard." + ), + json_schema_extra={"ui_type": GuardrailParamUITypes.BOOL}, + ) + + send_user_api_key_team_id: Optional[bool] = Field( + default=False, + description=( + "Send user API key team_id in request headers as 'user-api-key-team-id'. " + "Enables team-level tracking and analytics in Zscaler AI Guard." + ), + json_schema_extra={"ui_type": GuardrailParamUITypes.BOOL}, + ) + + @model_validator(mode="after") + def validate_endpoint_configuration(self) -> "ZscalerAIGuardConfigModel": + """ + Validate configuration consistency between api_base and other fields. + Provides warnings but doesn't block (since env vars might provide values). + """ + import os + + # Resolve actual api_base value (including env fallback) + api_base = self.api_base or os.getenv( + "ZSCALER_AI_GUARD_URL", + "https://api.us1.zseclipse.net/v1/detection/execute-policy", + ) + + # Resolve actual policy_id value + policy_id = self.policy_id + if policy_id is None: + env_policy = os.getenv("ZSCALER_AI_GUARD_POLICY_ID") + if env_policy: + try: + policy_id = int(env_policy) + except ValueError: + verbose_proxy_logger.warning( + f"ZSCALER_AI_GUARD_POLICY_ID env var is not a valid integer: {env_policy}" + ) + + # Check for configuration issues + is_resolve_policy = api_base.endswith("/resolve-and-execute-policy") + is_execute_policy = api_base.endswith("/execute-policy") and not is_resolve_policy + + # Scenario A: execute-policy without policy_id + if is_execute_policy and (policy_id is None or policy_id < 1): + verbose_proxy_logger.warning( + "Using /execute-policy endpoint without a valid policy_id. " + "Ensure ZSCALER_AI_GUARD_POLICY_ID environment variable is set, " + "or provide policy_id via request/key/team metadata." + ) + + # Scenario B: resolve-and-execute-policy without user_api_key_alias + if is_resolve_policy and not self.send_user_api_key_alias: + verbose_proxy_logger.warning( + "Using /resolve-and-execute-policy endpoint without send_user_api_key_alias=true. " + "The endpoint requires user-api-key-alias header to resolve the policy. " + "Set send_user_api_key_alias to true or ensure the header is sent via other means." + ) + + return self + + @staticmethod + def ui_friendly_name() -> str: + return "Zscaler AI Guard" diff --git a/litellm/utils.py b/litellm/utils.py index 0fa5436d98d..0fd21f09919 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1406,7 +1406,7 @@ def client(original_function): # noqa: PLR0915 # [OPTIONAL] CHECK MAX RETRIES / REQUEST if litellm.num_retries_per_request is not None: # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] - previous_models = kwargs.get("metadata", {}).get( + previous_models = (kwargs.get("metadata") or {}).get( "previous_models", None ) if previous_models is not None: @@ -1483,7 +1483,7 @@ def client(original_function): # noqa: PLR0915 # [OPTIONAL] CHECK MAX RETRIES / REQUEST if litellm.num_retries_per_request is not None: # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] - previous_models = kwargs.get("metadata", {}).get( + previous_models = (kwargs.get("metadata") or {}).get( "previous_models", None ) if previous_models is not None: @@ -1678,8 +1678,8 @@ def client(original_function): # noqa: PLR0915 "context_window_fallback_dict", {} ) - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( num_retries and not _is_litellm_router_call @@ -1724,8 +1724,8 @@ def client(original_function): # noqa: PLR0915 None # set retries to None to prevent infinite loops ) - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( num_retries and not _is_litellm_router_call @@ -1974,8 +1974,8 @@ def client(original_function): # noqa: PLR0915 "context_window_fallback_dict", {} ) - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( @@ -2008,8 +2008,8 @@ def client(original_function): # noqa: PLR0915 kwargs["model"] = context_window_fallback_dict[model] return await original_function(*args, **kwargs) elif call_type == CallTypes.aresponses.value: - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( @@ -7337,7 +7337,7 @@ def _get_base_model_from_metadata(model_call_details=None): _base_model = litellm_params.get("base_model", None) if _base_model is not None: return _base_model - metadata = litellm_params.get("metadata", {}) + metadata = litellm_params.get("metadata") or {} _get_base_model_from_litellm_call_metadata = getattr( sys.modules[__name__], "_get_base_model_from_litellm_call_metadata" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f6edcf7efd0..e6b7cf17297 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5859,6 +5859,7 @@ "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, "azure_ai/ministral-3b": { @@ -6104,6 +6105,32 @@ "output_cost_per_token": 2.4e-05, "supports_tool_choice": true }, + "bedrock/ap-northeast-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-northeast-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6115,6 +6142,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6127,15 +6181,18 @@ "supports_reasoning": true }, "bedrock/moonshotai.kimi-k2.5": { - "input_cost_per_token": 7.3e-07, + "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.03e-06, + "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_video_input": true, + "supports_vision": true }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.18e-06, @@ -6155,6 +6212,32 @@ "mode": "chat", "output_cost_per_token": 7.2e-07 }, + "bedrock/ap-south-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-south-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", @@ -6166,6 +6249,86 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/ap-south-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-south-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.05e-06, "litellm_provider": "bedrock", @@ -6184,6 +6347,46 @@ "mode": "chat", "output_cost_per_token": 6.9e-07 }, + "bedrock/eu-north-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-north-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-north-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "input_cost_per_second": 0.01635, "litellm_provider": "bedrock", @@ -6271,6 +6474,32 @@ "output_cost_per_token": 2.4e-05, "supports_tool_choice": true }, + "bedrock/eu-central-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-central-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.86e-06, "litellm_provider": "bedrock", @@ -6289,6 +6518,32 @@ "mode": "chat", "output_cost_per_token": 6.5e-07 }, + "bedrock/eu-west-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-west-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.45e-06, "litellm_provider": "bedrock", @@ -6307,6 +6562,32 @@ "mode": "chat", "output_cost_per_token": 7.8e-07 }, + "bedrock/eu-west-2/minimax.minimax-m2.1": { + "input_cost_per_token": 4.7e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.86e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-west-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 7.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.86e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", @@ -6337,6 +6618,32 @@ "output_cost_per_token": 9.1e-07, "supports_tool_choice": true }, + "bedrock/eu-south-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-south-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, "litellm_provider": "bedrock", @@ -6371,6 +6678,32 @@ "mode": "chat", "output_cost_per_token": 1.01e-06 }, + "bedrock/sa-east-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/sa-east-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6382,6 +6715,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/sa-east-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/sa-east-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "input_cost_per_second": 0.011, "litellm_provider": "bedrock", @@ -6518,6 +6878,32 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-east-1/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6529,6 +6915,59 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-east-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6540,6 +6979,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-east-2/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "input_cost_per_token": 9.6e-07, "litellm_provider": "bedrock", @@ -6946,6 +7412,32 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-west-2/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-west-2/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6957,6 +7449,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-west-2/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-west-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, "cache_read_input_token_cost": 8e-08, @@ -10870,6 +11389,19 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "dolphin": { "input_cost_per_token": 5e-07, "litellm_provider": "nlp_cloud", @@ -15784,44 +16316,14 @@ "tpm": 250000 }, "gemini/gemini-2.5-flash-preview-tts": { - "cache_read_input_token_cost": 3.75e-08, - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 1.5e-07, + "input_cost_per_token": 3e-07, "litellm_provider": "gemini", - "max_audio_length_hours": 8.4, - "max_audio_per_prompt": 1, - "max_images_per_prompt": 3000, - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_pdf_size_mb": 30, - "max_tokens": 65535, - "max_video_length": 1, - "max_videos_per_prompt": 10, - "mode": "chat", - "output_cost_per_reasoning_token": 3.5e-06, - "output_cost_per_token": 6e-07, - "rpm": 10, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "mode": "audio_speech", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions" - ], - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "audio" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000 + "/v1/audio/speech" + ] }, "gemini/gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -21370,6 +21872,19 @@ "output_cost_per_token": 1.2e-06, "supports_system_messages": true }, + "minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, "litellm_provider": "minimax", @@ -21432,6 +21947,36 @@ "max_input_tokens": 1000000, "max_output_tokens": 8192 }, + "minimax/MiniMax-M2.5": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, + "minimax/MiniMax-M2.5-lightning": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, "minimax/MiniMax-M2": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, @@ -22096,6 +22641,20 @@ "supports_reasoning": true, "supports_system_messages": true }, + "moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "moonshot/kimi-k2-0711-preview": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 6e-07, @@ -22147,9 +22706,10 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat", + "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, "moonshot/kimi-latest": { @@ -22611,6 +23171,19 @@ "output_cost_per_token": 2.3e-07, "supports_system_messages": true }, + "nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 6e-08, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.4e-07, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "o1": { "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, @@ -23616,36 +24189,6 @@ "output_cost_per_token": 2e-07, "supports_system_messages": true }, - "openrouter/anthropic/claude-2": { - "input_cost_per_token": 1.102e-05, - "litellm_provider": "openrouter", - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 3.268e-05, - "supports_tool_choice": true - }, - "openrouter/anthropic/claude-3-5-haiku": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_tokens": 200000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "supports_function_calling": true, - "supports_tool_choice": true - }, - "openrouter/anthropic/claude-3-5-haiku-20241022": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "tool_use_system_prompt_tokens": 264 - }, "openrouter/anthropic/claude-3-haiku": { "input_cost_per_image": 0.0004, "input_cost_per_token": 2.5e-07, @@ -23657,43 +24200,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/anthropic/claude-3-haiku-20240307": { - "input_cost_per_token": 2.5e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 264 - }, - "openrouter/anthropic/claude-3-opus": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 395 - }, - "openrouter/anthropic/claude-3-sonnet": { - "input_cost_per_image": 0.0048, - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_tokens": 200000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/anthropic/claude-3.5-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "openrouter", @@ -23709,20 +24215,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-3.5-sonnet:beta": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 - }, "openrouter/anthropic/claude-3.7-sonnet": { "input_cost_per_image": 0.0048, "input_cost_per_token": 3e-06, @@ -23740,31 +24232,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-3.7-sonnet:beta": { - "input_cost_per_image": 0.0048, - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 - }, - "openrouter/anthropic/claude-instant-v1": { - "input_cost_per_token": 1.63e-06, - "litellm_provider": "openrouter", - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 5.51e-06, - "supports_tool_choice": true - }, "openrouter/anthropic/claude-opus-4": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 1.875e-05, @@ -23903,30 +24370,6 @@ "source": "https://openrouter.ai/api/v1/models/bytedance/ui-tars-1.5-7b", "supports_tool_choice": true }, - "openrouter/cognitivecomputations/dolphin-mixtral-8x7b": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 32769, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/cohere/command-r-plus": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_tool_choice": true - }, - "openrouter/databricks/dbrx-instruct": { - "input_cost_per_token": 6e-07, - "litellm_provider": "openrouter", - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_tool_choice": true - }, "openrouter/deepseek/deepseek-chat": { "input_cost_per_token": 1.4e-07, "litellm_provider": "openrouter", @@ -23994,17 +24437,6 @@ "supports_reasoning": false, "supports_tool_choice": true }, - "openrouter/deepseek/deepseek-coder": { - "input_cost_per_token": 1.4e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 66000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2.8e-07, - "supports_prompt_caching": true, - "supports_tool_choice": true - }, "openrouter/deepseek/deepseek-r1": { "input_cost_per_token": 5.5e-07, "input_cost_per_token_cache_hit": 1.4e-07, @@ -24035,14 +24467,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "openrouter/fireworks/firellava-13b": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, "openrouter/google/gemini-2.0-flash-001": { "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 7e-07, @@ -24204,46 +24628,6 @@ "supports_web_search": true, "tpm": 800000 }, - "openrouter/google/gemini-pro-1.5": { - "input_cost_per_image": 0.00265, - "input_cost_per_token": 2.5e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.5e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "openrouter/google/gemini-pro-vision": { - "input_cost_per_image": 0.0025, - "input_cost_per_token": 1.25e-07, - "litellm_provider": "openrouter", - "max_tokens": 45875, - "mode": "chat", - "output_cost_per_token": 3.75e-07, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "openrouter/google/palm-2-chat-bison": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 25804, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/google/palm-2-codechat-bison": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 20070, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, "openrouter/gryphe/mythomax-l2-13b": { "input_cost_per_token": 1.875e-06, "litellm_provider": "openrouter", @@ -24252,14 +24636,6 @@ "output_cost_per_token": 1.875e-06, "supports_tool_choice": true }, - "openrouter/jondurbin/airoboros-l2-70b-2.1": { - "input_cost_per_token": 1.3875e-05, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.3875e-05, - "supports_tool_choice": true - }, "openrouter/mancer/weaver": { "input_cost_per_token": 5.625e-06, "litellm_provider": "openrouter", @@ -24268,30 +24644,6 @@ "output_cost_per_token": 5.625e-06, "supports_tool_choice": true }, - "openrouter/meta-llama/codellama-34b-instruct": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-2-13b-chat": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-2-70b-chat": { - "input_cost_per_token": 1.5e-06, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_tool_choice": true - }, "openrouter/meta-llama/llama-3-70b-instruct": { "input_cost_per_token": 5.9e-07, "litellm_provider": "openrouter", @@ -24300,38 +24652,6 @@ "output_cost_per_token": 7.9e-07, "supports_tool_choice": true }, - "openrouter/meta-llama/llama-3-70b-instruct:nitro": { - "input_cost_per_token": 9e-07, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-3-8b-instruct:extended": { - "input_cost_per_token": 2.25e-07, - "litellm_provider": "openrouter", - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2.25e-06, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-3-8b-instruct:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 0.0, - "supports_tool_choice": true - }, - "openrouter/microsoft/wizardlm-2-8x22b:nitro": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1e-06, - "supports_tool_choice": true - }, "openrouter/minimax/minimax-m2": { "input_cost_per_token": 2.55e-07, "litellm_provider": "openrouter", @@ -24345,20 +24665,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "openrouter/mistralai/devstral-2512:free": { - "input_cost_per_image": 0, - "input_cost_per_token": 0, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 0, - "supports_function_calling": true, - "supports_prompt_caching": false, - "supports_tool_choice": true, - "supports_vision": false - }, "openrouter/mistralai/devstral-2512": { "input_cost_per_image": 0, "input_cost_per_token": 1.5e-07, @@ -24437,14 +24743,6 @@ "output_cost_per_token": 1.3e-07, "supports_tool_choice": true }, - "openrouter/mistralai/mistral-7b-instruct:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 0.0, - "supports_tool_choice": true - }, "openrouter/mistralai/mistral-large": { "input_cost_per_token": 8e-06, "litellm_provider": "openrouter", @@ -24489,16 +24787,9 @@ "source": "https://openrouter.ai/moonshotai/kimi-k2.5", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, - "openrouter/nousresearch/nous-hermes-llama2-13b": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, "openrouter/openai/gpt-3.5-turbo": { "input_cost_per_token": 1.5e-06, "litellm_provider": "openrouter", @@ -24523,17 +24814,6 @@ "output_cost_per_token": 6e-05, "supports_tool_choice": true }, - "openrouter/openai/gpt-4-vision-preview": { - "input_cost_per_image": 0.01445, - "input_cost_per_token": 1e-05, - "litellm_provider": "openrouter", - "max_tokens": 130000, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1": { "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, @@ -24551,23 +24831,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-2025-04-14": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 8e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1-mini": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, @@ -24585,23 +24848,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-mini-2025-04-14": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 4e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1.6e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1-nano": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, @@ -24619,23 +24865,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-nano-2025-04-14": { - "cache_read_input_token_cost": 2.5e-08, - "input_cost_per_token": 1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 4e-07, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4o": { "input_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", @@ -24707,11 +24936,8 @@ "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, - "mode": "responses", + "mode": "chat", "output_cost_per_token": 1.4e-05, - "supported_endpoints": [ - "/v1/responses" - ], "supported_modalities": [ "text", "image" @@ -24872,58 +25098,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/o1-mini": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-mini-2024-09-12": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-preview": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-preview-2024-09-12": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, "openrouter/openai/o3-mini": { "input_cost_per_token": 1.1e-06, "litellm_provider": "openrouter", @@ -24952,14 +25126,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "openrouter/pygmalionai/mythalion-13b": { - "input_cost_per_token": 1.875e-06, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.875e-06, - "supports_tool_choice": true - }, "openrouter/qwen/qwen-2.5-coder-32b-instruct": { "input_cost_per_token": 1.8e-07, "litellm_provider": "openrouter", @@ -25051,20 +25217,6 @@ "supports_tool_choice": true, "supports_web_search": true }, - "openrouter/x-ai/grok-4-fast:free": { - "input_cost_per_token": 0, - "litellm_provider": "openrouter", - "max_input_tokens": 2000000, - "max_output_tokens": 30000, - "max_tokens": 30000, - "mode": "chat", - "output_cost_per_token": 0, - "source": "https://openrouter.ai/x-ai/grok-4-fast:free", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": false - }, "openrouter/z-ai/glm-4.6": { "input_cost_per_token": 4e-07, "litellm_provider": "openrouter", @@ -25937,6 +26089,19 @@ "supports_system_messages": true, "supports_vision": true }, + "qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "recraft/recraftv2": { "litellm_provider": "recraft", "mode": "image_generation", @@ -28373,6 +28538,30 @@ "supports_reasoning": true, "supports_tool_choice": false }, + "us.deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "eu.deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "us.meta.llama3-1-405b-instruct-v1:0": { "input_cost_per_token": 5.32e-06, "litellm_provider": "bedrock", @@ -30799,6 +30988,21 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/zai-org/glm-5-maas": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "vertex_ai/mistral-medium-3": { "input_cost_per_token": 4e-07, "litellm_provider": "vertex_ai-mistral_models", @@ -32302,6 +32506,20 @@ "supports_vision": true, "supports_web_search": true }, + "zai.glm-4.7": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "zai/glm-4.7": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 1.1e-07, @@ -32454,6 +32672,23 @@ "1280x720" ] }, + "openai/sora-2-pro-high-res": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.5, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "1024x1792", + "1792x1024" + ] + }, "azure/sora-2": { "litellm_provider": "azure", "mode": "video_generation", @@ -35975,5 +36210,715 @@ "mode": "chat", "output_cost_per_token": 0, "supports_reasoning": true + }, + "tts-1-1106": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "openai", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "tts-1-hd-1106": { + "input_cost_per_character": 3e-05, + "litellm_provider": "openai", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "gpt-4o-mini-tts-2025-03-20": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "mode": "audio_speech", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_second": 0.00025, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "audio" + ] + }, + "gpt-4o-mini-tts-2025-12-15": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "mode": "audio_speech", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_second": 0.00025, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "audio" + ] + }, + "gpt-4o-mini-transcribe-2025-03-20": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 16000, + "max_output_tokens": 2000, + "mode": "audio_transcription", + "output_cost_per_token": 5e-06, + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "gpt-4o-mini-transcribe-2025-12-15": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 16000, + "max_output_tokens": 2000, + "mode": "audio_transcription", + "output_cost_per_token": 5e-06, + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "gpt-5-search-api": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "gpt-5-search-api-2025-10-14": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "gpt-realtime-mini-2025-10-06": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "gpt-realtime-mini-2025-12-15": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "sora-2": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.1, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "720x1280", + "1280x720" + ] + }, + "sora-2-pro": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.3, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "720x1280", + "1280x720" + ] + }, + "sora-2-pro-high-res": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.5, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "1024x1792", + "1792x1024" + ] + }, + "chatgpt-image-latest": { + "cache_read_input_image_token_cost": 2.5e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 1e-05, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 4e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "gemini-2.0-flash-exp-image-generation": { + "input_cost_per_token": 0.0, + "litellm_provider": "gemini", + "max_images_per_prompt": 3000, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.039, + "output_cost_per_token": 0.0, + "source": "https://ai.google.dev/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "gemini/gemini-2.0-flash-exp-image-generation": { + "input_cost_per_token": 0.0, + "litellm_provider": "gemini", + "max_images_per_prompt": 3000, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.039, + "output_cost_per_token": 0.0, + "source": "https://ai.google.dev/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "gemini/gemini-2.0-flash-lite-001": { + "cache_read_input_token_cost": 1.875e-08, + "deprecation_date": "2026-03-31", + "input_cost_per_audio_token": 7.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_pdf_size_mb": 50, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 3e-07, + "rpm": 4000, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 4000000 + }, + "gemini-2.5-flash-native-audio-latest": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-native-audio-preview-09-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-native-audio-preview-12-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-latest": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-preview-09-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-preview-12-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-preview-tts": { + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "mode": "audio_speech", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "gemini-flash-latest": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "rpm": 100000, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 8000000 + }, + "gemini-flash-lite-latest": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_audio_token": 3e-07, + "input_cost_per_token": 1e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 4e-07, + "output_cost_per_token": 4e-07, + "rpm": 15, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-lite", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 250000 + }, + "gemini-pro-latest": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "rpm": 2000, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, + "gemini/gemini-pro-latest": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "rpm": 2000, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, + "gemini-exp-1206": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "rpm": 100000, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 8000000 } } diff --git a/poetry.lock b/poetry.lock index 533b48b793c..d01baa854af 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "a2a-sdk" @@ -3459,15 +3459,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.33" +version = "0.4.36" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.33-py3-none-any.whl", hash = "sha256:bebea1b091490df19cfa773bd311f08254dee5bb53f92d282b7a5bdfba936334"}, - {file = "litellm_proxy_extras-0.4.33.tar.gz", hash = "sha256:133dc5476b540d99e75d4baef622267e7344ced97737c174679baff429e7f212"}, + {file = "litellm_proxy_extras-0.4.36-py3-none-any.whl", hash = "sha256:71c7b2b31453a5abc74cdfb36774b7438dda02052391ea5f19cd428a79dadb4b"}, + {file = "litellm_proxy_extras-0.4.36.tar.gz", hash = "sha256:a71ee667385a8a05375f6c0b2a85f8d4f5a2c89cfaea89a4de4d9e8007d4392a"}, ] [[package]] @@ -8523,4 +8523,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "4d88cb21bfd0bcc87b8894e2915d297c2d0b15f164a468e712872b1f1b87db43" +content-hash = "001d36df0a50be6c680744e96124bf562eb831a7026c2b0fd7d919c78f0f29bd" diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index fd17b5309e8..18d9f4e6eef 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2453,7 +2453,7 @@ }, "messages": { "docs_label": "anthropic_unified", - "display_name": "Anthropic /v1/messages API", + "display_name": "Anthropic Messages API", "leftnav_label": "/messages", "provider_json_field": "messages", "url": "https://docs.litellm.ai/docs/anthropic_unified", @@ -2461,7 +2461,7 @@ }, "anthropic_count_tokens": { "docs_label": "anthropic_count_tokens", - "display_name": "Anthropic /v1/messages/count_tokens API", + "display_name": "Anthropic Count Tokens API", "leftnav_label": "/count_tokens", "provider_json_field": "count_tokens", "url": "https://docs.litellm.ai/docs/anthropic_count_tokens" @@ -2482,14 +2482,14 @@ }, "audio_transcription": { "docs_label": "audio_transcription", - "display_name": "Audio Transcription API", + "display_name": "OpenAI Audio Transcription API", "leftnav_label": "/audio/transcriptions", "provider_json_field": "audio_transcriptions", "url": "https://docs.litellm.ai/docs/audio_transcription" }, "batches": { "docs_label": "batches", - "display_name": "Batches API", + "display_name": "OpenAI Batches API", "leftnav_label": "/batches", "provider_json_field": "batches", "url": "https://docs.litellm.ai/docs/batches" @@ -2510,7 +2510,7 @@ }, "chat_completions": { "docs_label": "chat_completions", - "display_name": "Chat Completions API", + "display_name": "OpenAI Chat Completions API", "leftnav_label": "/chat/completions", "provider_json_field": "chat_completions", "url": "https://docs.litellm.ai/docs/chat_completions" @@ -2531,7 +2531,7 @@ }, "embeddings": { "docs_label": "embedding/supported_embedding", - "display_name": "Embedding API (OpenAI Format)", + "display_name": "OpenAI Embeddings API", "leftnav_label": "/embeddings", "provider_json_field": "embeddings", "url": "https://docs.litellm.ai/docs/embedding/supported_embedding" @@ -2552,7 +2552,7 @@ }, "generateContent": { "docs_label": "generateContent", - "display_name": "Google's GenerateContent API", + "display_name": "Google GenerateContent API", "leftnav_label": "/generateContent", "provider_json_field": "generateContent", "url": "https://docs.litellm.ai/docs/generateContent", @@ -2596,14 +2596,14 @@ }, "moderation": { "docs_label": "moderation", - "display_name": "OpenAI Moderation API", + "display_name": "OpenAI Moderations API", "leftnav_label": "/moderations", "provider_json_field": "moderations", "url": "https://docs.litellm.ai/docs/moderation" }, "ocr": { "docs_label": "ocr", - "display_name": "OCR API (Mistral Format)", + "display_name": "Mistral OCR API", "leftnav_label": "/ocr", "provider_json_field": "ocr", "url": "https://docs.litellm.ai/docs/ocr" @@ -2631,14 +2631,14 @@ }, "rerank": { "docs_label": "rerank", - "display_name": "Rerank API (Cohere Format)", + "display_name": "Cohere Rerank API", "leftnav_label": "/rerank", "provider_json_field": "rerank", "url": "https://docs.litellm.ai/docs/rerank" }, "responses": { "docs_label": "response_api", - "display_name": "Responses API (OpenAI Format)", + "display_name": "OpenAI Responses API", "leftnav_label": "/responses", "provider_json_field": "responses", "url": "https://docs.litellm.ai/docs/response_api", @@ -2646,7 +2646,7 @@ }, "response_api_compact": { "docs_label": "response_api_compact", - "display_name": "Responses API (OpenAI Format)", + "display_name": "OpenAI Responses API", "leftnav_label": "/responses", "provider_json_field": "compact", "url": "https://docs.litellm.ai/docs/response_api" @@ -2667,7 +2667,7 @@ }, "text_completion": { "docs_label": "text_completion", - "display_name": "Completions API (OpenAI Format)", + "display_name": "OpenAI Completions API", "leftnav_label": "/completions", "provider_json_field": "text_completion", "url": "https://docs.litellm.ai/docs/text_completion", @@ -2675,7 +2675,7 @@ }, "text_to_speech": { "docs_label": "text_to_speech", - "display_name": "Text-to-Speech API (OpenAI Format)", + "display_name": "OpenAI Text-to-Speech API", "leftnav_label": "/audio/speech", "provider_json_field": "audio_speech", "url": "https://docs.litellm.ai/docs/text_to_speech" @@ -2703,7 +2703,7 @@ }, "videos": { "docs_label": "videos", - "display_name": "OpenAI Video Generation API", + "display_name": "OpenAI Videos API", "leftnav_label": "/videos", "provider_json_field": "video_generations", "url": "https://docs.litellm.ai/docs/videos" diff --git a/pyproject.toml b/pyproject.toml index acb8bc2ada3..6ed7618dd26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.81.10" +version = "1.81.11" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -61,7 +61,7 @@ boto3 = { version = "1.40.76", optional = true } redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"} a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.4.34", optional = true} +litellm-proxy-extras = {version = "0.4.36", optional = true} rich = {version = "13.7.1", optional = true} litellm-enterprise = {version = "0.1.31", optional = true} diskcache = {version = "^5.6.1", optional = true} @@ -175,7 +175,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.81.10" +version = "1.81.11" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index bd313b105e5..f31730e20f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,8 @@ urllib3>=2.6.0 # CVE-2025-66471, CVE-2025-66418, CVE-2026-21441 tornado>=6.5.3 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724 filelock>=3.20.1 # CVE-2025-68146 +Pillow==12.1.1 #GHSA-cfh3-3jmp-rvhc +cryptography==46.0.5 #GHSA-r6ph-v2qm-q3c2 anyio==4.8.0 # openai + http req. httpx==0.28.1 @@ -38,7 +40,6 @@ apscheduler==3.10.4 # for resetting budget in background fastapi-sso==0.19.0 # admin UI, SSO pyjwt[crypto]==2.10.1 ; python_version >= "3.9" python-multipart==0.0.22 # admin UI -Pillow==11.0.0 jaraco.context>=6.1.0 azure-ai-contentsafety==1.0.0 # for azure content safety azure-identity==1.16.1 ; python_version >= "3.9" # for azure content safety @@ -53,9 +54,8 @@ grpcio>=1.62.3,!=1.68.*,!=1.69.*,!=1.70.*,!=1.71.0,!=1.71.1,!=1.72.0,!=1.72.1,!= grpcio>=1.75.0; python_version >= "3.14" sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests -cryptography==44.0.1 tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.34 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.36 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env diff --git a/schema.prisma b/schema.prisma index 4329f939a7b..2a11d0028fb 100644 --- a/schema.prisma +++ b/schema.prisma @@ -128,6 +128,7 @@ model LiteLLM_TeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team @@ -161,6 +162,7 @@ model LiteLLM_DeletedTeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) @@ -293,6 +295,7 @@ model LiteLLM_VerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") budget_id String? @@ -348,6 +351,7 @@ model LiteLLM_DeletedVerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") router_settings Json? @default("{}") @@ -363,7 +367,6 @@ model LiteLLM_DeletedVerificationToken { rotation_interval String? last_rotation_at DateTime? key_rotation_at DateTime? - // Deletion metadata deleted_at DateTime @default(now()) @map("deleted_at") deleted_by String? @map("deleted_by") // User who deleted the key @@ -919,3 +922,23 @@ model LiteLLM_PolicyAttachmentTable { updated_at DateTime @default(now()) @updatedAt updated_by String? } + +//Unified Access Groups table for storing unified access groups +model LiteLLM_AccessGroupTable { + access_group_id String @id @default(uuid()) + access_group_name String @unique + description String? + + // Resource memberships - explicit arrays per type + access_model_ids String[] @default([]) + access_mcp_server_ids String[] @default([]) + access_agent_ids String[] @default([]) + + assigned_team_ids String[] @default([]) + assigned_key_ids String[] @default([]) + + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? +} \ No newline at end of file diff --git a/tests/batches_tests/test_batch_rate_limits.py b/tests/batches_tests/test_batch_rate_limits.py index 13241e94d51..6bff3b82e52 100644 --- a/tests/batches_tests/test_batch_rate_limits.py +++ b/tests/batches_tests/test_batch_rate_limits.py @@ -619,3 +619,442 @@ async def test_batch_rate_limiter_without_user_context(): finally: os.unlink(file_path) + + +@pytest.mark.asyncio() +async def test_batch_rate_limiter_managed_files_regression(): + """ + Regression test for GEN-2166: Batch Rate Limiter Cannot Access User Files + + This test ensures that the batch rate limiter can properly access managed files + by verifying that: + 1. Managed files are detected correctly (base64 encoded unified file IDs) + 2. The _fetch_managed_file_content method uses the managed files hook + 3. User context (user_api_key_dict) is properly passed through + 4. No 403 errors occur when accessing files owned by the user + 5. The fix doesn't break non-managed file access + + This is a unit test that doesn't require external API calls. + """ + from unittest.mock import AsyncMock, MagicMock, patch + from litellm.llms.base_llm.files.transformation import BaseFileEndpoints + from litellm.types.llms.openai import HttpxBinaryResponseContent + import httpx + + print("\n=== Regression Test: GEN-2166 Batch Rate Limiter Managed Files ===") + + # Setup: Create batch rate limiter + dual_cache = DualCache() + internal_usage_cache = InternalUsageCache(dual_cache=dual_cache) + rate_limiter = _PROXY_MaxParallelRequestsHandler_v3( + internal_usage_cache=internal_usage_cache + ) + batch_limiter = rate_limiter._get_batch_rate_limiter() + assert batch_limiter is not None + + # Setup: Create user API key dict + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key-regression", + user_id="test-user-regression", + tpm_limit=1000, + rpm_limit=10, + ) + + # Setup: Create mock file content (batch input file) + batch_content = b'{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Test message for regression"}]}}' + + # Mock managed file ID (base64 encoded unified file ID format) + managed_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCxyZWdyZXNzaW9uLXRlc3QtZmlsZQ==" + + # Test 1: Verify managed file detection + print("\n1. Verifying managed file detection...") + from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, + ) + is_managed = _is_base64_encoded_unified_file_id(managed_file_id) + assert is_managed, "Managed file should be detected correctly" + print(" ✓ Managed file detected") + + # Test 2: Verify _fetch_managed_file_content uses managed files hook + print("\n2. Verifying managed files hook integration...") + + # Create mock managed files hook + class MockManagedFiles(BaseFileEndpoints): + def __init__(self): + self._afile_content_called = False + self._last_call_args = None + + async def acreate_file(self, *args, **kwargs): + pass + + async def afile_content(self, *args, **kwargs): + self._afile_content_called = True + self._last_call_args = kwargs + # Return mock file content + mock_response = httpx.Response( + status_code=200, + content=batch_content, + headers={"content-type": "application/octet-stream"}, + ) + return HttpxBinaryResponseContent(response=mock_response) + + async def afile_delete(self, *args, **kwargs): + pass + + async def afile_list(self, *args, **kwargs): + pass + + async def afile_retrieve(self, *args, **kwargs): + pass + + mock_managed_files = MockManagedFiles() + mock_llm_router = MagicMock() + mock_proxy_logging_obj = MagicMock() + mock_proxy_logging_obj.get_proxy_hook.return_value = mock_managed_files + + # Patch proxy_server imports + with patch.dict('sys.modules', { + 'litellm.proxy.proxy_server': MagicMock( + llm_router=mock_llm_router, + proxy_logging_obj=mock_proxy_logging_obj, + ) + }): + # Call _fetch_managed_file_content + result = await batch_limiter._fetch_managed_file_content( + file_id=managed_file_id, + user_api_key_dict=user_api_key_dict, + ) + + # Verify managed files hook was called + assert mock_managed_files._afile_content_called, \ + "REGRESSION: managed_files_obj.afile_content was not called! Bug GEN-2166 has returned." + + # Verify user context was passed + assert mock_managed_files._last_call_args is not None, \ + "REGRESSION: No arguments passed to afile_content" + assert 'file_id' in mock_managed_files._last_call_args, \ + "REGRESSION: file_id not passed to managed files hook" + assert mock_managed_files._last_call_args['file_id'] == managed_file_id, \ + "REGRESSION: Incorrect file_id passed" + assert 'llm_router' in mock_managed_files._last_call_args, \ + "REGRESSION: llm_router not passed to managed files hook" + + print(" ✓ Managed files hook called correctly") + print(" ✓ User context passed correctly") + + # Test 3: Verify count_input_file_usage uses managed files path + print("\n3. Verifying count_input_file_usage integration...") + + with patch.object(batch_limiter, '_fetch_managed_file_content') as mock_fetch: + mock_response = httpx.Response( + status_code=200, + content=batch_content, + headers={"content-type": "application/octet-stream"}, + ) + mock_fetch.return_value = HttpxBinaryResponseContent(response=mock_response) + + # Call count_input_file_usage with managed file + usage = await batch_limiter.count_input_file_usage( + file_id=managed_file_id, + custom_llm_provider="openai", + user_api_key_dict=user_api_key_dict, + ) + + # Verify _fetch_managed_file_content was called + assert mock_fetch.called, \ + "REGRESSION: _fetch_managed_file_content not called for managed files! Bug GEN-2166 has returned." + + # Verify correct parameters were passed + call_kwargs = mock_fetch.call_args.kwargs + assert call_kwargs['file_id'] == managed_file_id, \ + "REGRESSION: Incorrect file_id passed to _fetch_managed_file_content" + assert call_kwargs['user_api_key_dict'] == user_api_key_dict, \ + "REGRESSION: user_api_key_dict not passed! Bug GEN-2166 has returned." + + # Verify usage was calculated + assert usage.total_tokens > 0, "Token count should be greater than 0" + assert usage.request_count == 1, "Request count should be 1" + + print(" ✓ Managed file path used") + print(f" ✓ Token count: {usage.total_tokens}") + print(f" ✓ Request count: {usage.request_count}") + + # Test 4: Verify non-managed files still work + print("\n4. Verifying non-managed files still work...") + + non_managed_file_id = "file-abc123" # Standard OpenAI file ID + + with patch('litellm.afile_content') as mock_afile_content: + mock_response = httpx.Response( + status_code=200, + content=batch_content, + headers={"content-type": "application/octet-stream"}, + ) + mock_afile_content.return_value = HttpxBinaryResponseContent(response=mock_response) + + # Call count_input_file_usage with non-managed file + usage = await batch_limiter.count_input_file_usage( + file_id=non_managed_file_id, + custom_llm_provider="openai", + user_api_key_dict=user_api_key_dict, + ) + + # Verify litellm.afile_content was called + assert mock_afile_content.called, \ + "REGRESSION: litellm.afile_content not called for non-managed files" + + print(" ✓ Standard file path used") + print(f" ✓ Token count: {usage.total_tokens}") + + # Test 5: Verify the fix prevents 403 errors + print("\n5. Verifying 403 error prevention...") + + # Simulate the bug scenario: managed files hook not being used + with patch.object(batch_limiter, '_fetch_managed_file_content') as mock_fetch: + # If this is NOT called for managed files, the bug has returned + mock_fetch.side_effect = Exception("Should not be called if bug exists") + + # This should call _fetch_managed_file_content + try: + with patch('litellm.afile_content') as mock_afile_content: + # If litellm.afile_content is called for managed files, bug exists + mock_afile_content.side_effect = Exception( + "Error code: 403 - User does not have access to the file" + ) + + # Reset mock_fetch to return valid content + mock_response = httpx.Response( + status_code=200, + content=batch_content, + headers={"content-type": "application/octet-stream"}, + ) + mock_fetch.side_effect = None + mock_fetch.return_value = HttpxBinaryResponseContent(response=mock_response) + + # This should use _fetch_managed_file_content, not litellm.afile_content + usage = await batch_limiter.count_input_file_usage( + file_id=managed_file_id, + custom_llm_provider="openai", + user_api_key_dict=user_api_key_dict, + ) + + # Verify managed files path was used (not standard path that causes 403) + assert mock_fetch.called, \ + "REGRESSION: Managed files path not used! This would cause 403 errors." + assert not mock_afile_content.called, \ + "REGRESSION: Standard path used for managed files! This causes 403 errors." + + print(" ✓ 403 error prevention verified") + + except Exception as e: + if "403" in str(e): + pytest.fail( + f"REGRESSION: 403 error occurred! Bug GEN-2166 has returned. Error: {str(e)}" + ) + raise + + print("\n=== Regression Test Passed ===") + print("✓ Bug GEN-2166 is fixed and protected against regression") + print("✓ Managed files are properly accessed via managed files hook") + print("✓ User context is correctly passed through") + print("✓ No 403 errors occur") + print("✓ Non-managed files still work correctly\n") + + +@pytest.mark.asyncio() +async def test_batch_logging_azure_credentials_regression(): + """ + Regression test: LoggingWorker Missing Azure Credentials When Fetching Batch Output + + This test ensures that Azure credentials are properly passed when fetching batch + output files during logging, preventing "Missing credentials" errors. + + Bug: The LoggingWorker failed when processing completed Azure batches because + it attempted to fetch batch output file content without Azure credentials. + + Fix: Pass litellm_params (containing credentials) from the logging object + through to the file content retrieval functions. + """ + from unittest.mock import AsyncMock, MagicMock, patch + from litellm.batches.batch_utils import ( + _extract_file_access_credentials, + _get_batch_output_file_content_as_dictionary, + _handle_completed_batch, + ) + from litellm.types.llms.openai import Batch, HttpxBinaryResponseContent + import httpx + + print("\n=== Regression Test: Azure Batch Logging Credentials ===") + + # Setup: Create mock batch with output file + mock_batch = Batch( + id="batch-azure-test", + object="batch", + endpoint="/v1/chat/completions", + errors=None, + input_file_id="file-input-azure", + completion_window="24h", + status="completed", + output_file_id="file-output-azure", + error_file_id=None, + created_at=1234567890, + in_progress_at=1234567900, + expires_at=1234654290, + finalizing_at=1234568000, + completed_at=1234568100, + failed_at=None, + expired_at=None, + cancelling_at=None, + cancelled_at=None, + request_counts=None, + metadata=None, + ) + + # Setup: Azure credentials (as they would be in litellm_params) + azure_credentials = { + "api_key": "test-azure-key-regression", + "api_base": "https://test-regression.openai.azure.com", + "api_version": "2024-02-15-preview", + "organization": "test-org", + "timeout": 600, + } + + # Setup: Mock batch output content + batch_output = b'{"id": "batch_req_1", "custom_id": "request-1", "response": {"status_code": 200, "body": {"id": "chatcmpl-azure", "object": "chat.completion", "model": "gpt-4", "usage": {"prompt_tokens": 15, "completion_tokens": 25, "total_tokens": 40}}}}\n' + + # Test 1: Verify _extract_file_access_credentials works correctly + print("\n1. Testing credential extraction...") + + extracted_creds = _extract_file_access_credentials(azure_credentials) + assert "api_key" in extracted_creds, "api_key should be extracted" + assert extracted_creds["api_key"] == "test-azure-key-regression", "Incorrect api_key" + assert "api_base" in extracted_creds, "api_base should be extracted" + assert "api_version" in extracted_creds, "api_version should be extracted" + assert "timeout" in extracted_creds, "timeout should be extracted" + + print(" ✓ Credentials extracted correctly") + print(f" ✓ Extracted keys: {list(extracted_creds.keys())}") + + # Test 2: Verify credentials are passed to afile_content + print("\n2. Testing credentials passed to afile_content...") + + credentials_received = {"value": False, "params": None} + + async def mock_afile_content_tracker(**kwargs): + # Track if Azure credentials were passed + if "api_key" in kwargs and "api_base" in kwargs and "api_version" in kwargs: + credentials_received["value"] = True + credentials_received["params"] = { + "api_key": kwargs.get("api_key"), + "api_base": kwargs.get("api_base"), + "api_version": kwargs.get("api_version"), + } + mock_response = httpx.Response( + status_code=200, + content=batch_output, + headers={"content-type": "application/octet-stream"}, + ) + return HttpxBinaryResponseContent(response=mock_response) + + with patch('litellm.files.main.afile_content', side_effect=mock_afile_content_tracker): + result = await _get_batch_output_file_content_as_dictionary( + batch=mock_batch, + custom_llm_provider="azure", + litellm_params=azure_credentials, + ) + + # Verify credentials were passed + assert credentials_received["value"], \ + "REGRESSION: Azure credentials not passed to afile_content! This causes 'Missing credentials' error." + assert credentials_received["params"]["api_key"] == "test-azure-key-regression", \ + "REGRESSION: Incorrect api_key" + assert credentials_received["params"]["api_base"] == "https://test-regression.openai.azure.com", \ + "REGRESSION: Incorrect api_base" + + print(" ✓ Credentials passed to afile_content") + print(f" ✓ api_key: {credentials_received['params']['api_key']}") + print(f" ✓ api_base: {credentials_received['params']['api_base']}") + + # Test 3: Verify full flow through _handle_completed_batch + print("\n3. Testing full logging flow...") + + credentials_received["value"] = False + credentials_received["params"] = None + + with patch('litellm.files.main.afile_content', side_effect=mock_afile_content_tracker): + cost, usage, models = await _handle_completed_batch( + batch=mock_batch, + custom_llm_provider="azure", + litellm_params=azure_credentials, + ) + + # Verify credentials were passed through the entire flow + assert credentials_received["value"], \ + "REGRESSION: Credentials not passed through _handle_completed_batch" + + # Verify cost and usage were calculated + assert cost > 0, "Cost should be calculated" + assert usage.total_tokens == 40, "Usage should be calculated correctly" + + print(" ✓ Credentials passed through full flow") + print(f" ✓ Cost: {cost}") + print(f" ✓ Usage: {usage.total_tokens} tokens") + print(f" ✓ Models: {models}") + + # Test 4: Verify error prevention + print("\n4. Testing 'Missing credentials' error prevention...") + + # Simulate the bug: if credentials are NOT passed, Azure would fail + with patch('litellm.files.main.afile_content') as mock_afile_content_fail: + # This is what would happen without the fix + mock_afile_content_fail.side_effect = Exception( + "Missing credentials. Please pass one of `api_key`, `azure_ad_token`, " + "`azure_ad_token_provider`, or the `AZURE_OPENAI_API_KEY` or " + "`AZURE_OPENAI_AD_TOKEN` environment variables." + ) + + # Now test with the fix - should NOT raise the error + with patch('litellm.files.main.afile_content', side_effect=mock_afile_content_tracker): + try: + cost, usage, models = await _handle_completed_batch( + batch=mock_batch, + custom_llm_provider="azure", + litellm_params=azure_credentials, + ) + print(" ✓ No 'Missing credentials' error with fix") + except Exception as e: + if "Missing credentials" in str(e): + pytest.fail( + f"REGRESSION: 'Missing credentials' error occurred! " + f"Credentials not being passed. Error: {str(e)}" + ) + raise + + # Test 5: Verify backwards compatibility (works without credentials for OpenAI) + print("\n5. Testing backwards compatibility...") + + with patch('litellm.files.main.afile_content') as mock_afile_content: + mock_response = httpx.Response( + status_code=200, + content=batch_output, + headers={"content-type": "application/octet-stream"}, + ) + mock_afile_content.return_value = HttpxBinaryResponseContent(response=mock_response) + + # Call without litellm_params (should still work for OpenAI) + result = await _get_batch_output_file_content_as_dictionary( + batch=mock_batch, + custom_llm_provider="openai", + litellm_params=None, + ) + + assert len(result) > 0, "Should return file content" + print(" ✓ Backwards compatibility maintained") + print(" ✓ Works without litellm_params for OpenAI") + + print("\n=== Regression Test Passed ===") + print("✓ Azure credentials properly passed from logging to file retrieval") + print("✓ 'Missing credentials' error prevented") + print("✓ Batch output files can be fetched with Azure credentials") + print("✓ Cost and usage tracking works for Azure batches") + print("✓ Backwards compatibility maintained\n") diff --git a/tests/code_coverage_tests/check_guardrail_apply_decorator.py b/tests/code_coverage_tests/check_guardrail_apply_decorator.py index 18a86277aa9..523b9f5b35b 100644 --- a/tests/code_coverage_tests/check_guardrail_apply_decorator.py +++ b/tests/code_coverage_tests/check_guardrail_apply_decorator.py @@ -83,6 +83,12 @@ def test_guardrail_apply_decorator(): if python_file.name == "bedrock_guardrails.py": continue + # Skip content_filter.py - it implements its own detailed logging via + # _log_guardrail_information with detections, masked_entity_count, etc. + # Using the decorator would cause duplicate entries. + if python_file.name == "content_filter.py": + continue + results = find_apply_guardrail_methods(python_file) for class_name, line_num, has_decorator in results: diff --git a/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py b/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py index 706e3b71870..24e27977963 100644 --- a/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py +++ b/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py @@ -182,6 +182,76 @@ class TestEnterpriseRouteChecks: EnterpriseRouteChecks.should_call_route("/config/update") +@patch("litellm.proxy.proxy_server.premium_user", True) +class TestEnterpriseRouteChecksModelListExemption: + """Test that /models and /v1/models are exempt from DISABLE_LLM_API_ENDPOINTS""" + + @patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled") + @patch.object(EnterpriseRouteChecks, "is_management_routes_disabled") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_llm_api_route") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_management_route") + def test_models_route_allowed_when_llm_api_disabled( + self, + mock_is_management_route, + mock_is_llm_api_route, + mock_is_management_disabled, + mock_is_llm_api_disabled, + ): + """Test that /models is allowed even when LLM API routes are disabled""" + mock_is_management_route.return_value = False + mock_is_llm_api_route.return_value = True + mock_is_management_disabled.return_value = False + mock_is_llm_api_disabled.return_value = True + + # Should not raise exception for /models + EnterpriseRouteChecks.should_call_route("/models") + + @patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled") + @patch.object(EnterpriseRouteChecks, "is_management_routes_disabled") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_llm_api_route") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_management_route") + def test_v1_models_route_allowed_when_llm_api_disabled( + self, + mock_is_management_route, + mock_is_llm_api_route, + mock_is_management_disabled, + mock_is_llm_api_disabled, + ): + """Test that /v1/models is allowed even when LLM API routes are disabled""" + mock_is_management_route.return_value = False + mock_is_llm_api_route.return_value = True + mock_is_management_disabled.return_value = False + mock_is_llm_api_disabled.return_value = True + + # Should not raise exception for /v1/models + EnterpriseRouteChecks.should_call_route("/v1/models") + + @patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled") + @patch.object(EnterpriseRouteChecks, "is_management_routes_disabled") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_llm_api_route") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_management_route") + def test_chat_completions_still_blocked_when_llm_api_disabled( + self, + mock_is_management_route, + mock_is_llm_api_route, + mock_is_management_disabled, + mock_is_llm_api_disabled, + ): + """Test that non-exempt LLM routes like /v1/chat/completions are still blocked""" + mock_is_management_route.return_value = False + mock_is_llm_api_route.return_value = True + mock_is_management_disabled.return_value = False + mock_is_llm_api_disabled.return_value = True + + with pytest.raises(HTTPException) as exc_info: + EnterpriseRouteChecks.should_call_route("/v1/chat/completions") + + assert exc_info.value.status_code == 403 + assert "LLM API routes are disabled for this instance." in str( + exc_info.value.detail + ) + + class TestEnterpriseRouteChecksErrorMessages: """Test that error messages correctly identify which feature requires Enterprise license""" diff --git a/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py b/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py new file mode 100644 index 00000000000..89da8d87e63 --- /dev/null +++ b/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py @@ -0,0 +1,64 @@ +""" +Tests for _map_reasoning_effort in AnthropicConfig. + +Verifies that reasoning_effort=None returns None for all models, +including Claude Opus 4.6. +""" + +from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + +class TestMapReasoningEffort: + def test_none_returns_none_for_opus_4_6(self): + """reasoning_effort=None should return None for Opus 4.6, not adaptive.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort=None, model="claude-opus-4-6" + ) + assert result is None + + def test_none_returns_none_for_other_models(self): + """reasoning_effort=None should return None for non-Opus models.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort=None, model="claude-3-7-sonnet-20250219" + ) + assert result is None + + def test_opus_4_6_returns_adaptive_for_low(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="low", model="claude-opus-4-6" + ) + assert result["type"] == "adaptive" + + def test_opus_4_6_returns_adaptive_for_high(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="high", model="claude-opus-4-6" + ) + assert result["type"] == "adaptive" + + def test_other_model_low_returns_enabled_with_budget(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="low", model="claude-3-7-sonnet-20250219" + ) + assert result["type"] == "enabled" + assert "budget_tokens" in result + + def test_other_model_high_returns_enabled_with_budget(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="high", model="claude-3-7-sonnet-20250219" + ) + assert result["type"] == "enabled" + assert "budget_tokens" in result + + def test_none_string_returns_none_for_opus_4_6(self): + """reasoning_effort='none' should return None for Opus 4.6.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="none", model="claude-opus-4-6" + ) + assert result is None + + def test_none_string_returns_none_for_other_models(self): + """reasoning_effort='none' should return None for non-Opus models.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="none", model="claude-3-7-sonnet-20250219" + ) + assert result is None diff --git a/tests/litellm/proxy/management_endpoints/test_common_utils.py b/tests/litellm/proxy/management_endpoints/test_common_utils.py new file mode 100644 index 00000000000..f857db770d0 --- /dev/null +++ b/tests/litellm/proxy/management_endpoints/test_common_utils.py @@ -0,0 +1,159 @@ +""" +Tests for litellm/proxy/management_endpoints/common_utils.py + +Specifically tests that _update_metadata_fields does not trigger premium +user checks when premium fields are present but empty. + +Related: https://github.com/BerriAI/litellm/issues/20534 +""" + +from unittest.mock import patch + +import pytest + +from litellm.proxy.management_endpoints.common_utils import ( + _has_non_empty_value, + _update_metadata_fields, +) + + +class TestHasNonEmptyValue: + """Tests for the _has_non_empty_value helper.""" + + def test_none_is_empty(self): + assert _has_non_empty_value(None) is False + + def test_empty_list_is_empty(self): + assert _has_non_empty_value([]) is False + + def test_empty_string_is_empty(self): + assert _has_non_empty_value("") is False + + def test_blank_string_is_empty(self): + assert _has_non_empty_value(" ") is False + + def test_non_empty_list_has_value(self): + assert _has_non_empty_value(["policy-a"]) is True + + def test_non_empty_string_has_value(self): + assert _has_non_empty_value("30d") is True + + def test_dict_has_value(self): + assert _has_non_empty_value({"key": "val"}) is True + + def test_empty_dict_has_value(self): + # empty dict is not None/list/str, so it counts as non-empty + assert _has_non_empty_value({}) is True + + +class TestUpdateMetadataFieldsPremiumCheck: + """ + Tests that _update_metadata_fields skips premium user checks for empty + values but still enforces them for real values. + + Issue: The UI sends the full form on every team update, including premium + fields like `policies: []`. The backend was treating these empty values + as premium feature usage and returning 403. + """ + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_empty_policies_skips_premium_check(self, mock_check): + """policies: [] should NOT trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "team_alias": "my-team", + "policies": [], + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_empty_guardrails_skips_premium_check(self, mock_check): + """guardrails: [] should NOT trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "guardrails": [], + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_empty_string_team_member_key_duration_skips_premium_check( + self, mock_check + ): + """team_member_key_duration: '' should NOT trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "team_member_key_duration": "", + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_full_ui_payload_with_empty_premium_fields_skips_premium_check( + self, mock_check + ): + """A realistic UI payload with all empty premium fields should not 403.""" + updated_kv = { + "team_id": "team-123", + "team_alias": "renamed-team", + "models": ["gpt-4o"], + "max_budget": 200, + "policies": [], + "guardrails": [], + "logging": [], + "team_member_key_duration": "", + "prompts": [], + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + ) + def test_non_empty_policies_triggers_premium_check(self, mock_check): + """policies: ['real-policy'] SHOULD trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "policies": ["real-policy"], + } + _update_metadata_fields(updated_kv) + mock_check.assert_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + ) + def test_non_empty_guardrails_triggers_premium_check(self, mock_check): + """guardrails: ['my-guardrail'] SHOULD trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "guardrails": ["my-guardrail"], + } + _update_metadata_fields(updated_kv) + mock_check.assert_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + ) + def test_non_empty_team_member_key_duration_triggers_premium_check( + self, mock_check + ): + """team_member_key_duration: '30d' SHOULD trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "team_member_key_duration": "30d", + } + _update_metadata_fields(updated_kv) + mock_check.assert_called() diff --git a/tests/litellm/test_bedrock_extended_beta_models.py b/tests/litellm/test_bedrock_extended_beta_models.py new file mode 100644 index 00000000000..ebbbd6cab5c --- /dev/null +++ b/tests/litellm/test_bedrock_extended_beta_models.py @@ -0,0 +1,170 @@ +""" +Test suite for AWS Bedrock extended beta model support +Tests model configuration, pricing, and regional availability for: +- DeepSeek V3.2 +- Minimax M2.1 +- Moonshot AI Kimi K2.5 +- Qwen3 Coder Next +""" + +import os + +# Set env var to use local model cost map instead of fetching from remote +os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "true" + +import pytest + +from litellm import get_model_info + +# Model configurations: (model_name, regions, max_input, max_output) +MODEL_CONFIGS = [ + ( + "deepseek.v3.2", + [ + "ap-northeast-1", + "ap-south-1", + "ap-southeast-3", + "eu-north-1", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-2", + ], + 163840, + 163840, + ), + ( + "minimax.minimax-m2.1", + [ + "ap-northeast-1", + "ap-south-1", + "ap-southeast-3", + "eu-central-1", + "eu-north-1", + "eu-south-1", + "eu-west-1", + "eu-west-2", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-2", + ], + 196000, + 8192, + ), + ( + "moonshotai.kimi-k2.5", + [ + "ap-northeast-1", + "ap-south-1", + "ap-southeast-3", + "eu-north-1", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-2", + ], + 262144, + 262144, + ), + ( + "qwen.qwen3-coder-next", + [ + "ap-northeast-1", + "ap-south-1", + "ap-southeast-3", + "eu-central-1", + "eu-south-1", + "eu-west-1", + "eu-west-2", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-2", + ], + 262144, + 8192, + ), +] + + +class TestBedrockNewModels: + """Unified test suite for all new Bedrock models""" + + @pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS) + def test_model_info_primary_region( + self, model_name, regions, max_input, max_output + ): + """Test model configuration in primary region (us-east-1)""" + model = f"bedrock/us-east-1/{model_name}" + model_info = get_model_info(model) + + assert model_info is not None, f"Model {model_name} not found" + assert model_info["max_input_tokens"] == max_input + assert model_info["max_output_tokens"] == max_output + assert model_info["litellm_provider"] == "bedrock" + assert model_info["mode"] == "chat" + assert model_info["supports_function_calling"] is True + + @pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS) + def test_pricing_configured(self, model_name, regions, max_input, max_output): + """Verify pricing is set for all models""" + model = f"bedrock/us-east-1/{model_name}" + model_info = get_model_info(model) + + assert ( + model_info["input_cost_per_token"] > 0 + ), f"Missing input cost for {model_name}" + assert ( + model_info["output_cost_per_token"] > 0 + ), f"Missing output cost for {model_name}" + + @pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS) + def test_region_count(self, model_name, regions, max_input, max_output): + """Verify each bedrock/{region}/{model_name} resolves via get_model_info""" + for region in regions: + model = f"bedrock/{region}/{model_name}" + model_info = get_model_info(model) + assert model_info is not None, f"Model {model_name} not found in {region}" + assert model_info["max_input_tokens"] == max_input + assert model_info["max_output_tokens"] == max_output + + @pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS) + def test_sample_regional_variants(self, model_name, regions, max_input, max_output): + """Test sample regional variants (us-east-1, eu-west-1, ap-northeast-1)""" + for region in ["us-east-1", "ap-northeast-1"]: + if region in regions: + model = f"bedrock/{region}/{model_name}" + model_info = get_model_info(model) + assert ( + model_info is not None + ), f"Model {model_name} not found in {region}" + assert model_info["max_input_tokens"] == max_input + assert model_info["litellm_provider"] == "bedrock" + + +class TestModelSpecificFeatures: + """Model-specific capability tests""" + + def test_deepseek_v3_2_context_window(self): + """DeepSeek V3.2 has 163K context window""" + model_info = get_model_info("bedrock/us-east-1/deepseek.v3.2") + assert model_info["max_input_tokens"] == 163840 + + def test_minimax_m2_1_context_window(self): + """Minimax M2.1 has 196K input, 8K output""" + model_info = get_model_info("bedrock/us-east-1/minimax.minimax-m2.1") + assert model_info["max_input_tokens"] == 196000 + assert model_info["max_output_tokens"] == 8192 + + def test_moonshotai_kimi_k2_5_context_window(self): + """Moonshot AI Kimi K2.5 has 256K context window""" + model_info = get_model_info("bedrock/us-east-1/moonshotai.kimi-k2.5") + assert model_info["max_input_tokens"] == 262144 + assert model_info["max_output_tokens"] == 262144 + + def test_qwen3_coder_next_context_window(self): + """Qwen3 Coder Next has 256K input, 8K output""" + model_info = get_model_info("bedrock/us-east-1/qwen.qwen3-coder-next") + assert model_info["max_input_tokens"] == 262144 + assert model_info["max_output_tokens"] == 8192 diff --git a/tests/litellm/test_router_retry_backoff_headers.py b/tests/litellm/test_router_retry_backoff_headers.py new file mode 100644 index 00000000000..03c3af692ce --- /dev/null +++ b/tests/litellm/test_router_retry_backoff_headers.py @@ -0,0 +1,88 @@ +""" +Tests for router retry backoff behavior. +""" + +from unittest.mock import patch + +import httpx +import pytest + +import litellm +from litellm import Router + + +@pytest.mark.asyncio +async def test_retry_backoff_uses_current_exception_headers(): + """ + Ensure retry backoff uses the current retry exception, not the initial one. + """ + router = Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "sk-test", + }, + } + ], + num_retries=2, + ) + + first_error = litellm.RateLimitError( + message="Rate limited on first attempt", + model="gpt-3.5-turbo", + llm_provider="openai", + ) + first_error.litellm_response_headers = httpx.Headers({"retry-after": "1"}) + + second_error = litellm.RateLimitError( + message="Rate limited on second attempt", + model="gpt-3.5-turbo", + llm_provider="openai", + ) + second_error.litellm_response_headers = httpx.Headers({"retry-after": "15"}) + + third_error = litellm.RateLimitError( + message="Rate limited on third attempt", + model="gpt-3.5-turbo", + llm_provider="openai", + ) + third_error.litellm_response_headers = httpx.Headers({"retry-after": "30"}) + + raised_errors = [first_error, second_error, third_error] + captured_backoff_errors = [] + + async def mock_make_call(*args, **kwargs): + raise raised_errors.pop(0) + + def mock_time_to_sleep_before_retry(*args, **kwargs): + captured_backoff_errors.append(kwargs["e"]) + return 0.01 + + with patch.object(router, "make_call", side_effect=mock_make_call): + with patch.object( + router, + "_async_get_healthy_deployments", + return_value=( + [{"model_info": {"id": "test-id"}}], + [{"model_info": {"id": "test-id"}}], + ), + ): + with patch.object( + router, + "_time_to_sleep_before_retry", + side_effect=mock_time_to_sleep_before_retry, + ): + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello"}], + ) + + # Router computes backoff once after the initial failure, then once per failed retry. + # With num_retries=2 and all attempts failing, that's 1 + 2 = 3 invocations. + assert len(captured_backoff_errors) == router.num_retries + 1 + assert captured_backoff_errors[0] is first_error + assert captured_backoff_errors[1] is second_error + assert captured_backoff_errors[2] is third_error diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 37ed1a9b08c..7ac83753d17 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -2,7 +2,7 @@ import httpx import json import pytest import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from unittest.mock import MagicMock, Mock, patch import os from litellm._uuid import uuid @@ -114,6 +114,10 @@ class BaseResponsesAPITest(ABC): """Must return the base completion reasoning call args""" return None + def get_advanced_model_for_shell_tool(self) -> Optional[str]: + """If specified, overrides the model used by test_responses_api_shell_tool_streaming_sees_shell_output (e.g. openai/gpt-5.2 for shell support).""" + return None + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_basic_openai_responses_api(self, sync_mode): @@ -669,7 +673,7 @@ class BaseResponsesAPITest(ABC): async def test_cancel_responses_invalid_response_id(self, sync_mode): """Test cancel_responses with invalid response ID should raise appropriate error""" base_completion_call_args = self.get_base_completion_call_args() - + if sync_mode: with pytest.raises(Exception): litellm.cancel_responses( @@ -679,4 +683,128 @@ class BaseResponsesAPITest(ABC): with pytest.raises(Exception): await litellm.acancel_responses( response_id="invalid_response_id_12345", **base_completion_call_args - ) \ No newline at end of file + ) + + @pytest.mark.asyncio + async def test_responses_api_context_management_server_side_compaction(self): + """ + E2E test for server-side compaction (context_management) on OpenAI Responses API. + Passes context_management with compact_threshold; validates that the request is + accepted and returns a valid response. Compaction may not run for short inputs. + """ + base_completion_call_args = self.get_base_completion_call_args() + model = base_completion_call_args.get("model") or "" + # Only run with context_management for OpenAI (OAI) for now + if "openai/" not in str(model) and "azure/" not in str(model): + pytest.skip( + "context_management server-side compaction e2e is only run for OpenAI/Azure" + ) + context_management = [{"type": "compaction", "compact_threshold": 200000}] + try: + response = await litellm.aresponses( + input="Short ping to verify context_management is accepted.", + max_output_tokens=20, + context_management=context_management, + **base_completion_call_args, + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + validate_responses_api_response(response, final_chunk=True) + assert response.get("id") is not None + assert response.get("status") is not None + + @pytest.mark.asyncio + async def test_responses_api_shell_tool(self): + """ + E2E test for Shell tool on OpenAI Responses API. + Passes tools=[{"type": "shell", "environment": {"type": "container_auto"}}]; + validates that the request is accepted and returns a valid response. + Only runs for OpenAI/Azure (Responses API with shell support). + """ + base_completion_call_args = self.get_base_completion_call_args() + model = self.get_advanced_model_for_shell_tool() or base_completion_call_args.get( + "model" + ) or "" + if "openai/" not in str(model) and "azure/" not in str(model): + pytest.skip( + "Shell tool e2e is only run for OpenAI/Azure Responses API" + ) + tools = [{"type": "shell", "environment": {"type": "container_auto"}}] + input_msg = "List files in /mnt/data and show python --version." + try: + response = await litellm.aresponses( + **{**base_completion_call_args, "model": model}, + input=input_msg, + max_output_tokens=256, + tools=tools, + tool_choice="auto", + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + except litellm.BadRequestError as e: + if "shell" in str(e).lower() and "not supported" in str(e).lower(): + pytest.skip( + "Shell tool is not supported for this model (e.g. gpt-4o); use a model that supports shell" + ) + raise + validate_responses_api_response(response, final_chunk=True) + assert response.get("id") is not None + assert response.get("status") is not None + + @pytest.mark.asyncio + async def test_responses_api_shell_tool_streaming_sees_shell_output(self): + """ + E2E streaming call with Shell tool; validate we can see shell output in the stream. + + Calls aresponses(..., tools=[shell], stream=True), then iterates the stream and + asserts at least one event is shell-related or response output contains shell_call. + Skips when model does not support shell (e.g. gpt-4o). + """ + base_completion_call_args = self.get_base_completion_call_args() + model = self.get_advanced_model_for_shell_tool() or base_completion_call_args.get( + "model" + ) or "openai/gpt-5.2" + tools = [{"type": "shell", "environment": {"type": "container_auto"}}] + input_msg = "List files in /mnt/data and run python --version." + + stream = await litellm.aresponses( + **{**base_completion_call_args, "model": model}, + input=input_msg, + max_output_tokens=512, + tools=tools, + tool_choice="auto", + stream=True, + ) + + + event_types_seen = [] + output_items_with_shell = [] + + async for event in stream: + print("event=", json.dumps(event, indent=4, default=str)) + event_type = getattr(event, "type", None) or ( + event.get("type") if isinstance(event, dict) else None + ) + if event_type is not None: + event_types_seen.append(str(event_type)) + if "shell" in str(event_type or "").lower(): + output_items_with_shell.append(event_type) + response_obj = getattr(event, "response", None) or ( + event.get("response") if isinstance(event, dict) else None + ) + if response_obj is not None: + output = getattr(response_obj, "output", None) or ( + response_obj.get("output") if isinstance(response_obj, dict) else None + ) + if isinstance(output, list): + for item in output: + item_type = getattr(item, "type", None) or ( + item.get("type") if isinstance(item, dict) else None + ) + if item_type and "shell" in str(item_type).lower(): + output_items_with_shell.append(item_type) + + assert len(event_types_seen) > 0, "Expected at least one stream event" + assert len(output_items_with_shell) > 0, ( + f"Expected to see shell output in stream; event types seen: {event_types_seen!r}" + ) diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 5f35d6837c0..4972aa385ce 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -36,6 +36,9 @@ class TestOpenAIResponsesAPITest(BaseResponsesAPITest): "model": "openai/gpt-5-mini", } + def get_advanced_model_for_shell_tool(self): + return "openai/gpt-5.2" + class TestCustomLogger(CustomLogger): def __init__( diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 6a2ad406788..eff11c9cee3 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -519,7 +519,7 @@ async def test_openai_codex_stream(sync_mode): from litellm.main import stream_chunk_builder kwargs = { - "model": "openai/codex-mini-latest", + "model": "openai/gpt-5-codex-mini", "messages": [{"role": "user", "content": "Hey!"}], "stream": True, } @@ -549,16 +549,16 @@ async def test_openai_codex(sync_mode): router = Router( model_list=[ { - "model_name": "openai-codex-mini-latest", + "model_name": "openai-gpt-5-codex-mini", "litellm_params": { - "model": "openai/codex-mini-latest", + "model": "openai/gpt-5-codex-mini", }, } ] ) kwargs = { - "model": "openai-codex-mini-latest", + "model": "openai-gpt-5-codex-mini", "messages": [{"role": "user", "content": "Hey!"}], } diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index c322db157e9..26a93343caf 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -2189,7 +2189,7 @@ def test_completion_openrouter1(): try: litellm.set_verbose = True response = completion( - model="openrouter/mistralai/mistral-tiny", + model="openrouter/mistralai/ministral-8b", messages=messages, max_tokens=5, ) diff --git a/tests/local_testing/test_scheduler.py b/tests/local_testing/test_scheduler.py index f5b44224853..f198e572b21 100644 --- a/tests/local_testing/test_scheduler.py +++ b/tests/local_testing/test_scheduler.py @@ -117,3 +117,41 @@ async def test_scheduler_prioritized_requests(p0, p1, healthy_deployments): ) == False ) + + +@pytest.mark.asyncio +async def test_scheduler_queue_cleanup_on_timeout(): + """ + Test that a timed-out request is properly removed from the queue. + This prevents memory leaks from accumulating timed-out requests. + """ + scheduler = Scheduler() + + # Add multiple requests with different priorities + item1 = FlowItem(priority=0, request_id="req-0", model_name="gpt-3.5-turbo") + item2 = FlowItem(priority=1, request_id="req-1", model_name="gpt-3.5-turbo") + item3 = FlowItem(priority=2, request_id="req-2", model_name="gpt-3.5-turbo") + + await scheduler.add_request(item1) + await scheduler.add_request(item2) + await scheduler.add_request(item3) + + # Verify initial queue size + queue_before = await scheduler.get_queue(model_name="gpt-3.5-turbo") + assert len(queue_before) == 3, f"Expected 3 items in queue, got {len(queue_before)}" + + # Simulate timeout cleanup - remove a non-front request (item2) + await scheduler.remove_request(request_id="req-1", model_name="gpt-3.5-turbo") + + # Verify queue was cleaned up + queue_after = await scheduler.get_queue(model_name="gpt-3.5-turbo") + assert len(queue_after) == 2, f"Expected 2 items after cleanup, got {len(queue_after)}" + + # Verify the correct request was removed + remaining_ids = [item[1] for item in queue_after] + assert "req-1" not in remaining_ids, "Expected req-1 to be removed" + assert "req-0" in remaining_ids, "Expected req-0 to remain" + assert "req-2" in remaining_ids, "Expected req-2 to remain" + + # Verify remaining items are in correct priority order (0 should be first) + assert queue_after[0][1] == "req-0", "Expected req-0 (priority 0) to be at front" diff --git a/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py new file mode 100644 index 00000000000..d3c4ac80565 --- /dev/null +++ b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py @@ -0,0 +1,415 @@ +""" +Tests for standard_logging_payload_excluded_fields feature. + +This feature allows users to exclude specific fields from StandardLoggingPayload +before any callback receives it. This is useful for: +- Reducing log sizes (excluding large fields like 'response' or 'messages') +- Privacy compliance (excluding sensitive fields) +- Cost management (less data stored/transmitted) + +Example config: + litellm_settings: + success_callback: ["s3"] + standard_logging_payload_excluded_fields: ["response", "messages"] +""" + +import os +import sys +from copy import deepcopy +from typing import Dict, List, Optional +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import StandardLoggingPayload + + +def create_sample_standard_logging_payload() -> Dict: + """Create a sample StandardLoggingPayload for testing.""" + return { + "id": "test-id-123", + "trace_id": "trace-123", + "call_type": "completion", + "stream": False, + "response_cost": 0.001, + "cost_breakdown": None, + "response_cost_failure_debug_info": None, + "status": "success", + "status_fields": {}, + "custom_llm_provider": "openai", + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + "startTime": 1234567890.0, + "endTime": 1234567891.0, + "completionStartTime": 1234567890.5, + "response_time": 1.0, + "model_map_information": {}, + "model": "gpt-4", + "model_id": "model-123", + "model_group": None, + "api_base": "https://api.openai.com/v1", + "metadata": {}, + "cache_hit": False, + "cache_key": None, + "saved_cache_cost": 0.0, + "request_tags": [], + "end_user": None, + "requester_ip_address": None, + "user_agent": None, + "messages": [{"role": "user", "content": "Hello, this is sensitive data!"}], + "response": { + "choices": [ + {"message": {"content": "This is a sensitive response!"}} + ] + }, + "error_str": None, + "error_information": None, + "model_parameters": {}, + "hidden_params": {}, + "guardrail_information": None, + "standard_built_in_tools_params": None, + } + + +def create_model_call_details( + standard_logging_payload: Optional[Dict] = None, +) -> Dict: + """Create model_call_details dict with standard_logging_object.""" + if standard_logging_payload is None: + standard_logging_payload = create_sample_standard_logging_payload() + return { + "standard_logging_object": standard_logging_payload, + "other_key": "other_value", + } + + +class TestStandardLoggingPayloadExcludedFields: + """Test suite for standard_logging_payload_excluded_fields feature.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def test_no_excluded_fields_no_change(self): + """Test that payload is unchanged when no fields are excluded.""" + logger = CustomLogger() + model_call_details = create_model_call_details() + original_keys = set(model_call_details["standard_logging_object"].keys()) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + result_keys = set(result["standard_logging_object"].keys()) + assert result_keys == original_keys + + def test_exclude_single_field(self): + """Test excluding a single field (response).""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" in result["standard_logging_object"] + assert "model" in result["standard_logging_object"] + + def test_exclude_multiple_fields(self): + """Test excluding multiple fields (response, messages).""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" not in result["standard_logging_object"] + assert "model" in result["standard_logging_object"] + assert "model_parameters" in result["standard_logging_object"] + + def test_exclude_metadata_field(self): + """Test excluding the metadata field.""" + litellm.standard_logging_payload_excluded_fields = ["metadata"] + + logger = CustomLogger() + payload = create_sample_standard_logging_payload() + payload["metadata"] = {"sensitive_key": "sensitive_value"} + model_call_details = create_model_call_details(payload) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "metadata" not in result["standard_logging_object"] + + def test_exclude_hidden_params(self): + """Test excluding hidden_params field.""" + litellm.standard_logging_payload_excluded_fields = ["hidden_params"] + + logger = CustomLogger() + payload = create_sample_standard_logging_payload() + payload["hidden_params"] = {"api_key": "sk-secret-key"} + model_call_details = create_model_call_details(payload) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "hidden_params" not in result["standard_logging_object"] + + def test_exclude_nonexistent_field_no_error(self): + """Test that excluding a non-existent field doesn't cause an error.""" + litellm.standard_logging_payload_excluded_fields = [ + "nonexistent_field", + "response", + ] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + # Should not raise an exception + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" in result["standard_logging_object"] + + def test_original_payload_not_modified(self): + """Test that the original model_call_details is not modified.""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + original_payload = deepcopy(model_call_details) + + logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # Original should still have the fields + assert "response" in model_call_details["standard_logging_object"] + assert "messages" in model_call_details["standard_logging_object"] + assert model_call_details == original_payload + + def test_combined_with_turn_off_message_logging(self): + """Test that excluded_fields works together with turn_off_message_logging.""" + litellm.standard_logging_payload_excluded_fields = ["metadata", "hidden_params"] + + logger = CustomLogger(turn_off_message_logging=True) + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # excluded_fields should remove these + assert "metadata" not in result["standard_logging_object"] + assert "hidden_params" not in result["standard_logging_object"] + + # turn_off_message_logging should redact these + redacted_str = "redacted-by-litellm" + assert ( + result["standard_logging_object"]["messages"][0]["content"] == redacted_str + ) + assert ( + result["standard_logging_object"]["response"]["choices"][0]["message"][ + "content" + ] + == redacted_str + ) + + def test_excluded_fields_takes_precedence_over_redaction(self): + """Test that if a field is both excluded and would be redacted, it's excluded.""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger(turn_off_message_logging=True) + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # response should be excluded (not redacted) + assert "response" not in result["standard_logging_object"] + + # messages should still be redacted + redacted_str = "redacted-by-litellm" + assert ( + result["standard_logging_object"]["messages"][0]["content"] == redacted_str + ) + + def test_exclude_all_sensitive_fields(self): + """Test excluding all potentially sensitive fields.""" + litellm.standard_logging_payload_excluded_fields = [ + "messages", + "response", + "metadata", + "hidden_params", + "model_parameters", + "error_str", + "error_information", + ] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + standard_obj = result["standard_logging_object"] + + # All sensitive fields should be removed + assert "messages" not in standard_obj + assert "response" not in standard_obj + assert "metadata" not in standard_obj + assert "hidden_params" not in standard_obj + assert "model_parameters" not in standard_obj + assert "error_str" not in standard_obj + assert "error_information" not in standard_obj + + # Non-sensitive fields should remain + assert "id" in standard_obj + assert "model" in standard_obj + assert "response_cost" in standard_obj + assert "total_tokens" in standard_obj + + def test_empty_excluded_fields_list(self): + """Test that an empty list doesn't affect the payload.""" + litellm.standard_logging_payload_excluded_fields = [] + + logger = CustomLogger() + model_call_details = create_model_call_details() + original_keys = set(model_call_details["standard_logging_object"].keys()) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + result_keys = set(result["standard_logging_object"].keys()) + assert result_keys == original_keys + + def test_none_standard_logging_object(self): + """Test handling when standard_logging_object is None.""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger() + model_call_details = {"other_key": "other_value"} + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # Should return unchanged when no standard_logging_object + assert result == model_call_details + + +class TestExcludedFieldsIntegration: + """Integration tests for excluded fields with actual callbacks.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + litellm.callbacks = [] + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + litellm.callbacks = [] + + def test_custom_callback_receives_filtered_payload(self): + """Test that a custom callback receives the filtered payload.""" + captured_payloads = [] + + class TestCallback(CustomLogger): + def log_success_event(self, kwargs, response_obj, start_time, end_time): + captured_payloads.append(kwargs.get("standard_logging_object", {})) + + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + callback = TestCallback() + model_call_details = create_model_call_details() + + # Simulate what litellm_logging.py does + filtered_details = callback.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + callback.log_success_event( + kwargs=filtered_details, + response_obj=None, + start_time=None, + end_time=None, + ) + + assert len(captured_payloads) == 1 + assert "response" not in captured_payloads[0] + assert "messages" not in captured_payloads[0] + assert "model" in captured_payloads[0] + + +class TestExcludedFieldsConfigLoading: + """Test that the config is properly loaded from litellm_settings.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def test_config_attribute_exists(self): + """Test that the config attribute exists on litellm module.""" + assert hasattr(litellm, "standard_logging_payload_excluded_fields") + + def test_config_default_is_none(self): + """Test that the default value is None.""" + # Reset to ensure we're testing the default + litellm.standard_logging_payload_excluded_fields = None + assert litellm.standard_logging_payload_excluded_fields is None + + def test_config_can_be_set_to_list(self): + """Test that the config can be set to a list.""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + assert litellm.standard_logging_payload_excluded_fields == [ + "response", + "messages", + ] + + def test_config_setattr_simulates_proxy_loading(self): + """Test that setattr works as the proxy would use it.""" + # Simulating how proxy_server.py sets litellm_settings + config_value = ["response", "messages", "metadata"] + setattr(litellm, "standard_logging_payload_excluded_fields", config_value) + + assert litellm.standard_logging_payload_excluded_fields == config_value + + # Test it actually works in the logger + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" not in result["standard_logging_object"] + assert "metadata" not in result["standard_logging_object"] diff --git a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py index 345c5b2f2ac..f8dba78798d 100644 --- a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py +++ b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py @@ -293,3 +293,29 @@ def test_get_combined_callback_list(): assert "lago" in _logging.get_combined_callback_list( dynamic_success_callbacks=["langfuse"], global_callbacks=["lago"] ) + + +def test_get_combined_callback_list_returns_copy_when_dynamic_is_none(): + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + + _logging = LiteLLMLoggingObj( + model="claude-3-opus-20240229", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="123", + function_id="456", + ) + + global_callbacks = ["langfuse"] + combined_callbacks = _logging.get_combined_callback_list( + dynamic_success_callbacks=None, global_callbacks=global_callbacks + ) + + assert combined_callbacks == ["langfuse"] + assert combined_callbacks is not global_callbacks + + combined_callbacks.append("new_callback") + + assert global_callbacks == ["langfuse"] diff --git a/tests/mcp_tests/test_aresponses_api_with_mcp.py b/tests/mcp_tests/test_aresponses_api_with_mcp.py index a7bbfef14af..bae0b15dfec 100644 --- a/tests/mcp_tests/test_aresponses_api_with_mcp.py +++ b/tests/mcp_tests/test_aresponses_api_with_mcp.py @@ -683,8 +683,8 @@ async def test_streaming_responses_api_with_mcp_tools( Return the user the result of request 2 """ - # Skip test if required API keys are not set - if ("anthropic" in model.lower() or "claude" in model.lower()) and not os.getenv("ANTHROPIC_API_KEY"): + # Skip test if API keys are not set for the respective models + if ("claude" in model.lower() or "anthropic" in model.lower()) and not os.getenv("ANTHROPIC_API_KEY"): pytest.skip("ANTHROPIC_API_KEY not set, skipping anthropic model test") if ("gpt" in model.lower() or "openai" in model.lower()) and not os.getenv("OPENAI_API_KEY"): pytest.skip("OPENAI_API_KEY not set, skipping openai model test") diff --git a/tests/mcp_tests/test_mcp_chat_completions.py b/tests/mcp_tests/test_mcp_chat_completions.py index 0617dcb2e42..9010e8c0d29 100644 --- a/tests/mcp_tests/test_mcp_chat_completions.py +++ b/tests/mcp_tests/test_mcp_chat_completions.py @@ -19,7 +19,7 @@ async def test_acompletion_mcp_auto_exec(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -95,7 +95,7 @@ async def test_acompletion_mcp_respects_manual_approval(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -170,7 +170,7 @@ async def test_completion_mcp_with_streaming_no_timeout_error(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -470,7 +470,7 @@ async def test_mcp_metadata_in_streaming_final_chunk(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -793,7 +793,7 @@ async def test_mcp_streaming_metadata_ordering(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index ea823df1fb2..9718d714cfe 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -1053,6 +1053,7 @@ async def test_mcp_server_manager_access_groups_from_config(): mcp_server_manager_mod.global_mcp_server_manager = original_manager +@pytest.mark.asyncio async def test_mcp_server_manager_config_integration_with_database(): """ Test that config-based servers properly integrate with database servers, diff --git a/tests/proxy_unit_tests/test_ui_path_detection.py b/tests/proxy_unit_tests/test_ui_path_detection.py new file mode 100644 index 00000000000..72ee7770f94 --- /dev/null +++ b/tests/proxy_unit_tests/test_ui_path_detection.py @@ -0,0 +1,157 @@ +""" +Unit tests for UI path detection and configuration. + +Tests the new LITELLM_UI_PATH and LITELLM_ASSETS_PATH functionality +for read-only filesystem support. + +Note: Tests involving proxy_server imports are intentionally minimal +to avoid long module load times during testing. +""" + +import os +import tempfile +from pathlib import Path +from unittest import mock + +import pytest + + +class TestUIPathEnvironmentVariable: + """Test LITELLM_UI_PATH environment variable handling.""" + + def test_custom_ui_path_env_var(self): + """Test that LITELLM_UI_PATH overrides default.""" + custom_path = "/custom/ui/path" + + with mock.patch.dict( + os.environ, {"LITELLM_UI_PATH": custom_path, "LITELLM_NON_ROOT": "true"} + ): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_runtime_ui_path = ( + "/var/lib/litellm/ui" if is_non_root else "/default/packaged/path" + ) + runtime_ui_path = os.getenv("LITELLM_UI_PATH", default_runtime_ui_path) + + assert runtime_ui_path == custom_path + + def test_default_ui_path_non_root(self): + """Test default UI path in non-root mode.""" + with mock.patch.dict( + os.environ, {"LITELLM_NON_ROOT": "true"}, clear=False + ): + # Clear LITELLM_UI_PATH if it exists + env_copy = os.environ.copy() + if "LITELLM_UI_PATH" in env_copy: + del env_copy["LITELLM_UI_PATH"] + + with mock.patch.dict(os.environ, env_copy, clear=True): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_runtime_ui_path = ( + "/var/lib/litellm/ui" + if is_non_root + else "/default/packaged/path" + ) + runtime_ui_path = os.getenv( + "LITELLM_UI_PATH", default_runtime_ui_path + ) + + assert runtime_ui_path == "/var/lib/litellm/ui" + + +class TestAssetsPathEnvironmentVariable: + """Test LITELLM_ASSETS_PATH environment variable handling.""" + + def test_custom_assets_path_env_var(self): + """Test that LITELLM_ASSETS_PATH overrides default.""" + custom_path = "/custom/assets/path" + + with mock.patch.dict( + os.environ, + {"LITELLM_ASSETS_PATH": custom_path, "LITELLM_NON_ROOT": "true"}, + ): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_assets_dir = ( + "/var/lib/litellm/assets" if is_non_root else "/default/current/dir" + ) + assets_dir = os.getenv("LITELLM_ASSETS_PATH", default_assets_dir) + + assert assets_dir == custom_path + + def test_default_assets_path_non_root(self): + """Test default assets path in non-root mode.""" + env_copy = os.environ.copy() + env_copy["LITELLM_NON_ROOT"] = "true" + if "LITELLM_ASSETS_PATH" in env_copy: + del env_copy["LITELLM_ASSETS_PATH"] + + with mock.patch.dict(os.environ, env_copy, clear=True): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_assets_dir = ( + "/var/lib/litellm/assets" if is_non_root else "/default/current/dir" + ) + assets_dir = os.getenv("LITELLM_ASSETS_PATH", default_assets_dir) + + assert assets_dir == "/var/lib/litellm/assets" + + +class TestUIDetectionLogic: + """Test UI pre-restructured detection logic without importing proxy_server.""" + + def setup_method(self): + """Create temporary directory for testing.""" + self.temp_dir = tempfile.mkdtemp() + + def teardown_method(self): + """Clean up temporary directory.""" + import shutil + + if os.path.exists(self.temp_dir): + shutil.rmtree(self.temp_dir) + + def test_marker_file_exists(self): + """Test marker file detection logic.""" + marker_path = os.path.join(self.temp_dir, ".litellm_ui_ready") + Path(marker_path).touch() + + # Verify marker file exists + assert os.path.exists(marker_path) + + def test_structural_routes_exist(self): + """Test structural detection logic.""" + routes = ["login", "guardrails", "logs"] + for route in routes: + route_dir = os.path.join(self.temp_dir, route) + os.makedirs(route_dir, exist_ok=True) + index_html = os.path.join(route_dir, "index.html") + Path(index_html).touch() + + # Verify routes exist + found_routes = 0 + expected_routes = ["login", "guardrails", "logs", "api-reference"] + for route in expected_routes: + route_index = os.path.join(self.temp_dir, route, "index.html") + if os.path.exists(route_index): + found_routes += 1 + + assert found_routes >= 3 + + def test_writability_check(self): + """Test that os.access() correctly detects writable directories.""" + # Should be writable + assert os.access(self.temp_dir, os.W_OK) is True + + # Create a directory we can't write to (platform-dependent) + if os.name != "nt": # Skip on Windows + readonly_dir = os.path.join(self.temp_dir, "readonly") + os.makedirs(readonly_dir) + os.chmod(readonly_dir, 0o444) # Read-only + + # Should not be writable + assert os.access(readonly_dir, os.W_OK) is False + + # Restore permissions for cleanup + os.chmod(readonly_dir, 0o755) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py new file mode 100644 index 00000000000..9974c23e4b4 --- /dev/null +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -0,0 +1,58 @@ +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.caching.dual_cache import DualCache +from litellm.caching.redis_cache import RedisCache + + +@pytest.mark.asyncio +async def test_dual_cache_async_batch_get_cache_coalesces_concurrent_redis_reads(): + dual_cache = DualCache( + redis_cache=MagicMock(spec=RedisCache), default_redis_batch_cache_expiry=10 + ) + keys = ["shared_a", "shared_b"] + start_gate = asyncio.Event() + + async def _mock_async_batch_get_cache(key_list, parent_otel_span=None): + await asyncio.sleep(0.05) + return {k: None for k in key_list} + + with patch.object( + dual_cache.redis_cache, + "async_batch_get_cache", + new=AsyncMock(side_effect=_mock_async_batch_get_cache), + ) as mock_async_batch_get_cache: + + async def worker(): + await start_gate.wait() + return await dual_cache.async_batch_get_cache(keys=keys) + + tasks = [asyncio.create_task(worker()) for _ in range(50)] + start_gate.set() + await asyncio.gather(*tasks) + + assert mock_async_batch_get_cache.call_count == 1 + + +@pytest.mark.asyncio +async def test_dual_cache_async_batch_get_cache_rolls_back_redis_reservation_on_error(): + dual_cache = DualCache( + redis_cache=MagicMock(spec=RedisCache), default_redis_batch_cache_expiry=10 + ) + keys = ["shared_a", "shared_b"] + + with patch.object( + dual_cache.redis_cache, + "async_batch_get_cache", + new=AsyncMock(side_effect=RuntimeError("redis unavailable")), + ) as mock_async_batch_get_cache: + first_result = await dual_cache.async_batch_get_cache(keys=keys) + second_result = await dual_cache.async_batch_get_cache(keys=keys) + + assert first_result is None + assert second_result is None + assert mock_async_batch_get_cache.call_count == 2 + assert "shared_a" not in dual_cache.last_redis_batch_access_time + assert "shared_b" not in dual_cache.last_redis_batch_access_time diff --git a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py index 57352eafaf1..5458b466f68 100644 --- a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py @@ -156,3 +156,83 @@ def test_transform_request_includes_extra_headers(): litellm_logging_obj=MockLoggingObj(), ) assert result.get("extra_headers") == headers + + +def test_transform_request_strips_internal_metadata_to_litellm_metadata(): + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = { + "metadata": {"user_api_key_auth": {"id": "abc"}}, + "litellm_metadata": {"trace_id": "trace-1"}, + "api_key": "sk-test", + } + + class MockLoggingObj: + pass + + result = handler.transform_request( + model="gpt-5-pro", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + litellm_logging_obj=MockLoggingObj(), + ) + + assert "metadata" not in result + assert result["litellm_metadata"]["user_api_key_auth"]["id"] == "abc" + assert result["litellm_metadata"]["trace_id"] == "trace-1" + + +def test_transform_request_preserves_user_metadata(): + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {"metadata": {"customer_id": "cust-123"}} + litellm_params = {"metadata": {"internal_key": "secret"}} + + class MockLoggingObj: + pass + + result = handler.transform_request( + model="gpt-5-pro", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + litellm_logging_obj=MockLoggingObj(), + ) + + assert result["metadata"] == {"customer_id": "cust-123"} + assert "internal_key" not in result["metadata"] + assert result["litellm_metadata"]["internal_key"] == "secret" + + +def test_transform_request_drops_user_metadata_with_additional_drop_params(): + from litellm.utils import get_optional_params + + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": "Hello"}] + optional_params = get_optional_params( + model="gpt-4o", + messages=messages, + metadata={"customer_id": "cust-123"}, + additional_drop_params=["metadata"], + custom_llm_provider="openai", + ) + litellm_params = {"metadata": {"internal_key": "secret"}} + + class MockLoggingObj: + pass + + result = handler.transform_request( + model="gpt-4o", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + litellm_logging_obj=MockLoggingObj(), + ) + + assert "metadata" not in result + assert result["litellm_metadata"]["internal_key"] == "secret" diff --git a/tests/test_litellm/expected_responses_api_request/context_management_and_shell.json b/tests/test_litellm/expected_responses_api_request/context_management_and_shell.json new file mode 100644 index 00000000000..1e34b230182 --- /dev/null +++ b/tests/test_litellm/expected_responses_api_request/context_management_and_shell.json @@ -0,0 +1,20 @@ +{ + "model": "gpt-4o", + "input": "List files in /mnt/data and run python --version.", + "context_management": [ + { + "type": "compaction", + "compact_threshold": 200000 + } + ], + "tools": [ + { + "type": "shell", + "environment": { + "type": "container_auto" + } + } + ], + "tool_choice": "auto", + "max_output_tokens": 256 +} diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index c7ad18cfb0c..beb978584cb 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -77,6 +77,15 @@ context_window_test_cases = [ ("Rate limit reached for requests.", False), ("The context is large, but acceptable.", False), ("", False), # Empty string + # OpenAI user param length validation - not a context window error + ( + "Invalid 'user': string too long. Expected a string with maximum length 64, but got a string with length 123 instead.", + False, + ), + ( + '{"error": {"message": "Invalid \'user\': string too long.", "code": "string_above_max_length"}}', + False, + ), ] diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 0a397d116e7..a321a24540f 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1,84 +1,285 @@ """ -Tests for Anthropic OAuth token handling for Claude Code Max integration. +Tests for Anthropic OAuth token handling in common_utils. + +Verifies that OAuth tokens (sk-ant-oat*) are sent via Authorization: Bearer +instead of x-api-key, per Anthropic's OAuth specification. """ import os import sys -# Add litellm to path -sys.path.insert(0, os.path.abspath("../../../../..")) +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) # Fake OAuth token for testing (not a real secret) FAKE_OAUTH_TOKEN = "sk-ant-oat01-fake-token-for-testing-123456789abcdef" +FAKE_REGULAR_KEY = "sk-ant-api03-regular-key-for-testing-123456789" -def test_oauth_detection_in_common_utils(): - """Test 1: OAuth token detection in common_utils""" - from litellm.llms.anthropic.common_utils import optionally_handle_anthropic_oauth +class TestOptionallyHandleAnthropicOAuth: + """Tests for optionally_handle_anthropic_oauth function.""" - headers = {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}"} - updated_headers, extracted_api_key = optionally_handle_anthropic_oauth(headers, None) + def test_oauth_token_in_authorization_header(self): + """OAuth token in Authorization header should be detected and headers set correctly.""" + from litellm.llms.anthropic.common_utils import ( + optionally_handle_anthropic_oauth, + ) - assert extracted_api_key == FAKE_OAUTH_TOKEN - assert updated_headers["anthropic-beta"] == "oauth-2025-04-20" - assert updated_headers["anthropic-dangerous-direct-browser-access"] == "true" + headers = {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}"} + updated_headers, extracted_api_key = optionally_handle_anthropic_oauth( + headers, None + ) + + assert extracted_api_key == FAKE_OAUTH_TOKEN + assert updated_headers["anthropic-beta"] == "oauth-2025-04-20" + assert updated_headers["anthropic-dangerous-direct-browser-access"] == "true" + assert "x-api-key" not in updated_headers + + def test_oauth_token_in_api_key_directly(self): + """OAuth token passed as api_key should set Authorization: Bearer header.""" + from litellm.llms.anthropic.common_utils import ( + optionally_handle_anthropic_oauth, + ) + + headers = {} + updated_headers, returned_api_key = optionally_handle_anthropic_oauth( + headers, FAKE_OAUTH_TOKEN + ) + + assert returned_api_key == FAKE_OAUTH_TOKEN + assert updated_headers["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}" + assert updated_headers["anthropic-beta"] == "oauth-2025-04-20" + assert updated_headers["anthropic-dangerous-direct-browser-access"] == "true" + assert "x-api-key" not in updated_headers + + def test_oauth_removes_existing_x_api_key(self): + """When OAuth is detected, any existing x-api-key should be removed.""" + from litellm.llms.anthropic.common_utils import ( + optionally_handle_anthropic_oauth, + ) + + headers = {"x-api-key": FAKE_OAUTH_TOKEN} + updated_headers, _ = optionally_handle_anthropic_oauth( + headers, FAKE_OAUTH_TOKEN + ) + + assert "x-api-key" not in updated_headers + assert updated_headers["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}" + + def test_regular_api_key_unchanged(self): + """Regular API keys (non-OAuth) should pass through unmodified.""" + from litellm.llms.anthropic.common_utils import ( + optionally_handle_anthropic_oauth, + ) + + headers = {} + updated_headers, returned_api_key = optionally_handle_anthropic_oauth( + headers, FAKE_REGULAR_KEY + ) + + assert returned_api_key == FAKE_REGULAR_KEY + assert "authorization" not in updated_headers + assert "anthropic-dangerous-direct-browser-access" not in updated_headers + assert "anthropic-beta" not in updated_headers + + def test_regular_key_in_authorization_header(self): + """Non-OAuth token in Authorization header should not trigger OAuth handling.""" + from litellm.llms.anthropic.common_utils import ( + optionally_handle_anthropic_oauth, + ) + + headers = {"authorization": f"Bearer {FAKE_REGULAR_KEY}"} + updated_headers, returned_api_key = optionally_handle_anthropic_oauth( + headers, FAKE_REGULAR_KEY + ) + + assert returned_api_key == FAKE_REGULAR_KEY + assert "anthropic-dangerous-direct-browser-access" not in updated_headers + + def test_none_api_key_no_error(self): + """None api_key with empty headers should not raise errors.""" + from litellm.llms.anthropic.common_utils import ( + optionally_handle_anthropic_oauth, + ) + + headers = {} + updated_headers, returned_api_key = optionally_handle_anthropic_oauth( + headers, None + ) + + assert returned_api_key is None + assert "authorization" not in updated_headers -def test_oauth_integration_in_validate_environment(): - """Test 2: OAuth integration in AnthropicConfig validate_environment""" - from litellm.llms.anthropic.common_utils import AnthropicModelInfo +class TestGetAnthropicHeaders: + """Tests for get_anthropic_headers method with OAuth support.""" - config = AnthropicModelInfo() - headers = {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}"} + def test_oauth_token_uses_authorization_bearer(self): + """OAuth token should produce Authorization: Bearer header, not x-api-key.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo - updated_headers = config.validate_environment( - headers=headers, - model="claude-3-haiku-20240307", - messages=[{"role": "user", "content": "Hello"}], - optional_params={}, - litellm_params={}, - api_key=None, - api_base=None, - ) + config = AnthropicModelInfo() + headers = config.get_anthropic_headers( + api_key=FAKE_OAUTH_TOKEN, + computer_tool_used=False, + prompt_caching_set=False, + pdf_used=False, + is_vertex_request=False, + ) - assert updated_headers["x-api-key"] == FAKE_OAUTH_TOKEN - assert updated_headers["anthropic-dangerous-direct-browser-access"] == "true" + assert headers["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}" + assert headers["anthropic-dangerous-direct-browser-access"] == "true" + assert "oauth-2025-04-20" in headers.get("anthropic-beta", "") + assert "x-api-key" not in headers + + def test_regular_key_uses_x_api_key(self): + """Regular API key should produce x-api-key header, not Authorization.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + headers = config.get_anthropic_headers( + api_key=FAKE_REGULAR_KEY, + computer_tool_used=False, + prompt_caching_set=False, + pdf_used=False, + is_vertex_request=False, + ) + + assert headers["x-api-key"] == FAKE_REGULAR_KEY + assert "authorization" not in headers + assert "anthropic-dangerous-direct-browser-access" not in headers + + def test_oauth_includes_standard_headers(self): + """OAuth path should still include standard Anthropic headers.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + headers = config.get_anthropic_headers( + api_key=FAKE_OAUTH_TOKEN, + computer_tool_used=False, + prompt_caching_set=False, + pdf_used=False, + is_vertex_request=False, + ) + + assert headers["anthropic-version"] == "2023-06-01" + assert headers["accept"] == "application/json" + assert headers["content-type"] == "application/json" -def test_oauth_detection_in_messages_transformation(): - """Test 3: OAuth detection in messages transformation""" - from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( - AnthropicMessagesConfig, - ) +class TestValidateEnvironmentOAuth: + """Tests for validate_environment with OAuth tokens.""" - config = AnthropicMessagesConfig() - headers = {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}"} + def test_oauth_via_authorization_header(self): + """validate_environment should produce correct headers for OAuth tokens.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo - updated_headers, _ = config.validate_anthropic_messages_environment( - headers=headers, - model="claude-3-haiku-20240307", - messages=[{"role": "user", "content": "Hello"}], - optional_params={}, - litellm_params={}, - api_key=None, - api_base=None, - ) + config = AnthropicModelInfo() + headers = {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}"} - assert updated_headers["x-api-key"] == FAKE_OAUTH_TOKEN - assert "oauth-2025-04-20" in updated_headers["anthropic-beta"] - assert updated_headers["anthropic-dangerous-direct-browser-access"] == "true" + updated_headers = config.validate_environment( + headers=headers, + model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=None, + api_base=None, + ) + + assert updated_headers["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}" + assert updated_headers["anthropic-dangerous-direct-browser-access"] == "true" + assert "oauth-2025-04-20" in updated_headers.get("anthropic-beta", "") + assert "x-api-key" not in updated_headers + + def test_oauth_via_api_key_param(self): + """validate_environment with OAuth token as api_key should use Bearer auth.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + headers = {} + + updated_headers = config.validate_environment( + headers=headers, + model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=FAKE_OAUTH_TOKEN, + api_base=None, + ) + + assert updated_headers["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}" + assert updated_headers["anthropic-dangerous-direct-browser-access"] == "true" + assert "x-api-key" not in updated_headers + + def test_regular_key_via_api_key_param(self): + """validate_environment with regular API key should use x-api-key.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + headers = {} + + updated_headers = config.validate_environment( + headers=headers, + model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=FAKE_REGULAR_KEY, + api_base=None, + ) + + assert updated_headers["x-api-key"] == FAKE_REGULAR_KEY + assert "authorization" not in updated_headers + assert "anthropic-dangerous-direct-browser-access" not in updated_headers -def test_regular_api_keys_still_work(): - """Test 4: Regular API keys still work (regression test)""" - from litellm.llms.anthropic.common_utils import optionally_handle_anthropic_oauth +class TestPassthroughOAuth: + """Tests for passthrough messages endpoint with OAuth tokens.""" - regular_key = "sk-ant-api03-regular-key-123" - headers = {"authorization": f"Bearer {regular_key}"} + def test_passthrough_oauth_no_x_api_key(self): + """Passthrough endpoint should not add x-api-key for OAuth tokens.""" + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) - updated_headers, extracted_api_key = optionally_handle_anthropic_oauth(headers, regular_key) + config = AnthropicMessagesConfig() + headers = {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}"} - # Regular key should be unchanged - assert extracted_api_key == regular_key - # OAuth headers should NOT be added - assert "anthropic-dangerous-direct-browser-access" not in updated_headers \ No newline at end of file + updated_headers, _ = config.validate_anthropic_messages_environment( + headers=headers, + model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=None, + api_base=None, + ) + + assert "oauth-2025-04-20" in updated_headers.get("anthropic-beta", "") + assert updated_headers["anthropic-dangerous-direct-browser-access"] == "true" + assert "x-api-key" not in updated_headers + + def test_passthrough_regular_key_uses_x_api_key(self): + """Passthrough endpoint should still use x-api-key for regular API keys.""" + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) + + config = AnthropicMessagesConfig() + headers = {} + + updated_headers, _ = config.validate_anthropic_messages_environment( + headers=headers, + model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={}, + api_key=FAKE_REGULAR_KEY, + api_base=None, + ) + + assert updated_headers["x-api-key"] == FAKE_REGULAR_KEY + assert "authorization" not in updated_headers diff --git a/tests/test_litellm/llms/azure/test_azure_exception_mapping.py b/tests/test_litellm/llms/azure/test_azure_exception_mapping.py index f4abe7f2b9a..495ca958cf5 100644 --- a/tests/test_litellm/llms/azure/test_azure_exception_mapping.py +++ b/tests/test_litellm/llms/azure/test_azure_exception_mapping.py @@ -239,4 +239,149 @@ class TestAzureExceptionMapping: assert e.provider_specific_fields is not None assert e.provider_specific_fields["inner_error"]["code"] == "ResponsibleAIPolicyViolation" assert e.provider_specific_fields["inner_error"]["revised_prompt"] == "revised" - assert e.provider_specific_fields["inner_error"]["content_filter_results"]["violence"]["filtered"] is True \ No newline at end of file + assert e.provider_specific_fields["inner_error"]["content_filter_results"]["violence"]["filtered"] is True + + def test_azure_content_policy_violation_detected_via_inner_error_code(self): + """Regression test for #20811: Azure returns inner_error with + ResponsibleAIPolicyViolation but the top-level error message is + generic. Previously this fell through to the generic + BadRequestError handler and all error details were lost.""" + + mock_exception = Exception("Bad request") + # This body structure mirrors what Azure OpenAI Images API returns + # for DALL-E 3 content policy violations (issue #20811). + mock_exception.body = { + "error": { + "code": "content_policy_violation", + "inner_error": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "profanity": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": True, "severity": "low"}, + }, + "revised_prompt": ( + "A dark and intense illustration of a man " + "in a dramatic action scene." + ), + }, + "message": ( + "Your request was rejected as a result of our safety system." + ), + "type": "invalid_request_error", + } + } + + mock_response = MagicMock() + mock_response.status_code = 400 + mock_exception.response = mock_response + + with pytest.raises(ContentPolicyViolationError) as exc_info: + exception_type( + model="azure/dall-e-3", + original_exception=mock_exception, + custom_llm_provider="azure", + ) + + e = exc_info.value + # Must surface as ContentPolicyViolationError, not generic BadRequestError + assert "safety system" in str(e) + assert e.provider_specific_fields is not None + inner = e.provider_specific_fields["inner_error"] + assert inner["code"] == "ResponsibleAIPolicyViolation" + assert inner["content_filter_results"]["violence"]["filtered"] is True + assert inner["revised_prompt"] is not None + + def test_azure_policy_violation_detected_via_inner_error_without_top_code(self): + """When the top-level code is NOT 'content_policy_violation' but + inner_error.code IS 'ResponsibleAIPolicyViolation', the error + should still be recognized as a content policy violation.""" + + mock_exception = Exception("Some error") + mock_exception.body = { + "error": { + "code": "BadRequest", + "inner_error": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_results": { + "violence": {"filtered": True, "severity": "medium"}, + }, + }, + "message": "The request was rejected.", + "type": "invalid_request_error", + } + } + + mock_response = MagicMock() + mock_response.status_code = 400 + mock_exception.response = mock_response + + with pytest.raises(ContentPolicyViolationError) as exc_info: + exception_type( + model="azure/dall-e-3", + original_exception=mock_exception, + custom_llm_provider="azure", + ) + + e = exc_info.value + assert e.provider_specific_fields is not None + assert e.provider_specific_fields["inner_error"]["code"] == "ResponsibleAIPolicyViolation" + + def test_azure_image_polling_error_preserves_body(self): + """Verify that AzureOpenAIError raised from the DALL-E polling path + carries the structured body so exception_type() can inspect it.""" + from litellm.llms.azure.common_utils import AzureOpenAIError + + error_payload = { + "status": "failed", + "error": { + "code": "content_policy_violation", + "message": "Your request was rejected.", + "inner_error": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_results": { + "violence": {"filtered": True, "severity": "low"}, + }, + }, + }, + } + + # Simulate what the fixed polling path now does + _error_body = error_payload.get("error", error_payload) + _error_msg = ( + _error_body.get("message", "Image generation failed") + if isinstance(_error_body, dict) + else json.dumps(error_payload) + ) + exc = AzureOpenAIError( + status_code=400, + message=_error_msg, + body=error_payload, + ) + + assert exc.body is not None + assert isinstance(exc.body, dict) + assert exc.body["error"]["code"] == "content_policy_violation" + assert "Your request was rejected" in exc.message + + def test_azure_safety_system_message_detected_as_policy_violation(self): + """Azure's rejection message 'Your request was rejected as a result + of our safety system' should be detected by string matching even + when the structured body is unavailable.""" + + mock_exception = Exception( + "Your request was rejected as a result of our safety system. " + "The revised prompt may contain text that is not allowed." + ) + mock_response = MagicMock() + mock_response.status_code = 400 + mock_exception.response = mock_response + + with pytest.raises(ContentPolicyViolationError): + exception_type( + model="azure/dall-e-3", + original_exception=mock_exception, + custom_llm_provider="azure", + ) \ No newline at end of file diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index c249bd9970c..b0011fd8f76 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -480,6 +480,85 @@ async def test_session_reuse_integration(): await client2.close() +@pytest.mark.asyncio +async def test_shared_session_bypasses_cache(): + """ + Test that when shared_session is provided, the cache is bypassed. + + This is critical for aiohttp tracing support - users need their custom + ClientSession (with trace_configs) to be used, not a cached session. + + Related: GitHub issue #20174 + """ + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.utils import LlmProviders + + # First, get a cached client without shared_session + cached_client = get_async_httpx_client( + llm_provider=LlmProviders.ANTHROPIC, + shared_session=None + ) + + # Now create a mock shared session + mock_session = MockClientSession() + + # Get a client WITH shared_session - this should NOT return the cached client + client_with_session = get_async_httpx_client( + llm_provider=LlmProviders.ANTHROPIC, # Same provider! + shared_session=mock_session # type: ignore + ) + + # The clients should be DIFFERENT - cache should be bypassed when shared_session is provided + assert client_with_session is not cached_client, \ + "Cache should be bypassed when shared_session is provided" + + # Verify the shared_session handler is using our mock session + # The transport should have our mock_session as its client + transport = client_with_session.client._transport + if hasattr(transport, 'client'): + assert transport.client is mock_session, \ + "Handler should use the provided shared_session" + + # Clean up + await cached_client.close() + await client_with_session.close() + + +@pytest.mark.asyncio +async def test_shared_session_each_call_gets_new_handler(): + """ + Test that each call with shared_session creates a new handler. + + This ensures user sessions (with their trace_configs, etc.) are always + used and not affected by caching. + """ + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.utils import LlmProviders + + # Create two different mock sessions + mock_session1 = MockClientSession() + mock_session2 = MockClientSession() + + # Get clients with different sessions for the same provider + client1 = get_async_httpx_client( + llm_provider=LlmProviders.ANTHROPIC, + shared_session=mock_session1 # type: ignore + ) + + client2 = get_async_httpx_client( + llm_provider=LlmProviders.ANTHROPIC, # Same provider + shared_session=mock_session2 # type: ignore # Different session + ) + + # Should be different clients, each using their own session + assert client1 is not client2, \ + "Different shared_sessions should create different handlers" + + # Clean up + await client1.close() + await client2.close() + + @pytest.mark.asyncio async def test_session_validation(): """Test that session validation works correctly""" diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index c0695bf3588..e6ab199168d 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -9,7 +9,10 @@ import pytest sys.path.insert(0, os.path.abspath("../../../../..")) -from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.llms.openai.chat.gpt_transformation import ( + OpenAIGPTConfig, + OpenAIChatCompletionStreamingHandler, +) class TestOpenAIGPTConfig: @@ -136,6 +139,75 @@ class TestGetOptionalParamsIntegration: assert regular_params.get("user") == "my-end-user" assert responses_params.get("user") == "my-end-user" + +class TestOpenAIChatCompletionStreamingHandler: + """Tests for OpenAIChatCompletionStreamingHandler.chunk_parser()""" + + def test_chunk_parser_preserves_usage(self): + """ + Test that chunk_parser preserves the usage field from streaming chunks. + + """ + handler = OpenAIChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + usage_chunk = { + "id": "gen-123", + "created": 1234567890, + "model": "openai/gpt-4o-mini", + "object": "chat.completion.chunk", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": ""}, + "finish_reason": None, + } + ], + "usage": { + "prompt_tokens": 13797, + "completion_tokens": 350, + "total_tokens": 14147, + }, + } + + result = handler.chunk_parser(usage_chunk) + + assert result.usage is not None + assert result.usage.prompt_tokens == 13797 + assert result.usage.completion_tokens == 350 + assert result.usage.total_tokens == 14147 + + def test_chunk_parser_without_usage(self): + """Test that chunk_parser works normally for chunks without usage.""" + handler = OpenAIChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + chunk = { + "id": "gen-123", + "created": 1234567890, + "model": "openai/gpt-4o-mini", + "object": "chat.completion.chunk", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "Hello"}, + "finish_reason": None, + } + ], + } + + result = handler.chunk_parser(chunk) + + assert result.id == "gen-123" + assert result.choices[0].delta.content == "Hello" + assert not hasattr(result, "usage") or result.usage is None + + +class TestPromptCacheKeyIntegration: + """Tests for prompt_cache_key support""" + def test_prompt_cache_key_in_optional_params(self): """Test that 'prompt_cache_key' flows through get_optional_params for OpenAI models.""" from litellm.utils import get_optional_params diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 074378fd562..7c08716c04c 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -417,6 +417,129 @@ class TestOpenAIResponsesAPIConfig: assert event.error.code == "unknown_error" assert event.error.message == "Something went wrong" + def test_transform_streaming_response_missing_required_fields_response_created( + self, + ): + """Test that ResponseCreatedEvent with missing required fields (created_at, + output) does not crash but falls back to model_construct. + + Reproduces https://github.com/BerriAI/litellm/issues/20570 + """ + from litellm.types.llms.openai import ResponseCreatedEvent + + # Minimal payload an OpenAI-compatible provider might send, + # omitting `created_at` and `output` inside the response object. + parsed_chunk = { + "type": "response.created", + "response": { + "id": "resp_q7BOLpck7clq", + "model": "gpt-oss-120b", + "status": "in_progress", + }, + } + + result = self.config.transform_streaming_response( + model=self.model, parsed_chunk=parsed_chunk, logging_obj=self.logging_obj + ) + + assert isinstance(result, ResponseCreatedEvent) + assert result.type == ResponsesAPIStreamEvents.RESPONSE_CREATED + assert result.response["id"] == "resp_q7BOLpck7clq" + + def test_transform_streaming_response_missing_required_fields_output_text_delta( + self, + ): + """Test that OutputTextDeltaEvent with missing output_index and + content_index falls back to model_construct without crashing. + + Reproduces https://github.com/BerriAI/litellm/issues/20570 + """ + from litellm.types.llms.openai import OutputTextDeltaEvent + + # Provider omits output_index and content_index + parsed_chunk = { + "type": "response.output_text.delta", + "item_id": "item_456", + "delta": "Hello", + } + + result = self.config.transform_streaming_response( + model=self.model, parsed_chunk=parsed_chunk, logging_obj=self.logging_obj + ) + + assert isinstance(result, OutputTextDeltaEvent) + assert result.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA + assert result.delta == "Hello" + assert result.item_id == "item_456" + + def test_transform_streaming_response_missing_required_fields_content_part_added( + self, + ): + """Test that ContentPartAddedEvent with missing output_index and + content_index falls back to model_construct without crashing. + + Reproduces https://github.com/BerriAI/litellm/issues/20570 + """ + from litellm.types.llms.openai import ContentPartAddedEvent + + # Provider omits output_index and content_index + parsed_chunk = { + "type": "response.content_part.added", + "item_id": "item_789", + "part": {"type": "output_text", "text": ""}, + } + + result = self.config.transform_streaming_response( + model=self.model, parsed_chunk=parsed_chunk, logging_obj=self.logging_obj + ) + + assert isinstance(result, ContentPartAddedEvent) + assert result.type == ResponsesAPIStreamEvents.CONTENT_PART_ADDED + assert result.item_id == "item_789" + + def test_transform_streaming_response_missing_required_fields_output_item_added( + self, + ): + """Test that OutputItemAddedEvent with missing output_index falls back + to model_construct without crashing. + + Reproduces https://github.com/BerriAI/litellm/issues/20570 + """ + from litellm.types.llms.openai import OutputItemAddedEvent + + # Provider omits output_index + parsed_chunk = { + "type": "response.output_item.added", + "item": {"type": "message", "id": "msg_001", "role": "assistant"}, + } + + result = self.config.transform_streaming_response( + model=self.model, parsed_chunk=parsed_chunk, logging_obj=self.logging_obj + ) + + assert isinstance(result, OutputItemAddedEvent) + assert result.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + + def test_transform_streaming_response_valid_chunk_still_works(self): + """Ensure that fully valid chunks still go through normal Pydantic + validation (not model_construct) and work correctly.""" + parsed_chunk = { + "type": "response.output_text.delta", + "item_id": "item_123", + "output_index": 0, + "content_index": 0, + "delta": "World", + } + + result = self.config.transform_streaming_response( + model=self.model, parsed_chunk=parsed_chunk, logging_obj=self.logging_obj + ) + + assert isinstance(result, OutputTextDeltaEvent) + assert result.delta == "World" + assert result.output_index == 0 + assert result.content_index == 0 + class TestAzureResponsesAPIConfig: def setup_method(self): diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py index 989a06f80b4..42e62c75d63 100644 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py @@ -352,17 +352,17 @@ class TestErrorHandling: def test_hf_response_missing_embedding(self): """Test handling of HF response missing embedding field""" config = SagemakerEmbeddingConfig() - + # Mock response without embedding field mock_response = httpx.Response( status_code=200, content=json.dumps({"object": "list"}).encode('utf-8'), headers={"content-type": "application/json"} ) - + model_response = EmbeddingResponse() - - with pytest.raises(Exception, match="HF response missing 'embedding' field"): + + with pytest.raises(Exception, match="Unexpected response format"): config.transform_embedding_response( model="sentence-transformers-model", raw_response=mock_response, @@ -372,5 +372,99 @@ class TestErrorHandling: ) +class TestTEIEmbeddingResponse: + """Test HuggingFace Text Embeddings Inference (TEI) response format support""" + + def setup_method(self): + self.config = SagemakerEmbeddingConfig() + + def test_transform_embedding_response_tei_raw_array(self): + """Test TEI response transformation - raw array format [[...]]""" + # TEI returns raw embedding arrays without wrapper + tei_response = [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6] + ] + + mock_response = httpx.Response( + status_code=200, + content=json.dumps(tei_response).encode('utf-8'), + headers={"content-type": "application/json"} + ) + + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="tei-qwen-embedding", + raw_response=mock_response, + model_response=model_response, + logging_obj=None, + request_data={"inputs": ["Hello", "World"]} + ) + + # Verify response structure + assert result.object == "list" + assert result.model == "tei-qwen-embedding" + assert len(result.data) == 2 + assert result.data[0]["object"] == "embedding" + assert result.data[0]["index"] == 0 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert result.data[1]["object"] == "embedding" + assert result.data[1]["index"] == 1 + assert result.data[1]["embedding"] == [0.4, 0.5, 0.6] + assert isinstance(result.usage, Usage) + + def test_transform_embedding_response_tei_single_input(self): + """Test TEI response with single input""" + tei_response = [ + [0.1, 0.2, 0.3, 0.4, 0.5] + ] + + mock_response = httpx.Response( + status_code=200, + content=json.dumps(tei_response).encode('utf-8'), + headers={"content-type": "application/json"} + ) + + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="tei-model", + raw_response=mock_response, + model_response=model_response, + logging_obj=None, + request_data={"inputs": ["Hello"]} + ) + + assert len(result.data) == 1 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3, 0.4, 0.5] + + def test_transform_embedding_response_wrapped_format_still_works(self): + """Test that wrapped format {"embedding": [...]} still works""" + hf_response = { + "embedding": [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6] + ] + } + + mock_response = httpx.Response( + status_code=200, + content=json.dumps(hf_response).encode('utf-8'), + headers={"content-type": "application/json"} + ) + + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="hf-model", + raw_response=mock_response, + model_response=model_response, + logging_obj=None, + request_data={"inputs": ["Hello", "World"]} + ) + + assert len(result.data) == 2 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert result.data[1]["embedding"] == [0.4, 0.5, 0.6] + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 75fd597ffa1..581d1e603dd 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -3338,3 +3338,94 @@ def test_chunk_parser_handles_prompt_feedback_block_with_usage(): assert result.usage.completion_tokens == 0, f"completion_tokens should be 0, got {result.usage.completion_tokens}" assert result.usage.total_tokens == 8175, f"total_tokens should be 8175, got {result.usage.total_tokens}" + +def test_vertex_ai_traffic_type_preserved_in_hidden_params_streaming(): + """Test trafficType is preserved in _hidden_params for streaming.""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + ModelResponseIterator, + ) + + chunk = { + "candidates": [{"content": {"parts": [{"text": "Hello"}]}}], + "usageMetadata": { + "promptTokenCount": 100, + "candidatesTokenCount": 200, + "totalTokenCount": 300, + "trafficType": "ON_DEMAND", + }, + } + + iterator = ModelResponseIterator( + streaming_response=[], sync_stream=True, logging_obj=MagicMock() + ) + result = iterator.chunk_parser(chunk) + + assert result._hidden_params["provider_specific_fields"]["traffic_type"] == "ON_DEMAND" + + +def test_vertex_ai_traffic_type_preserved_in_hidden_params_non_streaming(): + """Test trafficType is preserved in _hidden_params for non-streaming.""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + completion_response = { + "candidates": [ + { + "content": {"parts": [{"text": "Hello"}], "role": "model"}, + "finishReason": "STOP", + } + ], + "usageMetadata": { + "promptTokenCount": 50, + "candidatesTokenCount": 100, + "totalTokenCount": 150, + "trafficType": "PROVISIONED_THROUGHPUT", + }, + } + + raw_response = MagicMock() + raw_response.json.return_value = completion_response + + result = VertexGeminiConfig().transform_response( + model="gemini-pro", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert result._hidden_params["provider_specific_fields"]["traffic_type"] == "PROVISIONED_THROUGHPUT" + + +def test_vertex_ai_traffic_type_surfaced_in_responses_api(): + """Test trafficType is surfaced as provider_specific_fields in ResponsesAPIResponse.""" + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + # Create a ModelResponse with provider_specific_fields in _hidden_params + from litellm.types.utils import Choices, Message + + model_response = ModelResponse() + model_response._hidden_params["provider_specific_fields"] = {"traffic_type": "ON_DEMAND"} + model_response.choices = [ + Choices( + message=Message(content="Hello", role="assistant"), + finish_reason="stop", + index=0, + ) + ] + + responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="test", + chat_completion_response=model_response, + responses_api_request={}, + ) + + assert responses_api_response.provider_specific_fields["traffic_type"] == "ON_DEMAND" + diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py new file mode 100644 index 00000000000..dde73016271 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py @@ -0,0 +1,182 @@ +import json +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../..") +) # Adds the parent directory to the system path + + +class TestMCPRegistryFile: + """Tests for the curated MCP registry JSON file.""" + + @pytest.fixture + def registry_path(self): + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "..", + "..", + "..", + "..", + "litellm", + "proxy", + "mcp_registry.json", + ) + + def test_registry_file_exists(self, registry_path): + assert os.path.exists(registry_path), f"Registry file not found at {registry_path}" + + def test_registry_file_is_valid_json(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + assert isinstance(data, dict) + assert "servers" in data + + def test_registry_servers_have_required_fields(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + servers = data["servers"] + assert len(servers) > 0, "Registry should have at least one server" + + required_fields = ["name", "title", "description", "category", "transport"] + for server in servers: + for field in required_fields: + assert field in server, f"Server {server.get('name', '?')} missing field '{field}'" + + def test_registry_server_names_are_unique(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + names = [s["name"] for s in data["servers"]] + assert len(names) == len(set(names)), f"Duplicate server names found: {[n for n in names if names.count(n) > 1]}" + + def test_registry_transport_values_are_valid(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + valid_transports = {"stdio", "http", "sse"} + for server in data["servers"]: + assert server["transport"] in valid_transports, ( + f"Server {server['name']} has invalid transport '{server['transport']}'" + ) + + def test_stdio_servers_have_command(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + for server in data["servers"]: + if server["transport"] == "stdio": + assert "command" in server and server["command"], ( + f"stdio server {server['name']} missing 'command'" + ) + + def test_http_servers_have_url(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + for server in data["servers"]: + if server["transport"] in ("http", "sse"): + assert "url" in server and server["url"], ( + f"HTTP/SSE server {server['name']} missing 'url'" + ) + + def test_well_known_servers_present(self, registry_path): + """Ensure key well-known MCPs are in the registry.""" + with open(registry_path, "r") as f: + data = json.load(f) + names = {s["name"] for s in data["servers"]} + expected = {"github", "slack", "postgresql", "snowflake", "atlassian"} + missing = expected - names + assert not missing, f"Missing well-known servers: {missing}" + + def test_env_vars_structure(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + for server in data["servers"]: + if "env_vars" in server: + assert isinstance(server["env_vars"], list) + for var in server["env_vars"]: + assert "name" in var, f"env_var in {server['name']} missing 'name'" + + +class TestDiscoverEndpointFiltering: + """Tests for the discover endpoint filtering logic (unit-level).""" + + @pytest.fixture + def sample_servers(self): + return [ + { + "name": "github", + "title": "GitHub", + "description": "Repository management", + "category": "Developer Tools", + "transport": "http", + "url": "https://mcp.github.com/sse", + }, + { + "name": "slack", + "title": "Slack", + "description": "Channel management and messaging", + "category": "Communication", + "transport": "stdio", + "command": "npx", + }, + { + "name": "postgresql", + "title": "PostgreSQL", + "description": "Query and manage databases", + "category": "Databases", + "transport": "stdio", + "command": "npx", + }, + ] + + def test_query_filter_by_name(self, sample_servers): + query = "github" + q = query.lower() + result = [ + s + for s in sample_servers + if q in s.get("name", "").lower() + or q in s.get("title", "").lower() + or q in s.get("description", "").lower() + ] + assert len(result) == 1 + assert result[0]["name"] == "github" + + def test_query_filter_by_description(self, sample_servers): + query = "messaging" + q = query.lower() + result = [ + s + for s in sample_servers + if q in s.get("name", "").lower() + or q in s.get("title", "").lower() + or q in s.get("description", "").lower() + ] + assert len(result) == 1 + assert result[0]["name"] == "slack" + + def test_category_filter(self, sample_servers): + category = "Databases" + result = [s for s in sample_servers if s.get("category") == category] + assert len(result) == 1 + assert result[0]["name"] == "postgresql" + + def test_no_filter_returns_all(self, sample_servers): + assert len(sample_servers) == 3 + + def test_query_filter_no_match(self, sample_servers): + query = "nonexistent" + q = query.lower() + result = [ + s + for s in sample_servers + if q in s.get("name", "").lower() + or q in s.get("title", "").lower() + or q in s.get("description", "").lower() + ] + assert len(result) == 0 + + def test_categories_extraction(self, sample_servers): + categories = sorted(set(s.get("category", "Other") for s in sample_servers)) + assert categories == ["Communication", "Databases", "Developer Tools"] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index b4b5811666b..1a50cacd308 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -112,7 +112,44 @@ class TestMCPServerManager: assert client.stdio_config is not None assert client.stdio_config["command"] == "node" assert client.stdio_config["args"] == ["server.js"] - assert client.stdio_config["env"] == {"NODE_ENV": "test"} + # NPM_CONFIG_CACHE is injected automatically for container compatibility + from litellm.constants import MCP_NPM_CACHE_DIR + + assert client.stdio_config["env"]["NODE_ENV"] == "test" + assert client.stdio_config["env"]["NPM_CONFIG_CACHE"] == MCP_NPM_CACHE_DIR + + async def test_create_mcp_client_stdio_injects_npm_config_cache(self): + """Test that _create_mcp_client injects NPM_CONFIG_CACHE when not already set, + and preserves user-provided NPM_CONFIG_CACHE when present.""" + from litellm.constants import MCP_NPM_CACHE_DIR + + manager = MCPServerManager() + + # Case 1: NPM_CONFIG_CACHE not set -> should be injected + server_no_cache = MCPServer( + server_id="stdio-npm-1", + name="test_npm_server", + url=None, + transport=MCPTransport.stdio, + command="npx", + args=["-y", "@modelcontextprotocol/server-everything"], + env={}, + ) + client = await manager._create_mcp_client(server_no_cache) + assert client.stdio_config["env"]["NPM_CONFIG_CACHE"] == MCP_NPM_CACHE_DIR + + # Case 2: NPM_CONFIG_CACHE already set -> should NOT be overwritten + server_with_cache = MCPServer( + server_id="stdio-npm-2", + name="test_npm_server_custom", + url=None, + transport=MCPTransport.stdio, + command="npx", + args=["-y", "@modelcontextprotocol/server-everything"], + env={"NPM_CONFIG_CACHE": "/custom/cache"}, + ) + client2 = await manager._create_mcp_client(server_with_cache) + assert client2.stdio_config["env"]["NPM_CONFIG_CACHE"] == "/custom/cache" def test_build_stdio_env_only_accepts_x_prefixed_placeholders(self): """Ensure only ${X-*} placeholders are substituted from headers.""" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py index a447ee6af01..5eb8c1e51ac 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py @@ -1,105 +1,193 @@ """ Tests for MCP stale session ID handling (Fixes #20292). -When VSCode reconnects to LiteLLM's MCP endpoint after a reload, it sends a stale -`mcp-session-id` header. The session manager returns a 404 because the old session -was cleaned up. This test verifies that stale session IDs are detected and stripped -so a new session is created automatically. +When clients reconnect to LiteLLM's MCP endpoint after a server restart or reload, +they may send a stale `mcp-session-id` header. This test verifies that: +1. For non-DELETE requests: stale session IDs are stripped so new sessions are created +2. For DELETE requests: idempotent behavior returns success even if session doesn't exist """ import pytest from unittest.mock import AsyncMock, MagicMock, patch -class TestStripStaleMcpSessionHeader: - """Unit tests for the _strip_stale_mcp_session_header helper.""" +class TestHandleStaleMcpSession: + """Unit tests for the _handle_stale_mcp_session helper.""" - def test_strips_stale_session_id(self): + @pytest.mark.asyncio + async def test_strips_stale_session_id_for_non_delete(self): + """Non-DELETE requests should have stale session IDs stripped.""" try: from litellm.proxy._experimental.mcp_server.server import ( - _strip_stale_mcp_session_header, + _handle_stale_mcp_session, ) except ImportError: pytest.skip("MCP server not available") scope = { + "method": "POST", "headers": [ (b"content-type", b"application/json"), (b"mcp-session-id", b"stale-id"), ], } + receive = AsyncMock() + send = AsyncMock() mgr = MagicMock() mgr._server_instances = {} # no active sessions - _strip_stale_mcp_session_header(scope, mgr) + handled = await _handle_stale_mcp_session(scope, receive, send, mgr) + # Should not be fully handled (returns False) + assert handled is False + # Header should be stripped header_names = [k for k, _ in scope["headers"]] assert b"mcp-session-id" not in header_names - def test_preserves_valid_session_id(self): + @pytest.mark.asyncio + async def test_delete_stale_session_returns_success(self): + """DELETE requests for non-existent sessions should return success (idempotent).""" try: from litellm.proxy._experimental.mcp_server.server import ( - _strip_stale_mcp_session_header, + _handle_stale_mcp_session, ) except ImportError: pytest.skip("MCP server not available") scope = { + "type": "http", + "method": "DELETE", + "headers": [ + (b"content-type", b"application/json"), + (b"mcp-session-id", b"stale-id"), + ], + } + receive = AsyncMock() + send = AsyncMock() + mgr = MagicMock() + mgr._server_instances = {} # no active sessions + + handled = await _handle_stale_mcp_session(scope, receive, send, mgr) + + # Should be fully handled (returns True) + assert handled is True + # Should have sent a success response + assert send.called + # Header should NOT be stripped (DELETE needs the session ID) + header_names = [k for k, _ in scope["headers"]] + assert b"mcp-session-id" in header_names + + @pytest.mark.asyncio + async def test_preserves_valid_session_id(self): + """Valid session IDs should not be modified.""" + try: + from litellm.proxy._experimental.mcp_server.server import ( + _handle_stale_mcp_session, + ) + except ImportError: + pytest.skip("MCP server not available") + + scope = { + "method": "POST", "headers": [ (b"content-type", b"application/json"), (b"mcp-session-id", b"valid-id"), ], } + receive = AsyncMock() + send = AsyncMock() mgr = MagicMock() mgr._server_instances = {"valid-id": MagicMock()} - _strip_stale_mcp_session_header(scope, mgr) + handled = await _handle_stale_mcp_session(scope, receive, send, mgr) + # Should not be handled (returns False) + assert handled is False + # Header should be preserved header_names = [k for k, _ in scope["headers"]] assert b"mcp-session-id" in header_names - def test_no_op_when_no_session_header(self): + @pytest.mark.asyncio + async def test_no_op_when_no_session_header(self): + """No session header should result in no-op.""" try: from litellm.proxy._experimental.mcp_server.server import ( - _strip_stale_mcp_session_header, + _handle_stale_mcp_session, ) except ImportError: pytest.skip("MCP server not available") scope = { + "method": "POST", "headers": [ (b"content-type", b"application/json"), ], } + receive = AsyncMock() + send = AsyncMock() mgr = MagicMock() mgr._server_instances = {} - _strip_stale_mcp_session_header(scope, mgr) + handled = await _handle_stale_mcp_session(scope, receive, send, mgr) + assert handled is False assert len(scope["headers"]) == 1 - def test_no_op_when_server_instances_missing(self): + @pytest.mark.asyncio + async def test_no_op_when_server_instances_missing(self): """If _server_instances attr doesn't exist, don't crash.""" try: from litellm.proxy._experimental.mcp_server.server import ( - _strip_stale_mcp_session_header, + _handle_stale_mcp_session, ) except ImportError: pytest.skip("MCP server not available") scope = { + "method": "POST", "headers": [ (b"mcp-session-id", b"some-id"), ], } + receive = AsyncMock() + send = AsyncMock() mgr = MagicMock(spec=[]) # no attributes - _strip_stale_mcp_session_header(scope, mgr) + handled = await _handle_stale_mcp_session(scope, receive, send, mgr) - # Should keep the header since we can't verify + # Should not be handled, header should be kept + assert handled is False header_names = [k for k, _ in scope["headers"]] assert b"mcp-session-id" in header_names + @pytest.mark.asyncio + async def test_delete_valid_session_not_handled(self): + """DELETE requests for existing sessions should not be intercepted.""" + try: + from litellm.proxy._experimental.mcp_server.server import ( + _handle_stale_mcp_session, + ) + except ImportError: + pytest.skip("MCP server not available") + + scope = { + "method": "DELETE", + "headers": [ + (b"mcp-session-id", b"valid-id"), + ], + } + receive = AsyncMock() + send = AsyncMock() + mgr = MagicMock() + mgr._server_instances = {"valid-id": MagicMock()} + + handled = await _handle_stale_mcp_session(scope, receive, send, mgr) + + # Should not be handled - let session manager handle it + assert handled is False + # Should not have sent any response + assert not send.called + @pytest.mark.asyncio async def test_stale_mcp_session_id_is_stripped(): @@ -166,6 +254,69 @@ async def test_stale_mcp_session_id_is_stripped(): ) +@pytest.mark.asyncio +async def test_delete_stale_mcp_session_returns_success(): + """ + When a DELETE request is made for a session that no longer exists, + handle_streamable_http_mcp should return success (200) immediately + without forwarding to the session manager (idempotent DELETE). + """ + try: + from litellm.proxy._experimental.mcp_server.server import ( + handle_streamable_http_mcp, + session_manager, + ) + except ImportError: + pytest.skip("MCP server not available") + + stale_session_id = "stale-session-id-12345" + + scope = { + "type": "http", + "method": "DELETE", + "path": "/mcp", + "headers": [ + (b"content-type", b"application/json"), + (b"mcp-session-id", stale_session_id.encode()), + (b"authorization", b"Bearer test-key"), + ], + } + + receive = AsyncMock() + send = AsyncMock() + + # Mock handle_request should NOT be called for stale DELETE + mock_handle_request = AsyncMock() + + with patch( + "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + new_callable=AsyncMock, + return_value=(MagicMock(), None, None, None, None, None), + ), patch( + "litellm.proxy._experimental.mcp_server.server.set_auth_context", + ), patch( + "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + True, + ), patch.object( + session_manager, + "handle_request", + side_effect=mock_handle_request, + ), patch.object( + session_manager, + "_server_instances", + {}, # Empty dict = no active sessions + ): + await handle_streamable_http_mcp(scope, receive, send) + + # Verify session manager was NOT called (request was handled early) + assert not mock_handle_request.called, ( + "Session manager should not be called for DELETE on non-existent session" + ) + + # Verify a success response was sent + assert send.called, "A response should have been sent" + + @pytest.mark.asyncio async def test_valid_mcp_session_id_is_preserved(): """ diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 65bb329e7ea..c339ed2b762 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -936,6 +936,109 @@ def test_proxy_admin_viewer_can_access_global_spend_tags(): ) +class TestModelsRouteExemptFromDisableLLMEndpoints: + """ + Test that /models and /v1/models are exempt from DISABLE_LLM_API_ENDPOINTS. + + When DISABLE_LLM_API_ENDPOINTS is set, inference routes like /v1/chat/completions + should be blocked, but /models and /v1/models should remain accessible because + they are read-only model listing routes needed by the Admin UI. + + Relevant issue: https://github.com/BerriAI/litellm/issues/new (UI breaks with DISABLE_LLM_ENDPOINTS) + """ + + def _get_enterprise_route_checks(self): + """Import EnterpriseRouteChecks from the local enterprise source file.""" + import importlib.util + + local_file = os.path.join( + os.path.dirname(__file__), + "..", "..", "..", "..", "enterprise", + "litellm_enterprise", "proxy", "auth", "route_checks.py", + ) + local_file = os.path.abspath(local_file) + + spec = importlib.util.spec_from_file_location( + "local_enterprise_route_checks", local_file + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.EnterpriseRouteChecks + + @patch("litellm.proxy.proxy_server.premium_user", True) + def test_should_models_route_allowed_when_llm_api_disabled(self): + """Test that /models is allowed even when LLM API routes are disabled""" + EnterpriseRouteChecks = self._get_enterprise_route_checks() + + with patch.object( + EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True + ), patch.object( + EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False + ): + # /models should NOT raise - it's exempt + EnterpriseRouteChecks.should_call_route("/models") + + @patch("litellm.proxy.proxy_server.premium_user", True) + def test_should_v1_models_route_allowed_when_llm_api_disabled(self): + """Test that /v1/models is allowed even when LLM API routes are disabled""" + EnterpriseRouteChecks = self._get_enterprise_route_checks() + + with patch.object( + EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True + ), patch.object( + EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False + ): + # /v1/models should NOT raise - it's exempt + EnterpriseRouteChecks.should_call_route("/v1/models") + + @patch("litellm.proxy.proxy_server.premium_user", True) + def test_should_chat_completions_still_blocked_when_llm_api_disabled(self): + """Test that non-exempt LLM routes like /v1/chat/completions are still blocked""" + EnterpriseRouteChecks = self._get_enterprise_route_checks() + + with patch.object( + EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True + ), patch.object( + EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False + ): + with pytest.raises(HTTPException) as exc_info: + EnterpriseRouteChecks.should_call_route("/v1/chat/completions") + + assert exc_info.value.status_code == 403 + assert "LLM API routes are disabled for this instance." in str( + exc_info.value.detail + ) + + @patch("litellm.proxy.proxy_server.premium_user", True) + def test_should_embeddings_still_blocked_when_llm_api_disabled(self): + """Test that /v1/embeddings is still blocked when LLM API routes are disabled""" + EnterpriseRouteChecks = self._get_enterprise_route_checks() + + with patch.object( + EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True + ), patch.object( + EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False + ): + with pytest.raises(HTTPException) as exc_info: + EnterpriseRouteChecks.should_call_route("/v1/embeddings") + + assert exc_info.value.status_code == 403 + + @patch("litellm.proxy.proxy_server.premium_user", True) + def test_should_models_route_allowed_when_llm_api_not_disabled(self): + """Test that /models works normally when LLM API routes are not disabled""" + EnterpriseRouteChecks = self._get_enterprise_route_checks() + + with patch.object( + EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=False + ), patch.object( + EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False + ): + # Should not raise + EnterpriseRouteChecks.should_call_route("/models") + EnterpriseRouteChecks.should_call_route("/v1/models") + + def test_route_in_additional_public_routes_wildcard_match(): """ Test that route_in_additonal_public_routes supports wildcard patterns. diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index 324a58acfa9..af366b082a0 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -697,3 +697,67 @@ def test_populate_request_with_path_params_does_not_overwrite_existing_values(): assert result["organization_id"] == "org-existing" # Should keep original, not "org-query-param" # Verify other data is preserved assert result["messages"] == [{"role": "user", "content": "Hello"}] + + +@pytest.mark.asyncio +async def test_request_body_with_html_script_tags(): + """ + Test that JSON request bodies containing HTML tags like ", + }, + { + "role": "user", + "content": "", + }, + { + "role": "user", + "content": "Can you explain what ", + }, + { + "role": "user", + "content": "", + }, + { + "role": "user", + "content": "", + }, + ] + + for msg in test_messages: + test_payload = { + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + msg, + ], + } + + mock_request = MagicMock() + mock_request.body = AsyncMock(return_value=orjson.dumps(test_payload)) + mock_request.headers = {"content-type": "application/json"} + mock_request.scope = {} + + result = await _read_request_body(mock_request) + + assert result["model"] == "gpt-4o" + assert len(result["messages"]) == 3 + assert result["messages"][2]["content"] == msg["content"], ( + f"Message content with HTML was modified during parsing: " + f"expected={msg['content']!r}, got={result['messages'][2]['content']!r}" + ) diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py index 9993b25dfdd..0ed5940dd75 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py @@ -225,6 +225,39 @@ async def test_aggregate_queue_updates_accuracy(spend_queue): assert aggregated["team_list_transactions"]["team1"] == 5.0 +def test_get_aggregated_spend_update_queue_item_does_not_mutate_original_updates( + spend_queue, +): + original_update: SpendUpdateQueueItem = { + "entity_type": Litellm_EntityType.USER, + "entity_id": "user1", + "response_cost": 10.0, + } + duplicate_key_update: SpendUpdateQueueItem = { + "entity_type": Litellm_EntityType.USER, + "entity_id": "user1", + "response_cost": 20.0, + } + + aggregated_updates = spend_queue._get_aggregated_spend_update_queue_item( + [original_update, duplicate_key_update] + ) + user1_aggregated_update = next( + ( + update + for update in aggregated_updates + if update.get("entity_type") == Litellm_EntityType.USER + and update.get("entity_id") == "user1" + ), + None, + ) + + assert original_update["response_cost"] == 10.0 + assert user1_aggregated_update is not None + assert user1_aggregated_update["response_cost"] == 30.0 + assert user1_aggregated_update is not original_update + + @pytest.mark.asyncio async def test_queue_size_reduction_with_large_volume(monkeypatch, spend_queue): """Test that queue size is actually reduced when dealing with many items""" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py index a5ce7deb8a9..a0d92a0fa6f 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py @@ -986,424 +986,146 @@ class TestContentFilterGuardrail: assert detail.get("category") == "harm_toxic_abuse" else: assert "harm_toxic_abuse" in str(detail) + async def test_html_tags_in_messages_not_blocked(self): + """ + Test that HTML tags like ", + "", + "Can you explain what ", + "", + "", + "The ", + "
click
", ] - guardrail = ContentFilterGuardrail( - guardrail_name="test-iban", - patterns=patterns, - ) - - # Test with UK IBAN - result = guardrail._check_patterns("IBAN: GB82WEST12345698765432") - assert result is not None - assert result[1] == "iban" - assert result[2] == ContentFilterAction.MASK - - # Test with German IBAN - result = guardrail._check_patterns("IBAN: DE89370400440532013000") - assert result is not None - assert result[1] == "iban" - - # Test with French IBAN - result = guardrail._check_patterns("IBAN: FR1420041010050500013M02606") - assert result is not None - assert result[1] == "iban" - - # Test without IBAN - result = guardrail._check_patterns("This is a normal message") - assert result is None - - @pytest.mark.asyncio - async def test_iban_pattern_mask(self): - """ - Test that IBAN patterns are properly masked in apply_guardrail - """ - patterns = [ - ContentFilterPattern( - pattern_type="prebuilt", - pattern_name="iban", - action=ContentFilterAction.MASK, - ), - ] - - guardrail = ContentFilterGuardrail( - guardrail_name="test-iban-mask", - patterns=patterns, - ) - - # Test masking UK IBAN - guardrailed_inputs = await guardrail.apply_guardrail( - inputs={"texts": ["Bank account: GB82WEST12345698765432"]}, - request_data={}, - input_type="request", - ) - result = guardrailed_inputs.get("texts", []) - assert result is not None - assert len(result) == 1 - assert "[IBAN_REDACTED]" in result[0] - assert "GB82WEST12345698765432" not in result[0] - - # Test masking German IBAN - guardrailed_inputs = await guardrail.apply_guardrail( - inputs={"texts": ["Account: DE89370400440532013000"]}, - request_data={}, - input_type="request", - ) - result = guardrailed_inputs.get("texts", []) - assert result is not None - assert len(result) == 1 - assert "[IBAN_REDACTED]" in result[0] - assert "DE89370400440532013000" not in result[0] - - @pytest.mark.asyncio - async def test_iban_multiple_matches(self): - """ - Test that multiple IBAN matches are all masked - """ - patterns = [ - ContentFilterPattern( - pattern_type="prebuilt", - pattern_name="iban", - action=ContentFilterAction.MASK, - ), - ] - - guardrail = ContentFilterGuardrail( - guardrail_name="test-iban-multiple", - patterns=patterns, - ) - - # Test with multiple IBANs - guardrailed_inputs = await guardrail.apply_guardrail( - inputs={ - "texts": [ - "Transfer from GB82WEST12345698765432 to DE89370400440532013000" - ] - }, - request_data={}, - input_type="request", - ) - result = guardrailed_inputs.get("texts", []) - assert result is not None - assert len(result) == 1 - assert result[0].count("[IBAN_REDACTED]") == 2 - assert "GB82WEST12345698765432" not in result[0] - assert "DE89370400440532013000" not in result[0] - - def test_street_address_pattern(self): - """ - Test street address pattern detection for AU/US/UK formats - """ - patterns = [ - ContentFilterPattern( - pattern_type="prebuilt", - pattern_name="street_address", - action=ContentFilterAction.MASK, - ), - ] - - guardrail = ContentFilterGuardrail( - guardrail_name="test-street-address", - patterns=patterns, - ) - - # Test with Australian address - result = guardrail._check_patterns( - "Office at 123 Main Street, Sydney NSW 2000, Australia" - ) - assert result is not None - assert result[1] == "street_address" - assert result[2] == ContentFilterAction.MASK - - # Test with US address - result = guardrail._check_patterns("Located at 1600 Pennsylvania Avenue") - assert result is not None - assert result[1] == "street_address" - - # Test with UK address - result = guardrail._check_patterns("Visit 10 Downing Street, Westminster") - assert result is not None - assert result[1] == "street_address" - - # Test without address - result = guardrail._check_patterns("This is a normal message") - assert result is None - - @pytest.mark.asyncio - async def test_street_address_pattern_mask(self): - """ - Test that street addresses are properly masked in apply_guardrail - """ - patterns = [ - ContentFilterPattern( - pattern_type="prebuilt", - pattern_name="street_address", - action=ContentFilterAction.MASK, - ), - ] - - guardrail = ContentFilterGuardrail( - guardrail_name="test-address-mask", - patterns=patterns, - ) - - # Test masking Australian address - guardrailed_inputs = await guardrail.apply_guardrail( - inputs={"texts": ["Employee lives at 123 Main Street, Sydney NSW 2000"]}, - request_data={}, - input_type="request", - ) - result = guardrailed_inputs.get("texts", []) - assert result is not None - assert len(result) == 1 - assert "[STREET_ADDRESS_REDACTED]" in result[0] - assert "123 Main Street" not in result[0] - - # Test masking US address - guardrailed_inputs = await guardrail.apply_guardrail( - inputs={"texts": ["Office at 350 Fifth Ave., New York"]}, - request_data={}, - input_type="request", - ) - result = guardrailed_inputs.get("texts", []) - assert result is not None - assert len(result) == 1 - assert "[STREET_ADDRESS_REDACTED]" in result[0] - assert "350 Fifth Ave" not in result[0] - - @pytest.mark.asyncio - async def test_street_address_multiple_matches(self): - """ - Test that multiple street addresses are all masked - """ - patterns = [ - ContentFilterPattern( - pattern_type="prebuilt", - pattern_name="street_address", - action=ContentFilterAction.MASK, - ), - ] - - guardrail = ContentFilterGuardrail( - guardrail_name="test-address-multiple", - patterns=patterns, - ) - - # Test with multiple addresses - guardrailed_inputs = await guardrail.apply_guardrail( - inputs={ - "texts": [ - "Moving from 123 Main Street, Sydney to 456 Oak Avenue, Melbourne" - ] - }, - request_data={}, - input_type="request", - ) - result = guardrailed_inputs.get("texts", []) - assert result is not None - assert len(result) == 1 - assert result[0].count("[STREET_ADDRESS_REDACTED]") == 2 - assert "123 Main Street" not in result[0] - assert "456 Oak Avenue" not in result[0] + for pattern_name in PREBUILT_PATTERNS: + compiled = get_compiled_pattern(pattern_name) + for test_string in html_test_strings: + match = compiled.search(test_string) + if match: + # Some patterns may legitimately match substrings + # (e.g., URL pattern matching src='https://...') + # but they should not match the script/HTML tag itself + matched_text = match.group() + assert "' in " + f"test string: {test_string!r}. " + f"LLM message content should not be blocked for HTML tags." + ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py index cebba2ff5e1..3a17bbd0025 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py @@ -7,7 +7,6 @@ import sys sys.path.insert(0, os.path.abspath("../../../../../..")) -import asyncio from unittest.mock import MagicMock, patch import pytest @@ -26,7 +25,7 @@ async def test_openai_moderation_guardrail_init(): guardrail = OpenAIModerationGuardrail( guardrail_name="test-openai-moderation", ) - + assert guardrail.guardrail_name == "test-openai-moderation" assert guardrail.api_key == "test-key" assert guardrail.model == "omni-moderation-latest" @@ -49,27 +48,27 @@ async def test_openai_moderation_guardrail_adds_to_litellm_callbacks(): # Clear existing callbacks for clean test original_callbacks = litellm.callbacks.copy() litellm.logging_callback_manager._reset_all_callbacks() - + try: with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): guardrail_litellm_params = LitellmParams( guardrail=SupportedGuardrailIntegrations.OPENAI_MODERATION, api_key="test-key", model="omni-moderation-latest", - mode="pre_call" + mode="pre_call", ) guardrail = openai_initialize_guardrail( litellm_params=guardrail_litellm_params, guardrail=Guardrail( guardrail_name="test-openai-moderation", - litellm_params=guardrail_litellm_params - ) + litellm_params=guardrail_litellm_params, + ), ) - + # Check that the guardrail was added to litellm callbacks assert guardrail in litellm.callbacks assert len(litellm.callbacks) == 1 - + # Verify it's the correct guardrail callback = litellm.callbacks[0] assert isinstance(callback, OpenAIModerationGuardrail) @@ -85,12 +84,12 @@ async def test_openai_moderation_guardrail_adds_to_litellm_callbacks(): async def test_openai_moderation_guardrail_safe_content(): """Test OpenAI moderation guardrail with safe content via apply_guardrail""" from litellm.types.utils import GenericGuardrailAPIInputs - + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): guardrail = OpenAIModerationGuardrail( guardrail_name="test-openai-moderation", ) - + # Mock safe moderation response mock_response = OpenAIModerationResponse( id="modr-123", @@ -118,25 +117,29 @@ async def test_openai_moderation_guardrail_safe_content(): "harassment": [], "self-harm": [], "violence": [], - } + }, ) - ] + ], ) - - with patch.object(guardrail, 'async_make_request', return_value=mock_response): + + with patch.object(guardrail, "async_make_request", return_value=mock_response): # Test apply_guardrail with safe content using structured_messages inputs = GenericGuardrailAPIInputs( structured_messages=[ {"role": "user", "content": "Hello, how are you today?"} ] ) - + result = await guardrail.apply_guardrail( inputs=inputs, - request_data={"messages": [{"role": "user", "content": "Hello, how are you today?"}]}, - input_type="request" + request_data={ + "messages": [ + {"role": "user", "content": "Hello, how are you today?"} + ] + }, + input_type="request", ) - + # Should return the original inputs unchanged assert result == inputs @@ -145,12 +148,12 @@ async def test_openai_moderation_guardrail_safe_content(): async def test_openai_moderation_guardrail_apply_guardrail(): """Test OpenAI moderation guardrail apply_guardrail method (unified guardrail interface)""" from litellm.types.utils import GenericGuardrailAPIInputs - + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): guardrail = OpenAIModerationGuardrail( guardrail_name="test-openai-moderation", ) - + # Mock safe moderation response mock_response = OpenAIModerationResponse( id="modr-123", @@ -178,37 +181,37 @@ async def test_openai_moderation_guardrail_apply_guardrail(): "harassment": [], "self-harm": [], "violence": [], - } + }, ) - ] + ], ) - - with patch.object(guardrail, 'async_make_request', return_value=mock_response): + + with patch.object(guardrail, "async_make_request", return_value=mock_response): # Test apply_guardrail with texts (embeddings-style input) inputs = GenericGuardrailAPIInputs( texts=["Hello, how are you?", "What is the weather?"] ) - + result = await guardrail.apply_guardrail( inputs=inputs, request_data={}, input_type="request", ) - + # Should return inputs unchanged (moderation doesn't modify, only blocks) assert result == inputs -@pytest.mark.asyncio +@pytest.mark.asyncio async def test_openai_moderation_guardrail_harmful_content(): """Test OpenAI moderation guardrail with harmful content via apply_guardrail""" from litellm.types.utils import GenericGuardrailAPIInputs - + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): guardrail = OpenAIModerationGuardrail( guardrail_name="test-openai-moderation", ) - + # Mock harmful moderation response mock_response = OpenAIModerationResponse( id="modr-123", @@ -236,40 +239,51 @@ async def test_openai_moderation_guardrail_harmful_content(): "harassment": [], "self-harm": [], "violence": [], - } + }, ) - ] + ], ) - - with patch.object(guardrail, 'async_make_request', return_value=mock_response): + + with patch.object(guardrail, "async_make_request", return_value=mock_response): # Test apply_guardrail with harmful content using structured_messages inputs = GenericGuardrailAPIInputs( structured_messages=[ {"role": "user", "content": "This is hateful content"} ] ) - + # Should raise HTTPException from fastapi import HTTPException + with pytest.raises(HTTPException) as exc_info: await guardrail.apply_guardrail( inputs=inputs, - request_data={"messages": [{"role": "user", "content": "This is hateful content"}]}, - input_type="request" + request_data={ + "messages": [ + {"role": "user", "content": "This is hateful content"} + ] + }, + input_type="request", ) - + assert exc_info.value.status_code == 400 assert "Violated OpenAI moderation policy" in str(exc_info.value.detail) @pytest.mark.asyncio async def test_openai_moderation_guardrail_streaming_safe_content(): - """Test OpenAI moderation guardrail with streaming safe content""" + """Test OpenAI moderation guardrail with streaming safe content via UnifiedLLMGuardrails""" + from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( + UnifiedLLMGuardrails, + ) + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): guardrail = OpenAIModerationGuardrail( guardrail_name="test-openai-moderation", + event_hook="post_call", ) - + unified_guardrail = UnifiedLLMGuardrails() + # Mock safe moderation response mock_response = OpenAIModerationResponse( id="modr-123", @@ -297,72 +311,85 @@ async def test_openai_moderation_guardrail_streaming_safe_content(): "harassment": [], "self-harm": [], "violence": [], - } + }, ) - ] + ], ) - + # Mock streaming chunks async def mock_stream(): # Simulate streaming chunks with safe content - chunks = [ - MagicMock(choices=[MagicMock(delta=MagicMock(content="Hello "))]), - MagicMock(choices=[MagicMock(delta=MagicMock(content="world"))]), - MagicMock(choices=[MagicMock(delta=MagicMock(content="!"))]) - ] - for chunk in chunks: + chunk1 = MagicMock() + chunk1.model = "gpt-4" + chunk1.choices = [MagicMock()] + chunk1.choices[0].delta = MagicMock() + chunk1.choices[0].delta.content = "Hello " + chunk1.choices[0].finish_reason = None + + chunk2 = MagicMock() + chunk2.model = "gpt-4" + chunk2.choices = [MagicMock()] + chunk2.choices[0].delta = MagicMock() + chunk2.choices[0].delta.content = "world" + chunk2.choices[0].finish_reason = None + + # Last chunk with finish_reason + chunk3 = MagicMock() + chunk3.model = "gpt-4" + chunk3.choices = [MagicMock()] + chunk3.choices[0].delta = MagicMock() + chunk3.choices[0].delta.content = "!" + chunk3.choices[0].finish_reason = "stop" + + for chunk in [chunk1, chunk2, chunk3]: yield chunk - - # Mock the stream_chunk_builder to return a proper ModelResponse + + # Mock for stream_chunk_builder mock_model_response = MagicMock() - mock_model_response.choices = [ - MagicMock(message=MagicMock(content="Hello world!")) - ] - - with patch.object(guardrail, 'async_make_request', return_value=mock_response), \ - patch('litellm.main.stream_chunk_builder', return_value=mock_model_response), \ - patch('litellm.llms.base_llm.base_model_iterator.MockResponseIterator') as mock_iterator: - - # Mock the iterator to yield the original chunks - async def mock_yield_chunks(): - chunks = [ - MagicMock(choices=[MagicMock(delta=MagicMock(content="Hello "))]), - MagicMock(choices=[MagicMock(delta=MagicMock(content="world"))]), - MagicMock(choices=[MagicMock(delta=MagicMock(content="!"))]) - ] - for chunk in chunks: - yield chunk - - mock_iterator.return_value.__aiter__ = lambda self: mock_yield_chunks() - - user_api_key_dict = UserAPIKeyAuth(api_key="test") + mock_model_response.choices = [MagicMock()] + mock_model_response.choices[0].message = MagicMock() + mock_model_response.choices[0].message.content = "Hello world!" + + with patch.object(guardrail, "async_make_request", return_value=mock_response), patch( + "litellm.llms.openai.chat.guardrail_translation.handler.stream_chunk_builder", + return_value=mock_model_response, + ): + user_api_key_dict = UserAPIKeyAuth( + api_key="test", request_route="/chat/completions" + ) request_data = { - "messages": [ - {"role": "user", "content": "Hello, how are you today?"} - ] + "messages": [{"role": "user", "content": "Hello, how are you today?"}], + "guardrail_to_apply": guardrail, + "metadata": {"guardrails": ["test-openai-moderation"]}, } - - # Test streaming hook with safe content + + # Test streaming hook with safe content via UnifiedLLMGuardrails result_chunks = [] - async for chunk in guardrail.async_post_call_streaming_iterator_hook( + async for chunk in unified_guardrail.async_post_call_streaming_iterator_hook( user_api_key_dict=user_api_key_dict, response=mock_stream(), - request_data=request_data + request_data=request_data, ): result_chunks.append(chunk) - + # Should return all chunks without blocking assert len(result_chunks) == 3 @pytest.mark.asyncio async def test_openai_moderation_guardrail_streaming_harmful_content(): - """Test OpenAI moderation guardrail with streaming harmful content""" + """Test OpenAI moderation guardrail with streaming harmful content via UnifiedLLMGuardrails""" + from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( + UnifiedLLMGuardrails, + ) + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): guardrail = OpenAIModerationGuardrail( guardrail_name="test-openai-moderation", + event_hook="post_call", ) - + unified_guardrail = UnifiedLLMGuardrails() + # Mock harmful moderation response mock_response = OpenAIModerationResponse( id="modr-123", @@ -390,46 +417,74 @@ async def test_openai_moderation_guardrail_streaming_harmful_content(): "harassment": [], "self-harm": [], "violence": [], - } + }, ) - ] + ], ) - + # Mock streaming chunks with harmful content async def mock_stream(): - chunks = [ - MagicMock(choices=[MagicMock(delta=MagicMock(content="This is "))]), - MagicMock(choices=[MagicMock(delta=MagicMock(content="harmful content"))]) - ] - for chunk in chunks: + # First chunk - no finish_reason + chunk1 = MagicMock() + chunk1.model = "gpt-4" + chunk1.choices = [MagicMock()] + chunk1.choices[0].delta = MagicMock() + chunk1.choices[0].delta.content = "This is " + chunk1.choices[0].finish_reason = None + + # Last chunk - with finish_reason to signal end of stream + chunk2 = MagicMock() + chunk2.model = "gpt-4" + chunk2.choices = [MagicMock()] + chunk2.choices[0].delta = MagicMock() + chunk2.choices[0].delta.content = "harmful content" + chunk2.choices[0].finish_reason = "stop" + + for chunk in [chunk1, chunk2]: yield chunk - - # Mock the stream_chunk_builder to return a ModelResponse with harmful content - mock_model_response = MagicMock() - mock_model_response.choices = [ - MagicMock(message=MagicMock(content="This is harmful content")) - ] - - with patch.object(guardrail, 'async_make_request', return_value=mock_response), \ - patch('litellm.main.stream_chunk_builder', return_value=mock_model_response): - - user_api_key_dict = UserAPIKeyAuth(api_key="test") + + # Mock for stream_chunk_builder - use real litellm types so isinstance checks pass + from litellm.types.utils import ModelResponse + import litellm + mock_model_response = ModelResponse( + id="mock-response", + model="gpt-4", + choices=[ + litellm.Choices( + index=0, + message=litellm.Message( + role="assistant", + content="This is harmful content", + ), + finish_reason="stop", + ) + ], + ) + + with patch.object(guardrail, "async_make_request", return_value=mock_response), patch( + "litellm.llms.openai.chat.guardrail_translation.handler.stream_chunk_builder", + return_value=mock_model_response, + ): + user_api_key_dict = UserAPIKeyAuth( + api_key="test", request_route="/chat/completions" + ) request_data = { - "messages": [ - {"role": "user", "content": "Generate harmful content"} - ] + "messages": [{"role": "user", "content": "Generate harmful content"}], + "guardrail_to_apply": guardrail, + "metadata": {"guardrails": ["test-openai-moderation"]}, } - + # Should raise HTTPException when processing streaming harmful content from fastapi import HTTPException + with pytest.raises(HTTPException) as exc_info: result_chunks = [] - async for chunk in guardrail.async_post_call_streaming_iterator_hook( + async for chunk in unified_guardrail.async_post_call_streaming_iterator_hook( user_api_key_dict=user_api_key_dict, response=mock_stream(), - request_data=request_data + request_data=request_data, ): result_chunks.append(chunk) - + assert exc_info.value.status_code == 400 - assert "Violated OpenAI moderation policy" in str(exc_info.value.detail) \ No newline at end of file + assert "Violated OpenAI moderation policy" in str(exc_info.value.detail) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py new file mode 100644 index 00000000000..c77a5d07b3b --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_openai_moderation_streaming.py @@ -0,0 +1,172 @@ +import pytest +from unittest.mock import MagicMock, patch +import os +from litellm.proxy.guardrails.guardrail_hooks.openai.moderations import ( + OpenAIModerationGuardrail, +) +from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( + UnifiedLLMGuardrails, +) +from litellm.types.utils import ModelResponseStream, ModelResponse +from litellm.proxy._types import UserAPIKeyAuth + + +@pytest.mark.asyncio +async def test_openai_moderation_guardrail_streaming_latency(): + """ + Test that the OpenAI Moderation guardrail, when run via UnifiedLLMGuardrails, + supports streaming (fast time-to-first-token) instead of buffering. + """ + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): + # 1. Initialize the specific guardrail with proper event_hook + openai_guardrail = OpenAIModerationGuardrail( + guardrail_name="test-openai-moderation", + event_hook="post_call", + ) + + # 2. Initialize the Unified Guardrail system (which invokes the specific guardrail) + unified_guardrail = UnifiedLLMGuardrails() + + # Mock safe moderation response + mock_mod_response = MagicMock() + mock_mod_response.results = [] + + # Mock streaming chunks (no artificial delay - test deterministically) + async def mock_stream(): + chunks_data = ["Hello", " ", "world", "!", " Goodbye"] + for i, content in enumerate(chunks_data): + chunk = MagicMock(spec=ModelResponseStream) + chunk.model = "gpt-4" + choice = MagicMock() + choice.delta = MagicMock() + choice.delta.content = content + # Last chunk gets finish_reason + choice.finish_reason = "stop" if i == len(chunks_data) - 1 else None + chunk.choices = [choice] + yield chunk + + # Mock for stream_chunk_builder to return a simple ModelResponse + mock_model_response = MagicMock(spec=ModelResponse) + mock_model_response.choices = [MagicMock()] + mock_model_response.choices[0].message = MagicMock() + mock_model_response.choices[0].message.content = "Hello world! Goodbye" + + # Patch the network call in the specific guardrail + with patch.object( + openai_guardrail, "async_make_request", return_value=mock_mod_response + ), patch( + "litellm.llms.openai.chat.guardrail_translation.handler.stream_chunk_builder", + return_value=mock_model_response, + ): + user_api_key_dict = UserAPIKeyAuth( + api_key="test", request_route="/chat/completions" + ) + request_data = { + "messages": [{"role": "user", "content": "hi"}], + "guardrail_to_apply": openai_guardrail, + "metadata": { + "guardrails": ["test-openai-moderation"], + "guardrail_config": {"streaming_sampling_rate": 1}, + }, # Check every chunk for test + } + + chunks_received = 0 + first_chunk_yielded = False + + # Call the hook on UnifiedLLMGuardrails + async for chunk in unified_guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=user_api_key_dict, + response=mock_stream(), + request_data=request_data, + ): + if not first_chunk_yielded: + first_chunk_yielded = True + chunks_received += 1 + + # Deterministic assertions (no flaky timing checks) + assert first_chunk_yielded, "Expected at least one chunk to be yielded" + assert chunks_received == 5, f"Expected 5 chunks, got {chunks_received}" + + +@pytest.mark.asyncio +async def test_openai_moderation_guardrail_streaming_harmful_content(): + """ + Test that harmful content is caught during streaming via UnifiedLLMGuardrails + """ + from fastapi import HTTPException + + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): + openai_guardrail = OpenAIModerationGuardrail( + guardrail_name="test-openai-moderation", + event_hook="post_call", + ) + unified_guardrail = UnifiedLLMGuardrails() + + # Mock harmful moderation response + mock_mod_response = MagicMock() + mock_mod_response.results = [ + MagicMock( + flagged=True, categories={"hate": True}, category_scores={"hate": 0.99} + ) + ] + + async def mock_stream(): + chunks_data = ["This ", "is ", "harmful ", "content"] + for i, content in enumerate(chunks_data): + chunk = MagicMock(spec=ModelResponseStream) + chunk.model = "gpt-4" + choice = MagicMock() + choice.delta = MagicMock() + choice.delta.content = content + # Last chunk gets finish_reason + choice.finish_reason = "stop" if i == len(chunks_data) - 1 else None + chunk.choices = [choice] + yield chunk + + # Mock for stream_chunk_builder - use real litellm types so isinstance checks pass + import litellm + + mock_model_response = ModelResponse( + id="mock-response", + model="gpt-4", + choices=[ + litellm.Choices( + index=0, + message=litellm.Message( + role="assistant", + content="This is harmful content", + ), + finish_reason="stop", + ) + ], + ) + + with patch.object( + openai_guardrail, "async_make_request", return_value=mock_mod_response + ), patch( + "litellm.llms.openai.chat.guardrail_translation.handler.stream_chunk_builder", + return_value=mock_model_response, + ): + user_api_key_dict = UserAPIKeyAuth( + api_key="test", request_route="/chat/completions" + ) + request_data = { + "messages": [{"role": "user", "content": "generate hate"}], + "guardrail_to_apply": openai_guardrail, + "metadata": { + "guardrails": ["test-openai-moderation"], + "guardrail_config": {"streaming_sampling_rate": 1}, + }, + } + + # Should raise HTTPException + with pytest.raises(HTTPException) as exc_info: + async for _ in unified_guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=user_api_key_dict, + response=mock_stream(), + request_data=request_data, + ): + pass + + assert exc_info.value.status_code == 400 + assert "Violated OpenAI moderation policy" in str(exc_info.value.detail) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index 5c039141928..7d2b6e84de7 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -14,10 +14,14 @@ import pytest import litellm from litellm import ModelResponse from litellm.exceptions import GuardrailRaisedException +from litellm._version import version as litellm_version from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( GenericGuardrailAPI, ) +from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api.generic_guardrail_api import ( + _HEADER_PRESENT_PLACEHOLDER, +) from litellm.types.utils import Choices, Message @@ -351,6 +355,58 @@ class TestMetadataExtraction: # Should be empty dict assert request_metadata == {} + @pytest.mark.asyncio + async def test_inbound_headers_and_litellm_version_forwarded_and_sanitized( + self, generic_guardrail, mock_request_data_input + ): + """ + Ensure inbound proxy request headers are forwarded in JSON payload with allowlist: + allowed headers show their value; all other headers show presence only ([present]). + """ + # Add proxy_server_request headers as they exist in proxy request context + request_data = dict(mock_request_data_input) + request_data["proxy_server_request"] = { + "headers": { + "User-Agent": "OpenAI/Python 2.17.0", + "Authorization": "Bearer should-not-forward", + "Cookie": "session=should-not-forward", + "X-Request-Id": "req_123", + } + } + + mock_response = MagicMock() + mock_response.json.return_value = { + "action": "NONE", + "texts": ["test"], + } + mock_response.raise_for_status = MagicMock() + + with patch.object( + generic_guardrail.async_handler, "post", return_value=mock_response + ) as mock_post: + await generic_guardrail.apply_guardrail( + inputs={"texts": ["test"]}, + request_data=request_data, + input_type="request", + ) + + call_args = mock_post.call_args + json_payload = call_args.kwargs["json"] + + # New fields should exist + assert json_payload["litellm_version"] == litellm_version + assert "request_headers" in json_payload + assert isinstance(json_payload["request_headers"], dict) + req_headers = json_payload["request_headers"] + + # Allowed: value forwarded + assert req_headers.get("User-Agent") == "OpenAI/Python 2.17.0" + + # Not on allowlist: key present, value is placeholder only + assert req_headers.get("Authorization") == _HEADER_PRESENT_PLACEHOLDER + assert req_headers.get("Cookie") == _HEADER_PRESENT_PLACEHOLDER + assert req_headers.get("X-Request-Id") == _HEADER_PRESENT_PLACEHOLDER + class TestGuardrailActions: """Test different guardrail action responses""" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index 987388a80c7..8080491f662 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -1122,6 +1122,83 @@ async def test_model_armor_non_model_response(): assert not guardrail.async_handler.post.called +@pytest.mark.asyncio +async def test_model_armor_guardrail_status_intervened_vs_failed(): + """ + regression test for bug where _process_error always set 'guardrail_failed_to_respond' + even for intentional blocks (error 400). + """ + mock_user_api_key_dict = UserAPIKeyAuth() + mock_cache = MagicMock(spec=DualCache) + + #1: Blocked content should raise exception and show guardrail status: guardrail_intervened" + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.json = AsyncMock(return_value={ + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "rai": { + "raiFilterResult": { + "matchState": "MATCH_FOUND", + } + } + } + } + }) + + guardrail._ensure_access_token_async = AsyncMock(return_value=("token", "test-project")) + with patch.object(guardrail.async_handler, "post", AsyncMock(return_value=mock_response)): + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "bad content"}], + "metadata": {"guardrails": ["model-armor-test"]}, + } + with pytest.raises(HTTPException): + await guardrail.async_pre_call_hook( + user_api_key_dict=mock_user_api_key_dict, + cache=mock_cache, + data=request_data, + call_type="completion", + ) + + info = request_data["metadata"]["standard_logging_guardrail_information"] + assert info[0]["guardrail_status"] == "guardrail_intervened" + + #2: if an API error - guardrail status should be guardrail_failed_to_respond" + guardrail2 = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test2", + fail_on_error=True, + ) + + guardrail2._ensure_access_token_async = AsyncMock(side_effect=ConnectionError("timeout")) + request_data2 = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "hello"}], + "metadata": {"guardrails": ["model-armor-test2"]}, + } + with pytest.raises(ConnectionError): + await guardrail2.async_pre_call_hook( + user_api_key_dict=mock_user_api_key_dict, + cache=mock_cache, + data=request_data2, + call_type="completion", + ) + + info2 = request_data2["metadata"]["standard_logging_guardrail_information"] + assert info2[0]["guardrail_status"] == "guardrail_failed_to_respond" + + def mock_open(read_data=''): """Helper to create a mock file object""" import io diff --git a/tests/test_litellm/proxy/image_endpoints/test_endpoints.py b/tests/test_litellm/proxy/image_endpoints/test_endpoints.py index a3b6a9c6022..c35630176bc 100644 --- a/tests/test_litellm/proxy/image_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/image_endpoints/test_endpoints.py @@ -40,10 +40,14 @@ async def test_image_generation_prompt_rerouting(monkeypatch): async def fake_post_call_failure_hook(**_: Any) -> None: return None + async def fake_post_call_success_hook(*, data, user_api_key_dict, response): + return response + fake_proxy_logger = SimpleNamespace( pre_call_hook=fake_pre_call_hook, update_request_status=fake_update_request_status, post_call_failure_hook=fake_post_call_failure_hook, + post_call_success_hook=fake_post_call_success_hook, ) captured_route_request_data: Dict[str, Any] = {} diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py new file mode 100644 index 00000000000..5f204918d08 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -0,0 +1,583 @@ +""" +Tests for access group management endpoints. +""" + +import os +import sys +import types +from contextlib import asynccontextmanager +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi.testclient import TestClient +from prisma.errors import PrismaError + +import litellm.proxy.proxy_server as ps +from litellm.proxy.proxy_server import app +from litellm.proxy._types import ( + CommonProxyErrors, + LitellmUserRoles, + UserAPIKeyAuth, +) + +sys.path.insert(0, os.path.abspath("../../../")) + + +def _make_access_group_record( + access_group_id: str = "ag-123", + access_group_name: str = "test-group", + description: str | None = "Test description", + access_model_ids: list | None = None, + access_mcp_server_ids: list | None = None, + access_agent_ids: list | None = None, + assigned_team_ids: list | None = None, + assigned_key_ids: list | None = None, + created_by: str | None = "admin-user", + updated_by: str | None = "admin-user", + created_at: datetime | None = None, +): + record = MagicMock() + record.access_group_id = access_group_id + record.access_group_name = access_group_name + record.description = description + record.access_model_ids = access_model_ids or [] + record.access_mcp_server_ids = access_mcp_server_ids or [] + record.access_agent_ids = access_agent_ids or [] + record.assigned_team_ids = assigned_team_ids or [] + record.assigned_key_ids = assigned_key_ids or [] + record.created_at = created_at or datetime.now() + record.created_by = created_by + record.updated_at = datetime.now() + record.updated_by = updated_by + return record + + +@pytest.fixture +def client_and_mocks(monkeypatch): + """Setup mock prisma and admin auth for access group endpoints.""" + mock_access_group_table = MagicMock() + mock_prisma = MagicMock() + + def _create_side_effect(*, data): + return _make_access_group_record( + access_group_id="ag-new", + access_group_name=data.get("access_group_name", "new"), + description=data.get("description"), + access_model_ids=data.get("access_model_ids", []), + access_mcp_server_ids=data.get("access_mcp_server_ids", []), + access_agent_ids=data.get("access_agent_ids", []), + assigned_team_ids=data.get("assigned_team_ids", []), + assigned_key_ids=data.get("assigned_key_ids", []), + created_by=data.get("created_by"), + updated_by=data.get("updated_by"), + ) + + mock_access_group_table.create = AsyncMock(side_effect=_create_side_effect) + mock_access_group_table.find_unique = AsyncMock(return_value=None) + mock_access_group_table.find_many = AsyncMock(return_value=[]) + mock_access_group_table.update = AsyncMock(side_effect=lambda *, where, data: _make_access_group_record( + access_group_id=where.get("access_group_id", "ag-123"), + access_group_name=data.get("access_group_name", "updated"), + description=data.get("description"), + access_model_ids=data.get("access_model_ids", []), + access_mcp_server_ids=data.get("access_mcp_server_ids", []), + access_agent_ids=data.get("access_agent_ids", []), + assigned_team_ids=data.get("assigned_team_ids", []), + assigned_key_ids=data.get("assigned_key_ids", []), + updated_by=data.get("updated_by"), + )) + mock_access_group_table.delete = AsyncMock(return_value=None) + + mock_team_table = MagicMock() + mock_team_table.find_many = AsyncMock(return_value=[]) + mock_team_table.update = AsyncMock(return_value=None) + + mock_key_table = MagicMock() + mock_key_table.find_many = AsyncMock(return_value=[]) + mock_key_table.update = AsyncMock(return_value=None) + + @asynccontextmanager + async def mock_tx(): + tx = types.SimpleNamespace( + litellm_accessgrouptable=mock_access_group_table, + litellm_teamtable=mock_team_table, + litellm_verificationtoken=mock_key_table, + ) + yield tx + + mock_db = types.SimpleNamespace( + litellm_accessgrouptable=mock_access_group_table, + litellm_teamtable=mock_team_table, + litellm_verificationtoken=mock_key_table, + tx=mock_tx, + ) + mock_prisma.db = mock_db + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + admin_user = UserAPIKeyAuth( + user_id="admin_user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: admin_user + + client = TestClient(app) + + yield client, mock_prisma, mock_access_group_table + + app.dependency_overrides.clear() + monkeypatch.setattr(ps, "prisma_client", ps.prisma_client) + + +# Paths for primary and alias endpoints (alias: /v1/unified_access_group) +ACCESS_GROUP_PATHS = ["/v1/access_group", "/v1/unified_access_group"] + + +# --------------------------------------------------------------------------- +# CREATE +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize( + "payload", + [ + {"access_group_name": "group-a"}, + { + "access_group_name": "group-b", + "description": "Group B description", + "access_model_ids": ["model-1"], + "access_mcp_server_ids": ["mcp-1"], + "assigned_team_ids": ["team-1"], + }, + ], +) +def test_create_access_group_success(client_and_mocks, base_path, payload): + """Create access group with various payloads returns 201.""" + client, _, mock_table = client_and_mocks + + resp = client.post(base_path, json=payload) + assert resp.status_code == 201 + body = resp.json() + assert body["access_group_name"] == payload["access_group_name"] + assert body.get("access_group_id") is not None + mock_table.create.assert_awaited_once() + + +def test_create_access_group_duplicate_name_conflict(client_and_mocks): + """Create with duplicate name returns 409.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_name="existing-group") + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.post("/v1/access_group", json={"access_group_name": "existing-group"}) + assert resp.status_code == 409 + assert "already exists" in resp.json()["detail"] + + +@pytest.mark.parametrize( + "error_message", + [ + "Unique constraint failed on the fields: (`access_group_name`)", + "P2002: Unique constraint failed", + "unique constraint violation", + ], +) +def test_create_access_group_race_condition_returns_409(client_and_mocks, error_message): + """Create race condition: Prisma unique constraint surfaces as 409, not 500.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + mock_table.create = AsyncMock(side_effect=Exception(error_message)) + + resp = client.post("/v1/access_group", json={"access_group_name": "race-group"}) + assert resp.status_code == 409 + assert "already exists" in resp.json()["detail"] + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_create_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot create access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.post("/v1/access_group", json={"access_group_name": "forbidden"}) + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +def test_create_access_group_validation_missing_name(client_and_mocks): + """Create with missing access_group_name returns 422.""" + client, _, _ = client_and_mocks + + resp = client.post("/v1/access_group", json={}) + assert resp.status_code == 422 + + +def test_create_access_group_500_on_non_constraint_prisma_error(client_and_mocks): + """Create with non-unique-constraint Prisma error returns 500.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + mock_table.create = AsyncMock(side_effect=Exception("Some other database error")) + + # Use raise_server_exceptions=False so unhandled exceptions become 500 responses + test_client = TestClient(app, raise_server_exceptions=False) + resp = test_client.post("/v1/access_group", json={"access_group_name": "test-group"}) + assert resp.status_code == 500 + + +# --------------------------------------------------------------------------- +# LIST +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +def test_list_access_groups_success_empty(client_and_mocks, base_path): + """List access groups returns empty list when none exist.""" + client, _, mock_table = client_and_mocks + + resp = client.get(base_path) + assert resp.status_code == 200 + assert resp.json() == [] + mock_table.find_many.assert_awaited_once() + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +def test_list_access_groups_success_with_items(client_and_mocks, base_path): + """List access groups returns items when they exist.""" + client, _, mock_table = client_and_mocks + + records = [ + _make_access_group_record(access_group_id="ag-1", access_group_name="group-1"), + _make_access_group_record(access_group_id="ag-2", access_group_name="group-2"), + ] + mock_table.find_many = AsyncMock(return_value=records) + + resp = client.get(base_path) + assert resp.status_code == 200 + body = resp.json() + assert len(body) == 2 + assert body[0]["access_group_name"] == "group-1" + assert body[1]["access_group_name"] == "group-2" + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +def test_list_access_groups_ordered_by_created_at_desc(client_and_mocks, base_path): + """List access groups calls find_many with created_at desc order.""" + client, _, mock_table = client_and_mocks + + older = datetime(2025, 1, 1, 12, 0, 0) + newer = datetime(2025, 1, 2, 12, 0, 0) + records = [ + _make_access_group_record( + access_group_id="ag-newer", + access_group_name="newer-group", + created_at=newer, + ), + _make_access_group_record( + access_group_id="ag-older", + access_group_name="older-group", + created_at=older, + ), + ] + mock_table.find_many = AsyncMock(return_value=records) + + resp = client.get(base_path) + assert resp.status_code == 200 + body = resp.json() + assert len(body) == 2 + # Mock returns newest first (simulating Prisma order desc) + assert body[0]["access_group_name"] == "newer-group" + assert body[1]["access_group_name"] == "older-group" + mock_table.find_many.assert_awaited_once_with(order={"created_at": "desc"}) + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_list_access_groups_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot list access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.get("/v1/access_group") + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +# --------------------------------------------------------------------------- +# GET +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize("access_group_id", ["ag-123", "ag-other-id"]) +def test_get_access_group_success(client_and_mocks, base_path, access_group_id): + """Get access group by id returns record when found.""" + client, _, mock_table = client_and_mocks + + record = _make_access_group_record(access_group_id=access_group_id) + mock_table.find_unique = AsyncMock(return_value=record) + + resp = client.get(f"{base_path}/{access_group_id}") + assert resp.status_code == 200 + assert resp.json()["access_group_id"] == access_group_id + + +def test_get_access_group_not_found(client_and_mocks): + """Get access group returns 404 when not found.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + + resp = client.get("/v1/access_group/nonexistent-id") + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_get_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot get access group.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.get("/v1/access_group/ag-123") + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +# --------------------------------------------------------------------------- +# UPDATE +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize( + "update_payload", + [ + {"description": "Updated description"}, + {"access_model_ids": ["model-1", "model-2"]}, + {"assigned_team_ids": [], "assigned_key_ids": ["key-1"]}, + ], +) +def test_update_access_group_success(client_and_mocks, base_path, update_payload): + """Update access group with various payloads returns 200.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-update") + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.put(f"{base_path}/ag-update", json=update_payload) + assert resp.status_code == 200 + mock_table.update.assert_awaited_once() + + +def test_update_access_group_not_found(client_and_mocks): + """Update access group returns 404 when not found.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + + resp = client.put( + "/v1/access_group/nonexistent-id", + json={"description": "Updated"}, + ) + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + mock_table.update.assert_not_awaited() + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_update_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot update access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.put("/v1/access_group/ag-123", json={"description": "Updated"}) + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +def test_update_access_group_empty_body(client_and_mocks): + """Update with empty body succeeds; only updated_by is set.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-update", access_group_name="unchanged") + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.put("/v1/access_group/ag-update", json={}) + assert resp.status_code == 200 + mock_table.update.assert_awaited_once() + call_kwargs = mock_table.update.call_args.kwargs + assert call_kwargs["where"] == {"access_group_id": "ag-update"} + assert "updated_by" in call_kwargs["data"] + assert call_kwargs["data"]["updated_by"] == "admin_user" + + +# --------------------------------------------------------------------------- +# DELETE +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize("access_group_id", ["ag-123", "ag-delete-me"]) +def test_delete_access_group_success(client_and_mocks, base_path, access_group_id): + """Delete access group returns 204 when found.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id=access_group_id) + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.delete(f"{base_path}/{access_group_id}") + assert resp.status_code == 204 + mock_table.delete.assert_awaited_once() + + +def test_delete_access_group_not_found(client_and_mocks): + """Delete access group returns 404 when not found.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + + resp = client.delete("/v1/access_group/nonexistent-id") + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + mock_table.delete.assert_not_awaited() + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_delete_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot delete access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.delete("/v1/access_group/ag-123") + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +def test_delete_access_group_cleans_up_teams_and_keys(client_and_mocks): + """Delete removes access_group_id from teams and keys before deleting the group.""" + client, mock_prisma, mock_access_group_table = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + team_with_group = MagicMock() + team_with_group.team_id = "team-1" + team_with_group.access_group_ids = ["ag-to-delete", "ag-other"] + mock_team_table.find_many = AsyncMock(return_value=[team_with_group]) + + key_with_group = MagicMock() + key_with_group.token = "key-token-1" + key_with_group.access_group_ids = ["ag-to-delete"] + mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 204 + + mock_team_table.update.assert_awaited_once_with( + where={"team_id": "team-1"}, + data={"access_group_ids": ["ag-other"]}, + ) + mock_key_table.update.assert_awaited_once_with( + where={"token": "key-token-1"}, + data={"access_group_ids": []}, + ) + mock_access_group_table.delete.assert_awaited_once_with( + where={"access_group_id": "ag-to-delete"} + ) + + +def test_delete_access_group_503_on_db_connection_error(client_and_mocks): + """Delete returns 503 when DB connection error occurs during transaction.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_table.find_unique = AsyncMock(return_value=existing) + mock_table.delete = AsyncMock(side_effect=PrismaError()) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 503 + assert resp.json()["detail"] == CommonProxyErrors.db_not_connected_error.value + + +def test_delete_access_group_404_on_p2025_or_record_not_found(client_and_mocks): + """Delete returns 404 when Prisma raises P2025 or record-not-found error.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_table.find_unique = AsyncMock(return_value=existing) + mock_table.delete = AsyncMock(side_effect=Exception("P2025: Record to delete does not exist")) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + + +def test_delete_access_group_500_on_generic_exception(client_and_mocks): + """Delete returns 500 when generic exception occurs during transaction.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_table.find_unique = AsyncMock(return_value=existing) + mock_table.delete = AsyncMock(side_effect=RuntimeError("Unexpected error")) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 500 + assert "Failed to delete access group" in resp.json()["detail"] + + +# --------------------------------------------------------------------------- +# DB NOT CONNECTED +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "method,url,factory", + [ + ("post", "/v1/access_group", lambda: {"json": {"access_group_name": "test"}}), + ("get", "/v1/access_group", lambda: {}), + ("get", "/v1/access_group/ag-123", lambda: {}), + ("put", "/v1/access_group/ag-123", lambda: {"json": {"description": "x"}}), + ("delete", "/v1/access_group/ag-123", lambda: {}), + # Alias: /v1/unified_access_group + ("post", "/v1/unified_access_group", lambda: {"json": {"access_group_name": "test"}}), + ("get", "/v1/unified_access_group", lambda: {}), + ("get", "/v1/unified_access_group/ag-123", lambda: {}), + ("put", "/v1/unified_access_group/ag-123", lambda: {"json": {"description": "x"}}), + ("delete", "/v1/unified_access_group/ag-123", lambda: {}), + ], +) +def test_access_group_endpoints_db_not_connected(client_and_mocks, monkeypatch, method, url, factory): + """All endpoints return 500 when DB is not connected.""" + client, _, _ = client_and_mocks + + monkeypatch.setattr(ps, "prisma_client", None) + + resp = getattr(client, method)(url, **factory()) + assert resp.status_code == 500 + assert resp.json()["detail"]["error"] == CommonProxyErrors.db_not_connected_error.value diff --git a/tests/test_litellm/proxy/test_api_key_masking_in_errors.py b/tests/test_litellm/proxy/test_api_key_masking_in_errors.py new file mode 100644 index 00000000000..2c16a2fd8bd --- /dev/null +++ b/tests/test_litellm/proxy/test_api_key_masking_in_errors.py @@ -0,0 +1,136 @@ +""" +Tests that API keys are masked in error responses. + +When an invalid/malformed API key is sent (e.g., with a leading space or +wrong prefix), the error response must NOT return the key in plain text. +Instead, it should show only the first 4 and last 4 characters with **** +in the middle. +""" + +import pytest + + +class TestKeyMaskingInAuthErrors: + """Test that user_api_key_auth masks keys in validation error messages.""" + + def test_assert_message_masks_key_without_sk_prefix(self): + """ + When a key doesn't start with 'sk-', the AssertionError message + should contain a masked version, not the full key. + """ + from litellm.proxy.auth.auth_utils import abbreviate_api_key + + # Simulate the logic from user_api_key_auth.py + api_key = "my-secret-api-key-1234567890abcdef" + _masked_key = ( + "{}****{}".format(api_key[:4], api_key[-4:]) + if len(api_key) > 8 + else "****" + ) + + # The masked key should NOT contain the full original key + assert api_key not in _masked_key + # Should show first 4 and last 4 chars + assert _masked_key == "my-s****cdef" + + def test_assert_message_masks_key_with_leading_space(self): + """ + Reported case: key with leading space like ' sk-abc123...' + """ + api_key = " sk-abc123def456ghi789jkl012mno345pqr" + _masked_key = ( + "{}****{}".format(api_key[:4], api_key[-4:]) + if len(api_key) > 8 + else "****" + ) + + assert api_key not in _masked_key + assert _masked_key == " sk-****5pqr" + + def test_assert_message_masks_short_key(self): + """Short keys (<=8 chars) should be fully masked.""" + api_key = "short" + _masked_key = ( + "{}****{}".format(api_key[:4], api_key[-4:]) + if len(api_key) > 8 + else "****" + ) + assert _masked_key == "****" + + def test_key_not_starting_with_sk_raises_masked_error(self): + """ + Verify the assert message format contains masked key, not the original. + + Note: Python's AssertionError str(e) includes the expression + message, + but the *message* part (which is what gets passed to ProxyException) + should only contain the masked key. + """ + api_key = "bad-key-format-1234567890abcdefghijklmnop" + _masked_key = ( + "{}****{}".format(api_key[:4], api_key[-4:]) + if len(api_key) > 8 + else "****" + ) + + # Build the same message string that user_api_key_auth.py would produce + error_message = "LiteLLM Virtual Key expected. Received={}, expected to start with 'sk-'.".format( + _masked_key + ) + # The full key must NOT appear in the message + assert api_key not in error_message + # The masked version should appear + assert _masked_key in error_message + # Should still have helpful context + assert "expected to start with 'sk-'" in error_message + + +class TestKeyMaskingInKeyManagement: + """Test that key_management_endpoints masks keys in validation errors.""" + + def test_invalid_key_format_error_is_masked(self): + """ + When creating a key that doesn't start with 'sk-', the error + should not include the full key value. + """ + key_value = "bad-prefix-1234567890abcdefghijklmnop" + _masked = ( + "{}****{}".format(key_value[:4], key_value[-4:]) + if len(key_value) > 8 + else "****" + ) + + error_msg = f"Invalid key format. LiteLLM Virtual Key must start with 'sk-'. Received: {_masked}" + + # Full key must not appear + assert key_value not in error_msg + # Masked version should appear + assert _masked in error_msg + assert "bad-****mnop" in error_msg + + +class TestPresidioErrorSanitization: + """Test that Presidio errors don't leak request text containing keys.""" + + def test_analyze_text_error_does_not_leak_text(self): + """ + If Presidio analyzer fails, the error message should NOT contain + the original text that was being analyzed. + """ + # Simulate what happens: user message contains an API key, + # Presidio fails, error message should be sanitized + original_text = "Please use this key: sk-secret1234567890abcdefghijklmnop" + + # The sanitized exception from our fix + sanitized_error = f"Presidio PII analysis failed: ConnectionError" + + assert original_text not in sanitized_error + assert "sk-secret1234567890abcdefghijklmnop" not in sanitized_error + + def test_anonymize_text_error_does_not_leak_text(self): + """ + If Presidio anonymizer fails, the error should be sanitized. + """ + sanitized_error = f"Presidio PII anonymization failed: ClientError" + + assert "sk-" not in sanitized_error + assert "api_key" not in sanitized_error diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index e7da4256182..d65df0087ad 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -2996,9 +2996,13 @@ async def test_get_image_non_root_uses_var_lib_assets_dir(monkeypatch): monkeypatch.setenv("LITELLM_NON_ROOT", "true") monkeypatch.delenv("UI_LOGO_PATH", raising=False) - # Mock os.path operations + # Mock os.path operations - exists=False for assets_dir so makedirs gets called + def exists_side_effect(path): + return False if path == "/var/lib/litellm/assets" else True + with patch("litellm.proxy.proxy_server.os.makedirs") as mock_makedirs, \ - patch("litellm.proxy.proxy_server.os.path.exists", return_value=True), \ + patch("litellm.proxy.proxy_server.os.path.exists", side_effect=exists_side_effect), \ + patch("litellm.proxy.proxy_server.os.access", return_value=True), \ patch("litellm.proxy.proxy_server.os.getenv") as mock_getenv, \ patch("litellm.proxy.proxy_server.FileResponse") as mock_file_response: @@ -3038,14 +3042,16 @@ async def test_get_image_non_root_fallback_to_default_logo(monkeypatch): def exists_side_effect(path): exists_calls.append(path) - # Return False for /var/lib/litellm/assets/logo.jpg to trigger fallback - if "/var/lib/litellm/assets/logo.jpg" in path: + # Return False for /var/lib/litellm/assets* so: makedirs is called, logo fallback + # triggers, and we don't return early with cached file + if "/var/lib/litellm/assets" in path: return False return True # Mock os.path operations with patch("litellm.proxy.proxy_server.os.makedirs") as mock_makedirs, \ patch("litellm.proxy.proxy_server.os.path.exists", side_effect=exists_side_effect), \ + patch("litellm.proxy.proxy_server.os.access", return_value=True), \ patch("litellm.proxy.proxy_server.os.getenv") as mock_getenv, \ patch("litellm.proxy.proxy_server.FileResponse") as mock_file_response: diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py new file mode 100644 index 00000000000..9c20d630a1b --- /dev/null +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -0,0 +1,103 @@ +""" +Test that litellm.responses() / litellm.aresponses() send the expected request body +over the wire. Expected JSON bodies are stored in expected_responses_api_request/. +""" +import json +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +import litellm + + +def _expected_dir() -> Path: + """Path to expected_responses_api_request folder (sibling of test_litellm/responses).""" + return Path(__file__).resolve().parent.parent / "expected_responses_api_request" + + +@pytest.mark.asyncio +async def test_aresponses_context_management_and_shell_request_body_matches_expected(): + """ + Call litellm.aresponses() with context_management and shell tool; + assert the httpx POST request body matches the expected JSON. + """ + expected_path = _expected_dir() / "context_management_and_shell.json" + assert expected_path.exists(), f"Expected file not found: {expected_path}" + with open(expected_path) as f: + expected_body = json.load(f) + + # Minimal Responses API response so parsing succeeds + mock_response = { + "id": "resp_ctx_shell_test", + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "gpt-4o", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "Done.", "annotations": []} + ], + } + ], + "parallel_tool_calls": True, + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "truncation": None, + "user": None, + } + + class MockResponse: + def __init__(self, json_data, status_code=200): + self._json_data = json_data + self.status_code = status_code + self.text = json.dumps(json_data) + self.headers = httpx.Headers({}) + + def json(self): + return self._json_data + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(mock_response, 200) + + await litellm.aresponses( + model="openai/gpt-4o", + input=expected_body["input"], + context_management=expected_body["context_management"], + tools=expected_body["tools"], + tool_choice=expected_body["tool_choice"], + max_output_tokens=expected_body["max_output_tokens"], + ) + + mock_post.assert_called_once() + request_body = mock_post.call_args.kwargs["json"] + + for key, expected_value in expected_body.items(): + assert key in request_body, f"Missing key in request body: {key}" + assert request_body[key] == expected_value, ( + f"Mismatch for key {key}: got {request_body[key]!r}, expected {expected_value!r}" + ) diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 8f7acb6c120..7feab9c6035 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -2,6 +2,7 @@ import base64 import json import os import sys +from unittest.mock import MagicMock, patch import pytest from fastapi.testclient import TestClient @@ -352,3 +353,34 @@ class TestResponsesAPIProviderSpecificParams: # Should not raise any exception result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params) assert "temperature" in result + + +def test_responses_extra_body_forwarded_to_completion_transformation_handler(): + """ + Regression test: extra_body must be forwarded to response_api_handler + when responses_api_provider_config is None (completion transformation path). + + Before the fix, extra_body was a named parameter of responses() but was + not passed to litellm_completion_transformation_handler.response_api_handler(), + so it was silently dropped. + """ + with patch( + "litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config", + return_value=None, + ), patch( + "litellm.responses.main.litellm_completion_transformation_handler.response_api_handler", + ) as mock_handler: + mock_handler.return_value = MagicMock() + + litellm.responses( + model="openai/gpt-4o", + input="Hello", + extra_body={"custom_key": "custom_value"}, + ) + + mock_handler.assert_called_once() + call_kwargs = mock_handler.call_args + # extra_body can be a positional or keyword arg; check both + assert call_kwargs.kwargs.get("extra_body") == { + "custom_key": "custom_value" + } diff --git a/tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py b/tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py new file mode 100644 index 00000000000..82b7fc4d42c --- /dev/null +++ b/tests/test_litellm/router_strategy/test_budget_limiter_hotpath.py @@ -0,0 +1,232 @@ +import pytest + +import litellm +from litellm.caching.caching import DualCache +from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.types.router import LiteLLM_Params +from litellm.types.utils import BudgetConfig + + +@pytest.fixture +def disable_budget_sync(monkeypatch): + async def noop(*args, **kwargs): + return None + + monkeypatch.setattr( + "litellm.router_strategy.budget_limiter.RouterBudgetLimiting.periodic_sync_in_memory_spend_with_redis", + noop, + ) + + +@pytest.mark.asyncio +async def test_get_llm_provider_for_deployment_dict_does_not_require_litellm_params_instantiation( + disable_budget_sync, monkeypatch +): + class RaiseOnInit: + def __init__(self, *args, **kwargs): + raise AssertionError("LiteLLM_Params should not be instantiated in hot path") + + monkeypatch.setattr( + "litellm.router_strategy.budget_limiter.LiteLLM_Params", + RaiseOnInit, + ) + + provider_budget = RouterBudgetLimiting( + dual_cache=DualCache(), + provider_budget_config={}, + ) + + deployment = {"litellm_params": {"model": "openai/gpt-4o-mini"}} + provider = provider_budget._get_llm_provider_for_deployment(deployment) + + assert provider == "openai" + + +@pytest.mark.asyncio +async def test_get_llm_provider_for_deployment_dict_view_supports_mapping_and_attr_access( + disable_budget_sync, monkeypatch +): + observed = {} + + def _future_style_get_llm_provider( + model, + custom_llm_provider=None, + api_base=None, + api_key=None, + litellm_params=None, + ): + assert litellm_params is not None + observed["model_attr"] = litellm_params.model + observed["provider_get"] = litellm_params.get("custom_llm_provider") + observed["api_base_item"] = litellm_params["api_base"] + observed["has_api_key"] = "api_key" in litellm_params + observed["model_dump"] = litellm_params.model_dump() + return model, "openai", None, None + + monkeypatch.setattr( + "litellm.router_strategy.budget_limiter.litellm.get_llm_provider", + _future_style_get_llm_provider, + ) + + provider_budget = RouterBudgetLimiting( + dual_cache=DualCache(), + provider_budget_config={}, + ) + + deployment = { + "litellm_params": { + "model": "openai/gpt-4o-mini", + "custom_llm_provider": "openai", + "api_base": "https://api.openai.com/v1", + } + } + provider = provider_budget._get_llm_provider_for_deployment(deployment) + + assert provider == "openai" + assert observed["model_attr"] == "openai/gpt-4o-mini" + assert observed["provider_get"] == "openai" + assert observed["api_base_item"] == "https://api.openai.com/v1" + assert observed["has_api_key"] is False + assert observed["model_dump"]["model"] == "openai/gpt-4o-mini" + + +@pytest.mark.asyncio +async def test_async_filter_deployments_resolves_provider_once_per_deployment( + disable_budget_sync, monkeypatch +): + provider_budget = RouterBudgetLimiting( + dual_cache=DualCache(), + provider_budget_config={ + "openai": BudgetConfig(budget_duration="1d", max_budget=100.0), + }, + ) + + healthy_deployments = [ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"id": "deployment-1"}, + }, + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"id": "deployment-2"}, + }, + ] + + provider_resolution_calls = 0 + + def _count_provider_calls(deployment): + nonlocal provider_resolution_calls + provider_resolution_calls += 1 + return "openai" + + monkeypatch.setattr( + provider_budget, + "_get_llm_provider_for_deployment", + _count_provider_calls, + ) + + filtered_deployments = await provider_budget.async_filter_deployments( + model="gpt-4o-mini", + healthy_deployments=healthy_deployments, + messages=[], + request_kwargs={}, + parent_otel_span=None, + ) + + assert len(filtered_deployments) == len(healthy_deployments) + assert provider_resolution_calls == len(healthy_deployments) + + +@pytest.mark.asyncio +async def test_async_filter_deployments_does_not_recompute_provider_when_resolved_none( + disable_budget_sync, monkeypatch +): + provider_budget = RouterBudgetLimiting( + dual_cache=DualCache(), + provider_budget_config={ + "openai": BudgetConfig(budget_duration="1d", max_budget=100.0), + }, + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "max_budget": 100.0, + "budget_duration": "1d", + }, + "model_info": {"id": "deployment-1"}, + } + ], + ) + + healthy_deployments = [ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "unknown-provider/model"}, + "model_info": {"id": "deployment-1"}, + } + ] + + provider_resolution_calls = 0 + + def _provider_returns_none(deployment): + nonlocal provider_resolution_calls + provider_resolution_calls += 1 + return None + + monkeypatch.setattr( + provider_budget, + "_get_llm_provider_for_deployment", + _provider_returns_none, + ) + + filtered_deployments = await provider_budget.async_filter_deployments( + model="gpt-4o-mini", + healthy_deployments=healthy_deployments, + messages=[], + request_kwargs={}, + parent_otel_span=None, + ) + + assert len(filtered_deployments) == len(healthy_deployments) + assert provider_resolution_calls == len(healthy_deployments) + + +def _legacy_provider_resolution(deployment): + """ + Reference implementation used before hot-path optimization. + """ + try: + _litellm_params = LiteLLM_Params(**deployment.get("litellm_params", {"model": ""})) + _, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=_litellm_params.model, + litellm_params=_litellm_params, + ) + except Exception: + return None + return custom_llm_provider + + +@pytest.mark.parametrize( + "deployment", + [ + {"litellm_params": {"model": "openai/gpt-4o-mini"}}, + {"litellm_params": {"model": "gpt-4o-mini", "custom_llm_provider": "openai"}}, + {"litellm_params": {"model": "unknown-provider/model"}}, + ], +) +@pytest.mark.asyncio +async def test_get_llm_provider_for_deployment_matches_legacy_behavior( + disable_budget_sync, deployment +): + provider_budget = RouterBudgetLimiting( + dual_cache=DualCache(), + provider_budget_config={}, + ) + + current_provider = provider_budget._get_llm_provider_for_deployment(deployment) + legacy_provider = _legacy_provider_resolution(deployment) + + assert current_provider == legacy_provider diff --git a/tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py b/tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py new file mode 100644 index 00000000000..83982482623 --- /dev/null +++ b/tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py @@ -0,0 +1,109 @@ +""" +Regression tests for AWS Secrets Manager same-name in-place rotation fix. + +When current_secret_name == new_secret_name (e.g. key alias preserved during +rotation), AWS must use PutSecretValue to update in place instead of +create+delete, which would fail with ResourceExistsException. +""" +from unittest.mock import AsyncMock, patch + +import pytest + +from litellm.secret_managers.aws_secret_manager_v2 import AWSSecretsManagerV2 + + +@pytest.mark.asyncio +async def test_rotate_secret_same_name_uses_put_secret_value(): + """ + When current_secret_name == new_secret_name, async_rotate_secret should + call PutSecretValue (async_put_secret_value) instead of create+delete. + """ + secret_name = "litellm/tenant/litellm-metis-key" + new_value = "sk-new-rotated-key-value" + + with patch.object( + AWSSecretsManagerV2, + "async_put_secret_value", + new_callable=AsyncMock, + return_value={"ARN": "arn:aws:secretsmanager:us-east-1:123:secret:test"}, + ) as mock_put: + with patch.object( + AWSSecretsManagerV2, + "async_write_secret", + new_callable=AsyncMock, + ) as mock_write: + with patch.object( + AWSSecretsManagerV2, + "async_delete_secret", + new_callable=AsyncMock, + ) as mock_delete: + manager = AWSSecretsManagerV2() + result = await manager.async_rotate_secret( + current_secret_name=secret_name, + new_secret_name=secret_name, + new_secret_value=new_value, + ) + + # PutSecretValue (in-place update) should be called + mock_put.assert_called_once_with( + secret_name=secret_name, + secret_value=new_value, + optional_params=None, + timeout=None, + ) + # Create + delete should NOT be called + mock_write.assert_not_called() + mock_delete.assert_not_called() + assert result["ARN"] == "arn:aws:secretsmanager:us-east-1:123:secret:test" + + +@pytest.mark.asyncio +async def test_rotate_secret_different_names_uses_create_delete(): + """ + When current_secret_name != new_secret_name, async_rotate_secret should + use base class logic (create new, delete old). + """ + current_name = "litellm/old-key-alias" + new_name = "litellm/virtual-key-new-token-id" + new_value = "sk-new-key-value" + + with patch.object( + AWSSecretsManagerV2, + "async_read_secret", + new_callable=AsyncMock, + side_effect=["sk-old-value", new_value], # read old, then read new + ): + with patch.object( + AWSSecretsManagerV2, + "async_write_secret", + new_callable=AsyncMock, + return_value={"ARN": "arn:new"}, + ) as mock_write: + with patch.object( + AWSSecretsManagerV2, + "async_delete_secret", + new_callable=AsyncMock, + return_value={}, + ) as mock_delete: + with patch.object( + AWSSecretsManagerV2, + "async_put_secret_value", + new_callable=AsyncMock, + ) as mock_put: + manager = AWSSecretsManagerV2() + await manager.async_rotate_secret( + current_secret_name=current_name, + new_secret_name=new_name, + new_secret_value=new_value, + ) + + # PutSecretValue should NOT be called (different names) + mock_put.assert_not_called() + # Create + delete should be called + mock_write.assert_called_once() + mock_delete.assert_called_once_with( + secret_name=current_name, + recovery_window_in_days=7, + optional_params=None, + timeout=None, + ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 794b3b87187..7374a605798 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -3305,3 +3305,73 @@ class TestIsStreamingRequest: def test_stream_true_overrides_non_streaming_call_type(self): assert _is_streaming_request(kwargs={"stream": True}, call_type=CallTypes.acompletion) is True + + +class TestMetadataNoneHandling: + """ + Test that metadata=None in kwargs doesn't cause TypeError. + + When metadata key exists with value None (e.g., from Azure OpenAI streaming), + dict.get("metadata", {}) returns None (key exists, so default is ignored). + The fix uses (kwargs.get("metadata") or {}) which handles both missing key + and explicit None value. + + Related: #20871 + """ + + def test_metadata_none_get_previous_models(self): + """kwargs.get("metadata") or {} should return {} when metadata is None.""" + kwargs = {"metadata": None} + previous_models = (kwargs.get("metadata") or {}).get( + "previous_models", None + ) + assert previous_models is None + + def test_metadata_none_model_group_check(self): + """'model_group' in (kwargs.get("metadata") or {}) should not raise TypeError.""" + kwargs = {"metadata": None} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} + ) + assert _is_litellm_router_call is False + + def test_metadata_missing_key(self): + """Should work when metadata key is completely absent.""" + kwargs = {} + previous_models = (kwargs.get("metadata") or {}).get( + "previous_models", None + ) + assert previous_models is None + + def test_metadata_present_with_values(self): + """Should work when metadata has actual values.""" + kwargs = {"metadata": {"previous_models": ["model1"], "model_group": "test"}} + previous_models = (kwargs.get("metadata") or {}).get( + "previous_models", None + ) + assert previous_models == ["model1"] + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} + ) + assert _is_litellm_router_call is True + + def test_metadata_none_causes_error_with_old_pattern(self): + """Demonstrate the bug: dict.get('metadata', {}) returns None when key exists with None value.""" + kwargs = {"metadata": None} + # Old pattern: kwargs.get("metadata", {}) returns None because key exists + result = kwargs.get("metadata", {}) + assert result is None # This is the root cause of the bug + + # Attempting to use .get() on None raises AttributeError or TypeError + with pytest.raises((TypeError, AttributeError)): + kwargs.get("metadata", {}).get("previous_models", None) + + # Attempting 'in' on None raises TypeError + with pytest.raises(TypeError): + "model_group" in kwargs.get("metadata", {}) + + def test_litellm_params_metadata_none(self): + """litellm_params.get("metadata") or {} should handle None value.""" + litellm_params = {"metadata": None} + metadata = litellm_params.get("metadata") or {} + assert metadata == {} diff --git a/tests/vector_store_tests/rag/test_rag_vertex_ai.py b/tests/vector_store_tests/rag/test_rag_vertex_ai.py index 76baa749ae2..dc076596f4a 100644 --- a/tests/vector_store_tests/rag/test_rag_vertex_ai.py +++ b/tests/vector_store_tests/rag/test_rag_vertex_ai.py @@ -1,14 +1,19 @@ """ Vertex AI RAG Engine ingestion tests. +Tests the Vertex AI RAG ingestion implementation that: +- Creates RAG corpora automatically (or uses existing ones) +- Uploads files directly to Vertex AI RAG Engine +- Handles long-running operations for corpus creation +- Supports both file upload and GCS import + Requires: - gcloud auth application-default login (for ADC authentication) Environment variables: - VERTEX_PROJECT: GCP project ID (required) -- VERTEX_LOCATION: GCP region (optional, defaults to europe-west1) -- VERTEX_CORPUS_ID: Existing RAG corpus ID (required for Vertex AI) -- GCS_BUCKET_NAME: GCS bucket for file uploads (required) +- VERTEX_LOCATION: GCP region (optional, defaults to us-central1) +- VERTEX_CORPUS_ID: Existing RAG corpus ID (optional - will create if not provided) """ import os @@ -31,37 +36,24 @@ class TestRAGVertexAI(BaseRAGTest): def check_env_vars(self): """Check required environment variables before each test.""" vertex_project = os.environ.get("VERTEX_PROJECT") - corpus_id = os.environ.get("VERTEX_CORPUS_ID") - gcs_bucket = os.environ.get("GCS_BUCKET_NAME") if not vertex_project: pytest.skip("Skipping Vertex AI test: VERTEX_PROJECT required") - if not corpus_id: - pytest.skip("Skipping Vertex AI test: VERTEX_CORPUS_ID required") - - if not gcs_bucket: - pytest.skip("Skipping Vertex AI test: GCS_BUCKET_NAME required") - - # Check if vertexai is installed - try: - from vertexai import rag - except ImportError: - pytest.skip("Skipping Vertex AI test: google-cloud-aiplatform>=1.60.0 required") - def get_base_ingest_options(self) -> RAGIngestOptions: """ Return Vertex AI-specific ingest options. Chunking is configured via chunking_strategy (unified interface), not inside vector_store. + + If VERTEX_CORPUS_ID is not set, a new corpus will be created automatically. """ - corpus_id = os.environ.get("VERTEX_CORPUS_ID") vertex_project = os.environ.get("VERTEX_PROJECT") - vertex_location = os.environ.get("VERTEX_LOCATION", "europe-west1") - gcs_bucket = os.environ.get("GCS_BUCKET_NAME") + vertex_location = os.environ.get("VERTEX_LOCATION", "us-central1") + corpus_id = os.environ.get("VERTEX_CORPUS_ID") # Optional - return { + options: RAGIngestOptions = { "chunking_strategy": { "chunk_size": 512, "chunk_overlap": 100, @@ -70,61 +62,174 @@ class TestRAGVertexAI(BaseRAGTest): "custom_llm_provider": "vertex_ai", "vertex_project": vertex_project, "vertex_location": vertex_location, - "vector_store_id": corpus_id, - "gcs_bucket": gcs_bucket, - "wait_for_import": True, }, } + + # Add corpus ID if provided (otherwise will create new corpus) + if corpus_id: + options["vector_store"]["vector_store_id"] = corpus_id + + return options async def query_vector_store( self, vector_store_id: str, query: str, ) -> Optional[Dict[str, Any]]: - """Query Vertex AI RAG corpus.""" - try: - from vertexai import init as vertexai_init - from vertexai import rag - except ImportError: - pytest.skip("vertexai required for Vertex AI tests") - + """ + Query Vertex AI RAG corpus using LiteLLM's vector store search. + + Args: + vector_store_id: The RAG corpus ID (can be full path or just the ID) + query: The search query + + Returns: + Search results dict or None if no results found + """ vertex_project = os.environ.get("VERTEX_PROJECT") - vertex_location = os.environ.get("VERTEX_LOCATION", "europe-west1") + vertex_location = os.environ.get("VERTEX_LOCATION", "us-central1") - # Initialize Vertex AI - vertexai_init(project=vertex_project, location=vertex_location) + try: + # Use LiteLLM's vector store search + search_response = await litellm.vector_stores.asearch( + vector_store_id=vector_store_id, + query=query, + max_num_results=5, + custom_llm_provider="vertex_ai", + vertex_project=vertex_project, + vertex_location=vertex_location, + ) - # Build corpus name - corpus_name = f"projects/{vertex_project}/locations/{vertex_location}/ragCorpora/{vector_store_id}" + # Check if we got results + if search_response and search_response.get("data"): + results = [] + for item in search_response["data"]: + # Extract text from content + text = "" + if item.get("content"): + for content_item in item["content"]: + if content_item.get("text"): + text += content_item["text"] + + results.append({ + "text": text, + "score": item.get("score", 0.0), + "file_id": item.get("file_id", ""), + "filename": item.get("filename", ""), + }) - # Query the corpus - response = rag.retrieval_query( - rag_resources=[ - rag.RagResource(rag_corpus=corpus_name) - ], - text=query, - rag_retrieval_config=rag.RagRetrievalConfig( - top_k=5, - ), - ) + # Check if query terms appear in results + for result in results: + if query.lower() in result["text"].lower(): + return {"results": results} - if hasattr(response, 'contexts') and response.contexts.contexts: - # Convert to dict format - results = [] - for ctx in response.contexts.contexts: - results.append({ - "text": ctx.text, - "score": ctx.score, - "source_uri": ctx.source_uri, - }) + # Return results even if exact match not found + return {"results": results} - # Check if query terms appear in results - for result in results: - if query.lower() in result["text"].lower(): - return {"results": results} + return None - # Return results even if exact match not found - return {"results": results} + except Exception as e: + print(f"Query failed: {e}") + return None - return None + @pytest.mark.asyncio + async def test_create_corpus_and_ingest(self): + """ + Test creating a new RAG corpus and ingesting a file. + + This test specifically validates: + - Automatic corpus creation when vector_store_id is not provided + - Long-running operation polling for corpus creation + - File upload to the newly created corpus + """ + litellm._turn_on_debug() + + filename, unique_id = self.get_unique_filename("create_corpus") + text_content = f""" + Test document {unique_id} for Vertex AI RAG corpus creation. + This tests the automatic corpus creation feature. + The corpus should be created and the file should be uploaded successfully. + """.encode("utf-8") + file_data = (filename, text_content, "text/plain") + + # Get base options WITHOUT corpus_id to trigger creation + ingest_options = self.get_base_ingest_options() + # Remove corpus_id if it was set from env var + if "vector_store_id" in ingest_options.get("vector_store", {}): + del ingest_options["vector_store"]["vector_store_id"] + + ingest_options["name"] = f"test-create-corpus-{unique_id}" + + try: + response = await litellm.rag.aingest( + ingest_options=ingest_options, + file_data=file_data, + ) + + print(f"Create Corpus Response: {response}") + + # Validate response + assert "id" in response + assert response["id"].startswith("ingest_") + assert "status" in response + assert response["status"] == "completed", f"Expected completed, got {response['status']}" + assert "vector_store_id" in response + assert response["vector_store_id"], "vector_store_id should not be empty" + + # The vector_store_id should be a full corpus path + corpus_id = response["vector_store_id"] + assert "projects/" in corpus_id, "Corpus ID should be a full resource path" + assert "ragCorpora/" in corpus_id, "Corpus ID should contain ragCorpora" + + print(f"✓ Successfully created corpus: {corpus_id}") + print(f"✓ Successfully uploaded file: {response.get('file_id')}") + + except litellm.InternalServerError as e: + pytest.skip(f"Skipping test due to litellm.InternalServerError: {e}") + except Exception as e: + print(f"Test failed with error: {e}") + raise + + @pytest.mark.asyncio + async def test_ingest_with_existing_corpus(self): + """ + Test ingesting a file to an existing RAG corpus. + + This test validates: + - Using an existing corpus_id from environment variable + - Direct file upload without corpus creation + """ + corpus_id = os.environ.get("VERTEX_CORPUS_ID") + if not corpus_id: + pytest.skip("Skipping test: VERTEX_CORPUS_ID not set") + + litellm._turn_on_debug() + + filename, unique_id = self.get_unique_filename("existing_corpus") + text_content = f""" + Test document {unique_id} for existing Vertex AI RAG corpus. + This tests file upload to a pre-existing corpus. + """.encode("utf-8") + file_data = (filename, text_content, "text/plain") + + ingest_options = self.get_base_ingest_options() + ingest_options["name"] = f"test-existing-corpus-{unique_id}" + + try: + response = await litellm.rag.aingest( + ingest_options=ingest_options, + file_data=file_data, + ) + + print(f"Existing Corpus Ingest Response: {response}") + + assert response["status"] == "completed" + assert response["vector_store_id"] == corpus_id or corpus_id in response["vector_store_id"] + assert response.get("file_id"), "file_id should be present" + + print(f"✓ Successfully uploaded to existing corpus: {corpus_id}") + print(f"✓ File ID: {response.get('file_id')}") + + except litellm.InternalServerError as e: + pytest.skip(f"Skipping test due to litellm.InternalServerError: {e}") diff --git a/ui/litellm-dashboard/public/assets/logos/zscaler.svg b/ui/litellm-dashboard/public/assets/logos/zscaler.svg new file mode 100644 index 00000000000..2a95cb02aed --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/zscaler.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx index 8a7119aa4a3..94bfbd54426 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx @@ -14,6 +14,10 @@ vi.mock("../../../playground/llm_calls/fetch_models", () => ({ fetchAvailableModels: vi.fn(), })); +vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ + useModelCostMap: vi.fn().mockReturnValue({ data: null }), +})); + vi.mock("openai", () => ({ default: { OpenAI: vi.fn().mockImplementation(() => ({ @@ -97,20 +101,9 @@ describe("Fallbacks", () => { modelData: mockModelData, }; - const findDeleteButton = (container: HTMLElement) => { - const tableRows = container.querySelectorAll("tbody tr"); - if (tableRows.length === 0) return null; - const firstRow = tableRows[0]; - const actionCells = firstRow.querySelectorAll("td"); - const lastCell = actionCells[actionCells.length - 1]; - const buttons = lastCell.querySelectorAll("button"); - if (buttons.length >= 2) { - return buttons[buttons.length - 1]; - } - const clickableElements = lastCell.querySelectorAll("[class*='cursor-pointer'], button"); - return Array.from(clickableElements).find((el) => - el.className.includes("red") || el.className.includes("hover:text-red") - ) || clickableElements[clickableElements.length - 1]; + const getFirstRowDeleteButton = () => { + const deleteButtons = screen.getAllByTestId("delete-fallback-button"); + return deleteButtons.length > 0 ? deleteButtons[0] : null; }; beforeEach(() => { @@ -156,20 +149,31 @@ describe("Fallbacks", () => { await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); - expect(screen.getByText("gpt-3.5-turbo, claude-3-opus")).toBeInTheDocument(); - expect(screen.getByText("claude-3-opus")).toBeInTheDocument(); + expect(screen.getAllByText(/gpt-3\.5-turbo/).length).toBeGreaterThan(0); + expect(screen.getAllByText(/claude-3-opus/).length).toBeGreaterThan(0); }); }); - it("should open delete modal when delete icon is clicked", async () => { - const user = userEvent.setup(); - const { container } = render(); + it("should show delete button for each fallback row when fallbacks exist", async () => { + render(); await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); }); - const deleteButton = findDeleteButton(container); + const deleteButtons = screen.getAllByTestId("delete-fallback-button"); + expect(deleteButtons.length).toBe(2); + }); + + it("should open delete modal when delete icon is clicked", async () => { + const user = userEvent.setup(); + render(); + + await waitFor(() => { + expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); + }); + + const deleteButton = getFirstRowDeleteButton(); expect(deleteButton).not.toBeNull(); await user.click(deleteButton as HTMLElement); @@ -182,13 +186,13 @@ describe("Fallbacks", () => { it("should delete fallback when confirmed", async () => { const user = userEvent.setup(); - const { container } = render(); + render(); await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); }); - const deleteButton = findDeleteButton(container); + const deleteButton = getFirstRowDeleteButton(); expect(deleteButton).not.toBeNull(); await user.click(deleteButton as HTMLElement); @@ -210,13 +214,13 @@ describe("Fallbacks", () => { it("should close delete modal when cancel is clicked", async () => { const user = userEvent.setup(); - const { container } = render(); + render(); await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); }); - const deleteButton = findDeleteButton(container); + const deleteButton = getFirstRowDeleteButton(); expect(deleteButton).not.toBeNull(); await user.click(deleteButton as HTMLElement); @@ -237,13 +241,13 @@ describe("Fallbacks", () => { const user = userEvent.setup(); const error = new Error("Delete failed"); vi.mocked(networkingModule.setCallbacksCall).mockRejectedValueOnce(error); - const { container } = render(); + render(); await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); }); - const deleteButton = findDeleteButton(container); + const deleteButton = getFirstRowDeleteButton(); expect(deleteButton).not.toBeNull(); await user.click(deleteButton as HTMLElement); @@ -264,13 +268,13 @@ describe("Fallbacks", () => { const user = userEvent.setup(); const error = new Error("Delete failed"); vi.mocked(networkingModule.setCallbacksCall).mockRejectedValueOnce(error); - const { container } = render(); + render(); await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); }); - const deleteButton = findDeleteButton(container); + const deleteButton = getFirstRowDeleteButton(); expect(deleteButton).not.toBeNull(); await user.click(deleteButton as HTMLElement); @@ -296,6 +300,9 @@ describe("Fallbacks", () => { await waitFor(() => { expect(screen.getByTestId("add-fallbacks-button")).toBeInTheDocument(); + expect( + screen.getByText(/No fallbacks configured. Add fallbacks to automatically try another model/), + ).toBeInTheDocument(); }); expect(screen.queryByText("gpt-4")).not.toBeInTheDocument(); @@ -309,6 +316,9 @@ describe("Fallbacks", () => { await waitFor(() => { expect(screen.getByTestId("add-fallbacks-button")).toBeInTheDocument(); + expect( + screen.getByText(/No fallbacks configured. Add fallbacks to automatically try another model/), + ).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx index f493cc51323..9c5933aba3a 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx @@ -1,9 +1,11 @@ -import { PlayIcon, TrashIcon } from "@heroicons/react/outline"; +import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; +import { ArrowRightIcon, PlayIcon, TrashIcon } from "@heroicons/react/outline"; import { Icon, Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow } from "@tremor/react"; -import { Tooltip } from "antd"; +import { Tooltip, Typography } from "antd"; import openai from "openai"; import React, { useEffect, useState } from "react"; import DeleteResourceModal from "../../../common_components/DeleteResourceModal"; +import { ProviderLogo } from "../../../molecules/models/ProviderLogo"; import NotificationsManager from "../../../molecules/notifications_manager"; import { getCallbacksCall, setCallbacksCall } from "../../../networking"; import AddFallbacks from "./AddFallbacks"; @@ -11,6 +13,61 @@ import AddFallbacks from "./AddFallbacks"; type FallbackEntry = { [modelName: string]: string[] }; type Fallbacks = FallbackEntry[]; +const modelCardClass = + "inline-flex items-center gap-2 px-2.5 py-1 rounded-md border border-gray-200 bg-gray-50 text-sm font-medium text-gray-800 shrink-0"; + +function renderModelNameCell( + modelName: string, + getProviderFromModel?: (modelName: string) => string, +): React.ReactNode { + const provider = getProviderFromModel?.(modelName) ?? modelName; + return ( + + + {modelName} + + ); +} + +function renderFallbacksChain( + _primaryModel: string, + fallbackModels: string[], + getProviderFromModel?: (modelName: string) => string, +): React.ReactNode { + const list = Array.isArray(fallbackModels) ? fallbackModels : []; + if (list.length === 0) return null; + + const ChainCard = ({ modelName }: { modelName: string }) => { + const provider = getProviderFromModel?.(modelName) ?? modelName; + return ( + + + {modelName} + + ); + }; + return ( + + + + + + {list.map((model, i) => ( + + {i > 0 && ( + + )} + + + ))} + + + ); +} + interface FallbacksProps { accessToken: string | null; userRole: string | null; @@ -71,6 +128,14 @@ const Fallbacks: React.FC = ({ accessToken, userRole, userID, mo const [fallbackToDelete, setFallbackToDelete] = useState(null); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); + const { data: modelCostMapData } = useModelCostMap(); + const getProviderFromModel = (model: string): string => { + if (modelCostMapData != null && typeof modelCostMapData === "object" && model in modelCostMapData) { + return modelCostMapData[model]["litellm_provider"] ?? ""; + } + return ""; + }; + useEffect(() => { if (!accessToken || !userRole || !userID) { return; @@ -177,6 +242,8 @@ const Fallbacks: React.FC = ({ accessToken, userRole, userID, mo } }; + const hasFallbacks = Array.isArray(routerSettings.fallbacks) && routerSettings.fallbacks.length > 0; + return ( <> = ({ accessToken, userRole, userID, mo value={routerSettings.fallbacks || []} onChange={handleFallbacksChange} /> - - - - Model Name - Fallbacks - Actions - - + {!hasFallbacks ? ( +
+ + No fallbacks configured. Add fallbacks to automatically try another model when the primary + fails. + +
+ ) : ( +
+ + + Model Name + Fallbacks + Actions + + - - {routerSettings["fallbacks"] && - routerSettings["fallbacks"].map((item: FallbackEntry, index: number) => + + {routerSettings["fallbacks"].map((item: FallbackEntry, index: number) => Object.entries(item).map(([key, value]) => ( - {key} - {Array.isArray(value) ? value.join(", ") : value} - + + {renderModelNameCell(key, getProviderFromModel)} + + + {renderFallbacksChain(key, Array.isArray(value) ? value : [], getProviderFromModel)} + + = ({ accessToken, userRole, userID, mo /> - handleDeleteClick(item)} - className="cursor-pointer hover:text-red-600" - /> + onKeyDown={(e) => e.key === "Enter" && handleDeleteClick(item)} + className="cursor-pointer inline-flex" + > + + )), )} - -
+ + + )} { + it("should render", () => { + render(); + + expect(screen.getByText("Routes Configuration")).toBeInTheDocument(); + }); + + it("should display Add Route button", () => { + render(); + + expect(screen.getByRole("button", { name: /add route/i })).toBeInTheDocument(); + }); + + it("should show empty state when no routes are configured", () => { + render(); + + expect(screen.getByText(/no routes configured/i)).toBeInTheDocument(); + }); + + it("should add a route when Add Route is clicked", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: /add route/i })); + + expect(screen.getByText("Route 1: Unnamed")).toBeInTheDocument(); + }); + + it("should call onChange when a route is added", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: /add route/i })); + + expect(onChange).toHaveBeenCalledWith({ + routes: [ + expect.objectContaining({ + name: "", + utterances: [], + description: "", + score_threshold: 0.5, + }), + ], + }); + }); + + it("should initialize routes from value prop", async () => { + const value = { + routes: [ + { + name: "gpt-4", + utterances: ["hello", "hi"], + description: "For greetings", + score_threshold: 0.7, + }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + }); + + it("should support both name and model fields in value prop", async () => { + const value = { + routes: [{ model: "gpt-3.5-turbo", utterances: [], description: "", score_threshold: 0.5 }], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-3.5-turbo")).toBeInTheDocument(); + }); + }); + + it("should remove a route when delete button is clicked", async () => { + const user = userEvent.setup(); + const value = { + routes: [ + { + name: "gpt-4", + utterances: [], + description: "", + score_threshold: 0.5, + }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + const deleteButton = screen.getByRole("button", { name: "delete" }); + await user.click(deleteButton); + + await waitFor(() => { + expect(screen.queryByText("Route 1: gpt-4")).not.toBeInTheDocument(); + expect(screen.getByText(/no routes configured/i)).toBeInTheDocument(); + }); + }); + + it("should call onChange when route is removed", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + const value = { + routes: [ + { + name: "gpt-4", + utterances: [], + description: "", + score_threshold: 0.5, + }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + const deleteButton = screen.getByRole("button", { name: "delete" }); + await user.click(deleteButton); + + await waitFor(() => { + expect(onChange).toHaveBeenCalledWith({ routes: [] }); + }); + }); + + + + + it("should update route when description is changed", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + const value = { + routes: [ + { + name: "gpt-4", + utterances: [], + description: "", + score_threshold: 0.5, + }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + const descriptionInput = screen.getByPlaceholderText("Describe when this route should be used..."); + await user.type(descriptionInput, "For code generation"); + + await waitFor(() => { + const lastCall = onChange.mock.calls[onChange.mock.calls.length - 1]; + expect(lastCall[0].routes[0].description).toBe("For code generation"); + }); + }); + + it("should update route when score threshold is changed", async () => { + const onChange = vi.fn(); + const value = { + routes: [ + { + name: "gpt-4", + utterances: [], + description: "", + score_threshold: 0.5, + }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + const scoreInput = screen.getByRole("spinbutton"); + fireEvent.change(scoreInput, { target: { value: "0.9" } }); + + await waitFor(() => { + const lastCall = onChange.mock.calls[onChange.mock.calls.length - 1]; + expect(lastCall[0].routes[0].score_threshold).toBe(0.9); + }); + }); + + it("should add multiple routes", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: /add route/i })); + await user.click(screen.getByRole("button", { name: /add route/i })); + + expect(screen.getByText("Route 1: Unnamed")).toBeInTheDocument(); + expect(screen.getByText("Route 2: Unnamed")).toBeInTheDocument(); + }); + + it("should toggle JSON preview visibility", async () => { + const user = userEvent.setup(); + const { container } = render(); + + expect(screen.getByText("JSON Preview")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Show" })).toBeInTheDocument(); + expect(container.querySelector("pre")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Show" })); + + expect(screen.getByRole("button", { name: "Hide" })).toBeInTheDocument(); + expect(container.querySelector("pre")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Hide" })); + + expect(screen.getByRole("button", { name: "Show" })).toBeInTheDocument(); + expect(container.querySelector("pre")).not.toBeInTheDocument(); + }); + + it("should display JSON preview with route data when routes exist", async () => { + const user = userEvent.setup(); + const { container } = render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: "Show" })); + + const preElement = container.querySelector("pre"); + expect(preElement).toBeInTheDocument(); + expect(preElement?.textContent).toContain("gpt-4"); + expect(preElement?.textContent).toContain("hello"); + expect(preElement?.textContent).toContain("0.8"); + }); + + it("should display model selector with options from modelInfo", async () => { + const value = { + routes: [ + { name: "", utterances: [], description: "", score_threshold: 0.5 }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: Unnamed")).toBeInTheDocument(); + }); + + expect(screen.getByText("Model")).toBeInTheDocument(); + const comboboxes = screen.getAllByRole("combobox"); + expect(comboboxes.length).toBeGreaterThan(0); + }); + + it("should clear routes when value prop changes to empty", async () => { + const value = { + routes: [ + { + name: "gpt-4", + utterances: [], + description: "", + score_threshold: 0.5, + }, + ], + }; + const { rerender } = render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + rerender(); + + await waitFor(() => { + expect(screen.getByText(/no routes configured/i)).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.tsx b/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.tsx new file mode 100644 index 00000000000..9b93287b36d --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.tsx @@ -0,0 +1,277 @@ +import { DeleteOutlined, InfoCircleOutlined, PlusOutlined } from "@ant-design/icons"; +import { Select as AntdSelect, Button, Card, Collapse, Divider, Empty, Flex, Input, InputNumber, Space, Tooltip, Typography } from "antd"; +import React, { useEffect, useState } from "react"; +import { ModelGroup } from "../playground/llm_calls/fetch_models"; + +const { Text } = Typography; + +const { TextArea } = Input; + +interface Route { + id: string; + model: string; + utterances: string[]; + description: string; + score_threshold: number; +} + +interface SavedRoute { + id?: string; + name?: string; + model?: string; + utterances?: string[]; + description?: string; + score_threshold?: number; +} + +interface RouterConfig { + routes?: SavedRoute[]; +} + +interface RouterConfigBuilderProps { + modelInfo: ModelGroup[]; + value?: RouterConfig; + onChange?: (config: any) => void; +} + +const RouterConfigBuilder: React.FC = ({ modelInfo, value, onChange }) => { + const [routes, setRoutes] = useState([]); + const [showJsonPreview, setShowJsonPreview] = useState(false); + const [expandedRoutes, setExpandedRoutes] = useState([]); + + // Initialize routes from value prop - preserve existing route IDs to avoid focus loss when parent re-renders + useEffect(() => { + const routesFromValue = value?.routes; + if (routesFromValue) { + const routeIds: string[] = []; + setRoutes((prevRoutes) => { + const initializedRoutes = routesFromValue.map((route: SavedRoute, index: number) => { + const existingRoute = prevRoutes[index]; + const id = existingRoute?.id || route.id || `route-${index}-${Date.now()}`; + routeIds.push(id); + return { + id, + model: route.name || route.model || "", // handle both 'name' and 'model' fields + utterances: route.utterances || [], + description: route.description || "", + score_threshold: route.score_threshold ?? 0.5, + }; + }); + return initializedRoutes; + }); + setExpandedRoutes(routeIds); + } else { + setRoutes([]); + setExpandedRoutes([]); + } + }, [value]); + + // Handle adding a new route + const addRoute = () => { + const newRouteId = `route-${Date.now()}`; + const newRoute: Route = { + id: newRouteId, + model: "", + utterances: [], + description: "", + score_threshold: 0.5, + }; + const updatedRoutes = [...routes, newRoute]; + setRoutes(updatedRoutes); + updateConfig(updatedRoutes); + // Automatically expand the new route + setExpandedRoutes((prev) => [...prev, newRouteId]); + }; + + // Handle removing a route + const removeRoute = (routeId: string) => { + const updatedRoutes = routes.filter((route) => route.id !== routeId); + setRoutes(updatedRoutes); + updateConfig(updatedRoutes); + // Remove from expanded routes as well + setExpandedRoutes((prev) => prev.filter((id) => id !== routeId)); + }; + + // Handle updating a route + const updateRoute = (routeId: string, field: keyof Route, value: any) => { + const updatedRoutes = routes.map((route) => (route.id === routeId ? { ...route, [field]: value } : route)); + setRoutes(updatedRoutes); + updateConfig(updatedRoutes); + }; + + // Update the overall configuration + const updateConfig = (updatedRoutes: Route[]) => { + const config = { + routes: updatedRoutes.map((route) => ({ + name: route.model, + utterances: route.utterances, + description: route.description, + score_threshold: route.score_threshold, + })), + }; + onChange?.(config); + }; + + // Handle utterances change (convert textarea string to array) + const handleUtterancesChange = (routeId: string, utterancesText: string) => { + const utterancesArray = utterancesText + .split("\n") + .map((line) => line.trim()) // Only trims leading/trailing whitespace, preserves internal spaces + .filter((line) => line.length > 0); + updateRoute(routeId, "utterances", utterancesArray); + }; + + // Prepare model options for dropdowns + const modelOptions = modelInfo.map((model) => ({ + value: model.model_group, + label: model.model_group, + })); + + const generateConfig = () => { + return { + routes: routes.map((route) => ({ + name: route.model, + utterances: route.utterances, + description: route.description, + score_threshold: route.score_threshold, + })), + }; + }; + + return ( +
+ + + Routes Configuration + + + + + + + + {/* Routes */} + {routes.length === 0 ? ( + + + + ) : ( + setExpandedRoutes(Array.isArray(keys) ? keys : [keys].filter(Boolean))} + style={{ width: "100%" }} + items={routes.map((route, index) => ({ + key: route.id, + label: ( + + Route {index + 1}: {route.model || "Unnamed"} + + ), + extra: ( +