diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml new file mode 100644 index 00000000000..b442c7dd5f5 --- /dev/null +++ b/.github/workflows/test-litellm-matrix.yml @@ -0,0 +1,109 @@ +name: LiteLLM Unit Tests (Matrix) + +on: + pull_request: + branches: [main] + +# Cancel in-progress runs for the same PR +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + test-group: + # tests/test_litellm split by subdirectory (~560 files total) + - name: "llms" + path: "tests/test_litellm/llms" + workers: 4 + # tests/test_litellm/proxy split by subdirectory (~180 files total) + - name: "proxy-guardrails" + path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers" + workers: 4 + - name: "proxy-core" + path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine" + workers: 4 + - name: "proxy-misc" + path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py" + workers: 4 + - name: "integrations" + path: "tests/test_litellm/integrations" + workers: 4 + - name: "core-utils" + path: "tests/test_litellm/litellm_core_utils" + workers: 2 + - name: "other" + path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types" + workers: 4 + - name: "root" + path: "tests/test_litellm/test_*.py" + workers: 4 + # tests/proxy_unit_tests split alphabetically (~48 files total) + - name: "proxy-unit-a" + path: "tests/proxy_unit_tests/test_[a-o]*.py" + workers: 2 + - name: "proxy-unit-b" + path: "tests/proxy_unit_tests/test_[p-z]*.py" + workers: 2 + + name: test (${{ matrix.test-group.name }}) + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Poetry + uses: snok/install-poetry@v1 + + - name: Cache Poetry dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install pytest-retry==1.6.3 pytest-xdist google-genai==1.22.0 \ + google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core + + - name: Setup litellm-enterprise + run: | + cd enterprise && poetry run pip install -e . && cd .. + + - name: Run tests - ${{ matrix.test-group.name }} + run: | + poetry run pytest ${{ matrix.test-group.path }} \ + --tb=short -vv \ + --maxfail=10 \ + -n ${{ matrix.test-group.workers }} \ + --durations=20 + + # Aggregate job to require all matrix jobs pass + test-complete: + needs: test + runs-on: ubuntu-latest + if: always() + steps: + - name: Check test results + run: | + if [ "${{ needs.test.result }}" != "success" ]; then + echo "Some test groups failed" + exit 1 + fi + echo "All test groups passed!" diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index d9cf2e74a11..dc9b48c28f6 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -1,8 +1,12 @@ name: LiteLLM Mock Tests (folder - tests/test_litellm) +# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs +# the same tests in parallel across 10 jobs for faster CI times. +# Kept for manual debugging only. on: - pull_request: - branches: [ main ] + workflow_dispatch: # Manual trigger only + # pull_request: + # branches: [ main ] jobs: test: diff --git a/.semgrep/rules/README.md b/.semgrep/rules/README.md index 6cffcc32963..0dbb77cdd48 100644 --- a/.semgrep/rules/README.md +++ b/.semgrep/rules/README.md @@ -1,52 +1,22 @@ -# Custom Semgrep Rules +# Custom Semgrep rules for LiteLLM -All `.yml` files under `.semgrep/rules/` run in CI (CircleCI `semgrep` job). +Add custom rule YAML files here. Semgrep loads all `.yml`/`.yaml` files under this directory. -## Add a Rule - -* Add a `.yml` file under `.semgrep/rules///` - - -[Rule syntax →](https://semgrep.dev/docs/writing-rules/rule-syntax/) - -## Organizing Rules - -### Structure: language → domain - -``` -.semgrep/rules///.yml -``` - -Examples: - -- `python/security/unsafe-yaml-load.yml` -- `python/reliability/missing-timeout-http.yml` -- `python/performance/blocking-io-in-async.yml` - -### Rule metadata - -Match tags to the folder for consistent filtering: - -```yaml -metadata: - tags: [python, security] -``` - -### Severity expectations - -All rules must fail CI on findings. No warn-only rules. - -- Use `severity: ERROR` in rule metadata -- If a rule is noisy → refine until low false positives before adding - -## Run Locally +**Run only custom rules (CI / fail on findings):** ```bash semgrep scan --config .semgrep/rules . --error ``` -With Semgrep registry: +**Run with registry + custom rules:** ```bash semgrep scan --config auto --config .semgrep/rules . ``` + +**Layout:** + +- `python/` – Python-specific rules (security, patterns) +- Add more subdirs as needed (e.g. `generic/` for language-agnostic rules) + +See [Semgrep rule syntax](https://semgrep.dev/docs/writing-rules/rule-syntax/). diff --git a/.semgrep/rules/python/unbounded-memory.yml b/.semgrep/rules/python/unbounded-memory.yml new file mode 100644 index 00000000000..811ef689344 --- /dev/null +++ b/.semgrep/rules/python/unbounded-memory.yml @@ -0,0 +1,14 @@ +# Unbounded memory growth – data structures without a clear max limit +# Can lead to OOM under load. + +rules: + - id: unbounded-asyncio-queue + message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues). + severity: ERROR + languages: [python] + pattern-either: + - pattern: asyncio.Queue() + - pattern: asyncio.Queue(maxsize=0) + metadata: + category: correctness + cwe: "CWE-400: Uncontrolled Resource Consumption" \ No newline at end of file diff --git a/Makefile b/Makefile index b867d7ea35e..74031f418d6 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ # LiteLLM Makefile # Simple Makefile for running tests and basic development tasks -.PHONY: help test test-unit test-integration test-unit-helm \ +.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \ + test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \ + test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ info lint lint-dev format \ install-dev install-proxy-dev install-test-deps \ install-helm-unittest check-circular-imports check-import-safety @@ -25,6 +27,16 @@ help: @echo " make check-import-safety - Check import safety" @echo " make test - Run all tests" @echo " make test-unit - Run unit tests (tests/test_litellm)" + @echo " make test-unit-llms - Run LLM provider tests (~225 files)" + @echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)" + @echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)" + @echo " make test-unit-proxy-misc - Run proxy misc tests (~77 files)" + @echo " make test-unit-integrations - Run integration tests (~60 files)" + @echo " make test-unit-core-utils - Run core utils tests (~32 files)" + @echo " make test-unit-other - Run other tests (caching, responses, etc., ~69 files)" + @echo " make test-unit-root - Run root-level tests (~34 files)" + @echo " make test-proxy-unit-a - Run proxy_unit_tests (a-o, ~20 files)" + @echo " make test-proxy-unit-b - Run proxy_unit_tests (p-z, ~28 files)" @echo " make test-integration - Run integration tests" @echo " make test-unit-helm - Run helm unit tests" @@ -129,6 +141,38 @@ test: test-unit: install-test-deps poetry run pytest tests/test_litellm -x -vv -n 4 +# Matrix test targets (matching CI workflow groups) +test-unit-llms: install-test-deps + poetry run pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-guardrails: install-test-deps + poetry run pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-core: install-test-deps + poetry run pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-misc: install-test-deps + poetry run pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 + +test-unit-integrations: install-test-deps + poetry run pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20 + +test-unit-core-utils: install-test-deps + poetry run pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20 + +test-unit-other: install-test-deps + poetry run pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20 + +test-unit-root: install-test-deps + poetry run pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20 + +# Proxy unit tests (tests/proxy_unit_tests split alphabetically) +test-proxy-unit-a: install-test-deps + poetry run pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20 + +test-proxy-unit-b: install-test-deps + poetry run pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20 + test-integration: poetry run pytest tests/ -k "not test_litellm" diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 64126bb0292..004377e19b3 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -59,7 +59,8 @@ RUN mkdir -p /var/lib/litellm/ui && \ mkdir -p "$folder_name" && \ mv "$html_file" "$folder_name/index.html"; \ fi; \ - done ) && \ + done && \ + touch .litellm_ui_ready ) && \ cd /app/ui/litellm-dashboard && rm -rf ./out # Build litellm wheel and place it in wheels dir (replace any PyPI wheels) diff --git a/docker/README.md b/docker/README.md index 6d81276bb4b..7027a30fdd7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -70,9 +70,12 @@ docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d This setup: - Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image. -- Runs the proxy as a non-root user with a read-only rootfs and only two writable tmpfs mounts: +- Runs the proxy as a non-root user with a read-only rootfs and only writable tmpfs mounts: - `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`) - `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`) +- Pre-builds and serves the admin UI from read-only paths: + - `/var/lib/litellm/ui` (pre-restructured Next.js UI with `.litellm_ui_ready` marker) + - `/var/lib/litellm/assets` (UI logos and assets) - Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines. You should also verify offline Prisma behaviour with: diff --git a/docs/my-website/blog/claude_opus_4_6/index.md b/docs/my-website/blog/claude_opus_4_6/index.md index 3fd70661543..82320472e13 100644 --- a/docs/my-website/blog/claude_opus_4_6/index.md +++ b/docs/my-website/blog/claude_opus_4_6/index.md @@ -389,6 +389,10 @@ Compaction blocks are also supported in streaming mode. You'll receive: ### Adaptive Thinking +:::note +When using `reasoning_effort` with Claude Opus 4.6, all values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets with `type: "enabled"`, pass the native `thinking` parameter directly (see "Native thinking param" tab below). +::: + @@ -434,6 +438,21 @@ curl --location 'http://0.0.0.0:4000/v1/messages' \ }' ``` + + + +Use the `thinking` parameter directly for adaptive thinking via the SDK: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-opus-4-6", + messages=[{"role": "user", "content": "Solve this complex problem: What is the optimal strategy for..."}], + thinking={"type": "adaptive"}, +) +``` + diff --git a/docs/my-website/blog/minimax_m2_5/index.md b/docs/my-website/blog/minimax_m2_5/index.md new file mode 100644 index 00000000000..50084fcc1e5 --- /dev/null +++ b/docs/my-website/blog/minimax_m2_5/index.md @@ -0,0 +1,394 @@ +--- +slug: minimax_m2_5 +title: "Day 0 Support: MiniMax-M2.5" +date: 2026-02-12T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +description: "Day 0 support for MiniMax-M2.5 on LiteLLM" +tags: [minimax, M2.5, llm] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway. + +## Supported Models + +LiteLLM supports the following MiniMax models: + +| Model | Description | Input Cost | Output Cost | Context Window | +|-------|-------------|------------|-------------|----------------| +| **MiniMax-M2.5** | Advanced reasoning, Agentic capabilities | $0.3/M tokens | $1.2/M tokens | 1M tokens | +| **MiniMax-M2.5-lightning** | Faster and More Agile (~100 tps) | $0.3/M tokens | $2.4/M tokens | 1M tokens | + +## Features Supported + +- **Prompt Caching**: Reduce costs with cached prompts ($0.03/M tokens for cache read, $0.375/M tokens for cache write) +- **Function Calling**: Built-in tool calling support +- **Reasoning**: Advanced reasoning capabilities with thinking support +- **System Messages**: Full system message support +- **Cost Tracking**: Automatic cost calculation for all requests + +## Docker Image + +```bash +docker pull litellm/litellm:v1.81.3-stable +``` + +## Usage - OpenAI Compatible API (/v1/chat/completions) + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: minimax-m2-5 + litellm_params: + model: minimax/MiniMax-M2.5 + api_key: os.environ/MINIMAX_API_KEY + api_base: https://api.minimax.io/v1 +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e MINIMAX_API_KEY=$MINIMAX_API_KEY \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:v1.81.3-stable \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +### With Reasoning Split + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "messages": [ + { + "role": "user", + "content": "Solve: 2+2=?" + } + ], + "extra_body": { + "reasoning_split": true + } +}' +``` + +## Usage - Anthropic Compatible API (/v1/messages) + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: minimax-m2-5 + litellm_params: + model: minimax/MiniMax-M2.5 + api_key: os.environ/MINIMAX_API_KEY + api_base: https://api.minimax.io/anthropic/v1/messages +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e MINIMAX_API_KEY=$MINIMAX_API_KEY \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:v1.81.3-stable \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "max_tokens": 1000, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +### With Thinking + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "max_tokens": 1000, + "thinking": { + "type": "enabled", + "budget_tokens": 1000 + }, + "messages": [ + { + "role": "user", + "content": "Solve: 2+2=?" + } + ] +}' +``` + +## Usage - LiteLLM SDK + +### OpenAI-compatible API + +```python +import litellm + +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[ + {"role": "user", "content": "Hello, how are you?"} + ], + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +print(response.choices[0].message.content) +``` + +### Anthropic-compatible API + +```python +import litellm + +response = litellm.anthropic.messages.acreate( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Hello, how are you?"}], + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/anthropic/v1/messages", + max_tokens=1000 +) + +print(response.choices[0].message.content) +``` + +### With Thinking + +```python +response = litellm.anthropic.messages.acreate( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Solve: 2+2=?"}], + thinking={"type": "enabled", "budget_tokens": 1000}, + api_key="your-minimax-api-key" +) + +# Access thinking content +for block in response.choices[0].message.content: + if hasattr(block, 'type') and block.type == 'thinking': + print(f"Thinking: {block.thinking}") +``` + +### With Reasoning Split (OpenAI API) + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[ + {"role": "user", "content": "Solve: 2+2=?"} + ], + extra_body={"reasoning_split": True}, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +# Access thinking and response +if hasattr(response.choices[0].message, 'reasoning_details'): + print(f"Thinking: {response.choices[0].message.reasoning_details}") +print(f"Response: {response.choices[0].message.content}") +``` + +## Cost Tracking + +LiteLLM automatically tracks costs for MiniMax-M2.5 requests. The pricing is: + +- **Input**: $0.3 per 1M tokens +- **Output**: $1.2 per 1M tokens +- **Cache Read**: $0.03 per 1M tokens +- **Cache Write**: $0.375 per 1M tokens + +### Accessing Cost Information + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Hello!"}], + api_key="your-minimax-api-key" +) + +# Access cost information +print(f"Cost: ${response._hidden_params.get('response_cost', 0)}") +``` + +## Streaming Support + +### OpenAI API + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Tell me a story"}], + stream=True, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + +### Streaming with Reasoning Split + +```python +stream = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[ + {"role": "user", "content": "Tell me a story"}, + ], + extra_body={"reasoning_split": True}, + stream=True, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +reasoning_buffer = "" +text_buffer = "" + +for chunk in stream: + if hasattr(chunk.choices[0].delta, "reasoning_details") and chunk.choices[0].delta.reasoning_details: + for detail in chunk.choices[0].delta.reasoning_details: + if "text" in detail: + reasoning_text = detail["text"] + new_reasoning = reasoning_text[len(reasoning_buffer):] + if new_reasoning: + print(new_reasoning, end="", flush=True) + reasoning_buffer = reasoning_text + + if chunk.choices[0].delta.content: + content_text = chunk.choices[0].delta.content + new_text = content_text[len(text_buffer):] if text_buffer else content_text + if new_text: + print(new_text, end="", flush=True) + text_buffer = content_text +``` + +## Using with Native SDKs + +### Anthropic SDK via LiteLLM Proxy + +```python +import os +os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000" +os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM proxy key + +import anthropic + +client = anthropic.Anthropic() + +message = client.messages.create( + model="minimax-m2-5", + max_tokens=1000, + system="You are a helpful assistant.", + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hi, how are you?" + } + ] + } + ] +) + +for block in message.content: + if block.type == "thinking": + print(f"Thinking:\n{block.thinking}\n") + elif block.type == "text": + print(f"Text:\n{block.text}\n") +``` + +### OpenAI SDK via LiteLLM Proxy + +```python +import os +os.environ["OPENAI_BASE_URL"] = "http://localhost:4000" +os.environ["OPENAI_API_KEY"] = "sk-1234" # Your LiteLLM proxy key + +from openai import OpenAI + +client = OpenAI() + +response = client.chat.completions.create( + model="minimax-m2-5", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hi, how are you?"}, + ], + extra_body={"reasoning_split": True}, +) + +# Access thinking and response +if hasattr(response.choices[0].message, 'reasoning_details'): + print(f"Thinking:\n{response.choices[0].message.reasoning_details[0]['text']}\n") +print(f"Text:\n{response.choices[0].message.content}\n") +``` diff --git a/docs/my-website/docs/adding_provider/generic_guardrail_api.md b/docs/my-website/docs/adding_provider/generic_guardrail_api.md index 482dedaa8a9..0931c349e48 100644 --- a/docs/my-website/docs/adding_provider/generic_guardrail_api.md +++ b/docs/my-website/docs/adding_provider/generic_guardrail_api.md @@ -93,6 +93,12 @@ Implement `POST /beta/litellm_basic_guardrail_api` "user_api_key_end_user_id": "end user id associated with the litellm virtual key used", "user_api_key_org_id": "org id associated with the litellm virtual key used" }, + "request_headers": { // optional: inbound request headers (allowlist). Allowed headers show their value; all others show "[present]" to indicate the header existed. + "User-Agent": "OpenAI/Python 2.17.0", + "Content-Type": "application/json", + "X-Request-Id": "[present]" + }, + "litellm_version": "1.x.y", // optional: LiteLLM library version running this proxy "input_type": "request", // "request" or "response" "litellm_call_id": "unique_call_id", // the call id of the individual LLM call "litellm_trace_id": "trace_id", // the trace id of the LLM call - useful if there are multiple LLM calls for the same conversation diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 446d663c5ac..de5a4dc610c 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -1473,6 +1473,20 @@ LiteLLM translates OpenAI's `reasoning_effort` to Anthropic's `thinking` paramet | "medium" | "budget_tokens": 2048 | | "high" | "budget_tokens": 4096 | +:::note +For Claude Opus 4.6, all `reasoning_effort` values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets, pass the native `thinking` parameter directly: + +```python +from litellm import completion + +resp = completion( + model="anthropic/claude-opus-4-6", + messages=[{"role": "user", "content": "What is the capital of France?"}], + thinking={"type": "enabled", "budget_tokens": 1024}, +) +``` +::: + @@ -1614,8 +1628,65 @@ curl http://0.0.0.0:4000/v1/chat/completions \ +#### Adaptive Thinking (Claude Opus 4.6) + + +```python +response = litellm.completion( + model="anthropic/claude-opus-4-6", + messages=[{"role": "user", "content": "What is the optimal strategy for solving this problem?"}], + thinking={"type": "adaptive"}, +) +``` + + + + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "anthropic/claude-opus-4-6", + "messages": [{"role": "user", "content": "What is the optimal strategy for solving this problem?"}], + "thinking": {"type": "adaptive"} + }' +``` + + + + +#### Enabled Thinking with Budget + + + + +```python +response = litellm.completion( + model="anthropic/claude-opus-4-6", + messages=[{"role": "user", "content": "What is the capital of France?"}], + thinking={"type": "enabled", "budget_tokens": 5000}, +) +``` + + + + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "anthropic/claude-opus-4-6", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "thinking": {"type": "enabled", "budget_tokens": 5000} + }' +``` + + + ## **Passing Extra Headers to Anthropic API** diff --git a/docs/my-website/docs/providers/dashscope.md b/docs/my-website/docs/providers/dashscope.md index 565776d6c4c..3df0fbab1ba 100644 --- a/docs/my-website/docs/providers/dashscope.md +++ b/docs/my-website/docs/providers/dashscope.md @@ -1,7 +1,7 @@ -# Dashscope (Qwen API) +# Dashscope API (Qwen models) https://dashscope.console.aliyun.com/ -**We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests** +**We support ALL Qwen models (from Alibaba Cloud), just set `dashscope/` as a prefix when sending completion requests** ## API Key ```python @@ -9,6 +9,26 @@ https://dashscope.console.aliyun.com/ os.environ['DASHSCOPE_API_KEY'] ``` +## API Base +You can optionally specify the API base URL depending on your region: + +| Region | API Base | +|--------|----------| +| **International** | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` | +| **China/Beijing** | `https://dashscope.aliyuncs.com/compatible-mode/v1` | + +```python +# Set via environment variable +os.environ['DASHSCOPE_API_BASE'] = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + +# Or pass directly in the completion call +response = completion( + model="dashscope/qwen-turbo", + messages=[{"role": "user", "content": "hello"}], + api_base="https://dashscope-intl.aliyuncs.com/compatible-mode/v1" +) +``` + ## Sample Usage ```python from litellm import completion @@ -43,9 +63,7 @@ for chunk in response: ``` -## Supported Models - ALL Qwen Models Supported! -We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests - +## All supported Models [DashScope Model List](https://help.aliyun.com/zh/model-studio/compatibility-of-openai-with-dashscope?spm=a2c4g.11186623.help-menu-2400256.d_2_8_0.1efd516e2tTXBn&scm=20140722.H_2833609._.OR_help-T_cn~zh-V_1#7f9c78ae99pwz) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 38ad9bdd0ee..05e490a8b46 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -746,6 +746,7 @@ router_settings: | LITERAL_API_URL | API URL for Literal service | LITERAL_BATCH_SIZE | Batch size for Literal operations | LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints +| LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker. | LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours | LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API | LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518 @@ -760,6 +761,7 @@ router_settings: | LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems. | LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM | LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset. +| LITELLM_UI_PATH | Path to directory for Admin UI files. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/ui` in Docker. | LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval. | LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false. | LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours). diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index a5674bf2bc5..56a8b9566db 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -469,6 +469,7 @@ credential_list: api_version: "2023-05-15" credential_info: description: "Production credentials for EU region" + custom_llm_provider: "azure" ``` #### Key Parameters diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index a42d91a7d5f..994788a3ad9 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -250,11 +250,133 @@ The migrate deploy command: ### Read-only File System -If you see a `Permission denied` error, it means the LiteLLM pod is running with a read-only file system. +Running LiteLLM with `readOnlyRootFilesystem: true` is a Kubernetes security best practice that prevents container processes from writing to the root filesystem. LiteLLM fully supports this configuration. -To fix this, just set `LITELLM_MIGRATION_DIR="/path/to/writeable/directory"` in your environment. +#### Quick Fix for Permission Errors -LiteLLM will use this directory to write migration files. +If you see a `Permission denied` error, it means the LiteLLM pod is running with a read-only file system. LiteLLM needs writable directories for: +- **Database migrations**: Set `LITELLM_MIGRATION_DIR="/path/to/writable/directory"` +- **Admin UI**: Set `LITELLM_UI_PATH="/path/to/writable/directory"` +- **UI assets/logos**: Set `LITELLM_ASSETS_PATH="/path/to/writable/directory"` + +#### Complete Read-Only Filesystem Setup (Kubernetes) + +For production deployments with enhanced security, use this configuration: + +**Option 1: Using EmptyDir Volumes with InitContainer (Recommended)** + +This approach copies the pre-built UI from the Docker image to writable emptyDir volumes at pod startup. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm-proxy +spec: + template: + spec: + initContainers: + - name: setup-ui + image: ghcr.io/berriai/litellm:main-stable + command: + - sh + - -c + - | + cp -r /var/lib/litellm/ui/* /app/var/litellm/ui/ && \ + cp -r /var/lib/litellm/assets/* /app/var/litellm/assets/ + volumeMounts: + - name: ui-volume + mountPath: /app/var/litellm/ui + - name: assets-volume + mountPath: /app/var/litellm/assets + + containers: + - name: litellm + image: ghcr.io/berriai/litellm:main-stable + env: + - name: LITELLM_NON_ROOT + value: "true" + - name: LITELLM_UI_PATH + value: "/app/var/litellm/ui" + - name: LITELLM_ASSETS_PATH + value: "/app/var/litellm/assets" + - name: LITELLM_MIGRATION_DIR + value: "/app/migrations" + - name: PRISMA_BINARY_CACHE_DIR + value: "/app/cache/prisma-python/binaries" + - name: XDG_CACHE_HOME + value: "/app/cache" + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 101 + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: /app/config.yaml + subPath: config.yaml + readOnly: true + - name: ui-volume + mountPath: /app/var/litellm/ui + - name: assets-volume + mountPath: /app/var/litellm/assets + - name: cache + mountPath: /app/cache + - name: migrations + mountPath: /app/migrations + + volumes: + - name: config + configMap: + name: litellm-config + - name: ui-volume + emptyDir: + sizeLimit: 100Mi + - name: assets-volume + emptyDir: + sizeLimit: 10Mi + - name: cache + emptyDir: + sizeLimit: 500Mi + - name: migrations + emptyDir: + sizeLimit: 64Mi +``` + +**Option 2: Without UI (API-only deployment)** + +If you don't need the admin UI, you can run with minimal configuration: + +```yaml +env: + - name: LITELLM_NON_ROOT + value: "true" + - name: LITELLM_MIGRATION_DIR + value: "/app/migrations" +securityContext: + readOnlyRootFilesystem: true +``` + +The proxy will log a warning about the UI but API endpoints will work normally. + +#### Environment Variables for Read-Only Filesystems + +| Variable | Purpose | Default | +|----------|---------|---------| +| `LITELLM_UI_PATH` | Admin UI directory | `/var/lib/litellm/ui` (Docker) | +| `LITELLM_ASSETS_PATH` | UI assets/logos | `/var/lib/litellm/assets` (Docker) | +| `LITELLM_MIGRATION_DIR` | Database migrations | Package directory | +| `PRISMA_BINARY_CACHE_DIR` | Prisma binary cache | System default | +| `XDG_CACHE_HOME` | General cache directory | System default | + +#### Important Notes + +1. **Migrations**: Always set `LITELLM_MIGRATION_DIR` to a writable emptyDir path +2. **Prisma Cache**: Set `PRISMA_BINARY_CACHE_DIR` and `XDG_CACHE_HOME` to writable paths +3. **Server Root Path**: If using a custom `server_root_path`, you must pre-process UI files in your Dockerfile as the proxy cannot modify files at runtime with read-only filesystem +4. **Automatic Detection**: The UI is automatically detected as pre-restructured if it contains a `.litellm_ui_ready` marker file (created by the official Docker images) ## 10. Use a Separate Health Check App :::info diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index 140dfd4faf8..dd2b77712c4 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -1023,6 +1023,134 @@ curl http://localhost:4000/v1/responses \ +## Server-side compaction + +For long-running conversations, you can enable **server-side compaction** so that when the rendered context size crosses a threshold, the server automatically runs compaction in-stream and emits a compaction item—no separate `POST /v1/responses/compact` call is required. + +Supported on the OpenAI Responses API when using the `openai` or `azure` provider. Pass `context_management` with a compaction entry and `compact_threshold` (token count; minimum 1000). When the context crosses the threshold, the server compacts in-stream and continues. Chain turns with `previous_response_id` or by appending output items to your next input array. See [OpenAI Compaction guide](https://developers.openai.com/api/docs/guides/compaction) for details. + +For explicit control over when compaction runs, use the standalone compact endpoint (`POST /v1/responses/compact`) instead. + +### Python SDK + +```python showLineNumbers title="Server-side compaction with LiteLLM Python SDK" +import litellm + +# Non-streaming: enable compaction when context exceeds 200k tokens +response = litellm.responses( + model="openai/gpt-4o", + input="Your conversation input...", + context_management=[{"type": "compaction", "compact_threshold": 200000}], + max_output_tokens=1024, +) +print(response) + +# Streaming: same context_management, compaction runs in-stream if threshold is crossed +stream = litellm.responses( + model="openai/gpt-4o", + input="Your conversation input...", + context_management=[{"type": "compaction", "compact_threshold": 200000}], + stream=True, +) +for event in stream: + print(event) +``` + +### LiteLLM Proxy (AI Gateway) + +Use the OpenAI SDK with your proxy as `base_url`, or call the proxy with curl. The proxy forwards `context_management` to the provider. + +**OpenAI Python SDK (proxy as base_url):** + +```python showLineNumbers title="Server-side compaction via LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", # LiteLLM Proxy (AI Gateway) + api_key="your-proxy-api-key", +) + +response = client.responses.create( + model="openai/gpt-4o", + input="Your conversation input...", + context_management=[{"type": "compaction", "compact_threshold": 200000}], + max_output_tokens=1024, +) +print(response) +``` + +**curl (proxy):** + +```bash title="Server-side compaction via curl to LiteLLM Proxy" +curl -X POST "http://localhost:4000/v1/responses" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "openai/gpt-4o", + "input": "Your conversation input...", + "context_management": [{"type": "compaction", "compact_threshold": 200000}], + "max_output_tokens": 1024 + }' +``` + +## Shell tool + +The **Shell tool** lets the model run commands in a hosted container or local runtime (OpenAI Responses API). You pass `tools=[{"type": "shell", "environment": {...}}]`; the `environment` object configures the runtime (e.g. `type: "container_auto"` for auto-provisioned containers). See [OpenAI Shell tool guide](https://developers.openai.com/api/docs/guides/tools-shell) for full options. + +Supported when using the `openai` or `azure` provider with a model that supports the Shell tool. + +### Python SDK + +```python showLineNumbers title="Shell tool with LiteLLM Python SDK" +import litellm + +response = litellm.responses( + model="openai/gpt-5.2", + input="List files in /mnt/data and run python --version.", + tools=[{"type": "shell", "environment": {"type": "container_auto"}}], + tool_choice="auto", + max_output_tokens=1024, +) +``` + +### LiteLLM Proxy (AI Gateway) + +Use the OpenAI SDK with your proxy as `base_url`, or call the proxy with curl. The proxy forwards `tools` (including `type: "shell"`) to the provider. + +**OpenAI Python SDK (proxy as base_url):** + +```python showLineNumbers title="Shell tool via LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-proxy-api-key", +) + +response = client.responses.create( + model="openai/gpt-5.2", + input="List files in /mnt/data.", + tools=[{"type": "shell", "environment": {"type": "container_auto"}}], + tool_choice="auto", + max_output_tokens=1024, +) +``` + +**curl:** + +```bash title="Shell tool via curl to LiteLLM Proxy" +curl -X POST "http://localhost:4000/v1/responses" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "openai/gpt-5.2", + "input": "List files in /mnt/data.", + "tools": [{"type": "shell", "environment": {"type": "container_auto"}}], + "tool_choice": "auto", + "max_output_tokens": 1024 + }' +``` + ## Session Management LiteLLM Proxy supports session management for all supported models. This allows you to store and fetch conversation history (state) in LiteLLM Proxy. diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35-py3-none-any.whl new file mode 100644 index 00000000000..8a443f38ef5 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35.tar.gz new file mode 100644 index 00000000000..4dde13b32e2 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35.tar.gz differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl new file mode 100644 index 00000000000..c98d9cfcfac Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz new file mode 100644 index 00000000000..c8c33404620 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz differ diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql new file mode 100644 index 00000000000..f3a0821d37f --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "LiteLLM_PolicyAttachmentTable" ADD COLUMN "tags" TEXT[] DEFAULT ARRAY[]::TEXT[]; + diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql new file mode 100644 index 00000000000..67e75e84c4a --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql @@ -0,0 +1,33 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- AlterTable +ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- AlterTable +ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- AlterTable +ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- CreateTable +CREATE TABLE "LiteLLM_AccessGroupTable" ( + "access_group_id" TEXT NOT NULL, + "access_group_name" TEXT NOT NULL, + "description" TEXT, + "access_model_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "access_mcp_server_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "access_agent_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "assigned_team_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "assigned_key_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_by" TEXT, + + CONSTRAINT "LiteLLM_AccessGroupTable_pkey" PRIMARY KEY ("access_group_id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_AccessGroupTable_access_group_name_key" ON "LiteLLM_AccessGroupTable"("access_group_name"); + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 558dfcc9517..2a0faee2808 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -128,6 +128,7 @@ model LiteLLM_TeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team @@ -161,6 +162,7 @@ model LiteLLM_DeletedTeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) @@ -293,6 +295,7 @@ model LiteLLM_VerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") budget_id String? @@ -348,6 +351,7 @@ model LiteLLM_DeletedVerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") router_settings Json? @default("{}") @@ -920,3 +924,23 @@ model LiteLLM_PolicyAttachmentTable { updated_at DateTime @default(now()) @updatedAt updated_by String? } + +//Unified Access Groups table for storing unified access groups +model LiteLLM_AccessGroupTable { + access_group_id String @id @default(uuid()) + access_group_name String @unique + description String? + + // Resource memberships - explicit arrays per type + access_model_ids String[] @default([]) + access_mcp_server_ids String[] @default([]) + access_agent_ids String[] @default([]) + + assigned_team_ids String[] @default([]) + assigned_key_ids String[] @default([]) + + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? +} \ No newline at end of file diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index e0a769a5edf..eda49bfb9fa 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.34" +version = "0.4.36" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.34" +version = "0.4.36" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index 538ee727612..0fdbac63feb 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -175,6 +175,7 @@ _async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # Custo pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False +standard_logging_payload_excluded_fields: Optional[List[str]] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 753a94295b3..e546a0dbb02 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -227,6 +227,84 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return input_items, instructions + def _map_optional_params_to_responses_api_request( + self, + optional_params: dict, + responses_api_request: "ResponsesAPIOptionalRequestParams", + ) -> None: + """Map optional_params into responses_api_request (mutates in place).""" + for key, value in optional_params.items(): + if value is None: + continue + if key in ("max_tokens", "max_completion_tokens"): + responses_api_request["max_output_tokens"] = value + elif key == "tools" and value is not None: + responses_api_request["tools"] = ( + self._convert_tools_to_responses_format( + cast(List[Dict[str, Any]], value) + ) + ) + elif key == "response_format": + text_format = self._transform_response_format_to_text_format(value) + if text_format: + responses_api_request["text"] = text_format # type: ignore + elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): + responses_api_request[key] = value # type: ignore + elif key == "previous_response_id": + responses_api_request["previous_response_id"] = value + elif key == "reasoning_effort": + responses_api_request["reasoning"] = self._map_reasoning_effort(value) + elif key == "web_search_options": + self._add_web_search_tool(responses_api_request, value) + + def _build_sanitized_litellm_params( + self, litellm_params: dict + ) -> Dict[str, Any]: + """Build sanitized litellm_params with merged metadata.""" + responses_optional_param_keys = set( + ResponsesAPIOptionalRequestParams.__annotations__.keys() + ) + sanitized: Dict[str, Any] = { + key: value + for key, value in litellm_params.items() + if key not in responses_optional_param_keys + } + legacy_metadata = litellm_params.get("metadata") + existing_litellm_metadata = litellm_params.get("litellm_metadata") + merged_litellm_metadata: Dict[str, Any] = {} + if isinstance(legacy_metadata, dict): + merged_litellm_metadata.update(legacy_metadata) + if isinstance(existing_litellm_metadata, dict): + merged_litellm_metadata.update(existing_litellm_metadata) + if merged_litellm_metadata: + sanitized["litellm_metadata"] = merged_litellm_metadata + else: + sanitized.pop("litellm_metadata", None) + return sanitized + + def _merge_responses_api_request_into_request_data( + self, + request_data: Dict[str, Any], + responses_api_request: "ResponsesAPIOptionalRequestParams", + instructions: Optional[str], + ) -> None: + """Add non-None values from responses_api_request into request_data.""" + for key, value in responses_api_request.items(): + if value is None: + continue + if key == "instructions" and instructions: + request_data["instructions"] = instructions + elif key == "stream_options" and isinstance(value, dict): + request_data["stream_options"] = value.get("include_obfuscation") + elif key == "user" and isinstance(value, str): + # OpenAI API requires user param to be max 64 chars - truncate if longer + if len(value) <= 64: + request_data["user"] = value + else: + request_data["user"] = value[:64] + else: + request_data[key] = value + def transform_request( self, model: str, @@ -251,36 +329,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if instructions: responses_api_request["instructions"] = instructions - # Map optional parameters - for key, value in optional_params.items(): - if value is None: - continue - if key in ("max_tokens", "max_completion_tokens"): - responses_api_request["max_output_tokens"] = value - elif key == "tools" and value is not None: - # Convert chat completion tools to responses API tools format - responses_api_request["tools"] = ( - self._convert_tools_to_responses_format( - cast(List[Dict[str, Any]], value) - ) - ) - elif key == "response_format": - # Convert response_format to text.format - text_format = self._transform_response_format_to_text_format(value) - if text_format: - responses_api_request["text"] = text_format # type: ignore - elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): - responses_api_request[key] = value # type: ignore - elif key == "metadata": - responses_api_request["metadata"] = value - elif key == "previous_response_id": - responses_api_request["previous_response_id"] = value - elif key == "reasoning_effort": - responses_api_request["reasoning"] = self._map_reasoning_effort(value) - elif key == "web_search_options": - self._add_web_search_tool(responses_api_request, value) + self._map_optional_params_to_responses_api_request( + optional_params, responses_api_request + ) - # Get stream parameter from litellm_params if not in optional_params stream = optional_params.get("stream") or litellm_params.get("stream", False) verbose_logger.debug(f"Chat provider: Stream parameter: {stream}") @@ -304,11 +356,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): setattr(litellm_logging_obj, "call_type", CallTypes.responses.value) + sanitized_litellm_params = self._build_sanitized_litellm_params( + litellm_params + ) + request_data = { "model": api_model, "input": input_items, "litellm_logging_obj": litellm_logging_obj, - **litellm_params, + **sanitized_litellm_params, "client": client, } @@ -316,18 +372,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): f"Chat provider: Final request model={api_model}, input_items={len(input_items)}" ) - # Add non-None values from responses_api_request - for key, value in responses_api_request.items(): - if value is not None: - if key == "instructions" and instructions: - request_data["instructions"] = instructions - elif key == "stream_options" and isinstance(value, dict): - request_data["stream_options"] = value.get("include_obfuscation") - elif key == "user": # string can't be longer than 64 characters - if isinstance(value, str) and len(value) <= 64: - request_data["user"] = value - else: - request_data[key] = value + self._merge_responses_api_request_into_request_data( + request_data, responses_api_request, instructions + ) if headers: request_data["extra_headers"] = headers diff --git a/litellm/constants.py b/litellm/constants.py index 88c57d3ce4c..4e89ddd7bd9 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -101,6 +101,11 @@ MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE = int( MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL = int( os.getenv("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", "3600") ) + +# Default npm cache directory for STDIO MCP servers. +# npm/npx needs a writable cache dir; in containers the default (~/.npm) +# may not exist or be read-only. /tmp is always writable. +MCP_NPM_CACHE_DIR = os.getenv("MCP_NPM_CACHE_DIR", "/tmp/.npm_mcp_cache") MCP_OAUTH2_TOKEN_CACHE_MIN_TTL = int( os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10") ) diff --git a/litellm/integrations/cloudzero/transform.py b/litellm/integrations/cloudzero/transform.py index c36833a6dbf..b40a71da1c6 100644 --- a/litellm/integrations/cloudzero/transform.py +++ b/litellm/integrations/cloudzero/transform.py @@ -141,7 +141,7 @@ class CBFTransformer: # Required CBF fields 'time/usage_start': usage_date.isoformat() if usage_date else None, # Required: ISO-formatted UTC datetime 'cost/cost': float(row.get('spend', 0.0)), # Required: billed cost - 'resource/id': model, # Send model name + 'resource/id': resource_id, # CZRN (CloudZero Resource Name) # Usage metrics for token consumption 'usage/amount': total_tokens, # Numeric value of tokens consumed diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 407bc581f71..fb395366004 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -624,7 +624,9 @@ class CustomGuardrail(CustomLogger): This gets logged on downsteam Langfuse, DataDog, etc. """ # Convert None to empty dict to satisfy type requirements - guardrail_response = {} if response is None else response + guardrail_response: Union[Dict[str, Any], str] = ( + {} if response is None else response + ) # For apply_guardrail functions in custom_code_guardrail scenario, # simplify the logged response to "allow", "deny", or "mask" diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 4a341863d4b..c244363e389 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -774,15 +774,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac self, model_call_details: Dict ) -> Dict: """ - Only redacts messages and responses when self.turn_off_message_logging is True + Redacts or excludes fields from StandardLoggingPayload before callbacks receive it. + This method handles two features: + 1. turn_off_message_logging: When True, redacts messages and responses + 2. standard_logging_payload_excluded_fields: Removes specified fields entirely - By default, self.turn_off_message_logging is False and this does nothing. - - Return a redacted deepcopy of the provided logging payload. + Return a modified copy of the provided logging payload. This is useful for logging payloads that contain sensitive information. """ + import litellm from copy import copy from litellm import Choices, Message, ModelResponse @@ -790,14 +792,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac turn_off_message_logging: bool = getattr( self, "turn_off_message_logging", False ) + excluded_fields: Optional[List[str]] = getattr( + litellm, "standard_logging_payload_excluded_fields", None + ) - if turn_off_message_logging is False: + # Early return if no processing needed + if turn_off_message_logging is False and not excluded_fields: return model_call_details # Only make a shallow copy of the top-level dict to avoid deepcopy issues # with complex objects like AuthenticationError that may be present model_call_details_copy = copy(model_call_details) - redacted_str = "redacted-by-litellm" standard_logging_object = model_call_details.get("standard_logging_object") if standard_logging_object is None: return model_call_details_copy @@ -805,39 +810,58 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac # Make a copy of just the standard_logging_object to avoid modifying the original standard_logging_object_copy = copy(standard_logging_object) - if standard_logging_object_copy.get("messages") is not None: - standard_logging_object_copy["messages"] = [ - Message(content=redacted_str).model_dump() - ] + # Handle excluded fields - remove them entirely from the payload + if excluded_fields: + for field in excluded_fields: + if field in standard_logging_object_copy: + del standard_logging_object_copy[field] - if standard_logging_object_copy.get("response") is not None: - response = standard_logging_object_copy["response"] - # Check if this is a ResponsesAPIResponse (has "output" field) - if isinstance(response, dict) and "output" in response: - # Make a copy to avoid modifying the original - from copy import deepcopy + # Handle turn_off_message_logging - redact messages and responses (if not already excluded) + if turn_off_message_logging: + redacted_str = "redacted-by-litellm" - response_copy = deepcopy(response) - # Redact content in output array - if isinstance(response_copy.get("output"), list): - for output_item in response_copy["output"]: - if isinstance(output_item, dict) and "content" in output_item: - if isinstance(output_item["content"], list): - # Redact text in content items - for content_item in output_item["content"]: - if ( - isinstance(content_item, dict) - and "text" in content_item - ): - content_item["text"] = redacted_str - standard_logging_object_copy["response"] = response_copy - else: - # Standard ModelResponse format - model_response = ModelResponse( - choices=[Choices(message=Message(content=redacted_str))] - ) - model_response_dict = model_response.model_dump() - standard_logging_object_copy["response"] = model_response_dict + if ( + "messages" not in (excluded_fields or []) + and standard_logging_object_copy.get("messages") is not None + ): + standard_logging_object_copy["messages"] = [ + Message(content=redacted_str).model_dump() + ] + + if ( + "response" not in (excluded_fields or []) + and standard_logging_object_copy.get("response") is not None + ): + response = standard_logging_object_copy["response"] + # Check if this is a ResponsesAPIResponse (has "output" field) + if isinstance(response, dict) and "output" in response: + # Make a copy to avoid modifying the original + from copy import deepcopy + + response_copy = deepcopy(response) + # Redact content in output array + if isinstance(response_copy.get("output"), list): + for output_item in response_copy["output"]: + if ( + isinstance(output_item, dict) + and "content" in output_item + ): + if isinstance(output_item["content"], list): + # Redact text in content items + for content_item in output_item["content"]: + if ( + isinstance(content_item, dict) + and "text" in content_item + ): + content_item["text"] = redacted_str + standard_logging_object_copy["response"] = response_copy + else: + # Standard ModelResponse format + model_response = ModelResponse( + choices=[Choices(message=Message(content=redacted_str))] + ) + model_response_dict = model_response.model_dump() + standard_logging_object_copy["response"] = model_response_dict model_call_details_copy["standard_logging_object"] = ( standard_logging_object_copy diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 3ddcae69315..dde44cced36 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -70,6 +70,11 @@ class ExceptionCheckers: Check if an error string indicates a context window exceeded error. """ _error_str_lowercase = error_str.lower() + # Exclude param validation errors (e.g. OpenAI "user" param max 64 chars) + if "string_above_max_length" in _error_str_lowercase: + return False + if "invalid 'user'" in _error_str_lowercase and "string too long" in _error_str_lowercase: + return False known_exception_substrings = [ "exceed context limit", "this model's maximum context length is", @@ -98,16 +103,18 @@ class ExceptionCheckers: """ Check if an error string indicates a content policy violation error. """ + _lower = error_str.lower() known_exception_substrings = [ - "invalid_request_error", "content_policy_violation", + "responsibleaipolicyviolation", "the response was filtered due to the prompt triggering azure openai's content management", "your task failed as a result of our safety system", "the model produced invalid content", "content_filter_policy", + "your request was rejected as a result of our safety system", ] for substring in known_exception_substrings: - if substring in error_str.lower(): + if substring in _lower: return True return False @@ -2060,6 +2067,19 @@ def exception_type( # type: ignore # noqa: PLR0915 if isinstance(body_dict, dict): if isinstance(body_dict.get("error"), dict): azure_error_code = body_dict["error"].get("code") # type: ignore[index] + # Also check inner_error for + # ResponsibleAIPolicyViolation which indicates a + # content policy violation even when the top-level + # code is generic (e.g. "invalid_request_error"). + if azure_error_code != "content_policy_violation": + _inner = ( + body_dict["error"].get("inner_error") # type: ignore[index] + or body_dict["error"].get("innererror") # type: ignore[index] + ) + if isinstance(_inner, dict) and _inner.get( + "code" + ) == "ResponsibleAIPolicyViolation": + azure_error_code = "content_policy_violation" else: azure_error_code = body_dict.get("code") except Exception: diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 718773a1b16..8ab4ec15b07 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -51,7 +51,7 @@ def handle_cohere_chat_model_custom_llm_provider( if custom_llm_provider == "cohere" and model in litellm.cohere_chat_models: return model, "cohere_chat" - if "/" in model: + if model and "/" in model: _custom_llm_provider, _model = model.split("/", 1) if ( _custom_llm_provider @@ -84,7 +84,7 @@ def handle_anthropic_text_model_custom_llm_provider( ): return model, "anthropic_text" - if "/" in model: + if model and "/" in model: _custom_llm_provider, _model = model.split("/", 1) if ( _custom_llm_provider @@ -113,6 +113,12 @@ def get_llm_provider( # noqa: PLR0915 Return model, custom_llm_provider, dynamic_api_key, api_base """ try: + # Early validation - model is required + if model is None: + raise ValueError( + "model parameter is required but was None. Please provide a valid model name." + ) + if litellm.LiteLLMProxyChatConfig._should_use_litellm_proxy_by_default( litellm_params=litellm_params ): diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index acd08f0a569..9938cd7979b 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -664,35 +664,34 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): reasoning_effort: Optional[Union[REASONING_EFFORT, str]], model: str, ) -> Optional[AnthropicThinkingParam]: + if reasoning_effort is None or reasoning_effort == "none": + return None if AnthropicConfig._is_claude_opus_4_6(model): return AnthropicThinkingParam( type="adaptive", ) + elif reasoning_effort == "low": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, + ) + elif reasoning_effort == "medium": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + ) + elif reasoning_effort == "high": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, + ) + elif reasoning_effort == "minimal": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, + ) else: - if reasoning_effort is None: - return None - elif reasoning_effort == "low": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, - ) - elif reasoning_effort == "medium": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - ) - elif reasoning_effort == "high": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, - ) - elif reasoning_effort == "minimal": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, - ) - else: - raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}") + raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}") def _extract_json_schema_from_response_format( self, value: Optional[dict] diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 76fa713ca8c..44ee51d14ab 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -901,7 +901,20 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): if response.json()["status"] == "failed": error_data = response.json() - raise AzureOpenAIError(status_code=400, message=json.dumps(error_data)) + # Preserve Azure error details (e.g. content_policy_violation, + # inner_error, content_filter_results) as structured body so + # exception_type() can route them correctly. + _error_body = error_data.get("error", error_data) + _error_msg = ( + _error_body.get("message", "Image generation failed") + if isinstance(_error_body, dict) + else json.dumps(error_data) + ) + raise AzureOpenAIError( + status_code=400, + message=_error_msg, + body=error_data, + ) result = response.json()["result"] return httpx.Response( @@ -999,7 +1012,20 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): if response.json()["status"] == "failed": error_data = response.json() - raise AzureOpenAIError(status_code=400, message=json.dumps(error_data)) + # Preserve Azure error details (e.g. content_policy_violation, + # inner_error, content_filter_results) as structured body so + # exception_type() can route them correctly. + _error_body = error_data.get("error", error_data) + _error_msg = ( + _error_body.get("message", "Image generation failed") + if isinstance(_error_body, dict) + else json.dumps(error_data) + ) + raise AzureOpenAIError( + status_code=400, + message=_error_msg, + body=error_data, + ) result = response.json()["result"] return httpx.Response( diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 2c041adba7a..477fa3316d1 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -246,6 +246,11 @@ class AmazonAnthropicClaudeMessagesConfig( "sonnet_4.5", "sonnet-4-5", "sonnet_4_5", + # Opus 4.6 + "opus-4.6", + "opus_4.6", + "opus-4-6", + "opus_4_6", ] return any(pattern in model_lower for pattern in supported_patterns) diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index 1b03ec47643..fb98006c7e4 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -4,7 +4,7 @@ import os import ssl import typing import urllib.request -from typing import Callable, Dict, Optional, Union +from typing import Any, Callable, Dict, Optional, Union import aiohttp import aiohttp.client_exceptions @@ -248,26 +248,25 @@ class LiteLLMAiohttpTransport(AiohttpTransport): # Only pass ssl kwarg when explicitly configured, to avoid # overriding the session/connector defaults with None (which is # not a valid value for aiohttp's ssl parameter). - ssl_kwargs: Dict[str, Union[bool, ssl.SSLContext]] = {} - if ssl_verify is not None: - ssl_kwargs["ssl"] = ssl_verify - - response = await client_session.request( - method=request.method, - url=YarlURL(str(request.url), encoded=True), - headers=request.headers, - data=data, - allow_redirects=False, - auto_decompress=False, - timeout=ClientTimeout( + request_kwargs: Dict[str, Any] = { + "method": request.method, + "url": YarlURL(str(request.url), encoded=True), + "headers": request.headers, + "data": data, + "allow_redirects": False, + "auto_decompress": False, + "timeout": ClientTimeout( sock_connect=timeout.get("connect"), sock_read=timeout.get("read"), connect=timeout.get("pool"), ), - proxy=proxy, - server_hostname=sni_hostname, - **ssl_kwargs, - ).__aenter__() + "proxy": proxy, + "server_hostname": sni_hostname, + } + if ssl_verify is not None: + request_kwargs["ssl"] = ssl_verify + + response = await client_session.request(**request_kwargs).__aenter__() return response diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index cc2439b431a..5c870711062 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -240,24 +240,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): event_pydantic_model = OpenAIResponsesAPIConfig.get_event_model_class( event_type=event_type ) - # Defensive: Some OpenAI-compatible providers may send `error.code: null`. - # Pydantic will raise a ValidationError when it expects a string but gets None. - # Coalesce a None `error.code` to a stable default string so streaming - # iteration does not crash (see issue report). This keeps behavior similar - # to previous fixes (coalesce before validation) and lets higher-level - # handlers still receive an `ErrorEvent` object. + # Some OpenAI-compatible providers send error.code: null; coalesce so validation succeeds. try: error_obj = parsed_chunk.get("error") if isinstance(error_obj, dict) and error_obj.get("code") is None: - # Preserve other fields, but ensure `code` is a non-null string parsed_chunk = dict(parsed_chunk) parsed_chunk["error"] = dict(error_obj) parsed_chunk["error"]["code"] = "unknown_error" except Exception: - # If anything unexpected happens here, fall back to attempting - # instantiation and let higher-level handlers manage errors. verbose_logger.debug("Failed to coalesce error.code in parsed_chunk") - return event_pydantic_model(**parsed_chunk) @staticmethod @@ -307,6 +298,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ResponsesAPIStreamEvents.MCP_CALL_FAILED: MCPCallFailedEvent, ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE: ImageGenerationPartialImageEvent, ResponsesAPIStreamEvents.ERROR: ErrorEvent, + # Shell tool events: passthrough as GenericEvent so payload is preserved + ResponsesAPIStreamEvents.SHELL_CALL_IN_PROGRESS: GenericEvent, + ResponsesAPIStreamEvents.SHELL_CALL_COMPLETED: GenericEvent, + ResponsesAPIStreamEvents.SHELL_CALL_OUTPUT: GenericEvent, } model_class = event_models.get(cast(ResponsesAPIStreamEvents, event_type)) diff --git a/litellm/llms/sagemaker/embedding/transformation.py b/litellm/llms/sagemaker/embedding/transformation.py index bd8abc5e01a..04b201380fc 100644 --- a/litellm/llms/sagemaker/embedding/transformation.py +++ b/litellm/llms/sagemaker/embedding/transformation.py @@ -102,11 +102,18 @@ class SagemakerEmbeddingConfig(BaseEmbeddingConfig): status_code=raw_response.status_code ) - if "embedding" not in response_data: + # Handle both raw array format (TEI) and wrapped format (standard HF) + if isinstance(response_data, list): + # TEI and some HF models return raw embedding arrays directly + embeddings = response_data + elif isinstance(response_data, dict) and "embedding" in response_data: + # Standard HF format with "embedding" key + embeddings = response_data["embedding"] + else: raise SagemakerError( - status_code=500, message="HF response missing 'embedding' field" + status_code=500, + message=f"Unexpected response format. Expected list or dict with 'embedding' key, got: {type(response_data).__name__}", ) - embeddings = response_data["embedding"] if not isinstance(embeddings, list): raise SagemakerError( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f6edcf7efd0..71fb82b2d78 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5859,6 +5859,7 @@ "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, "azure_ai/ministral-3b": { @@ -6134,8 +6135,11 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.03e-06, + "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_video_input": true, + "supports_vision": true }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.18e-06, @@ -15784,44 +15788,14 @@ "tpm": 250000 }, "gemini/gemini-2.5-flash-preview-tts": { - "cache_read_input_token_cost": 3.75e-08, - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 1.5e-07, + "input_cost_per_token": 3e-07, "litellm_provider": "gemini", - "max_audio_length_hours": 8.4, - "max_audio_per_prompt": 1, - "max_images_per_prompt": 3000, - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_pdf_size_mb": 30, - "max_tokens": 65535, - "max_video_length": 1, - "max_videos_per_prompt": 10, - "mode": "chat", - "output_cost_per_reasoning_token": 3.5e-06, - "output_cost_per_token": 6e-07, - "rpm": 10, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "mode": "audio_speech", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions" - ], - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "audio" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000 + "/v1/audio/speech" + ] }, "gemini/gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -21432,6 +21406,36 @@ "max_input_tokens": 1000000, "max_output_tokens": 8192 }, + "minimax/MiniMax-M2.5": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, + "minimax/MiniMax-M2.5-lightning": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, "minimax/MiniMax-M2": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, @@ -22147,9 +22151,10 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat", + "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, "moonshot/kimi-latest": { @@ -22611,6 +22616,19 @@ "output_cost_per_token": 2.3e-07, "supports_system_messages": true }, + "nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 6e-08, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.4e-07, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "o1": { "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, @@ -23616,36 +23634,6 @@ "output_cost_per_token": 2e-07, "supports_system_messages": true }, - "openrouter/anthropic/claude-2": { - "input_cost_per_token": 1.102e-05, - "litellm_provider": "openrouter", - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 3.268e-05, - "supports_tool_choice": true - }, - "openrouter/anthropic/claude-3-5-haiku": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_tokens": 200000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "supports_function_calling": true, - "supports_tool_choice": true - }, - "openrouter/anthropic/claude-3-5-haiku-20241022": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "tool_use_system_prompt_tokens": 264 - }, "openrouter/anthropic/claude-3-haiku": { "input_cost_per_image": 0.0004, "input_cost_per_token": 2.5e-07, @@ -23657,43 +23645,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/anthropic/claude-3-haiku-20240307": { - "input_cost_per_token": 2.5e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 264 - }, - "openrouter/anthropic/claude-3-opus": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 395 - }, - "openrouter/anthropic/claude-3-sonnet": { - "input_cost_per_image": 0.0048, - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_tokens": 200000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/anthropic/claude-3.5-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "openrouter", @@ -23709,20 +23660,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-3.5-sonnet:beta": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 - }, "openrouter/anthropic/claude-3.7-sonnet": { "input_cost_per_image": 0.0048, "input_cost_per_token": 3e-06, @@ -23740,31 +23677,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-3.7-sonnet:beta": { - "input_cost_per_image": 0.0048, - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 - }, - "openrouter/anthropic/claude-instant-v1": { - "input_cost_per_token": 1.63e-06, - "litellm_provider": "openrouter", - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 5.51e-06, - "supports_tool_choice": true - }, "openrouter/anthropic/claude-opus-4": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 1.875e-05, @@ -23903,30 +23815,6 @@ "source": "https://openrouter.ai/api/v1/models/bytedance/ui-tars-1.5-7b", "supports_tool_choice": true }, - "openrouter/cognitivecomputations/dolphin-mixtral-8x7b": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 32769, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/cohere/command-r-plus": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_tool_choice": true - }, - "openrouter/databricks/dbrx-instruct": { - "input_cost_per_token": 6e-07, - "litellm_provider": "openrouter", - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_tool_choice": true - }, "openrouter/deepseek/deepseek-chat": { "input_cost_per_token": 1.4e-07, "litellm_provider": "openrouter", @@ -23994,17 +23882,6 @@ "supports_reasoning": false, "supports_tool_choice": true }, - "openrouter/deepseek/deepseek-coder": { - "input_cost_per_token": 1.4e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 66000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2.8e-07, - "supports_prompt_caching": true, - "supports_tool_choice": true - }, "openrouter/deepseek/deepseek-r1": { "input_cost_per_token": 5.5e-07, "input_cost_per_token_cache_hit": 1.4e-07, @@ -24035,14 +23912,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "openrouter/fireworks/firellava-13b": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, "openrouter/google/gemini-2.0-flash-001": { "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 7e-07, @@ -24204,46 +24073,6 @@ "supports_web_search": true, "tpm": 800000 }, - "openrouter/google/gemini-pro-1.5": { - "input_cost_per_image": 0.00265, - "input_cost_per_token": 2.5e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.5e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "openrouter/google/gemini-pro-vision": { - "input_cost_per_image": 0.0025, - "input_cost_per_token": 1.25e-07, - "litellm_provider": "openrouter", - "max_tokens": 45875, - "mode": "chat", - "output_cost_per_token": 3.75e-07, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "openrouter/google/palm-2-chat-bison": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 25804, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/google/palm-2-codechat-bison": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 20070, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, "openrouter/gryphe/mythomax-l2-13b": { "input_cost_per_token": 1.875e-06, "litellm_provider": "openrouter", @@ -24252,14 +24081,6 @@ "output_cost_per_token": 1.875e-06, "supports_tool_choice": true }, - "openrouter/jondurbin/airoboros-l2-70b-2.1": { - "input_cost_per_token": 1.3875e-05, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.3875e-05, - "supports_tool_choice": true - }, "openrouter/mancer/weaver": { "input_cost_per_token": 5.625e-06, "litellm_provider": "openrouter", @@ -24268,30 +24089,6 @@ "output_cost_per_token": 5.625e-06, "supports_tool_choice": true }, - "openrouter/meta-llama/codellama-34b-instruct": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-2-13b-chat": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-2-70b-chat": { - "input_cost_per_token": 1.5e-06, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_tool_choice": true - }, "openrouter/meta-llama/llama-3-70b-instruct": { "input_cost_per_token": 5.9e-07, "litellm_provider": "openrouter", @@ -24300,38 +24097,6 @@ "output_cost_per_token": 7.9e-07, "supports_tool_choice": true }, - "openrouter/meta-llama/llama-3-70b-instruct:nitro": { - "input_cost_per_token": 9e-07, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-3-8b-instruct:extended": { - "input_cost_per_token": 2.25e-07, - "litellm_provider": "openrouter", - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2.25e-06, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-3-8b-instruct:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 0.0, - "supports_tool_choice": true - }, - "openrouter/microsoft/wizardlm-2-8x22b:nitro": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1e-06, - "supports_tool_choice": true - }, "openrouter/minimax/minimax-m2": { "input_cost_per_token": 2.55e-07, "litellm_provider": "openrouter", @@ -24345,20 +24110,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "openrouter/mistralai/devstral-2512:free": { - "input_cost_per_image": 0, - "input_cost_per_token": 0, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 0, - "supports_function_calling": true, - "supports_prompt_caching": false, - "supports_tool_choice": true, - "supports_vision": false - }, "openrouter/mistralai/devstral-2512": { "input_cost_per_image": 0, "input_cost_per_token": 1.5e-07, @@ -24437,14 +24188,6 @@ "output_cost_per_token": 1.3e-07, "supports_tool_choice": true }, - "openrouter/mistralai/mistral-7b-instruct:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 0.0, - "supports_tool_choice": true - }, "openrouter/mistralai/mistral-large": { "input_cost_per_token": 8e-06, "litellm_provider": "openrouter", @@ -24489,16 +24232,9 @@ "source": "https://openrouter.ai/moonshotai/kimi-k2.5", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, - "openrouter/nousresearch/nous-hermes-llama2-13b": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, "openrouter/openai/gpt-3.5-turbo": { "input_cost_per_token": 1.5e-06, "litellm_provider": "openrouter", @@ -24523,17 +24259,6 @@ "output_cost_per_token": 6e-05, "supports_tool_choice": true }, - "openrouter/openai/gpt-4-vision-preview": { - "input_cost_per_image": 0.01445, - "input_cost_per_token": 1e-05, - "litellm_provider": "openrouter", - "max_tokens": 130000, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1": { "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, @@ -24551,23 +24276,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-2025-04-14": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 8e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1-mini": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, @@ -24585,23 +24293,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-mini-2025-04-14": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 4e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1.6e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1-nano": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, @@ -24619,23 +24310,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-nano-2025-04-14": { - "cache_read_input_token_cost": 2.5e-08, - "input_cost_per_token": 1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 4e-07, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4o": { "input_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", @@ -24872,58 +24546,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/o1-mini": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-mini-2024-09-12": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-preview": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-preview-2024-09-12": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, "openrouter/openai/o3-mini": { "input_cost_per_token": 1.1e-06, "litellm_provider": "openrouter", @@ -24952,14 +24574,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "openrouter/pygmalionai/mythalion-13b": { - "input_cost_per_token": 1.875e-06, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.875e-06, - "supports_tool_choice": true - }, "openrouter/qwen/qwen-2.5-coder-32b-instruct": { "input_cost_per_token": 1.8e-07, "litellm_provider": "openrouter", @@ -25051,20 +24665,6 @@ "supports_tool_choice": true, "supports_web_search": true }, - "openrouter/x-ai/grok-4-fast:free": { - "input_cost_per_token": 0, - "litellm_provider": "openrouter", - "max_input_tokens": 2000000, - "max_output_tokens": 30000, - "max_tokens": 30000, - "mode": "chat", - "output_cost_per_token": 0, - "source": "https://openrouter.ai/x-ai/grok-4-fast:free", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": false - }, "openrouter/z-ai/glm-4.6": { "input_cost_per_token": 4e-07, "litellm_provider": "openrouter", @@ -30799,6 +30399,21 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/zai-org/glm-5-maas": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "vertex_ai/mistral-medium-3": { "input_cost_per_token": 4e-07, "litellm_provider": "vertex_ai-mistral_models", @@ -32302,6 +31917,20 @@ "supports_vision": true, "supports_web_search": true }, + "zai.glm-4.7": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "zai/glm-4.7": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 1.1e-07, @@ -35975,5 +35604,694 @@ "mode": "chat", "output_cost_per_token": 0, "supports_reasoning": true + }, + "tts-1-1106": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "openai", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "tts-1-hd-1106": { + "input_cost_per_character": 3e-05, + "litellm_provider": "openai", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "gpt-4o-mini-tts-2025-03-20": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "mode": "audio_speech", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_second": 0.00025, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "audio" + ] + }, + "gpt-4o-mini-tts-2025-12-15": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "mode": "audio_speech", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_second": 0.00025, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "audio" + ] + }, + "gpt-4o-mini-transcribe-2025-03-20": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 16000, + "max_output_tokens": 2000, + "mode": "audio_transcription", + "output_cost_per_token": 5e-06, + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "gpt-4o-mini-transcribe-2025-12-15": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 16000, + "max_output_tokens": 2000, + "mode": "audio_transcription", + "output_cost_per_token": 5e-06, + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "gpt-5-search-api": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "gpt-5-search-api-2025-10-14": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "gpt-realtime-mini-2025-10-06": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_token": 6e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "gpt-realtime-mini-2025-12-15": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_token": 6e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "sora-2": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.1, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "720x1280", + "1280x720" + ] + }, + "sora-2-pro": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.3, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "720x1280", + "1280x720" + ] + }, + "chatgpt-image-latest": { + "cache_read_input_image_token_cost": 2.5e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 1e-05, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 4e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "gemini-2.0-flash-exp-image-generation": { + "input_cost_per_token": 0.0, + "litellm_provider": "gemini", + "max_images_per_prompt": 3000, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.039, + "output_cost_per_token": 0.0, + "source": "https://ai.google.dev/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "gemini/gemini-2.0-flash-exp-image-generation": { + "input_cost_per_token": 0.0, + "litellm_provider": "gemini", + "max_images_per_prompt": 3000, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.039, + "output_cost_per_token": 0.0, + "source": "https://ai.google.dev/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "gemini/gemini-2.0-flash-lite-001": { + "cache_read_input_token_cost": 1.875e-08, + "deprecation_date": "2026-03-31", + "input_cost_per_audio_token": 7.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_pdf_size_mb": 50, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 3e-07, + "rpm": 4000, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 4000000 + }, + "gemini-2.5-flash-native-audio-latest": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-native-audio-preview-09-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-native-audio-preview-12-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-latest": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-preview-09-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-preview-12-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-preview-tts": { + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "mode": "audio_speech", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "gemini-flash-latest": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "rpm": 100000, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 8000000 + }, + "gemini-flash-lite-latest": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_audio_token": 3e-07, + "input_cost_per_token": 1e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 4e-07, + "output_cost_per_token": 4e-07, + "rpm": 15, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-lite", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 250000 + }, + "gemini-pro-latest": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "rpm": 2000, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, + "gemini/gemini-pro-latest": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "rpm": 2000, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, + "gemini-exp-1206": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "rpm": 100000, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 8000000 } } diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 548e3bc3dbf..ed4fb133478 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -536,10 +536,25 @@ class MCPRequestHandler: user_api_key_auth: Optional[UserAPIKeyAuth] = None, ) -> List[str]: try: - # Get key object permission (already loaded in main auth flow) + # Get key object permission (already loaded in main auth flow, or fetch from DB) key_object_permission = MCPRequestHandler._get_key_object_permission( user_api_key_auth ) + if key_object_permission is None and user_api_key_auth and user_api_key_auth.object_permission_id: + from litellm.proxy.auth.auth_checks import get_object_permission + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + if prisma_client is not None: + key_object_permission = await get_object_permission( + object_permission_id=user_api_key_auth.object_permission_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) if key_object_permission is None: return [] diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index fd251488db4..62a06468e9a 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -14,6 +14,7 @@ import re from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Union, cast from urllib.parse import urlparse +import anyio from fastapi import HTTPException from httpx import HTTPStatusError from mcp import ReadResourceResult, Resource @@ -70,7 +71,9 @@ try: from mcp.shared.tool_name_validation import ( validate_tool_name, # pyright: ignore[reportAssignmentType] ) - from mcp.shared.tool_name_validation import SEP_986_URL + from mcp.shared.tool_name_validation import ( + SEP_986_URL, + ) except ImportError: from pydantic import BaseModel @@ -887,8 +890,15 @@ class MCPServerManager: # Handle stdio transport if transport == MCPTransport.stdio: - # For stdio, we need to get the stdio config from the server - resolved_env = stdio_env if stdio_env is not None else server.env or {} + resolved_env = stdio_env if stdio_env is not None else dict(server.env or {}) + + # Ensure npm-based STDIO MCP servers have a writable cache dir. + # In containers the default (~/.npm or /app/.npm) may not exist + # or be read-only, causing npx to fail with ENOENT. + if "NPM_CONFIG_CACHE" not in resolved_env: + from litellm.constants import MCP_NPM_CACHE_DIR + + resolved_env["NPM_CONFIG_CACHE"] = MCP_NPM_CACHE_DIR stdio_config: Optional[MCPStdioConfig] = None if server.command and server.args is not None: stdio_config = MCPStdioConfig( @@ -1437,6 +1447,9 @@ class MCPServerManager: """ Fetch tools from MCP client with timeout and error handling. + Uses anyio.fail_after() instead of asyncio.wait_for() to avoid conflicts + with the MCP SDK's anyio TaskGroup. See GitHub issue #20715 for details. + Args: client: MCP client instance server_name: Name of the server for logging @@ -1444,24 +1457,12 @@ class MCPServerManager: Returns: List of tools from the server """ - - async def _list_tools_task(): - try: + try: + with anyio.fail_after(30.0): tools = await client.list_tools() verbose_logger.debug(f"Tools from {server_name}: {tools}") return tools - except asyncio.CancelledError: - verbose_logger.warning(f"Client operation cancelled for {server_name}") - return [] - except Exception as e: - verbose_logger.warning( - f"Client operation failed for {server_name}: {str(e)}" - ) - return [] - - try: - return await asyncio.wait_for(_list_tools_task(), timeout=30.0) - except asyncio.TimeoutError: + except TimeoutError: verbose_logger.warning(f"Timeout while listing tools from {server_name}") return [] except asyncio.CancelledError: @@ -2481,6 +2482,9 @@ class MCPServerManager: except asyncio.TimeoutError: health_check_error = "Health check timed out after 10 seconds" status = "unhealthy" + except asyncio.CancelledError: + health_check_error = "Health check was cancelled" + status = "unknown" except Exception as e: health_check_error = str(e) status = "unhealthy" diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 58cd8c99e7b..fa28b6070b3 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -41,6 +41,9 @@ from litellm.proxy._experimental.mcp_server.utils import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, +) from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall @@ -842,6 +845,7 @@ if MCP_AVAILABLE: raw_headers: Optional[Dict[str, str]] = None, log_list_tools_to_spendlogs: bool = False, list_tools_log_source: Optional[str] = None, + litellm_trace_id: Optional[str] = None, ) -> List[MCPTool]: """ Helper method to fetch tools from MCP servers based on server filtering criteria. @@ -879,6 +883,7 @@ if MCP_AVAILABLE: "model": "MCP: list_tools", "call_type": CallTypes.list_mcp_tools.value, "litellm_call_id": list_tools_call_id, + "litellm_trace_id": litellm_trace_id, "metadata": { "spend_logs_metadata": spend_logs_metadata, }, @@ -894,13 +899,14 @@ if MCP_AVAILABLE: ], } - # Attach user identifiers when available (matches call_mcp_tool style) + # Attach user identifiers using the standard helper if user_api_key_auth is not None: - user_api_key = getattr(user_api_key_auth, "api_key", None) - if user_api_key: - cast(dict, list_tools_request_data["metadata"])[ - "user_api_key" - ] = user_api_key + + LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( + data=list_tools_request_data, + user_api_key_dict=user_api_key_auth, + _metadata_variable_name="metadata", + ) user_identifier = getattr( user_api_key_auth, "end_user_id", None diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 87ff4a66e08..de785b221bc 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -514,6 +514,8 @@ class LiteLLMRoutes(enum.Enum): "/user/delete", "/user/info", "/user/list", + "/user/daily/activity", + "/user/daily/activity/aggregated", # team "/team/new", "/team/update", @@ -526,6 +528,7 @@ class LiteLLMRoutes(enum.Enum): "/team/available", "/team/permissions_list", "/team/permissions_update", + "/team/daily/activity", # model "/model/new", "/model/update", @@ -893,6 +896,7 @@ class KeyRequestBase(GenerateRequestBase): Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] ] = None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm router_settings: Optional[UpdateRouterConfig] = None + access_group_ids: Optional[List[str]] = None class LiteLLMKeyType(str, enum.Enum): @@ -1502,6 +1506,7 @@ class TeamBase(LiteLLMPydanticObjectBase): models: list = [] blocked: bool = False router_settings: Optional[dict] = None + access_group_ids: Optional[List[str]] = None class NewTeamRequest(TeamBase): @@ -1589,6 +1594,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): model_tpm_limit: Optional[Dict[str, int]] = None allowed_vector_store_indexes: Optional[List[AllowedVectorStoreIndexItem]] = None router_settings: Optional[dict] = None + access_group_ids: Optional[List[str]] = None class ResetTeamBudgetRequest(LiteLLMPydanticObjectBase): @@ -2177,6 +2183,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase): updated_by: Optional[str] = None object_permission_id: Optional[str] = None object_permission: Optional[LiteLLM_ObjectPermissionTable] = None + access_group_ids: Optional[List[str]] = None rotation_count: Optional[int] = 0 # Number of times key has been rotated auto_rotate: Optional[bool] = False # Whether this key should be auto-rotated rotation_interval: Optional[str] = None # How often to rotate (e.g., "30d", "90d") diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f33b2412260..a02bc7f9e51 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -282,7 +282,7 @@ def _override_openai_response_model( if isinstance(response_obj, dict): downstream_model = response_obj.get("model") if downstream_model != requested_model: - verbose_proxy_logger.warning( + verbose_proxy_logger.debug( "%s: response model mismatch - requested=%r downstream=%r. Overriding response['model'] to requested model.", log_context, requested_model, @@ -301,7 +301,7 @@ def _override_openai_response_model( downstream_model = getattr(response_obj, "model", None) if downstream_model != requested_model: - verbose_proxy_logger.warning( + verbose_proxy_logger.debug( "%s: response model mismatch - requested=%r downstream=%r. Overriding response.model to requested model.", log_context, requested_model, diff --git a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py index c96564252d0..b41ff121622 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py @@ -112,7 +112,8 @@ class SpendUpdateQueue(BaseUpdateQueue): for update in updates: _key = f"{update.get('entity_type')}:{update.get('entity_id')}" if _key not in _in_memory_map: - _in_memory_map[_key] = update + # avoid mutating caller-owned dicts while aggregating queue entries + _in_memory_map[_key] = update.copy() else: current_cost = _in_memory_map[_key].get("response_cost", 0) or 0 update_cost = update.get("response_cost", 0) or 0 diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index 9018675d7a5..9cded6f0ac2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -5,10 +5,12 @@ # +-------------------------------------------------------------+ # Thank you users! We ❤️ you! - Krrish & Ishaan +import fnmatch import os from typing import TYPE_CHECKING, Any, Dict, Literal, Optional from litellm._logging import verbose_proxy_logger +from litellm._version import version as litellm_version from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_guardrail import ( CustomGuardrail, @@ -31,6 +33,110 @@ if TYPE_CHECKING: GUARDRAIL_NAME = "generic_guardrail_api" +# Headers whose values are forwarded as-is (case-insensitive). Glob patterns supported (e.g. x-stainless-*, x-litellm*). +_HEADER_VALUE_ALLOWLIST = frozenset({ + "host", + "accept-encoding", + "connection", + "accept", + "content-type", + "user-agent", + "x-stainless-*", + "x-litellm-*", + "content-length", +}) + +# Placeholder for headers that exist but are not on the allowlist (we don't expose their value). +_HEADER_PRESENT_PLACEHOLDER = "[present]" + + +def _header_value_allowed(header_name: str) -> bool: + """Return True if this header's value may be forwarded (allowlist, including globs).""" + lower = header_name.lower() + if lower in _HEADER_VALUE_ALLOWLIST: + return True + for pattern in _HEADER_VALUE_ALLOWLIST: + if "*" in pattern and fnmatch.fnmatch(lower, pattern): + return True + return False + + +def _sanitize_inbound_headers(headers: Any) -> Optional[Dict[str, str]]: + """ + Sanitize inbound headers before passing them to a 3rd party guardrail service. + + - Allowlist: only headers in the allowlist have their values forwarded (exact + glob: x-stainless-*, x-litellm-*). + - All other headers are included with value "[present]" so the guardrail knows the header existed. + - Coerces values to str (for JSON serialization). + """ + if not headers or not isinstance(headers, dict): + return None + + sanitized: Dict[str, str] = {} + for k, v in headers.items(): + if k is None: + continue + key = str(k) + if _header_value_allowed(key): + try: + sanitized[key] = str(v) + except Exception: + continue + else: + sanitized[key] = _HEADER_PRESENT_PLACEHOLDER + + return sanitized or None + + +def _extract_inbound_headers( + request_data: dict, logging_obj: Optional["LiteLLMLoggingObj"] +) -> Optional[Dict[str, str]]: + """ + Extract inbound headers from available request context. + + We try multiple locations to support different call paths: + - proxy endpoints: request_data["proxy_server_request"]["headers"] + - if the guardrail is passed the proxy_server_request object directly + - metadata headers captured in litellm_pre_call_utils + - response hooks: fallback to logging_obj.model_call_details + """ + # 1) Most common path (proxy): full request context in proxy_server_request + headers = request_data.get("proxy_server_request", {}).get("headers") + if headers: + return _sanitize_inbound_headers(headers) + + # 2) Some guardrails pass proxy_server_request as request_data itself + headers = request_data.get("headers") + if headers: + return _sanitize_inbound_headers(headers) + + # 3) Pre-call: headers stored in request metadata + metadata_headers = (request_data.get("metadata") or {}).get("headers") + if metadata_headers: + return _sanitize_inbound_headers(metadata_headers) + + litellm_metadata_headers = (request_data.get("litellm_metadata") or {}).get( + "headers" + ) + if litellm_metadata_headers: + return _sanitize_inbound_headers(litellm_metadata_headers) + + # 4) Post-call: headers not present on response; fallback to logging object + if logging_obj and getattr(logging_obj, "model_call_details", None): + try: + details = logging_obj.model_call_details or {} + headers = ( + details.get("litellm_params", {}) + .get("metadata", {}) + .get("headers", None) + ) + if headers: + return _sanitize_inbound_headers(headers) + except Exception: + pass + + return None + class GenericGuardrailAPI(CustomGuardrail): """ @@ -207,6 +313,7 @@ class GenericGuardrailAPI(CustomGuardrail): # Extract user API key metadata user_metadata = self._extract_user_api_key_metadata(request_data) + inbound_headers = _extract_inbound_headers(request_data=request_data, logging_obj=logging_obj) # Create request payload guardrail_request = GenericGuardrailAPIRequest( @@ -214,6 +321,8 @@ class GenericGuardrailAPI(CustomGuardrail): litellm_trace_id=logging_obj.litellm_trace_id if logging_obj else None, texts=texts, request_data=user_metadata, + request_headers=inbound_headers, + litellm_version=litellm_version, images=images, tools=tools, structured_messages=structured_messages, diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index 9d64aea8910..9b2b43c6fcb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -29,10 +29,7 @@ from fastapi import HTTPException from litellm import Router from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import ( - CustomGuardrail, - log_guardrail_information, -) +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ModelResponseStream @@ -1056,7 +1053,6 @@ class ContentFilterGuardrail(CustomGuardrail): masked_entity_count=masked_entity_count, ) - @log_guardrail_information async def apply_guardrail( self, inputs: "GenericGuardrailAPIInputs", diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 38462094b11..2d3f048f81b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -330,6 +330,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): end_time: Optional[float] = None, duration: Optional[float] = None, event_type: Optional[GuardrailEventHooks] = None, + original_inputs: Optional[dict] = None, ): """ Override to store only the Model Armor API response, not the entire data dict. diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py new file mode 100644 index 00000000000..33b81e85654 --- /dev/null +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -0,0 +1,264 @@ +from typing import List + +from fastapi import APIRouter, Depends, HTTPException, status + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.utils import get_prisma_client_or_throw +from litellm.types.access_group import ( + AccessGroupCreateRequest, + AccessGroupResponse, + AccessGroupUpdateRequest, +) + +router = APIRouter( + tags=["access group management"], +) + + +def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={"error": CommonProxyErrors.not_allowed_access.value}, + ) + + +def _record_to_response(record) -> AccessGroupResponse: + return AccessGroupResponse( + access_group_id=record.access_group_id, + access_group_name=record.access_group_name, + description=record.description, + access_model_ids=record.access_model_ids, + access_mcp_server_ids=record.access_mcp_server_ids, + access_agent_ids=record.access_agent_ids, + assigned_team_ids=record.assigned_team_ids, + assigned_key_ids=record.assigned_key_ids, + created_at=record.created_at, + created_by=record.created_by, + updated_at=record.updated_at, + updated_by=record.updated_by, + ) + + +@router.post( + "/v1/access_group", + response_model=AccessGroupResponse, + status_code=status.HTTP_201_CREATED, +) +async def create_access_group( + data: AccessGroupCreateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> AccessGroupResponse: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + existing = await prisma_client.db.litellm_accessgrouptable.find_unique( + where={"access_group_name": data.access_group_name} + ) + if existing is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Access group '{data.access_group_name}' already exists", + ) + + try: + record = await prisma_client.db.litellm_accessgrouptable.create( + data={ + "access_group_name": data.access_group_name, + "description": data.description, + "access_model_ids": data.access_model_ids or [], + "access_mcp_server_ids": data.access_mcp_server_ids or [], + "access_agent_ids": data.access_agent_ids or [], + "assigned_team_ids": data.assigned_team_ids or [], + "assigned_key_ids": data.assigned_key_ids or [], + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, + } + ) + except Exception as e: + # Race condition: another request created the same name between find_unique and create. + # Prisma raises UniqueViolationError (P2002) or similar for unique constraint. + if "unique constraint" in str(e).lower() or "P2002" in str(e): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Access group '{data.access_group_name}' already exists", + ) + raise + return _record_to_response(record) + + +@router.get( + "/v1/access_group", + response_model=List[AccessGroupResponse], +) +async def list_access_groups( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> List[AccessGroupResponse]: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + records = await prisma_client.db.litellm_accessgrouptable.find_many( + order={"created_at": "desc"} + ) + return [_record_to_response(r) for r in records] + + +@router.get( + "/v1/access_group/{access_group_id}", + response_model=AccessGroupResponse, +) +async def get_access_group( + access_group_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> AccessGroupResponse: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + record = await prisma_client.db.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if record is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + return _record_to_response(record) + + +@router.put( + "/v1/access_group/{access_group_id}", + response_model=AccessGroupResponse, +) +async def update_access_group( + access_group_id: str, + data: AccessGroupUpdateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> AccessGroupResponse: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + existing = await prisma_client.db.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if existing is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + + update_data: dict = {"updated_by": user_api_key_dict.user_id} + for field, value in data.model_dump(exclude_unset=True).items(): + update_data[field] = value + + record = await prisma_client.db.litellm_accessgrouptable.update( + where={"access_group_id": access_group_id}, + data=update_data, + ) + return _record_to_response(record) + + +@router.delete( + "/v1/access_group/{access_group_id}", + status_code=status.HTTP_204_NO_CONTENT, +) +async def delete_access_group( + access_group_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> None: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + try: + async with prisma_client.db.tx() as tx: + existing = await tx.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if existing is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + + # Remove access_group_id from teams and keys that reference it + teams_with_group = await tx.litellm_teamtable.find_many( + where={"access_group_ids": {"hasSome": [access_group_id]}} + ) + for team in teams_with_group: + updated_ids = [tid for tid in (team.access_group_ids or []) if tid != access_group_id] + await tx.litellm_teamtable.update( + where={"team_id": team.team_id}, + data={"access_group_ids": updated_ids}, + ) + + keys_with_group = await tx.litellm_verificationtoken.find_many( + where={"access_group_ids": {"hasSome": [access_group_id]}} + ) + for key in keys_with_group: + updated_ids = [kid for kid in (key.access_group_ids or []) if kid != access_group_id] + await tx.litellm_verificationtoken.update( + where={"token": key.token}, + data={"access_group_ids": updated_ids}, + ) + + await tx.litellm_accessgrouptable.delete( + where={"access_group_id": access_group_id} + ) + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception( + "delete_access_group failed: access_group_id=%s error=%s", + access_group_id, + e, + ) + if PrismaDBExceptionHandler.is_database_connection_error(e): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=CommonProxyErrors.db_not_connected_error.value, + ) + if "P2025" in str(e) or ("record" in str(e).lower() and "not found" in str(e).lower()): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to delete access group. Please try again.", + ) + + +# Alias routes for /v1/unified_access_group +router.add_api_route( + "/v1/unified_access_group", + create_access_group, + methods=["POST"], + response_model=AccessGroupResponse, + status_code=status.HTTP_201_CREATED, +) +router.add_api_route( + "/v1/unified_access_group", + list_access_groups, + methods=["GET"], + response_model=List[AccessGroupResponse], +) +router.add_api_route( + "/v1/unified_access_group/{access_group_id}", + get_access_group, + methods=["GET"], + response_model=AccessGroupResponse, +) +router.add_api_route( + "/v1/unified_access_group/{access_group_id}", + update_access_group, + methods=["PUT"], + response_model=AccessGroupResponse, +) +router.add_api_route( + "/v1/unified_access_group/{access_group_id}", + delete_access_group, + methods=["DELETE"], + status_code=status.HTTP_204_NO_CONTENT, +) diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 942758e3bab..d6f6d1337a3 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -383,6 +383,17 @@ def _update_metadata_field(updated_kv: dict, field_name: str) -> None: updated_kv["metadata"] = {field_name: _value} +def _has_non_empty_value(value: Any) -> bool: + """Check if a value has real content (not None, not empty list, not blank string).""" + if value is None: + return False + if isinstance(value, list) and len(value) == 0: + return False + if isinstance(value, str) and value.strip() == "": + return False + return True + + def _update_metadata_fields(updated_kv: dict) -> None: """ Helper function to update all metadata fields (both premium and standard). @@ -391,7 +402,7 @@ def _update_metadata_fields(updated_kv: dict) -> None: updated_kv: The key-value dict being used for the update """ for field in LiteLLM_ManagementEndpoint_MetadataFields_Premium: - if field in updated_kv and updated_kv[field] is not None: + if field in updated_kv and _has_non_empty_value(updated_kv[field]): _update_metadata_field(updated_kv=updated_kv, field_name=field) for field in LiteLLM_ManagementEndpoint_MetadataFields: diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 597521ae773..8c4d4e7937e 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -10,10 +10,13 @@ Endpoints here: - DELETE `/v1/mcp/server/{server_id}` - Deletes the mcp server given `server_id`. - GET `/v1/mcp/tools - lists all the tools available for a key - GET `/v1/mcp/access_groups` - lists all available MCP access groups +- GET `/v1/mcp/discover` - Returns curated list of well-known MCP servers for discovery UI """ import importlib +import json +import os from dataclasses import dataclass from datetime import datetime, timedelta from typing import Any, Dict, Iterable, List, Literal, Optional @@ -1176,3 +1179,88 @@ if MCP_AVAILABLE: except Exception as e: verbose_proxy_logger.exception(f"Error making agent public: {e}") raise HTTPException(status_code=500, detail=str(e)) + + # --- MCP Discovery --- + + _MCP_REGISTRY_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "mcp_registry.json", + ) + + _mcp_registry_cache: Optional[Dict[str, Any]] = None + + def _load_mcp_registry() -> Dict[str, Any]: + """Load the curated MCP registry from disk. Cached after first read.""" + global _mcp_registry_cache + if _mcp_registry_cache is not None: + return _mcp_registry_cache + try: + with open(_MCP_REGISTRY_PATH, "r") as f: + data: Dict[str, Any] = json.load(f) + except Exception as e: + verbose_proxy_logger.warning( + f"Failed to load MCP registry from {_MCP_REGISTRY_PATH}: {e}" + ) + data = {"servers": []} + _mcp_registry_cache = data + return data + + @router.get( + "/discover", + description="Returns a curated list of well-known MCP servers for discovery UI", + dependencies=[Depends(user_api_key_auth)], + ) + async def discover_mcp_servers( + query: Optional[str] = Query( + None, description="Search filter for server names and descriptions" + ), + category: Optional[str] = Query( + None, description="Filter by category" + ), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + """ + Returns a curated list of well-known MCP servers that can be added to the proxy. + + Used by the UI to show a discovery grid when adding new MCP servers. + """ + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail={ + "error": "Only proxy admins can access MCP discovery. Your role={}".format( + user_api_key_dict.user_role + ) + }, + ) + + registry = _load_mcp_registry() + servers = registry.get("servers", []) + + # Apply query filter + if query: + query_lower = query.lower() + servers = [ + s + for s in servers + if query_lower in s.get("name", "").lower() + or query_lower in s.get("title", "").lower() + or query_lower in s.get("description", "").lower() + ] + + # Apply category filter + if category: + servers = [ + s for s in servers if s.get("category", "") == category + ] + + # Extract unique categories from the full list (before filtering) + all_servers = registry.get("servers", []) + categories = sorted( + set(s.get("category", "Other") for s in all_servers) + ) + + return { + "servers": servers, + "categories": categories, + } diff --git a/litellm/proxy/mcp_registry.json b/litellm/proxy/mcp_registry.json new file mode 100644 index 00000000000..2e1e8f64eae --- /dev/null +++ b/litellm/proxy/mcp_registry.json @@ -0,0 +1,426 @@ +{ + "servers": [ + { + "name": "github", + "title": "GitHub", + "description": "Manage repos, issues, PRs, and workflows through natural language", + "icon_url": "https://cdn.simpleicons.org/github", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.github%2Fgithub-mcp-server", + "transport": "http", + "url": "https://api.githubcopilot.com/mcp/", + "env_vars": [ + {"name": "GITHUB_PERSONAL_ACCESS_TOKEN", "description": "GitHub Personal Access Token", "secret": true} + ] + }, + { + "name": "gitlab", + "title": "GitLab", + "description": "Official GitLab MCP Server for project and repository management", + "icon_url": "https://cdn.simpleicons.org/gitlab", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.gitlab%2Fmcp", + "transport": "http", + "url": "https://gitlab.com/api/v4/mcp", + "env_vars": [ + {"name": "GITLAB_PERSONAL_ACCESS_TOKEN", "description": "GitLab Personal Access Token", "secret": true} + ] + }, + { + "name": "atlassian", + "title": "Atlassian (Jira & Confluence)", + "description": "Jira issues, Confluence pages, and Atlassian product integration", + "icon_url": "https://cdn.simpleicons.org/atlassian", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.atlassian%2Fatlassian-mcp-server", + "transport": "sse", + "url": "https://mcp.atlassian.com/v1/sse", + "env_vars": [] + }, + { + "name": "linear", + "title": "Linear", + "description": "Issue tracking, project management, and team workflow automation", + "icon_url": "https://cdn.simpleicons.org/linear", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/app.linear%2Flinear", + "transport": "sse", + "url": "https://mcp.linear.app/sse", + "env_vars": [] + }, + { + "name": "sentry", + "title": "Sentry", + "description": "Error monitoring, issue tracking, and debugging for AI assistants", + "icon_url": "https://cdn.simpleicons.org/sentry", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.getsentry%2Fsentry-mcp", + "transport": "stdio", + "command": "npx", + "args": ["-y", "@sentry/mcp-server"], + "env_vars": [ + {"name": "SENTRY_ACCESS_TOKEN", "description": "Sentry Access Token", "secret": true} + ] + }, + { + "name": "slack", + "title": "Slack", + "description": "Channel management, messaging, and Slack workspace integration", + "icon_url": "https://cdn.simpleicons.org/slack", + "category": "Communication", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-slack"], + "env_vars": [ + {"name": "SLACK_BOT_TOKEN", "description": "Slack Bot User OAuth Token", "secret": true}, + {"name": "SLACK_TEAM_ID", "description": "Slack Team/Workspace ID", "secret": false} + ] + }, + { + "name": "discord", + "title": "Discord", + "description": "Discord server management, messaging, and bot integration", + "icon_url": "https://cdn.simpleicons.org/discord", + "category": "Communication", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-discord"], + "env_vars": [ + {"name": "DISCORD_BOT_TOKEN", "description": "Discord Bot Token", "secret": true} + ] + }, + { + "name": "postgresql", + "title": "PostgreSQL", + "description": "Query and manage PostgreSQL databases with read-only access", + "icon_url": "https://cdn.simpleicons.org/postgresql", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-postgres"], + "env_vars": [ + {"name": "POSTGRES_CONNECTION_STRING", "description": "PostgreSQL connection string (e.g., postgresql://user:pass@host:5432/db)", "secret": true} + ] + }, + { + "name": "sqlite", + "title": "SQLite", + "description": "Query and manage SQLite databases", + "icon_url": "https://cdn.simpleicons.org/sqlite", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-sqlite"], + "env_vars": [ + {"name": "SQLITE_DB_PATH", "description": "Path to SQLite database file", "secret": false} + ] + }, + { + "name": "mysql", + "title": "MySQL", + "description": "Query and manage MySQL databases", + "icon_url": "https://cdn.simpleicons.org/mysql", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-mysql"], + "env_vars": [ + {"name": "MYSQL_HOST", "description": "MySQL host", "secret": false}, + {"name": "MYSQL_USER", "description": "MySQL username", "secret": false}, + {"name": "MYSQL_PASSWORD", "description": "MySQL password", "secret": true}, + {"name": "MYSQL_DATABASE", "description": "MySQL database name", "secret": false} + ] + }, + { + "name": "mongodb", + "title": "MongoDB", + "description": "Query and manage MongoDB databases and collections", + "icon_url": "https://cdn.simpleicons.org/mongodb", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-mongodb"], + "env_vars": [ + {"name": "MONGODB_CONNECTION_STRING", "description": "MongoDB connection string", "secret": true} + ] + }, + { + "name": "redis", + "title": "Redis", + "description": "Interact with Redis key-value stores", + "icon_url": "https://cdn.simpleicons.org/redis", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-redis"], + "env_vars": [ + {"name": "REDIS_URL", "description": "Redis connection URL (e.g., redis://localhost:6379)", "secret": true} + ] + }, + { + "name": "snowflake", + "title": "Snowflake", + "description": "MCP Server for Snowflake from Snowflake Labs", + "icon_url": "https://cdn.simpleicons.org/snowflake", + "category": "Databases", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.Snowflake-Labs%2Fmcp", + "transport": "stdio", + "command": "uvx", + "args": ["snowflake-labs-mcp"], + "env_vars": [ + {"name": "SNOWFLAKE_ACCOUNT", "description": "Snowflake account identifier (e.g., xy12345.us-east-1)", "secret": false}, + {"name": "SNOWFLAKE_USER", "description": "Snowflake username", "secret": false}, + {"name": "SNOWFLAKE_PASSWORD", "description": "Snowflake password", "secret": true} + ] + }, + { + "name": "notion", + "title": "Notion", + "description": "Official Notion MCP server for pages and databases", + "icon_url": "https://cdn.simpleicons.org/notion", + "category": "Productivity", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.notion%2Fmcp", + "transport": "sse", + "url": "https://mcp.notion.com/sse", + "env_vars": [] + }, + { + "name": "google_drive", + "title": "Google Drive", + "description": "Search and access files in Google Drive", + "icon_url": "https://cdn.simpleicons.org/googledrive", + "category": "Productivity", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-gdrive"], + "env_vars": [ + {"name": "GOOGLE_CLIENT_ID", "description": "Google OAuth Client ID", "secret": false}, + {"name": "GOOGLE_CLIENT_SECRET", "description": "Google OAuth Client Secret", "secret": true} + ] + }, + { + "name": "google_calendar", + "title": "Google Calendar", + "description": "Manage events and calendars in Google Calendar", + "icon_url": "https://cdn.simpleicons.org/googlecalendar", + "category": "Productivity", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-google-calendar"], + "env_vars": [ + {"name": "GOOGLE_CLIENT_ID", "description": "Google OAuth Client ID", "secret": false}, + {"name": "GOOGLE_CLIENT_SECRET", "description": "Google OAuth Client Secret", "secret": true} + ] + }, + { + "name": "obsidian", + "title": "Obsidian", + "description": "Read, search, and manage Obsidian vault notes and files", + "icon_url": "https://cdn.simpleicons.org/obsidian", + "category": "Productivity", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-obsidian"], + "env_vars": [ + {"name": "OBSIDIAN_VAULT_PATH", "description": "Path to Obsidian vault directory", "secret": false} + ] + }, + { + "name": "brave_search", + "title": "Brave Search", + "description": "Web results, images, videos, and AI summaries via Brave Search API", + "icon_url": "https://cdn.simpleicons.org/brave", + "category": "Search", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.brave%2Fbrave-search-mcp-server", + "transport": "stdio", + "command": "npx", + "args": ["-y", "@brave/brave-search-mcp-server"], + "env_vars": [ + {"name": "BRAVE_API_KEY", "description": "Brave Search API Key", "secret": true} + ] + }, + { + "name": "exa", + "title": "Exa", + "description": "Fast, intelligent web search and web crawling", + "icon_url": "https://cdn.simpleicons.org/exa", + "category": "Search", + "registry_url": "https://registry.modelcontextprotocol.io/servers/ai.exa%2Fexa", + "transport": "http", + "url": "https://mcp.exa.ai/mcp", + "env_vars": [ + {"name": "EXA_API_KEY", "description": "Exa API Key", "secret": true} + ] + }, + { + "name": "tavily", + "title": "Tavily", + "description": "AI-optimized search engine for research and retrieval", + "icon_url": "https://cdn.simpleicons.org/tavily", + "category": "Search", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-tavily"], + "env_vars": [ + {"name": "TAVILY_API_KEY", "description": "Tavily API Key", "secret": true} + ] + }, + { + "name": "puppeteer", + "title": "Puppeteer", + "description": "Browser automation, web scraping, and screenshot capture", + "icon_url": "https://cdn.simpleicons.org/puppeteer", + "category": "Web & Browser", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-puppeteer"], + "env_vars": [] + }, + { + "name": "playwright", + "title": "Playwright", + "description": "Browser automation and testing with Playwright", + "icon_url": "https://cdn.simpleicons.org/playwright", + "category": "Web & Browser", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-playwright"], + "env_vars": [] + }, + { + "name": "browserbase", + "title": "Browserbase", + "description": "Cloud browser automation and session management", + "icon_url": "https://cdn.simpleicons.org/browserbase", + "category": "Web & Browser", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-browserbase"], + "env_vars": [ + {"name": "BROWSERBASE_API_KEY", "description": "Browserbase API Key", "secret": true}, + {"name": "BROWSERBASE_PROJECT_ID", "description": "Browserbase Project ID", "secret": false} + ] + }, + { + "name": "aws", + "title": "AWS", + "description": "Interact with Amazon Web Services resources and APIs", + "icon_url": "https://cdn.simpleicons.org/amazonaws", + "category": "Cloud", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-aws"], + "env_vars": [ + {"name": "AWS_ACCESS_KEY_ID", "description": "AWS Access Key ID", "secret": true}, + {"name": "AWS_SECRET_ACCESS_KEY", "description": "AWS Secret Access Key", "secret": true}, + {"name": "AWS_REGION", "description": "AWS Region (e.g., us-east-1)", "secret": false} + ] + }, + { + "name": "cloudflare", + "title": "Cloudflare", + "description": "Manage Cloudflare Workers, KV, R2, D1, and more", + "icon_url": "https://cdn.simpleicons.org/cloudflare", + "category": "Cloud", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.cloudflare.mcp%2Fmcp", + "transport": "sse", + "url": "https://bindings.mcp.cloudflare.com/sse", + "env_vars": [] + }, + { + "name": "filesystem", + "title": "Filesystem", + "description": "Read, write, and manage files and directories on disk", + "icon_url": "https://cdn.simpleicons.org/files", + "category": "System", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem"], + "env_vars": [] + }, + { + "name": "docker", + "title": "Docker", + "description": "Manage Docker containers, images, and networks", + "icon_url": "https://cdn.simpleicons.org/docker", + "category": "System", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-docker"], + "env_vars": [] + }, + { + "name": "stripe", + "title": "Stripe", + "description": "Manage payments, customers, and subscriptions via Stripe", + "icon_url": "https://cdn.simpleicons.org/stripe", + "category": "Finance", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.stripe%2Fmcp", + "transport": "http", + "url": "https://mcp.stripe.com", + "env_vars": [] + }, + { + "name": "shopify", + "title": "Shopify", + "description": "Manage Shopify stores, products, orders, and customers", + "icon_url": "https://cdn.simpleicons.org/shopify", + "category": "E-Commerce", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-shopify"], + "env_vars": [ + {"name": "SHOPIFY_ACCESS_TOKEN", "description": "Shopify Admin API Access Token", "secret": true}, + {"name": "SHOPIFY_STORE_URL", "description": "Shopify Store URL (e.g., mystore.myshopify.com)", "secret": false} + ] + }, + { + "name": "twilio", + "title": "Twilio", + "description": "Send SMS, make calls, and manage communication via Twilio", + "icon_url": "https://cdn.simpleicons.org/twilio", + "category": "Communication", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-twilio"], + "env_vars": [ + {"name": "TWILIO_ACCOUNT_SID", "description": "Twilio Account SID", "secret": false}, + {"name": "TWILIO_AUTH_TOKEN", "description": "Twilio Auth Token", "secret": true} + ] + }, + { + "name": "supabase", + "title": "Supabase", + "description": "Manage Supabase projects, databases, and storage", + "icon_url": "https://cdn.simpleicons.org/supabase", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-supabase"], + "env_vars": [ + {"name": "SUPABASE_URL", "description": "Supabase Project URL", "secret": false}, + {"name": "SUPABASE_SERVICE_ROLE_KEY", "description": "Supabase Service Role Key", "secret": true} + ] + } + ] +} diff --git a/litellm/proxy/policy_engine/policy_resolve_endpoints.py b/litellm/proxy/policy_engine/policy_resolve_endpoints.py index eb4d3fc5845..318e990ff12 100644 --- a/litellm/proxy/policy_engine/policy_resolve_endpoints.py +++ b/litellm/proxy/policy_engine/policy_resolve_endpoints.py @@ -12,6 +12,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query from litellm._logging import verbose_proxy_logger from litellm.constants import MAX_POLICY_ESTIMATE_IMPACT_ROWS from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry @@ -85,7 +86,6 @@ def _filter_keys_by_tags(keys: list, tag_patterns: list) -> tuple: Returns (named_aliases, unnamed_count). """ - from litellm.proxy.auth.route_checks import RouteChecks affected: list = [] unnamed_count = 0 @@ -111,7 +111,6 @@ def _filter_teams_by_tags(teams: list, tag_patterns: list) -> tuple: Returns (named_aliases, unnamed_count). """ - from litellm.proxy.auth.route_checks import RouteChecks affected: list = [] unnamed_count = 0 @@ -141,7 +140,6 @@ async def _find_affected_by_team_patterns( Returns (new_teams, new_keys, unnamed_keys_count). """ - from litellm.proxy.auth.route_checks import RouteChecks new_teams: list = [] matched_team_ids: list = [] @@ -178,7 +176,6 @@ async def _find_affected_keys_by_alias( prisma_client: object, key_patterns: list, existing_keys: list ) -> list: """Find keys whose alias matches the given patterns.""" - from litellm.proxy.auth.route_checks import RouteChecks affected: list = [] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6286d6dd1ca..63109916ab1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -393,6 +393,9 @@ from litellm.proxy.management_endpoints.tag_management_endpoints import ( from litellm.proxy.management_endpoints.team_callback_endpoints import ( router as team_callback_router, ) +from litellm.proxy.management_endpoints.access_group_endpoints import ( + router as access_group_router, +) from litellm.proxy.management_endpoints.team_endpoints import router as team_router from litellm.proxy.management_endpoints.team_endpoints import ( update_team, @@ -1051,98 +1054,236 @@ try: except FileNotFoundError: return False + def _validate_ui_directory(ui_path: str) -> bool: + """ + Verify UI directory has minimum required structure. + + Checks for: + - Directory exists + - Has index.html (main entry point) + - Has _next directory (Next.js assets) + + Returns True if UI directory appears valid and servable. + """ + if not os.path.isdir(ui_path): + return False + + # Must have main index.html + if not os.path.exists(os.path.join(ui_path, "index.html")): + return False + + # Must have _next directory with Next.js assets + next_dir = os.path.join(ui_path, "_next") + if not os.path.isdir(next_dir): + return False + + return True + + def _is_ui_pre_restructured(ui_dir: str) -> bool: + """ + Detect if UI directory is already pre-restructured and ready to serve. + + Returns True if: + 1. Marker file .litellm_ui_ready exists (created by Dockerfile), OR + 2. Restructuring pattern detected (subdirectories with index.html inside) + + This allows skipping copy/restructure operations on read-only filesystems. + """ + if not os.path.isdir(ui_dir): + return False + + # Primary signal: marker file created by Dockerfile + marker_file = os.path.join(ui_dir, ".litellm_ui_ready") + if os.path.exists(marker_file): + verbose_proxy_logger.debug(f"Found UI ready marker: {marker_file}") + return True + + # Fallback signal: Detect restructuring pattern + # After restructuring, routes exist as directories with index.html inside + # (e.g., login/index.html instead of login.html) + # Check for main index.html first (basic UI structure requirement) + if not os.path.exists(os.path.join(ui_dir, "index.html")): + return False + + # Look for ANY subdirectory with index.html (proves restructuring happened) + # Ignore directories starting with _ (Next.js internals like _next) + try: + for entry in os.scandir(ui_dir): + if entry.is_dir() and not entry.name.startswith("_"): + index_path = os.path.join(entry.path, "index.html") + if os.path.exists(index_path): + # Found at least one restructured route - this proves the pattern + verbose_proxy_logger.debug( + f"Detected restructured UI via pattern: found {entry.name}/index.html" + ) + return True + except (PermissionError, OSError) as e: + verbose_proxy_logger.debug( + f"Could not scan {ui_dir} for restructuring detection: {e}" + ) + return False + + # No restructured routes found + return False + + def _try_populate_ui_directory( + source_path: str, target_path: str + ) -> tuple[bool, str]: + """ + Attempt to populate target UI directory from source. + + Returns: (success: bool, error_message: str) + """ + try: + os.makedirs(target_path, exist_ok=True) + if not _dir_has_content(target_path) and _dir_has_content(source_path): + shutil.copytree( + source_path, + target_path, + dirs_exist_ok=True, + ) + verbose_proxy_logger.info(f"Successfully populated UI at {target_path}") + return True, "" + else: + return False, "Source or target directory state invalid" + except (PermissionError, OSError) as e: + return False, str(e) + # Use a writable runtime UI directory whenever possible. # This prevents mutating the packaged UI directory (e.g. site-packages or the repo checkout) # and ensures extensionless routes like /ui/login work via /index.html. is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - # Only use runtime UI path in Docker/non-root environments - # In local development, use the packaged UI directly + # Determine runtime UI path + # Priority: LITELLM_UI_PATH env var > default path based on is_non_root if is_non_root: - # Use /var/lib/litellm/ui for Docker (more secure than /tmp) - runtime_ui_path = "/var/lib/litellm/ui" + default_runtime_ui_path = "/var/lib/litellm/ui" + else: + default_runtime_ui_path = packaged_ui_path - if _dir_has_content(runtime_ui_path): + runtime_ui_path = os.getenv("LITELLM_UI_PATH", default_runtime_ui_path) + + # Validate packaged UI before proceeding + if not _validate_ui_directory(packaged_ui_path): + verbose_proxy_logger.error( + f"Packaged UI at {packaged_ui_path} is invalid or incomplete. " + f"UI may not function correctly." + ) + + # Decision tree for UI path selection: + # 1. If runtime path == packaged path: use packaged UI directly + # 2. If runtime UI exists and is pre-restructured: use it + # 3. If runtime UI exists but not restructured: use it (will restructure later) + # 4. If runtime UI missing: try to populate from packaged UI + # 4a. If population succeeds: use runtime UI + # 4b. If population fails: fall back to packaged UI + + should_use_runtime_path = runtime_ui_path != packaged_ui_path + + if should_use_runtime_path: + is_pre_restructured = _is_ui_pre_restructured(runtime_ui_path) + has_content = _dir_has_content(runtime_ui_path) + + # Case 2: Runtime UI exists and is ready + if has_content and is_pre_restructured: verbose_proxy_logger.info( - f"Using pre-built UI for non-root Docker: {runtime_ui_path}" + f"Using pre-restructured UI at {runtime_ui_path}" ) ui_path = runtime_ui_path + + # Case 3: Runtime UI exists but needs restructuring + elif has_content and not is_pre_restructured: + verbose_proxy_logger.warning( + f"UI at {runtime_ui_path} has content but is not properly restructured. " + f"Will attempt to restructure in place." + ) + ui_path = runtime_ui_path + + # Case 4: Runtime UI missing - try to populate else: - verbose_proxy_logger.error( - f"UI not found at {runtime_ui_path}. Attempting to populate it from packaged UI." - ) - verbose_proxy_logger.error( - f"Path exists: {os.path.exists(runtime_ui_path)}, Has content: {_dir_has_content(runtime_ui_path)}" + verbose_proxy_logger.info( + f"UI not found at {runtime_ui_path}. Attempting to populate from packaged UI." ) - try: - os.makedirs(runtime_ui_path, exist_ok=True) - if not _dir_has_content(runtime_ui_path) and _dir_has_content( - packaged_ui_path - ): - shutil.copytree( - packaged_ui_path, - runtime_ui_path, - dirs_exist_ok=True, - ) - except Exception as e: - verbose_proxy_logger.exception( - f"Failed to populate runtime UI directory {runtime_ui_path} from {packaged_ui_path}: {e}" - ) + success, error = _try_populate_ui_directory( + packaged_ui_path, runtime_ui_path + ) + + if success: + # Case 4a: Population succeeded + ui_path = runtime_ui_path else: - if _dir_has_content(runtime_ui_path): - verbose_proxy_logger.info( - f"Using populated UI for non-root Docker: {runtime_ui_path}" - ) - ui_path = runtime_ui_path + # Case 4b: Population failed - fall back to packaged UI + verbose_proxy_logger.warning( + f"Failed to populate UI at {runtime_ui_path}: {error}. " + f"Falling back to packaged UI at {packaged_ui_path}. " + f"For read-only deployments, pre-build UI in Dockerfile " + f"or set LITELLM_UI_PATH to a writable emptyDir volume." + ) + ui_path = packaged_ui_path else: - # Local development: use packaged UI directly, no runtime copy needed - verbose_proxy_logger.info( - f"Using packaged UI directory for local development: {packaged_ui_path}" - ) + # Case 1: Using packaged UI directly (local development) + verbose_proxy_logger.info(f"Using packaged UI directory: {packaged_ui_path}") ui_path = packaged_ui_path - # Only modify files if a custom server root path is set + + # Validate final UI path + if not _validate_ui_directory(ui_path): + verbose_proxy_logger.error( + f"Selected UI path {ui_path} is invalid or incomplete. UI may not work correctly." + ) + + # Only modify files if a custom server root path is set AND filesystem is writable if server_root_path and server_root_path != "/": - # Iterate through files in the UI directory - for root, dirs, files in os.walk(ui_path): - for filename in files: - file_path = os.path.join(root, filename) - # Skip binary files and files that don't need path replacement - if filename.endswith( - ( - ".png", - ".jpg", - ".jpeg", - ".gif", - ".ico", - ".woff", - ".woff2", - ".ttf", - ".eot", - ) - ): - continue - try: - with open(file_path, "r", encoding="utf-8") as f: - content = f.read() + # Check if UI path is writable + is_writable = os.access(ui_path, os.W_OK) - # Replace the asset prefix with the server root path - modified_content = content.replace( - f"{litellm_asset_prefix}", - f"{server_root_path}", - ) + if not is_writable: + verbose_proxy_logger.warning( + f"Cannot apply server_root_path replacements to UI at {ui_path}: " + f"path is not writable. Ensure server_root_path is '/' or pre-process " + f"UI files in Dockerfile with custom server_root_path." + ) + else: + # Iterate through files in the UI directory + for root, dirs, files in os.walk(ui_path): + for filename in files: + file_path = os.path.join(root, filename) + # Skip binary files and files that don't need path replacement + if filename.endswith( + ( + ".png", + ".jpg", + ".jpeg", + ".gif", + ".ico", + ".woff", + ".woff2", + ".ttf", + ".eot", + ) + ): + continue + try: + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() - # Replace the /.well-known/litellm-ui-config with the server root path - modified_content = modified_content.replace( - "/litellm/.well-known/litellm-ui-config", - f"{server_root_path}/.well-known/litellm-ui-config", - ) + # Replace the asset prefix with the server root path + modified_content = content.replace( + f"{litellm_asset_prefix}", + f"{server_root_path}", + ) - with open(file_path, "w", encoding="utf-8") as f: - f.write(modified_content) - except UnicodeDecodeError: - # Skip binary files that can't be decoded - continue + # Replace the /.well-known/litellm-ui-config with the server root path + modified_content = modified_content.replace( + "/litellm/.well-known/litellm-ui-config", + f"{server_root_path}/.well-known/litellm-ui-config", + ) + + with open(file_path, "w", encoding="utf-8") as f: + f.write(modified_content) + except (UnicodeDecodeError, PermissionError, OSError): + # Skip binary files or files we can't write to + continue # # Mount the _next directory at the root level app.mount( @@ -1186,14 +1327,22 @@ try: continue # Handle HTML file restructuring - # Always restructure the directory we actually serve. - # This is critical for extensionless routes like /ui/login (expects login/index.html). - # In development, we restructure directly in _experimental/out. - # In non-root Docker, we restructure in /var/lib/litellm/ui. + # Only restructure if: + # 1. UI is not already pre-restructured + # 2. Filesystem is writable try: - if is_non_root and ui_path == "/var/lib/litellm/ui": + is_pre_restructured = _is_ui_pre_restructured(ui_path) + is_writable = os.access(ui_path, os.W_OK) + + if is_pre_restructured: verbose_proxy_logger.info( - f"Skipping runtime UI restructuring for non-root Docker. UI at {ui_path} is pre-restructured." + f"Skipping UI restructuring: {ui_path} is already pre-restructured" + ) + elif not is_writable: + verbose_proxy_logger.warning( + f"Cannot restructure UI at {ui_path}: path is not writable. " + f"UI may not work correctly for extensionless routes. " + f"Pre-build and restructure UI in Dockerfile for read-only deployments." ) else: _restructure_ui_html_files(ui_path) @@ -4716,8 +4865,10 @@ async def async_assistants_data_generator( if isinstance(e, HTTPException): raise e else: - error_traceback = traceback.format_exc() - error_msg = f"{str(e)}\n\n{error_traceback}" + # Only include the error message, not the traceback. + # The traceback is already logged above via verbose_proxy_logger.exception(). + # Including it in the SSE response leaks internal details to clients. + error_msg = str(e) proxy_exception = ProxyException( message=getattr(e, "message", error_msg), @@ -4764,7 +4915,7 @@ def _restamp_streaming_chunk_model( chunk.get("model") if isinstance(chunk, dict) else getattr(chunk, "model", None) ) if not model_mismatch_logged and downstream_model != requested_model_from_client: - verbose_proxy_logger.warning( + verbose_proxy_logger.debug( "litellm_call_id=%s: streaming chunk model mismatch - requested=%r downstream=%r. Overriding model to requested.", request_data.get("litellm_call_id"), requested_model_from_client, @@ -4867,8 +5018,10 @@ async def async_data_generator( elif isinstance(e, StreamingCallbackError): error_msg = str(e) else: - error_traceback = traceback.format_exc() - error_msg = f"{str(e)}\n\n{error_traceback}" + # Only include the error message, not the traceback. + # The traceback is already logged above via verbose_proxy_logger.exception(). + # Including it in the SSE response leaks internal details to clients. + error_msg = str(e) proxy_exception = ProxyException( message=getattr(e, "message", error_msg), @@ -10294,18 +10447,34 @@ async def get_image(): default_site_logo = os.path.join(current_dir, "logo.jpg") is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - assets_dir = "/var/lib/litellm/assets" if is_non_root else current_dir - if is_non_root: - os.makedirs(assets_dir, exist_ok=True) + # Determine assets directory + # Priority: LITELLM_ASSETS_PATH env var > default based on is_non_root + default_assets_dir = "/var/lib/litellm/assets" if is_non_root else current_dir + assets_dir = os.getenv("LITELLM_ASSETS_PATH", default_assets_dir) + # Try to create assets_dir if it doesn't exist (simple try/except approach) + if not os.path.exists(assets_dir): + try: + os.makedirs(assets_dir, exist_ok=True) + verbose_proxy_logger.debug(f"Created assets directory at {assets_dir}") + except (PermissionError, OSError) as e: + verbose_proxy_logger.warning( + f"Cannot create assets directory at {assets_dir}: {e}. " + f"Logo caching may not work. Using current directory for assets." + ) + assets_dir = current_dir + + # Determine default logo path default_logo = ( - os.path.join(assets_dir, "logo.jpg") if is_non_root else default_site_logo + os.path.join(assets_dir, "logo.jpg") + if assets_dir != current_dir + else default_site_logo ) - if is_non_root and not os.path.exists(default_logo): + if assets_dir != current_dir and not os.path.exists(default_logo): default_logo = default_site_logo - cache_dir = assets_dir if is_non_root else current_dir + cache_dir = assets_dir if os.access(assets_dir, os.W_OK) else current_dir cache_path = os.path.join(cache_dir, "cached_logo.jpg") # [OPTIMIZATION] Check if the cached image exists first @@ -11832,6 +12001,7 @@ app.include_router(enterprise_router) app.include_router(ui_discovery_endpoints_router) app.include_router(agent_endpoints_router) app.include_router(a2a_router) +app.include_router(access_group_router) ######################################################## # MCP Server ######################################################## diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 37ed0182663..b3caa46147b 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -128,6 +128,7 @@ model LiteLLM_TeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) @@ -160,6 +161,7 @@ model LiteLLM_DeletedTeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases @@ -291,6 +293,7 @@ model LiteLLM_VerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") budget_id String? @@ -346,6 +349,7 @@ model LiteLLM_DeletedVerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") router_settings Json? @default("{}") @@ -917,3 +921,23 @@ model LiteLLM_PolicyAttachmentTable { updated_at DateTime @default(now()) @updatedAt updated_by String? } + +//Unified Access Groups table for storing unified access groups +model LiteLLM_AccessGroupTable { + access_group_id String @id @default(uuid()) + access_group_name String @unique + description String? + + // Resource memberships - explicit arrays per type + access_model_ids String[] @default([]) + access_mcp_server_ids String[] @default([]) + access_agent_ids String[] @default([]) + + assigned_team_ids String[] @default([]) + assigned_key_ids String[] @default([]) + + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? +} \ No newline at end of file diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index afbc57360e2..9c0190bdfec 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1878,13 +1878,15 @@ async def ui_view_spend_logs( # noqa: PLR0915 verbose_proxy_logger.debug("data= %s", json.dumps(data, indent=4, default=str)) - return { - "data": data, - "total": total_records, - "page": page, - "page_size": page_size, - "total_pages": total_pages, - } + return await _build_ui_spend_logs_response( + prisma_client, + data, + total_records, + page, + page_size, + total_pages, + enrich_session_counts=not is_v2, + ) except Exception as e: verbose_proxy_logger.exception(f"Error in ui_view_spend_logs: {e}") raise handle_exception_on_proxy(e) @@ -3129,6 +3131,91 @@ async def ui_view_session_spend_logs( ) +async def _build_ui_spend_logs_response( + prisma_client: "PrismaClient", + data: list, + total_records: int, + page: int, + page_size: int, + total_pages: int, + enrich_session_counts: bool = True, +) -> dict: + """ + Build the paginated response for the UI spend-logs endpoint. + + When ``enrich_session_counts`` is ``True`` (the default for the v1/UI + endpoint), each row is enriched with ``session_total_count`` so the + frontend knows which sessions are expandable (multi-call sessions). + For every row that carries a ``session_id``, a single ``GROUP BY`` query + fetches the total number of logs in each referenced session. Rows without + a ``session_id`` default to ``1``. + + When ``enrich_session_counts`` is ``False`` (v2 endpoint), rows are + serialised without the extra query. + + Args: + prisma_client: The connected Prisma client instance. + data: A list of Prisma model instances (must support ``.model_dump()`` + and have a ``session_id`` attribute). + total_records: Total number of matching records (for pagination). + page: Current page number. + page_size: Number of items per page. + total_pages: Total number of pages. + enrich_session_counts: Whether to add ``session_total_count`` to each + row. Defaults to ``True``. + + Returns: + A dict with ``data`` (enriched rows), ``total``, ``page``, + ``page_size``, and ``total_pages``. + """ + count_map: dict[str, int] = {} + if enrich_session_counts: + session_ids = list( + {row.session_id for row in data if getattr(row, "session_id", None)} + ) + if session_ids: + # NOTE: This GROUP BY runs on every v1/UI page load. The IN clause + # is bounded by page_size (typically 25-50 distinct session IDs). + # If performance degrades at scale, consider short-lived caching or + # folding the count into the main query via a window function. + counts = await prisma_client.db.litellm_spendlogs.group_by( + by=["session_id"], + where={"session_id": {"in": session_ids}}, + count={"session_id": True}, + ) + count_map = { + r["session_id"]: r["_count"]["session_id"] + for r in counts + if r.get("session_id") + } + + if enrich_session_counts: + enriched: List[dict] = [] + for row in data: + row_dict = ( + dict(row) + if isinstance(row, dict) + else row.model_dump() + ) + sid = row_dict.get("session_id") + row_dict["session_total_count"] = count_map.get(sid, 1) if sid else 1 + enriched.append(row_dict) + response_data: list = enriched + else: + # v2 path: return raw Prisma model instances so FastAPI applies its + # own Pydantic-aware serialisation (preserves alias handling, custom + # serializers, etc.). + response_data = data # type: ignore[assignment] + + return { + "data": response_data, + "total": total_records, + "page": page, + "page_size": page_size, + "total_pages": total_pages, + } + + def _build_status_filter_condition(status_filter: Optional[str]) -> Dict[str, Any]: """ Helper function to build the status filter condition for database queries. diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 8f524690be1..7105f1ae6fb 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -181,6 +181,7 @@ async def aresponses_api_with_mcp( ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + litellm_trace_id=kwargs.get("litellm_trace_id"), ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( original_mcp_tools diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index 4a640a61064..377ce396457 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -127,6 +127,7 @@ async def acompletion_with_mcp( # noqa: PLR0915 ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + litellm_trace_id=kwargs.get("litellm_trace_id"), ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( @@ -235,7 +236,10 @@ async def acompletion_with_mcp( # noqa: PLR0915 def _add_mcp_list_tools_to_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: """Add mcp_list_tools to the first chunk.""" - from litellm.types.utils import StreamingChoices, add_provider_specific_fields + from litellm.types.utils import ( + StreamingChoices, + add_provider_specific_fields, + ) if not self.openai_tools: return chunk @@ -258,7 +262,10 @@ async def acompletion_with_mcp( # noqa: PLR0915 def _add_mcp_tool_metadata_to_final_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: """Add mcp_tool_calls and mcp_call_results to the final chunk.""" - from litellm.types.utils import StreamingChoices, add_provider_specific_fields + from litellm.types.utils import ( + StreamingChoices, + add_provider_specific_fields, + ) if hasattr(chunk, "choices") and chunk.choices: for choice in chunk.choices: diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 297ccf4355e..805a1958552 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -6,10 +6,10 @@ from typing import ( Dict, Iterable, List, + Literal, Optional, Tuple, Union, - Literal, ) from litellm._logging import verbose_logger @@ -29,6 +29,7 @@ from litellm.utils import Rules, function_setup if TYPE_CHECKING: from mcp.types import Tool as MCPTool + from litellm.proxy.utils import ProxyLogging else: MCPTool = Any @@ -97,6 +98,7 @@ class LiteLLM_Proxy_MCP_Handler: async def _get_mcp_tools_from_manager( user_api_key_auth: Any, mcp_tools_with_litellm_proxy: Optional[Iterable[ToolParam]], + litellm_trace_id: Optional[str] = None, ) -> tuple[List[MCPTool], List[str]]: """ Get available tools from the MCP server manager. @@ -109,13 +111,13 @@ class LiteLLM_Proxy_MCP_Handler: List of MCP tools List names of allowed MCP servers """ - from litellm.proxy._experimental.mcp_server.server import ( - _get_tools_from_mcp_servers, - _get_allowed_mcp_servers_from_mcp_server_names, - ) from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) + from litellm.proxy._experimental.mcp_server.server import ( + _get_allowed_mcp_servers_from_mcp_server_names, + _get_tools_from_mcp_servers, + ) mcp_servers: List[str] = [] if mcp_tools_with_litellm_proxy: @@ -136,6 +138,7 @@ class LiteLLM_Proxy_MCP_Handler: mcp_server_auth_headers=None, log_list_tools_to_spendlogs=True, list_tools_log_source="responses", + litellm_trace_id=litellm_trace_id, ) allowed_mcp_server_ids = ( await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) @@ -239,7 +242,9 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _process_mcp_tools_to_openai_format( - user_api_key_auth: Any, mcp_tools_with_litellm_proxy: List[ToolParam] + user_api_key_auth: Any, + mcp_tools_with_litellm_proxy: List[ToolParam], + litellm_trace_id: Optional[str] = None, ) -> tuple[List[Any], dict[str, str]]: """ Centralized method to process MCP tools through the complete pipeline. @@ -247,6 +252,7 @@ class LiteLLM_Proxy_MCP_Handler: Args: user_api_key_auth: User authentication info for access control mcp_tools_with_litellm_proxy: ToolParam objects with server_url starting with "litellm_proxy" + litellm_trace_id: Optional trace ID for linking list_mcp_tools spend logs to parent request Returns: List of tools in OpenAI format ready to be sent to the LLM @@ -258,6 +264,7 @@ class LiteLLM_Proxy_MCP_Handler: ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( user_api_key_auth, mcp_tools_with_litellm_proxy, + litellm_trace_id=litellm_trace_id, ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( @@ -268,7 +275,9 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _process_mcp_tools_without_openai_transform( - user_api_key_auth: Any, mcp_tools_with_litellm_proxy: List[ToolParam] + user_api_key_auth: Any, + mcp_tools_with_litellm_proxy: List[ToolParam], + litellm_trace_id: Optional[str] = None, ) -> tuple[List[Any], dict[str, str]]: """ Process MCP tools through filtering and deduplication pipeline without OpenAI transformation. @@ -291,6 +300,7 @@ class LiteLLM_Proxy_MCP_Handler: ) = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager( user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + litellm_trace_id=litellm_trace_id, ) # Step 2: Filter tools based on allowed_tools parameter @@ -495,14 +505,13 @@ class LiteLLM_Proxy_MCP_Handler: """Execute tool calls and return results.""" from fastapi import HTTPException + from litellm._uuid import uuid from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.proxy.proxy_server import proxy_logging_obj - from litellm._uuid import uuid - tool_results = [] tool_call_id: Optional[str] = None rules_obj = Rules() @@ -1025,7 +1034,6 @@ class LiteLLM_Proxy_MCP_Handler: List of MCP tool execution events for streaming """ from litellm._uuid import uuid - from litellm.responses.mcp.mcp_streaming_iterator import create_mcp_call_events tool_execution_events: List[Any] = [] @@ -1108,8 +1116,8 @@ class LiteLLM_Proxy_MCP_Handler: """Add custom output elements to the final response for MCP tool execution.""" # Import the required classes for creating output items import json - from litellm._uuid import uuid + from litellm._uuid import uuid from litellm.types.responses.main import GenericResponseOutputItem, OutputText # Create output element for initial MCP tools diff --git a/litellm/router.py b/litellm/router.py index 37fa3926b4d..7bba0902a5e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2283,7 +2283,7 @@ class Router: item = FlowItem( priority=priority, # 👈 SET PRIORITY FOR REQUEST request_id=_request_id, # 👈 SET REQUEST ID - model_name="gpt-3.5-turbo", # 👈 SAME as 'Router' + model_name=model, # 👈 SAME as 'Router' ) ### [fin] ### @@ -2325,6 +2325,10 @@ class Router: setattr(e, "priority", priority) raise e else: + # Clean up the request from the scheduler queue also before raising the timeout exception + await self.scheduler.remove_request( + request_id=item.request_id, model_name=item.model_name + ) raise litellm.Timeout( message="Request timed out while polling queue", model=model, @@ -2386,6 +2390,10 @@ class Router: setattr(e, "priority", priority) raise e else: + # Clean up the request from the scheduler queue also before raising the timeout exception + await self.scheduler.remove_request( + request_id=item.request_id, model_name=item.model_name + ) raise litellm.Timeout( message="Request timed out while polling queue", model=model, @@ -5039,7 +5047,7 @@ class Router: else: _healthy_deployments = [] _timeout = self._time_to_sleep_before_retry( - e=original_exception, + e=e, remaining_retries=remaining_retries, num_retries=num_retries, healthy_deployments=_healthy_deployments, diff --git a/litellm/scheduler.py b/litellm/scheduler.py index 5f3dd4cbf61..0221e249848 100644 --- a/litellm/scheduler.py +++ b/litellm/scheduler.py @@ -92,6 +92,17 @@ class Scheduler: return True + async def remove_request(self, request_id: str, model_name: str) -> None: + """ + Remove a specific request from the priority queue for a model. + Used when a request times out while waiting in the queue. + """ + queue = await self.get_queue(model_name=model_name) + filtered_queue = [item for item in queue if item[1] != request_id] + heapq.heapify(filtered_queue) # restore heap invariant after filtering + await self.save_queue(queue=filtered_queue, model_name=model_name) + print_verbose(f"Removed request_id: {request_id} from queue for model: {model_name}") + async def peek(self, id: str, model_name: str, health_deployments: list) -> bool: """Return if the id is at the top of the queue. Don't pop the value from heap.""" queue = await self.get_queue(model_name=model_name) diff --git a/litellm/types/access_group.py b/litellm/types/access_group.py new file mode 100644 index 00000000000..3a6b75768ef --- /dev/null +++ b/litellm/types/access_group.py @@ -0,0 +1,38 @@ +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel + + +class AccessGroupCreateRequest(BaseModel): + access_group_name: str + description: Optional[str] = None + access_model_ids: Optional[List[str]] = None + access_mcp_server_ids: Optional[List[str]] = None + access_agent_ids: Optional[List[str]] = None + assigned_team_ids: Optional[List[str]] = None + assigned_key_ids: Optional[List[str]] = None + + +class AccessGroupUpdateRequest(BaseModel): + description: Optional[str] = None + access_model_ids: Optional[List[str]] = None + access_mcp_server_ids: Optional[List[str]] = None + access_agent_ids: Optional[List[str]] = None + assigned_team_ids: Optional[List[str]] = None + assigned_key_ids: Optional[List[str]] = None + + +class AccessGroupResponse(BaseModel): + access_group_id: str + access_group_name: str + description: Optional[str] = None + access_model_ids: List[str] + access_mcp_server_ids: List[str] + access_agent_ids: List[str] + assigned_team_ids: List[str] + assigned_key_ids: List[str] + created_at: datetime + created_by: Optional[str] = None + updated_at: datetime + updated_by: Optional[str] = None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 299b47199ed..4ab81f8fd57 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1058,7 +1058,20 @@ class ComputerToolParam(TypedDict, total=False): type: Required[Union[Literal["computer_use_preview"], str]] -ALL_RESPONSES_API_TOOL_PARAMS = Union[ToolParam, ComputerToolParam] +class ShellToolParam(TypedDict, total=False): + """ + Shell tool for Responses API: run commands in hosted containers or local runtime. + See https://developers.openai.com/api/docs/guides/tools-shell. + """ + + type: Required[Union[Literal["shell"], str]] + """The type of tool. Use ``\"shell\"``.""" + + environment: Required[Dict[str, Any]] + """Environment config: ``type`` (e.g. ``\"container_auto\"``, ``\"container_reference\"``, ``\"local\"``), optional ``container_id``, ``network_policy``, ``domain_secrets``, ``skills``.""" + + +ALL_RESPONSES_API_TOOL_PARAMS = Union[ToolParam, ComputerToolParam, ShellToolParam] class PromptObject(TypedDict, total=False): @@ -1074,6 +1087,19 @@ class PromptObject(TypedDict, total=False): """Optional version of the prompt template.""" +class ContextManagementEntry(TypedDict, total=False): + """ + Context management configuration entry for a request. + See https://developers.openai.com/api/docs/guides/compaction. + """ + + type: str + """The context management entry type. Currently only ``'compaction'`` is supported.""" + + compact_threshold: int + """Token threshold at which compaction is triggered for this entry. Minimum 1000.""" + + class ResponsesAPIOptionalRequestParams(TypedDict, total=False): """TypedDict for Optional parameters supported by the responses API.""" @@ -1104,6 +1130,8 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): partial_images: Optional[ int ] # Number of partial images to generate (1-3) for streaming image generation + context_management: Optional[List[ContextManagementEntry]] + """Context management configuration. E.g. [{\"type\": \"compaction\", \"compact_threshold\": 200000}] for server-side compaction (minimum 1000).""" class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): @@ -1189,7 +1217,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): top_p: Optional[float] = None max_output_tokens: Optional[int] = None previous_response_id: Optional[str] = None - reasoning: Optional[Reasoning] = None + reasoning: Optional[Dict[str, Any]] = None status: Optional[str] = None text: Optional[Union["ResponseText", Dict[str, Any]]] = None truncation: Optional[Literal["auto", "disabled"]] = None @@ -1199,6 +1227,18 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): # Define private attributes using PrivateAttr _hidden_params: dict = PrivateAttr(default_factory=dict) + @field_validator("reasoning", mode="before") + @classmethod + def validate_reasoning_to_dict(cls, value: Any) -> Optional[Dict[str, Any]]: + """Accept API reasoning dict (including effort 'none'/'xhigh'); always store as dict.""" + if value is None: + return None + if isinstance(value, dict): + return value + if hasattr(value, "model_dump"): + return value.model_dump() + return value + @field_validator("usage", mode="before") @classmethod def validate_usage(cls, value): @@ -1307,6 +1347,11 @@ class ResponsesAPIStreamEvents(str, Enum): # Image generation events IMAGE_GENERATION_PARTIAL_IMAGE = "image_generation.partial_image" + # Shell tool events (Responses API; passthrough via GenericEvent) + SHELL_CALL_IN_PROGRESS = "response.shell_call.in_progress" + SHELL_CALL_COMPLETED = "response.shell_call.completed" + SHELL_CALL_OUTPUT = "response.shell_call_output.done" + # Error event ERROR = "error" @@ -1593,12 +1638,12 @@ class ImageGenerationPartialImageEvent(BaseLiteLLMOpenAIResponseObject): class ErrorEventError(BaseLiteLLMOpenAIResponseObject): - """Nested error object within ErrorEvent""" + """Nested error object within ErrorEvent.""" type: str # e.g., 'invalid_request_error' code: str # e.g., 'context_length_exceeded' message: str - param: Optional[str] + param: Optional[str] = None class ErrorEvent(BaseLiteLLMOpenAIResponseObject): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py index 96d78cf8827..21f6f5b3b4e 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py @@ -60,6 +60,14 @@ class GenericGuardrailAPIRequest(BaseModel): tools: Optional[List[ChatCompletionToolParam]] = None texts: Optional[List[str]] = None request_data: GenericGuardrailAPIMetadata + request_headers: Optional[Dict[str, str]] = Field( + default=None, + description="Sanitized inbound request headers from the original proxy request.", + ) + litellm_version: Optional[str] = Field( + default=None, + description="LiteLLM library version running this proxy.", + ) additional_provider_specific_params: Optional[Dict[str, Any]] = None tool_calls: Optional[ Union[List[ChatCompletionToolCallChunk], List[ChatCompletionMessageToolCall]] diff --git a/litellm/utils.py b/litellm/utils.py index 0fa5436d98d..0fd21f09919 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1406,7 +1406,7 @@ def client(original_function): # noqa: PLR0915 # [OPTIONAL] CHECK MAX RETRIES / REQUEST if litellm.num_retries_per_request is not None: # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] - previous_models = kwargs.get("metadata", {}).get( + previous_models = (kwargs.get("metadata") or {}).get( "previous_models", None ) if previous_models is not None: @@ -1483,7 +1483,7 @@ def client(original_function): # noqa: PLR0915 # [OPTIONAL] CHECK MAX RETRIES / REQUEST if litellm.num_retries_per_request is not None: # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] - previous_models = kwargs.get("metadata", {}).get( + previous_models = (kwargs.get("metadata") or {}).get( "previous_models", None ) if previous_models is not None: @@ -1678,8 +1678,8 @@ def client(original_function): # noqa: PLR0915 "context_window_fallback_dict", {} ) - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( num_retries and not _is_litellm_router_call @@ -1724,8 +1724,8 @@ def client(original_function): # noqa: PLR0915 None # set retries to None to prevent infinite loops ) - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( num_retries and not _is_litellm_router_call @@ -1974,8 +1974,8 @@ def client(original_function): # noqa: PLR0915 "context_window_fallback_dict", {} ) - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( @@ -2008,8 +2008,8 @@ def client(original_function): # noqa: PLR0915 kwargs["model"] = context_window_fallback_dict[model] return await original_function(*args, **kwargs) elif call_type == CallTypes.aresponses.value: - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( @@ -7337,7 +7337,7 @@ def _get_base_model_from_metadata(model_call_details=None): _base_model = litellm_params.get("base_model", None) if _base_model is not None: return _base_model - metadata = litellm_params.get("metadata", {}) + metadata = litellm_params.get("metadata") or {} _get_base_model_from_litellm_call_metadata = getattr( sys.modules[__name__], "_get_base_model_from_litellm_call_metadata" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f6edcf7efd0..f43e3dcbde5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5859,6 +5859,7 @@ "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, "azure_ai/ministral-3b": { @@ -6134,8 +6135,11 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.03e-06, + "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_video_input": true, + "supports_vision": true }, "bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.18e-06, @@ -15784,44 +15788,14 @@ "tpm": 250000 }, "gemini/gemini-2.5-flash-preview-tts": { - "cache_read_input_token_cost": 3.75e-08, - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 1.5e-07, + "input_cost_per_token": 3e-07, "litellm_provider": "gemini", - "max_audio_length_hours": 8.4, - "max_audio_per_prompt": 1, - "max_images_per_prompt": 3000, - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_pdf_size_mb": 30, - "max_tokens": 65535, - "max_video_length": 1, - "max_videos_per_prompt": 10, - "mode": "chat", - "output_cost_per_reasoning_token": 3.5e-06, - "output_cost_per_token": 6e-07, - "rpm": 10, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "mode": "audio_speech", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions" - ], - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "audio" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000 + "/v1/audio/speech" + ] }, "gemini/gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -21432,6 +21406,36 @@ "max_input_tokens": 1000000, "max_output_tokens": 8192 }, + "minimax/MiniMax-M2.5": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, + "minimax/MiniMax-M2.5-lightning": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, "minimax/MiniMax-M2": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, @@ -22147,9 +22151,10 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat", + "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, "moonshot/kimi-latest": { @@ -22611,6 +22616,19 @@ "output_cost_per_token": 2.3e-07, "supports_system_messages": true }, + "nvidia.nemotron-nano-3-30b": { + "input_cost_per_token": 6e-08, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.4e-07, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "o1": { "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, @@ -23616,36 +23634,6 @@ "output_cost_per_token": 2e-07, "supports_system_messages": true }, - "openrouter/anthropic/claude-2": { - "input_cost_per_token": 1.102e-05, - "litellm_provider": "openrouter", - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 3.268e-05, - "supports_tool_choice": true - }, - "openrouter/anthropic/claude-3-5-haiku": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_tokens": 200000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "supports_function_calling": true, - "supports_tool_choice": true - }, - "openrouter/anthropic/claude-3-5-haiku-20241022": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "tool_use_system_prompt_tokens": 264 - }, "openrouter/anthropic/claude-3-haiku": { "input_cost_per_image": 0.0004, "input_cost_per_token": 2.5e-07, @@ -23657,43 +23645,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/anthropic/claude-3-haiku-20240307": { - "input_cost_per_token": 2.5e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 264 - }, - "openrouter/anthropic/claude-3-opus": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 395 - }, - "openrouter/anthropic/claude-3-sonnet": { - "input_cost_per_image": 0.0048, - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_tokens": 200000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/anthropic/claude-3.5-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "openrouter", @@ -23709,20 +23660,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-3.5-sonnet:beta": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 - }, "openrouter/anthropic/claude-3.7-sonnet": { "input_cost_per_image": 0.0048, "input_cost_per_token": 3e-06, @@ -23740,31 +23677,6 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-3.7-sonnet:beta": { - "input_cost_per_image": 0.0048, - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "tool_use_system_prompt_tokens": 159 - }, - "openrouter/anthropic/claude-instant-v1": { - "input_cost_per_token": 1.63e-06, - "litellm_provider": "openrouter", - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 5.51e-06, - "supports_tool_choice": true - }, "openrouter/anthropic/claude-opus-4": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 1.875e-05, @@ -23903,30 +23815,6 @@ "source": "https://openrouter.ai/api/v1/models/bytedance/ui-tars-1.5-7b", "supports_tool_choice": true }, - "openrouter/cognitivecomputations/dolphin-mixtral-8x7b": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 32769, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/cohere/command-r-plus": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_tool_choice": true - }, - "openrouter/databricks/dbrx-instruct": { - "input_cost_per_token": 6e-07, - "litellm_provider": "openrouter", - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_tool_choice": true - }, "openrouter/deepseek/deepseek-chat": { "input_cost_per_token": 1.4e-07, "litellm_provider": "openrouter", @@ -23994,17 +23882,6 @@ "supports_reasoning": false, "supports_tool_choice": true }, - "openrouter/deepseek/deepseek-coder": { - "input_cost_per_token": 1.4e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 66000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2.8e-07, - "supports_prompt_caching": true, - "supports_tool_choice": true - }, "openrouter/deepseek/deepseek-r1": { "input_cost_per_token": 5.5e-07, "input_cost_per_token_cache_hit": 1.4e-07, @@ -24035,14 +23912,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "openrouter/fireworks/firellava-13b": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, "openrouter/google/gemini-2.0-flash-001": { "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 7e-07, @@ -24204,46 +24073,6 @@ "supports_web_search": true, "tpm": 800000 }, - "openrouter/google/gemini-pro-1.5": { - "input_cost_per_image": 0.00265, - "input_cost_per_token": 2.5e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1000000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 7.5e-06, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "openrouter/google/gemini-pro-vision": { - "input_cost_per_image": 0.0025, - "input_cost_per_token": 1.25e-07, - "litellm_provider": "openrouter", - "max_tokens": 45875, - "mode": "chat", - "output_cost_per_token": 3.75e-07, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "openrouter/google/palm-2-chat-bison": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 25804, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/google/palm-2-codechat-bison": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 20070, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, "openrouter/gryphe/mythomax-l2-13b": { "input_cost_per_token": 1.875e-06, "litellm_provider": "openrouter", @@ -24252,14 +24081,6 @@ "output_cost_per_token": 1.875e-06, "supports_tool_choice": true }, - "openrouter/jondurbin/airoboros-l2-70b-2.1": { - "input_cost_per_token": 1.3875e-05, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.3875e-05, - "supports_tool_choice": true - }, "openrouter/mancer/weaver": { "input_cost_per_token": 5.625e-06, "litellm_provider": "openrouter", @@ -24268,30 +24089,6 @@ "output_cost_per_token": 5.625e-06, "supports_tool_choice": true }, - "openrouter/meta-llama/codellama-34b-instruct": { - "input_cost_per_token": 5e-07, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-2-13b-chat": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-2-70b-chat": { - "input_cost_per_token": 1.5e-06, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_tool_choice": true - }, "openrouter/meta-llama/llama-3-70b-instruct": { "input_cost_per_token": 5.9e-07, "litellm_provider": "openrouter", @@ -24300,38 +24097,6 @@ "output_cost_per_token": 7.9e-07, "supports_tool_choice": true }, - "openrouter/meta-llama/llama-3-70b-instruct:nitro": { - "input_cost_per_token": 9e-07, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-3-8b-instruct:extended": { - "input_cost_per_token": 2.25e-07, - "litellm_provider": "openrouter", - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2.25e-06, - "supports_tool_choice": true - }, - "openrouter/meta-llama/llama-3-8b-instruct:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 0.0, - "supports_tool_choice": true - }, - "openrouter/microsoft/wizardlm-2-8x22b:nitro": { - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1e-06, - "supports_tool_choice": true - }, "openrouter/minimax/minimax-m2": { "input_cost_per_token": 2.55e-07, "litellm_provider": "openrouter", @@ -24345,20 +24110,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "openrouter/mistralai/devstral-2512:free": { - "input_cost_per_image": 0, - "input_cost_per_token": 0, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 0, - "supports_function_calling": true, - "supports_prompt_caching": false, - "supports_tool_choice": true, - "supports_vision": false - }, "openrouter/mistralai/devstral-2512": { "input_cost_per_image": 0, "input_cost_per_token": 1.5e-07, @@ -24437,14 +24188,6 @@ "output_cost_per_token": 1.3e-07, "supports_tool_choice": true }, - "openrouter/mistralai/mistral-7b-instruct:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 0.0, - "supports_tool_choice": true - }, "openrouter/mistralai/mistral-large": { "input_cost_per_token": 8e-06, "litellm_provider": "openrouter", @@ -24489,16 +24232,9 @@ "source": "https://openrouter.ai/moonshotai/kimi-k2.5", "supports_function_calling": true, "supports_tool_choice": true, + "supports_video_input": true, "supports_vision": true }, - "openrouter/nousresearch/nous-hermes-llama2-13b": { - "input_cost_per_token": 2e-07, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-07, - "supports_tool_choice": true - }, "openrouter/openai/gpt-3.5-turbo": { "input_cost_per_token": 1.5e-06, "litellm_provider": "openrouter", @@ -24523,17 +24259,6 @@ "output_cost_per_token": 6e-05, "supports_tool_choice": true }, - "openrouter/openai/gpt-4-vision-preview": { - "input_cost_per_image": 0.01445, - "input_cost_per_token": 1e-05, - "litellm_provider": "openrouter", - "max_tokens": 130000, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1": { "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, @@ -24551,23 +24276,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-2025-04-14": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 8e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1-mini": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, @@ -24585,23 +24293,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-mini-2025-04-14": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 4e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1.6e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4.1-nano": { "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, @@ -24619,23 +24310,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/gpt-4.1-nano-2025-04-14": { - "cache_read_input_token_cost": 2.5e-08, - "input_cost_per_token": 1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1047576, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 4e-07, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/openai/gpt-4o": { "input_cost_per_token": 2.5e-06, "litellm_provider": "openrouter", @@ -24872,58 +24546,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/openai/o1-mini": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-mini-2024-09-12": { - "input_cost_per_token": 3e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-preview": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, - "openrouter/openai/o1-preview-2024-09-12": { - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "supports_vision": false - }, "openrouter/openai/o3-mini": { "input_cost_per_token": 1.1e-06, "litellm_provider": "openrouter", @@ -24952,14 +24574,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "openrouter/pygmalionai/mythalion-13b": { - "input_cost_per_token": 1.875e-06, - "litellm_provider": "openrouter", - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.875e-06, - "supports_tool_choice": true - }, "openrouter/qwen/qwen-2.5-coder-32b-instruct": { "input_cost_per_token": 1.8e-07, "litellm_provider": "openrouter", @@ -25051,20 +24665,6 @@ "supports_tool_choice": true, "supports_web_search": true }, - "openrouter/x-ai/grok-4-fast:free": { - "input_cost_per_token": 0, - "litellm_provider": "openrouter", - "max_input_tokens": 2000000, - "max_output_tokens": 30000, - "max_tokens": 30000, - "mode": "chat", - "output_cost_per_token": 0, - "source": "https://openrouter.ai/x-ai/grok-4-fast:free", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": false - }, "openrouter/z-ai/glm-4.6": { "input_cost_per_token": 4e-07, "litellm_provider": "openrouter", @@ -30799,6 +30399,21 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/zai-org/glm-5-maas": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "vertex_ai/mistral-medium-3": { "input_cost_per_token": 4e-07, "litellm_provider": "vertex_ai-mistral_models", @@ -32302,6 +31917,20 @@ "supports_vision": true, "supports_web_search": true }, + "zai.glm-4.7": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "zai/glm-4.7": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 1.1e-07, @@ -32454,6 +32083,23 @@ "1280x720" ] }, + "openai/sora-2-pro-high-res": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.5, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "1024x1792", + "1792x1024" + ] + }, "azure/sora-2": { "litellm_provider": "azure", "mode": "video_generation", @@ -35975,5 +35621,715 @@ "mode": "chat", "output_cost_per_token": 0, "supports_reasoning": true + }, + "tts-1-1106": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "openai", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "tts-1-hd-1106": { + "input_cost_per_character": 3e-05, + "litellm_provider": "openai", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "gpt-4o-mini-tts-2025-03-20": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "mode": "audio_speech", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_second": 0.00025, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "audio" + ] + }, + "gpt-4o-mini-tts-2025-12-15": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "mode": "audio_speech", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_second": 0.00025, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "audio" + ] + }, + "gpt-4o-mini-transcribe-2025-03-20": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 16000, + "max_output_tokens": 2000, + "mode": "audio_transcription", + "output_cost_per_token": 5e-06, + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "gpt-4o-mini-transcribe-2025-12-15": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 16000, + "max_output_tokens": 2000, + "mode": "audio_transcription", + "output_cost_per_token": 5e-06, + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, + "gpt-5-search-api": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "gpt-5-search-api-2025-10-14": { + "cache_read_input_token_cost": 1.25e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "gpt-realtime-mini-2025-10-06": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "gpt-realtime-mini-2025-12-15": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "sora-2": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.1, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "720x1280", + "1280x720" + ] + }, + "sora-2-pro": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.3, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "720x1280", + "1280x720" + ] + }, + "sora-2-pro-high-res": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.5, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "1024x1792", + "1792x1024" + ] + }, + "chatgpt-image-latest": { + "cache_read_input_image_token_cost": 2.5e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_image_token": 1e-05, + "input_cost_per_token": 5e-06, + "litellm_provider": "openai", + "mode": "image_generation", + "output_cost_per_image_token": 4e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "gemini-2.0-flash-exp-image-generation": { + "input_cost_per_token": 0.0, + "litellm_provider": "gemini", + "max_images_per_prompt": 3000, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.039, + "output_cost_per_token": 0.0, + "source": "https://ai.google.dev/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "gemini/gemini-2.0-flash-exp-image-generation": { + "input_cost_per_token": 0.0, + "litellm_provider": "gemini", + "max_images_per_prompt": 3000, + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.039, + "output_cost_per_token": 0.0, + "source": "https://ai.google.dev/pricing", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_vision": true + }, + "gemini/gemini-2.0-flash-lite-001": { + "cache_read_input_token_cost": 1.875e-08, + "deprecation_date": "2026-03-31", + "input_cost_per_audio_token": 7.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_pdf_size_mb": 50, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 3e-07, + "rpm": 4000, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 4000000 + }, + "gemini-2.5-flash-native-audio-latest": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-native-audio-preview-09-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-native-audio-preview-12-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-latest": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-preview-09-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini/gemini-2.5-flash-native-audio-preview-12-2025": { + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true + }, + "gemini-2.5-flash-preview-tts": { + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "mode": "audio_speech", + "output_cost_per_token": 2.5e-06, + "source": "https://ai.google.dev/pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "gemini-flash-latest": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "rpm": 100000, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 8000000 + }, + "gemini-flash-lite-latest": { + "cache_read_input_token_cost": 1e-08, + "input_cost_per_audio_token": 3e-07, + "input_cost_per_token": 1e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 4e-07, + "output_cost_per_token": 4e-07, + "rpm": 15, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-lite", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 250000 + }, + "gemini-pro-latest": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "rpm": 2000, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, + "gemini/gemini-pro-latest": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_200k_tokens": 1.5e-05, + "rpm": 2000, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 800000 + }, + "gemini-exp-1206": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_token": 3e-07, + "litellm_provider": "gemini", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_reasoning_token": 2.5e-06, + "output_cost_per_token": 2.5e-06, + "rpm": 100000, + "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_output": false, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "tpm": 8000000 } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index fd17b5309e8..18d9f4e6eef 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2453,7 +2453,7 @@ }, "messages": { "docs_label": "anthropic_unified", - "display_name": "Anthropic /v1/messages API", + "display_name": "Anthropic Messages API", "leftnav_label": "/messages", "provider_json_field": "messages", "url": "https://docs.litellm.ai/docs/anthropic_unified", @@ -2461,7 +2461,7 @@ }, "anthropic_count_tokens": { "docs_label": "anthropic_count_tokens", - "display_name": "Anthropic /v1/messages/count_tokens API", + "display_name": "Anthropic Count Tokens API", "leftnav_label": "/count_tokens", "provider_json_field": "count_tokens", "url": "https://docs.litellm.ai/docs/anthropic_count_tokens" @@ -2482,14 +2482,14 @@ }, "audio_transcription": { "docs_label": "audio_transcription", - "display_name": "Audio Transcription API", + "display_name": "OpenAI Audio Transcription API", "leftnav_label": "/audio/transcriptions", "provider_json_field": "audio_transcriptions", "url": "https://docs.litellm.ai/docs/audio_transcription" }, "batches": { "docs_label": "batches", - "display_name": "Batches API", + "display_name": "OpenAI Batches API", "leftnav_label": "/batches", "provider_json_field": "batches", "url": "https://docs.litellm.ai/docs/batches" @@ -2510,7 +2510,7 @@ }, "chat_completions": { "docs_label": "chat_completions", - "display_name": "Chat Completions API", + "display_name": "OpenAI Chat Completions API", "leftnav_label": "/chat/completions", "provider_json_field": "chat_completions", "url": "https://docs.litellm.ai/docs/chat_completions" @@ -2531,7 +2531,7 @@ }, "embeddings": { "docs_label": "embedding/supported_embedding", - "display_name": "Embedding API (OpenAI Format)", + "display_name": "OpenAI Embeddings API", "leftnav_label": "/embeddings", "provider_json_field": "embeddings", "url": "https://docs.litellm.ai/docs/embedding/supported_embedding" @@ -2552,7 +2552,7 @@ }, "generateContent": { "docs_label": "generateContent", - "display_name": "Google's GenerateContent API", + "display_name": "Google GenerateContent API", "leftnav_label": "/generateContent", "provider_json_field": "generateContent", "url": "https://docs.litellm.ai/docs/generateContent", @@ -2596,14 +2596,14 @@ }, "moderation": { "docs_label": "moderation", - "display_name": "OpenAI Moderation API", + "display_name": "OpenAI Moderations API", "leftnav_label": "/moderations", "provider_json_field": "moderations", "url": "https://docs.litellm.ai/docs/moderation" }, "ocr": { "docs_label": "ocr", - "display_name": "OCR API (Mistral Format)", + "display_name": "Mistral OCR API", "leftnav_label": "/ocr", "provider_json_field": "ocr", "url": "https://docs.litellm.ai/docs/ocr" @@ -2631,14 +2631,14 @@ }, "rerank": { "docs_label": "rerank", - "display_name": "Rerank API (Cohere Format)", + "display_name": "Cohere Rerank API", "leftnav_label": "/rerank", "provider_json_field": "rerank", "url": "https://docs.litellm.ai/docs/rerank" }, "responses": { "docs_label": "response_api", - "display_name": "Responses API (OpenAI Format)", + "display_name": "OpenAI Responses API", "leftnav_label": "/responses", "provider_json_field": "responses", "url": "https://docs.litellm.ai/docs/response_api", @@ -2646,7 +2646,7 @@ }, "response_api_compact": { "docs_label": "response_api_compact", - "display_name": "Responses API (OpenAI Format)", + "display_name": "OpenAI Responses API", "leftnav_label": "/responses", "provider_json_field": "compact", "url": "https://docs.litellm.ai/docs/response_api" @@ -2667,7 +2667,7 @@ }, "text_completion": { "docs_label": "text_completion", - "display_name": "Completions API (OpenAI Format)", + "display_name": "OpenAI Completions API", "leftnav_label": "/completions", "provider_json_field": "text_completion", "url": "https://docs.litellm.ai/docs/text_completion", @@ -2675,7 +2675,7 @@ }, "text_to_speech": { "docs_label": "text_to_speech", - "display_name": "Text-to-Speech API (OpenAI Format)", + "display_name": "OpenAI Text-to-Speech API", "leftnav_label": "/audio/speech", "provider_json_field": "audio_speech", "url": "https://docs.litellm.ai/docs/text_to_speech" @@ -2703,7 +2703,7 @@ }, "videos": { "docs_label": "videos", - "display_name": "OpenAI Video Generation API", + "display_name": "OpenAI Videos API", "leftnav_label": "/videos", "provider_json_field": "video_generations", "url": "https://docs.litellm.ai/docs/videos" diff --git a/pyproject.toml b/pyproject.toml index acb8bc2ada3..6ed7618dd26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.81.10" +version = "1.81.11" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -61,7 +61,7 @@ boto3 = { version = "1.40.76", optional = true } redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"} a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.4.34", optional = true} +litellm-proxy-extras = {version = "0.4.36", optional = true} rich = {version = "13.7.1", optional = true} litellm-enterprise = {version = "0.1.31", optional = true} diskcache = {version = "^5.6.1", optional = true} @@ -175,7 +175,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.81.10" +version = "1.81.11" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index bd313b105e5..f31730e20f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,8 @@ urllib3>=2.6.0 # CVE-2025-66471, CVE-2025-66418, CVE-2026-21441 tornado>=6.5.3 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724 filelock>=3.20.1 # CVE-2025-68146 +Pillow==12.1.1 #GHSA-cfh3-3jmp-rvhc +cryptography==46.0.5 #GHSA-r6ph-v2qm-q3c2 anyio==4.8.0 # openai + http req. httpx==0.28.1 @@ -38,7 +40,6 @@ apscheduler==3.10.4 # for resetting budget in background fastapi-sso==0.19.0 # admin UI, SSO pyjwt[crypto]==2.10.1 ; python_version >= "3.9" python-multipart==0.0.22 # admin UI -Pillow==11.0.0 jaraco.context>=6.1.0 azure-ai-contentsafety==1.0.0 # for azure content safety azure-identity==1.16.1 ; python_version >= "3.9" # for azure content safety @@ -53,9 +54,8 @@ grpcio>=1.62.3,!=1.68.*,!=1.69.*,!=1.70.*,!=1.71.0,!=1.71.1,!=1.72.0,!=1.72.1,!= grpcio>=1.75.0; python_version >= "3.14" sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests -cryptography==44.0.1 tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.34 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.36 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env diff --git a/schema.prisma b/schema.prisma index 4329f939a7b..2a11d0028fb 100644 --- a/schema.prisma +++ b/schema.prisma @@ -128,6 +128,7 @@ model LiteLLM_TeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team @@ -161,6 +162,7 @@ model LiteLLM_DeletedTeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) @@ -293,6 +295,7 @@ model LiteLLM_VerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") budget_id String? @@ -348,6 +351,7 @@ model LiteLLM_DeletedVerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") router_settings Json? @default("{}") @@ -363,7 +367,6 @@ model LiteLLM_DeletedVerificationToken { rotation_interval String? last_rotation_at DateTime? key_rotation_at DateTime? - // Deletion metadata deleted_at DateTime @default(now()) @map("deleted_at") deleted_by String? @map("deleted_by") // User who deleted the key @@ -919,3 +922,23 @@ model LiteLLM_PolicyAttachmentTable { updated_at DateTime @default(now()) @updatedAt updated_by String? } + +//Unified Access Groups table for storing unified access groups +model LiteLLM_AccessGroupTable { + access_group_id String @id @default(uuid()) + access_group_name String @unique + description String? + + // Resource memberships - explicit arrays per type + access_model_ids String[] @default([]) + access_mcp_server_ids String[] @default([]) + access_agent_ids String[] @default([]) + + assigned_team_ids String[] @default([]) + assigned_key_ids String[] @default([]) + + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? +} \ No newline at end of file diff --git a/tests/code_coverage_tests/check_guardrail_apply_decorator.py b/tests/code_coverage_tests/check_guardrail_apply_decorator.py index 18a86277aa9..523b9f5b35b 100644 --- a/tests/code_coverage_tests/check_guardrail_apply_decorator.py +++ b/tests/code_coverage_tests/check_guardrail_apply_decorator.py @@ -83,6 +83,12 @@ def test_guardrail_apply_decorator(): if python_file.name == "bedrock_guardrails.py": continue + # Skip content_filter.py - it implements its own detailed logging via + # _log_guardrail_information with detections, masked_entity_count, etc. + # Using the decorator would cause duplicate entries. + if python_file.name == "content_filter.py": + continue + results = find_apply_guardrail_methods(python_file) for class_name, line_num, has_decorator in results: diff --git a/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py b/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py new file mode 100644 index 00000000000..89da8d87e63 --- /dev/null +++ b/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py @@ -0,0 +1,64 @@ +""" +Tests for _map_reasoning_effort in AnthropicConfig. + +Verifies that reasoning_effort=None returns None for all models, +including Claude Opus 4.6. +""" + +from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + +class TestMapReasoningEffort: + def test_none_returns_none_for_opus_4_6(self): + """reasoning_effort=None should return None for Opus 4.6, not adaptive.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort=None, model="claude-opus-4-6" + ) + assert result is None + + def test_none_returns_none_for_other_models(self): + """reasoning_effort=None should return None for non-Opus models.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort=None, model="claude-3-7-sonnet-20250219" + ) + assert result is None + + def test_opus_4_6_returns_adaptive_for_low(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="low", model="claude-opus-4-6" + ) + assert result["type"] == "adaptive" + + def test_opus_4_6_returns_adaptive_for_high(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="high", model="claude-opus-4-6" + ) + assert result["type"] == "adaptive" + + def test_other_model_low_returns_enabled_with_budget(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="low", model="claude-3-7-sonnet-20250219" + ) + assert result["type"] == "enabled" + assert "budget_tokens" in result + + def test_other_model_high_returns_enabled_with_budget(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="high", model="claude-3-7-sonnet-20250219" + ) + assert result["type"] == "enabled" + assert "budget_tokens" in result + + def test_none_string_returns_none_for_opus_4_6(self): + """reasoning_effort='none' should return None for Opus 4.6.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="none", model="claude-opus-4-6" + ) + assert result is None + + def test_none_string_returns_none_for_other_models(self): + """reasoning_effort='none' should return None for non-Opus models.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="none", model="claude-3-7-sonnet-20250219" + ) + assert result is None diff --git a/tests/litellm/proxy/management_endpoints/test_common_utils.py b/tests/litellm/proxy/management_endpoints/test_common_utils.py new file mode 100644 index 00000000000..f857db770d0 --- /dev/null +++ b/tests/litellm/proxy/management_endpoints/test_common_utils.py @@ -0,0 +1,159 @@ +""" +Tests for litellm/proxy/management_endpoints/common_utils.py + +Specifically tests that _update_metadata_fields does not trigger premium +user checks when premium fields are present but empty. + +Related: https://github.com/BerriAI/litellm/issues/20534 +""" + +from unittest.mock import patch + +import pytest + +from litellm.proxy.management_endpoints.common_utils import ( + _has_non_empty_value, + _update_metadata_fields, +) + + +class TestHasNonEmptyValue: + """Tests for the _has_non_empty_value helper.""" + + def test_none_is_empty(self): + assert _has_non_empty_value(None) is False + + def test_empty_list_is_empty(self): + assert _has_non_empty_value([]) is False + + def test_empty_string_is_empty(self): + assert _has_non_empty_value("") is False + + def test_blank_string_is_empty(self): + assert _has_non_empty_value(" ") is False + + def test_non_empty_list_has_value(self): + assert _has_non_empty_value(["policy-a"]) is True + + def test_non_empty_string_has_value(self): + assert _has_non_empty_value("30d") is True + + def test_dict_has_value(self): + assert _has_non_empty_value({"key": "val"}) is True + + def test_empty_dict_has_value(self): + # empty dict is not None/list/str, so it counts as non-empty + assert _has_non_empty_value({}) is True + + +class TestUpdateMetadataFieldsPremiumCheck: + """ + Tests that _update_metadata_fields skips premium user checks for empty + values but still enforces them for real values. + + Issue: The UI sends the full form on every team update, including premium + fields like `policies: []`. The backend was treating these empty values + as premium feature usage and returning 403. + """ + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_empty_policies_skips_premium_check(self, mock_check): + """policies: [] should NOT trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "team_alias": "my-team", + "policies": [], + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_empty_guardrails_skips_premium_check(self, mock_check): + """guardrails: [] should NOT trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "guardrails": [], + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_empty_string_team_member_key_duration_skips_premium_check( + self, mock_check + ): + """team_member_key_duration: '' should NOT trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "team_member_key_duration": "", + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_full_ui_payload_with_empty_premium_fields_skips_premium_check( + self, mock_check + ): + """A realistic UI payload with all empty premium fields should not 403.""" + updated_kv = { + "team_id": "team-123", + "team_alias": "renamed-team", + "models": ["gpt-4o"], + "max_budget": 200, + "policies": [], + "guardrails": [], + "logging": [], + "team_member_key_duration": "", + "prompts": [], + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + ) + def test_non_empty_policies_triggers_premium_check(self, mock_check): + """policies: ['real-policy'] SHOULD trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "policies": ["real-policy"], + } + _update_metadata_fields(updated_kv) + mock_check.assert_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + ) + def test_non_empty_guardrails_triggers_premium_check(self, mock_check): + """guardrails: ['my-guardrail'] SHOULD trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "guardrails": ["my-guardrail"], + } + _update_metadata_fields(updated_kv) + mock_check.assert_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + ) + def test_non_empty_team_member_key_duration_triggers_premium_check( + self, mock_check + ): + """team_member_key_duration: '30d' SHOULD trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "team_member_key_duration": "30d", + } + _update_metadata_fields(updated_kv) + mock_check.assert_called() diff --git a/tests/litellm/test_router_retry_backoff_headers.py b/tests/litellm/test_router_retry_backoff_headers.py new file mode 100644 index 00000000000..03c3af692ce --- /dev/null +++ b/tests/litellm/test_router_retry_backoff_headers.py @@ -0,0 +1,88 @@ +""" +Tests for router retry backoff behavior. +""" + +from unittest.mock import patch + +import httpx +import pytest + +import litellm +from litellm import Router + + +@pytest.mark.asyncio +async def test_retry_backoff_uses_current_exception_headers(): + """ + Ensure retry backoff uses the current retry exception, not the initial one. + """ + router = Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "sk-test", + }, + } + ], + num_retries=2, + ) + + first_error = litellm.RateLimitError( + message="Rate limited on first attempt", + model="gpt-3.5-turbo", + llm_provider="openai", + ) + first_error.litellm_response_headers = httpx.Headers({"retry-after": "1"}) + + second_error = litellm.RateLimitError( + message="Rate limited on second attempt", + model="gpt-3.5-turbo", + llm_provider="openai", + ) + second_error.litellm_response_headers = httpx.Headers({"retry-after": "15"}) + + third_error = litellm.RateLimitError( + message="Rate limited on third attempt", + model="gpt-3.5-turbo", + llm_provider="openai", + ) + third_error.litellm_response_headers = httpx.Headers({"retry-after": "30"}) + + raised_errors = [first_error, second_error, third_error] + captured_backoff_errors = [] + + async def mock_make_call(*args, **kwargs): + raise raised_errors.pop(0) + + def mock_time_to_sleep_before_retry(*args, **kwargs): + captured_backoff_errors.append(kwargs["e"]) + return 0.01 + + with patch.object(router, "make_call", side_effect=mock_make_call): + with patch.object( + router, + "_async_get_healthy_deployments", + return_value=( + [{"model_info": {"id": "test-id"}}], + [{"model_info": {"id": "test-id"}}], + ), + ): + with patch.object( + router, + "_time_to_sleep_before_retry", + side_effect=mock_time_to_sleep_before_retry, + ): + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello"}], + ) + + # Router computes backoff once after the initial failure, then once per failed retry. + # With num_retries=2 and all attempts failing, that's 1 + 2 = 3 invocations. + assert len(captured_backoff_errors) == router.num_retries + 1 + assert captured_backoff_errors[0] is first_error + assert captured_backoff_errors[1] is second_error + assert captured_backoff_errors[2] is third_error diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 37ed1a9b08c..7ac83753d17 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -2,7 +2,7 @@ import httpx import json import pytest import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from unittest.mock import MagicMock, Mock, patch import os from litellm._uuid import uuid @@ -114,6 +114,10 @@ class BaseResponsesAPITest(ABC): """Must return the base completion reasoning call args""" return None + def get_advanced_model_for_shell_tool(self) -> Optional[str]: + """If specified, overrides the model used by test_responses_api_shell_tool_streaming_sees_shell_output (e.g. openai/gpt-5.2 for shell support).""" + return None + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_basic_openai_responses_api(self, sync_mode): @@ -669,7 +673,7 @@ class BaseResponsesAPITest(ABC): async def test_cancel_responses_invalid_response_id(self, sync_mode): """Test cancel_responses with invalid response ID should raise appropriate error""" base_completion_call_args = self.get_base_completion_call_args() - + if sync_mode: with pytest.raises(Exception): litellm.cancel_responses( @@ -679,4 +683,128 @@ class BaseResponsesAPITest(ABC): with pytest.raises(Exception): await litellm.acancel_responses( response_id="invalid_response_id_12345", **base_completion_call_args - ) \ No newline at end of file + ) + + @pytest.mark.asyncio + async def test_responses_api_context_management_server_side_compaction(self): + """ + E2E test for server-side compaction (context_management) on OpenAI Responses API. + Passes context_management with compact_threshold; validates that the request is + accepted and returns a valid response. Compaction may not run for short inputs. + """ + base_completion_call_args = self.get_base_completion_call_args() + model = base_completion_call_args.get("model") or "" + # Only run with context_management for OpenAI (OAI) for now + if "openai/" not in str(model) and "azure/" not in str(model): + pytest.skip( + "context_management server-side compaction e2e is only run for OpenAI/Azure" + ) + context_management = [{"type": "compaction", "compact_threshold": 200000}] + try: + response = await litellm.aresponses( + input="Short ping to verify context_management is accepted.", + max_output_tokens=20, + context_management=context_management, + **base_completion_call_args, + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + validate_responses_api_response(response, final_chunk=True) + assert response.get("id") is not None + assert response.get("status") is not None + + @pytest.mark.asyncio + async def test_responses_api_shell_tool(self): + """ + E2E test for Shell tool on OpenAI Responses API. + Passes tools=[{"type": "shell", "environment": {"type": "container_auto"}}]; + validates that the request is accepted and returns a valid response. + Only runs for OpenAI/Azure (Responses API with shell support). + """ + base_completion_call_args = self.get_base_completion_call_args() + model = self.get_advanced_model_for_shell_tool() or base_completion_call_args.get( + "model" + ) or "" + if "openai/" not in str(model) and "azure/" not in str(model): + pytest.skip( + "Shell tool e2e is only run for OpenAI/Azure Responses API" + ) + tools = [{"type": "shell", "environment": {"type": "container_auto"}}] + input_msg = "List files in /mnt/data and show python --version." + try: + response = await litellm.aresponses( + **{**base_completion_call_args, "model": model}, + input=input_msg, + max_output_tokens=256, + tools=tools, + tool_choice="auto", + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + except litellm.BadRequestError as e: + if "shell" in str(e).lower() and "not supported" in str(e).lower(): + pytest.skip( + "Shell tool is not supported for this model (e.g. gpt-4o); use a model that supports shell" + ) + raise + validate_responses_api_response(response, final_chunk=True) + assert response.get("id") is not None + assert response.get("status") is not None + + @pytest.mark.asyncio + async def test_responses_api_shell_tool_streaming_sees_shell_output(self): + """ + E2E streaming call with Shell tool; validate we can see shell output in the stream. + + Calls aresponses(..., tools=[shell], stream=True), then iterates the stream and + asserts at least one event is shell-related or response output contains shell_call. + Skips when model does not support shell (e.g. gpt-4o). + """ + base_completion_call_args = self.get_base_completion_call_args() + model = self.get_advanced_model_for_shell_tool() or base_completion_call_args.get( + "model" + ) or "openai/gpt-5.2" + tools = [{"type": "shell", "environment": {"type": "container_auto"}}] + input_msg = "List files in /mnt/data and run python --version." + + stream = await litellm.aresponses( + **{**base_completion_call_args, "model": model}, + input=input_msg, + max_output_tokens=512, + tools=tools, + tool_choice="auto", + stream=True, + ) + + + event_types_seen = [] + output_items_with_shell = [] + + async for event in stream: + print("event=", json.dumps(event, indent=4, default=str)) + event_type = getattr(event, "type", None) or ( + event.get("type") if isinstance(event, dict) else None + ) + if event_type is not None: + event_types_seen.append(str(event_type)) + if "shell" in str(event_type or "").lower(): + output_items_with_shell.append(event_type) + response_obj = getattr(event, "response", None) or ( + event.get("response") if isinstance(event, dict) else None + ) + if response_obj is not None: + output = getattr(response_obj, "output", None) or ( + response_obj.get("output") if isinstance(response_obj, dict) else None + ) + if isinstance(output, list): + for item in output: + item_type = getattr(item, "type", None) or ( + item.get("type") if isinstance(item, dict) else None + ) + if item_type and "shell" in str(item_type).lower(): + output_items_with_shell.append(item_type) + + assert len(event_types_seen) > 0, "Expected at least one stream event" + assert len(output_items_with_shell) > 0, ( + f"Expected to see shell output in stream; event types seen: {event_types_seen!r}" + ) diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 5f35d6837c0..4972aa385ce 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -36,6 +36,9 @@ class TestOpenAIResponsesAPITest(BaseResponsesAPITest): "model": "openai/gpt-5-mini", } + def get_advanced_model_for_shell_tool(self): + return "openai/gpt-5.2" + class TestCustomLogger(CustomLogger): def __init__( diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 6a2ad406788..eff11c9cee3 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -519,7 +519,7 @@ async def test_openai_codex_stream(sync_mode): from litellm.main import stream_chunk_builder kwargs = { - "model": "openai/codex-mini-latest", + "model": "openai/gpt-5-codex-mini", "messages": [{"role": "user", "content": "Hey!"}], "stream": True, } @@ -549,16 +549,16 @@ async def test_openai_codex(sync_mode): router = Router( model_list=[ { - "model_name": "openai-codex-mini-latest", + "model_name": "openai-gpt-5-codex-mini", "litellm_params": { - "model": "openai/codex-mini-latest", + "model": "openai/gpt-5-codex-mini", }, } ] ) kwargs = { - "model": "openai-codex-mini-latest", + "model": "openai-gpt-5-codex-mini", "messages": [{"role": "user", "content": "Hey!"}], } diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index c322db157e9..26a93343caf 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -2189,7 +2189,7 @@ def test_completion_openrouter1(): try: litellm.set_verbose = True response = completion( - model="openrouter/mistralai/mistral-tiny", + model="openrouter/mistralai/ministral-8b", messages=messages, max_tokens=5, ) diff --git a/tests/local_testing/test_scheduler.py b/tests/local_testing/test_scheduler.py index f5b44224853..f198e572b21 100644 --- a/tests/local_testing/test_scheduler.py +++ b/tests/local_testing/test_scheduler.py @@ -117,3 +117,41 @@ async def test_scheduler_prioritized_requests(p0, p1, healthy_deployments): ) == False ) + + +@pytest.mark.asyncio +async def test_scheduler_queue_cleanup_on_timeout(): + """ + Test that a timed-out request is properly removed from the queue. + This prevents memory leaks from accumulating timed-out requests. + """ + scheduler = Scheduler() + + # Add multiple requests with different priorities + item1 = FlowItem(priority=0, request_id="req-0", model_name="gpt-3.5-turbo") + item2 = FlowItem(priority=1, request_id="req-1", model_name="gpt-3.5-turbo") + item3 = FlowItem(priority=2, request_id="req-2", model_name="gpt-3.5-turbo") + + await scheduler.add_request(item1) + await scheduler.add_request(item2) + await scheduler.add_request(item3) + + # Verify initial queue size + queue_before = await scheduler.get_queue(model_name="gpt-3.5-turbo") + assert len(queue_before) == 3, f"Expected 3 items in queue, got {len(queue_before)}" + + # Simulate timeout cleanup - remove a non-front request (item2) + await scheduler.remove_request(request_id="req-1", model_name="gpt-3.5-turbo") + + # Verify queue was cleaned up + queue_after = await scheduler.get_queue(model_name="gpt-3.5-turbo") + assert len(queue_after) == 2, f"Expected 2 items after cleanup, got {len(queue_after)}" + + # Verify the correct request was removed + remaining_ids = [item[1] for item in queue_after] + assert "req-1" not in remaining_ids, "Expected req-1 to be removed" + assert "req-0" in remaining_ids, "Expected req-0 to remain" + assert "req-2" in remaining_ids, "Expected req-2 to remain" + + # Verify remaining items are in correct priority order (0 should be first) + assert queue_after[0][1] == "req-0", "Expected req-0 (priority 0) to be at front" diff --git a/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py new file mode 100644 index 00000000000..d3c4ac80565 --- /dev/null +++ b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py @@ -0,0 +1,415 @@ +""" +Tests for standard_logging_payload_excluded_fields feature. + +This feature allows users to exclude specific fields from StandardLoggingPayload +before any callback receives it. This is useful for: +- Reducing log sizes (excluding large fields like 'response' or 'messages') +- Privacy compliance (excluding sensitive fields) +- Cost management (less data stored/transmitted) + +Example config: + litellm_settings: + success_callback: ["s3"] + standard_logging_payload_excluded_fields: ["response", "messages"] +""" + +import os +import sys +from copy import deepcopy +from typing import Dict, List, Optional +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import StandardLoggingPayload + + +def create_sample_standard_logging_payload() -> Dict: + """Create a sample StandardLoggingPayload for testing.""" + return { + "id": "test-id-123", + "trace_id": "trace-123", + "call_type": "completion", + "stream": False, + "response_cost": 0.001, + "cost_breakdown": None, + "response_cost_failure_debug_info": None, + "status": "success", + "status_fields": {}, + "custom_llm_provider": "openai", + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + "startTime": 1234567890.0, + "endTime": 1234567891.0, + "completionStartTime": 1234567890.5, + "response_time": 1.0, + "model_map_information": {}, + "model": "gpt-4", + "model_id": "model-123", + "model_group": None, + "api_base": "https://api.openai.com/v1", + "metadata": {}, + "cache_hit": False, + "cache_key": None, + "saved_cache_cost": 0.0, + "request_tags": [], + "end_user": None, + "requester_ip_address": None, + "user_agent": None, + "messages": [{"role": "user", "content": "Hello, this is sensitive data!"}], + "response": { + "choices": [ + {"message": {"content": "This is a sensitive response!"}} + ] + }, + "error_str": None, + "error_information": None, + "model_parameters": {}, + "hidden_params": {}, + "guardrail_information": None, + "standard_built_in_tools_params": None, + } + + +def create_model_call_details( + standard_logging_payload: Optional[Dict] = None, +) -> Dict: + """Create model_call_details dict with standard_logging_object.""" + if standard_logging_payload is None: + standard_logging_payload = create_sample_standard_logging_payload() + return { + "standard_logging_object": standard_logging_payload, + "other_key": "other_value", + } + + +class TestStandardLoggingPayloadExcludedFields: + """Test suite for standard_logging_payload_excluded_fields feature.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def test_no_excluded_fields_no_change(self): + """Test that payload is unchanged when no fields are excluded.""" + logger = CustomLogger() + model_call_details = create_model_call_details() + original_keys = set(model_call_details["standard_logging_object"].keys()) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + result_keys = set(result["standard_logging_object"].keys()) + assert result_keys == original_keys + + def test_exclude_single_field(self): + """Test excluding a single field (response).""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" in result["standard_logging_object"] + assert "model" in result["standard_logging_object"] + + def test_exclude_multiple_fields(self): + """Test excluding multiple fields (response, messages).""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" not in result["standard_logging_object"] + assert "model" in result["standard_logging_object"] + assert "model_parameters" in result["standard_logging_object"] + + def test_exclude_metadata_field(self): + """Test excluding the metadata field.""" + litellm.standard_logging_payload_excluded_fields = ["metadata"] + + logger = CustomLogger() + payload = create_sample_standard_logging_payload() + payload["metadata"] = {"sensitive_key": "sensitive_value"} + model_call_details = create_model_call_details(payload) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "metadata" not in result["standard_logging_object"] + + def test_exclude_hidden_params(self): + """Test excluding hidden_params field.""" + litellm.standard_logging_payload_excluded_fields = ["hidden_params"] + + logger = CustomLogger() + payload = create_sample_standard_logging_payload() + payload["hidden_params"] = {"api_key": "sk-secret-key"} + model_call_details = create_model_call_details(payload) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "hidden_params" not in result["standard_logging_object"] + + def test_exclude_nonexistent_field_no_error(self): + """Test that excluding a non-existent field doesn't cause an error.""" + litellm.standard_logging_payload_excluded_fields = [ + "nonexistent_field", + "response", + ] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + # Should not raise an exception + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" in result["standard_logging_object"] + + def test_original_payload_not_modified(self): + """Test that the original model_call_details is not modified.""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + original_payload = deepcopy(model_call_details) + + logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # Original should still have the fields + assert "response" in model_call_details["standard_logging_object"] + assert "messages" in model_call_details["standard_logging_object"] + assert model_call_details == original_payload + + def test_combined_with_turn_off_message_logging(self): + """Test that excluded_fields works together with turn_off_message_logging.""" + litellm.standard_logging_payload_excluded_fields = ["metadata", "hidden_params"] + + logger = CustomLogger(turn_off_message_logging=True) + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # excluded_fields should remove these + assert "metadata" not in result["standard_logging_object"] + assert "hidden_params" not in result["standard_logging_object"] + + # turn_off_message_logging should redact these + redacted_str = "redacted-by-litellm" + assert ( + result["standard_logging_object"]["messages"][0]["content"] == redacted_str + ) + assert ( + result["standard_logging_object"]["response"]["choices"][0]["message"][ + "content" + ] + == redacted_str + ) + + def test_excluded_fields_takes_precedence_over_redaction(self): + """Test that if a field is both excluded and would be redacted, it's excluded.""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger(turn_off_message_logging=True) + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # response should be excluded (not redacted) + assert "response" not in result["standard_logging_object"] + + # messages should still be redacted + redacted_str = "redacted-by-litellm" + assert ( + result["standard_logging_object"]["messages"][0]["content"] == redacted_str + ) + + def test_exclude_all_sensitive_fields(self): + """Test excluding all potentially sensitive fields.""" + litellm.standard_logging_payload_excluded_fields = [ + "messages", + "response", + "metadata", + "hidden_params", + "model_parameters", + "error_str", + "error_information", + ] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + standard_obj = result["standard_logging_object"] + + # All sensitive fields should be removed + assert "messages" not in standard_obj + assert "response" not in standard_obj + assert "metadata" not in standard_obj + assert "hidden_params" not in standard_obj + assert "model_parameters" not in standard_obj + assert "error_str" not in standard_obj + assert "error_information" not in standard_obj + + # Non-sensitive fields should remain + assert "id" in standard_obj + assert "model" in standard_obj + assert "response_cost" in standard_obj + assert "total_tokens" in standard_obj + + def test_empty_excluded_fields_list(self): + """Test that an empty list doesn't affect the payload.""" + litellm.standard_logging_payload_excluded_fields = [] + + logger = CustomLogger() + model_call_details = create_model_call_details() + original_keys = set(model_call_details["standard_logging_object"].keys()) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + result_keys = set(result["standard_logging_object"].keys()) + assert result_keys == original_keys + + def test_none_standard_logging_object(self): + """Test handling when standard_logging_object is None.""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger() + model_call_details = {"other_key": "other_value"} + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # Should return unchanged when no standard_logging_object + assert result == model_call_details + + +class TestExcludedFieldsIntegration: + """Integration tests for excluded fields with actual callbacks.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + litellm.callbacks = [] + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + litellm.callbacks = [] + + def test_custom_callback_receives_filtered_payload(self): + """Test that a custom callback receives the filtered payload.""" + captured_payloads = [] + + class TestCallback(CustomLogger): + def log_success_event(self, kwargs, response_obj, start_time, end_time): + captured_payloads.append(kwargs.get("standard_logging_object", {})) + + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + callback = TestCallback() + model_call_details = create_model_call_details() + + # Simulate what litellm_logging.py does + filtered_details = callback.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + callback.log_success_event( + kwargs=filtered_details, + response_obj=None, + start_time=None, + end_time=None, + ) + + assert len(captured_payloads) == 1 + assert "response" not in captured_payloads[0] + assert "messages" not in captured_payloads[0] + assert "model" in captured_payloads[0] + + +class TestExcludedFieldsConfigLoading: + """Test that the config is properly loaded from litellm_settings.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def test_config_attribute_exists(self): + """Test that the config attribute exists on litellm module.""" + assert hasattr(litellm, "standard_logging_payload_excluded_fields") + + def test_config_default_is_none(self): + """Test that the default value is None.""" + # Reset to ensure we're testing the default + litellm.standard_logging_payload_excluded_fields = None + assert litellm.standard_logging_payload_excluded_fields is None + + def test_config_can_be_set_to_list(self): + """Test that the config can be set to a list.""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + assert litellm.standard_logging_payload_excluded_fields == [ + "response", + "messages", + ] + + def test_config_setattr_simulates_proxy_loading(self): + """Test that setattr works as the proxy would use it.""" + # Simulating how proxy_server.py sets litellm_settings + config_value = ["response", "messages", "metadata"] + setattr(litellm, "standard_logging_payload_excluded_fields", config_value) + + assert litellm.standard_logging_payload_excluded_fields == config_value + + # Test it actually works in the logger + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" not in result["standard_logging_object"] + assert "metadata" not in result["standard_logging_object"] diff --git a/tests/mcp_tests/test_aresponses_api_with_mcp.py b/tests/mcp_tests/test_aresponses_api_with_mcp.py index a7bbfef14af..bae0b15dfec 100644 --- a/tests/mcp_tests/test_aresponses_api_with_mcp.py +++ b/tests/mcp_tests/test_aresponses_api_with_mcp.py @@ -683,8 +683,8 @@ async def test_streaming_responses_api_with_mcp_tools( Return the user the result of request 2 """ - # Skip test if required API keys are not set - if ("anthropic" in model.lower() or "claude" in model.lower()) and not os.getenv("ANTHROPIC_API_KEY"): + # Skip test if API keys are not set for the respective models + if ("claude" in model.lower() or "anthropic" in model.lower()) and not os.getenv("ANTHROPIC_API_KEY"): pytest.skip("ANTHROPIC_API_KEY not set, skipping anthropic model test") if ("gpt" in model.lower() or "openai" in model.lower()) and not os.getenv("OPENAI_API_KEY"): pytest.skip("OPENAI_API_KEY not set, skipping openai model test") diff --git a/tests/mcp_tests/test_mcp_chat_completions.py b/tests/mcp_tests/test_mcp_chat_completions.py index 0617dcb2e42..9010e8c0d29 100644 --- a/tests/mcp_tests/test_mcp_chat_completions.py +++ b/tests/mcp_tests/test_mcp_chat_completions.py @@ -19,7 +19,7 @@ async def test_acompletion_mcp_auto_exec(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -95,7 +95,7 @@ async def test_acompletion_mcp_respects_manual_approval(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -170,7 +170,7 @@ async def test_completion_mcp_with_streaming_no_timeout_error(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -470,7 +470,7 @@ async def test_mcp_metadata_in_streaming_final_chunk(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -793,7 +793,7 @@ async def test_mcp_streaming_metadata_ordering(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index ea823df1fb2..9718d714cfe 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -1053,6 +1053,7 @@ async def test_mcp_server_manager_access_groups_from_config(): mcp_server_manager_mod.global_mcp_server_manager = original_manager +@pytest.mark.asyncio async def test_mcp_server_manager_config_integration_with_database(): """ Test that config-based servers properly integrate with database servers, diff --git a/tests/proxy_unit_tests/test_ui_path_detection.py b/tests/proxy_unit_tests/test_ui_path_detection.py new file mode 100644 index 00000000000..72ee7770f94 --- /dev/null +++ b/tests/proxy_unit_tests/test_ui_path_detection.py @@ -0,0 +1,157 @@ +""" +Unit tests for UI path detection and configuration. + +Tests the new LITELLM_UI_PATH and LITELLM_ASSETS_PATH functionality +for read-only filesystem support. + +Note: Tests involving proxy_server imports are intentionally minimal +to avoid long module load times during testing. +""" + +import os +import tempfile +from pathlib import Path +from unittest import mock + +import pytest + + +class TestUIPathEnvironmentVariable: + """Test LITELLM_UI_PATH environment variable handling.""" + + def test_custom_ui_path_env_var(self): + """Test that LITELLM_UI_PATH overrides default.""" + custom_path = "/custom/ui/path" + + with mock.patch.dict( + os.environ, {"LITELLM_UI_PATH": custom_path, "LITELLM_NON_ROOT": "true"} + ): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_runtime_ui_path = ( + "/var/lib/litellm/ui" if is_non_root else "/default/packaged/path" + ) + runtime_ui_path = os.getenv("LITELLM_UI_PATH", default_runtime_ui_path) + + assert runtime_ui_path == custom_path + + def test_default_ui_path_non_root(self): + """Test default UI path in non-root mode.""" + with mock.patch.dict( + os.environ, {"LITELLM_NON_ROOT": "true"}, clear=False + ): + # Clear LITELLM_UI_PATH if it exists + env_copy = os.environ.copy() + if "LITELLM_UI_PATH" in env_copy: + del env_copy["LITELLM_UI_PATH"] + + with mock.patch.dict(os.environ, env_copy, clear=True): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_runtime_ui_path = ( + "/var/lib/litellm/ui" + if is_non_root + else "/default/packaged/path" + ) + runtime_ui_path = os.getenv( + "LITELLM_UI_PATH", default_runtime_ui_path + ) + + assert runtime_ui_path == "/var/lib/litellm/ui" + + +class TestAssetsPathEnvironmentVariable: + """Test LITELLM_ASSETS_PATH environment variable handling.""" + + def test_custom_assets_path_env_var(self): + """Test that LITELLM_ASSETS_PATH overrides default.""" + custom_path = "/custom/assets/path" + + with mock.patch.dict( + os.environ, + {"LITELLM_ASSETS_PATH": custom_path, "LITELLM_NON_ROOT": "true"}, + ): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_assets_dir = ( + "/var/lib/litellm/assets" if is_non_root else "/default/current/dir" + ) + assets_dir = os.getenv("LITELLM_ASSETS_PATH", default_assets_dir) + + assert assets_dir == custom_path + + def test_default_assets_path_non_root(self): + """Test default assets path in non-root mode.""" + env_copy = os.environ.copy() + env_copy["LITELLM_NON_ROOT"] = "true" + if "LITELLM_ASSETS_PATH" in env_copy: + del env_copy["LITELLM_ASSETS_PATH"] + + with mock.patch.dict(os.environ, env_copy, clear=True): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_assets_dir = ( + "/var/lib/litellm/assets" if is_non_root else "/default/current/dir" + ) + assets_dir = os.getenv("LITELLM_ASSETS_PATH", default_assets_dir) + + assert assets_dir == "/var/lib/litellm/assets" + + +class TestUIDetectionLogic: + """Test UI pre-restructured detection logic without importing proxy_server.""" + + def setup_method(self): + """Create temporary directory for testing.""" + self.temp_dir = tempfile.mkdtemp() + + def teardown_method(self): + """Clean up temporary directory.""" + import shutil + + if os.path.exists(self.temp_dir): + shutil.rmtree(self.temp_dir) + + def test_marker_file_exists(self): + """Test marker file detection logic.""" + marker_path = os.path.join(self.temp_dir, ".litellm_ui_ready") + Path(marker_path).touch() + + # Verify marker file exists + assert os.path.exists(marker_path) + + def test_structural_routes_exist(self): + """Test structural detection logic.""" + routes = ["login", "guardrails", "logs"] + for route in routes: + route_dir = os.path.join(self.temp_dir, route) + os.makedirs(route_dir, exist_ok=True) + index_html = os.path.join(route_dir, "index.html") + Path(index_html).touch() + + # Verify routes exist + found_routes = 0 + expected_routes = ["login", "guardrails", "logs", "api-reference"] + for route in expected_routes: + route_index = os.path.join(self.temp_dir, route, "index.html") + if os.path.exists(route_index): + found_routes += 1 + + assert found_routes >= 3 + + def test_writability_check(self): + """Test that os.access() correctly detects writable directories.""" + # Should be writable + assert os.access(self.temp_dir, os.W_OK) is True + + # Create a directory we can't write to (platform-dependent) + if os.name != "nt": # Skip on Windows + readonly_dir = os.path.join(self.temp_dir, "readonly") + os.makedirs(readonly_dir) + os.chmod(readonly_dir, 0o444) # Read-only + + # Should not be writable + assert os.access(readonly_dir, os.W_OK) is False + + # Restore permissions for cleanup + os.chmod(readonly_dir, 0o755) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py index 57352eafaf1..5458b466f68 100644 --- a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py @@ -156,3 +156,83 @@ def test_transform_request_includes_extra_headers(): litellm_logging_obj=MockLoggingObj(), ) assert result.get("extra_headers") == headers + + +def test_transform_request_strips_internal_metadata_to_litellm_metadata(): + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = { + "metadata": {"user_api_key_auth": {"id": "abc"}}, + "litellm_metadata": {"trace_id": "trace-1"}, + "api_key": "sk-test", + } + + class MockLoggingObj: + pass + + result = handler.transform_request( + model="gpt-5-pro", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + litellm_logging_obj=MockLoggingObj(), + ) + + assert "metadata" not in result + assert result["litellm_metadata"]["user_api_key_auth"]["id"] == "abc" + assert result["litellm_metadata"]["trace_id"] == "trace-1" + + +def test_transform_request_preserves_user_metadata(): + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {"metadata": {"customer_id": "cust-123"}} + litellm_params = {"metadata": {"internal_key": "secret"}} + + class MockLoggingObj: + pass + + result = handler.transform_request( + model="gpt-5-pro", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + litellm_logging_obj=MockLoggingObj(), + ) + + assert result["metadata"] == {"customer_id": "cust-123"} + assert "internal_key" not in result["metadata"] + assert result["litellm_metadata"]["internal_key"] == "secret" + + +def test_transform_request_drops_user_metadata_with_additional_drop_params(): + from litellm.utils import get_optional_params + + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": "Hello"}] + optional_params = get_optional_params( + model="gpt-4o", + messages=messages, + metadata={"customer_id": "cust-123"}, + additional_drop_params=["metadata"], + custom_llm_provider="openai", + ) + litellm_params = {"metadata": {"internal_key": "secret"}} + + class MockLoggingObj: + pass + + result = handler.transform_request( + model="gpt-4o", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + litellm_logging_obj=MockLoggingObj(), + ) + + assert "metadata" not in result + assert result["litellm_metadata"]["internal_key"] == "secret" diff --git a/tests/test_litellm/expected_responses_api_request/context_management_and_shell.json b/tests/test_litellm/expected_responses_api_request/context_management_and_shell.json new file mode 100644 index 00000000000..1e34b230182 --- /dev/null +++ b/tests/test_litellm/expected_responses_api_request/context_management_and_shell.json @@ -0,0 +1,20 @@ +{ + "model": "gpt-4o", + "input": "List files in /mnt/data and run python --version.", + "context_management": [ + { + "type": "compaction", + "compact_threshold": 200000 + } + ], + "tools": [ + { + "type": "shell", + "environment": { + "type": "container_auto" + } + } + ], + "tool_choice": "auto", + "max_output_tokens": 256 +} diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index c7ad18cfb0c..beb978584cb 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -77,6 +77,15 @@ context_window_test_cases = [ ("Rate limit reached for requests.", False), ("The context is large, but acceptable.", False), ("", False), # Empty string + # OpenAI user param length validation - not a context window error + ( + "Invalid 'user': string too long. Expected a string with maximum length 64, but got a string with length 123 instead.", + False, + ), + ( + '{"error": {"message": "Invalid \'user\': string too long.", "code": "string_above_max_length"}}', + False, + ), ] diff --git a/tests/test_litellm/llms/azure/test_azure_exception_mapping.py b/tests/test_litellm/llms/azure/test_azure_exception_mapping.py index f4abe7f2b9a..495ca958cf5 100644 --- a/tests/test_litellm/llms/azure/test_azure_exception_mapping.py +++ b/tests/test_litellm/llms/azure/test_azure_exception_mapping.py @@ -239,4 +239,149 @@ class TestAzureExceptionMapping: assert e.provider_specific_fields is not None assert e.provider_specific_fields["inner_error"]["code"] == "ResponsibleAIPolicyViolation" assert e.provider_specific_fields["inner_error"]["revised_prompt"] == "revised" - assert e.provider_specific_fields["inner_error"]["content_filter_results"]["violence"]["filtered"] is True \ No newline at end of file + assert e.provider_specific_fields["inner_error"]["content_filter_results"]["violence"]["filtered"] is True + + def test_azure_content_policy_violation_detected_via_inner_error_code(self): + """Regression test for #20811: Azure returns inner_error with + ResponsibleAIPolicyViolation but the top-level error message is + generic. Previously this fell through to the generic + BadRequestError handler and all error details were lost.""" + + mock_exception = Exception("Bad request") + # This body structure mirrors what Azure OpenAI Images API returns + # for DALL-E 3 content policy violations (issue #20811). + mock_exception.body = { + "error": { + "code": "content_policy_violation", + "inner_error": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "profanity": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": True, "severity": "low"}, + }, + "revised_prompt": ( + "A dark and intense illustration of a man " + "in a dramatic action scene." + ), + }, + "message": ( + "Your request was rejected as a result of our safety system." + ), + "type": "invalid_request_error", + } + } + + mock_response = MagicMock() + mock_response.status_code = 400 + mock_exception.response = mock_response + + with pytest.raises(ContentPolicyViolationError) as exc_info: + exception_type( + model="azure/dall-e-3", + original_exception=mock_exception, + custom_llm_provider="azure", + ) + + e = exc_info.value + # Must surface as ContentPolicyViolationError, not generic BadRequestError + assert "safety system" in str(e) + assert e.provider_specific_fields is not None + inner = e.provider_specific_fields["inner_error"] + assert inner["code"] == "ResponsibleAIPolicyViolation" + assert inner["content_filter_results"]["violence"]["filtered"] is True + assert inner["revised_prompt"] is not None + + def test_azure_policy_violation_detected_via_inner_error_without_top_code(self): + """When the top-level code is NOT 'content_policy_violation' but + inner_error.code IS 'ResponsibleAIPolicyViolation', the error + should still be recognized as a content policy violation.""" + + mock_exception = Exception("Some error") + mock_exception.body = { + "error": { + "code": "BadRequest", + "inner_error": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_results": { + "violence": {"filtered": True, "severity": "medium"}, + }, + }, + "message": "The request was rejected.", + "type": "invalid_request_error", + } + } + + mock_response = MagicMock() + mock_response.status_code = 400 + mock_exception.response = mock_response + + with pytest.raises(ContentPolicyViolationError) as exc_info: + exception_type( + model="azure/dall-e-3", + original_exception=mock_exception, + custom_llm_provider="azure", + ) + + e = exc_info.value + assert e.provider_specific_fields is not None + assert e.provider_specific_fields["inner_error"]["code"] == "ResponsibleAIPolicyViolation" + + def test_azure_image_polling_error_preserves_body(self): + """Verify that AzureOpenAIError raised from the DALL-E polling path + carries the structured body so exception_type() can inspect it.""" + from litellm.llms.azure.common_utils import AzureOpenAIError + + error_payload = { + "status": "failed", + "error": { + "code": "content_policy_violation", + "message": "Your request was rejected.", + "inner_error": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_results": { + "violence": {"filtered": True, "severity": "low"}, + }, + }, + }, + } + + # Simulate what the fixed polling path now does + _error_body = error_payload.get("error", error_payload) + _error_msg = ( + _error_body.get("message", "Image generation failed") + if isinstance(_error_body, dict) + else json.dumps(error_payload) + ) + exc = AzureOpenAIError( + status_code=400, + message=_error_msg, + body=error_payload, + ) + + assert exc.body is not None + assert isinstance(exc.body, dict) + assert exc.body["error"]["code"] == "content_policy_violation" + assert "Your request was rejected" in exc.message + + def test_azure_safety_system_message_detected_as_policy_violation(self): + """Azure's rejection message 'Your request was rejected as a result + of our safety system' should be detected by string matching even + when the structured body is unavailable.""" + + mock_exception = Exception( + "Your request was rejected as a result of our safety system. " + "The revised prompt may contain text that is not allowed." + ) + mock_response = MagicMock() + mock_response.status_code = 400 + mock_exception.response = mock_response + + with pytest.raises(ContentPolicyViolationError): + exception_type( + model="azure/dall-e-3", + original_exception=mock_exception, + custom_llm_provider="azure", + ) \ No newline at end of file diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py index 989a06f80b4..42e62c75d63 100644 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py @@ -352,17 +352,17 @@ class TestErrorHandling: def test_hf_response_missing_embedding(self): """Test handling of HF response missing embedding field""" config = SagemakerEmbeddingConfig() - + # Mock response without embedding field mock_response = httpx.Response( status_code=200, content=json.dumps({"object": "list"}).encode('utf-8'), headers={"content-type": "application/json"} ) - + model_response = EmbeddingResponse() - - with pytest.raises(Exception, match="HF response missing 'embedding' field"): + + with pytest.raises(Exception, match="Unexpected response format"): config.transform_embedding_response( model="sentence-transformers-model", raw_response=mock_response, @@ -372,5 +372,99 @@ class TestErrorHandling: ) +class TestTEIEmbeddingResponse: + """Test HuggingFace Text Embeddings Inference (TEI) response format support""" + + def setup_method(self): + self.config = SagemakerEmbeddingConfig() + + def test_transform_embedding_response_tei_raw_array(self): + """Test TEI response transformation - raw array format [[...]]""" + # TEI returns raw embedding arrays without wrapper + tei_response = [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6] + ] + + mock_response = httpx.Response( + status_code=200, + content=json.dumps(tei_response).encode('utf-8'), + headers={"content-type": "application/json"} + ) + + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="tei-qwen-embedding", + raw_response=mock_response, + model_response=model_response, + logging_obj=None, + request_data={"inputs": ["Hello", "World"]} + ) + + # Verify response structure + assert result.object == "list" + assert result.model == "tei-qwen-embedding" + assert len(result.data) == 2 + assert result.data[0]["object"] == "embedding" + assert result.data[0]["index"] == 0 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert result.data[1]["object"] == "embedding" + assert result.data[1]["index"] == 1 + assert result.data[1]["embedding"] == [0.4, 0.5, 0.6] + assert isinstance(result.usage, Usage) + + def test_transform_embedding_response_tei_single_input(self): + """Test TEI response with single input""" + tei_response = [ + [0.1, 0.2, 0.3, 0.4, 0.5] + ] + + mock_response = httpx.Response( + status_code=200, + content=json.dumps(tei_response).encode('utf-8'), + headers={"content-type": "application/json"} + ) + + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="tei-model", + raw_response=mock_response, + model_response=model_response, + logging_obj=None, + request_data={"inputs": ["Hello"]} + ) + + assert len(result.data) == 1 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3, 0.4, 0.5] + + def test_transform_embedding_response_wrapped_format_still_works(self): + """Test that wrapped format {"embedding": [...]} still works""" + hf_response = { + "embedding": [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6] + ] + } + + mock_response = httpx.Response( + status_code=200, + content=json.dumps(hf_response).encode('utf-8'), + headers={"content-type": "application/json"} + ) + + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="hf-model", + raw_response=mock_response, + model_response=model_response, + logging_obj=None, + request_data={"inputs": ["Hello", "World"]} + ) + + assert len(result.data) == 2 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert result.data[1]["embedding"] == [0.4, 0.5, 0.6] + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py new file mode 100644 index 00000000000..dde73016271 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py @@ -0,0 +1,182 @@ +import json +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../..") +) # Adds the parent directory to the system path + + +class TestMCPRegistryFile: + """Tests for the curated MCP registry JSON file.""" + + @pytest.fixture + def registry_path(self): + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "..", + "..", + "..", + "..", + "litellm", + "proxy", + "mcp_registry.json", + ) + + def test_registry_file_exists(self, registry_path): + assert os.path.exists(registry_path), f"Registry file not found at {registry_path}" + + def test_registry_file_is_valid_json(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + assert isinstance(data, dict) + assert "servers" in data + + def test_registry_servers_have_required_fields(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + servers = data["servers"] + assert len(servers) > 0, "Registry should have at least one server" + + required_fields = ["name", "title", "description", "category", "transport"] + for server in servers: + for field in required_fields: + assert field in server, f"Server {server.get('name', '?')} missing field '{field}'" + + def test_registry_server_names_are_unique(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + names = [s["name"] for s in data["servers"]] + assert len(names) == len(set(names)), f"Duplicate server names found: {[n for n in names if names.count(n) > 1]}" + + def test_registry_transport_values_are_valid(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + valid_transports = {"stdio", "http", "sse"} + for server in data["servers"]: + assert server["transport"] in valid_transports, ( + f"Server {server['name']} has invalid transport '{server['transport']}'" + ) + + def test_stdio_servers_have_command(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + for server in data["servers"]: + if server["transport"] == "stdio": + assert "command" in server and server["command"], ( + f"stdio server {server['name']} missing 'command'" + ) + + def test_http_servers_have_url(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + for server in data["servers"]: + if server["transport"] in ("http", "sse"): + assert "url" in server and server["url"], ( + f"HTTP/SSE server {server['name']} missing 'url'" + ) + + def test_well_known_servers_present(self, registry_path): + """Ensure key well-known MCPs are in the registry.""" + with open(registry_path, "r") as f: + data = json.load(f) + names = {s["name"] for s in data["servers"]} + expected = {"github", "slack", "postgresql", "snowflake", "atlassian"} + missing = expected - names + assert not missing, f"Missing well-known servers: {missing}" + + def test_env_vars_structure(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + for server in data["servers"]: + if "env_vars" in server: + assert isinstance(server["env_vars"], list) + for var in server["env_vars"]: + assert "name" in var, f"env_var in {server['name']} missing 'name'" + + +class TestDiscoverEndpointFiltering: + """Tests for the discover endpoint filtering logic (unit-level).""" + + @pytest.fixture + def sample_servers(self): + return [ + { + "name": "github", + "title": "GitHub", + "description": "Repository management", + "category": "Developer Tools", + "transport": "http", + "url": "https://mcp.github.com/sse", + }, + { + "name": "slack", + "title": "Slack", + "description": "Channel management and messaging", + "category": "Communication", + "transport": "stdio", + "command": "npx", + }, + { + "name": "postgresql", + "title": "PostgreSQL", + "description": "Query and manage databases", + "category": "Databases", + "transport": "stdio", + "command": "npx", + }, + ] + + def test_query_filter_by_name(self, sample_servers): + query = "github" + q = query.lower() + result = [ + s + for s in sample_servers + if q in s.get("name", "").lower() + or q in s.get("title", "").lower() + or q in s.get("description", "").lower() + ] + assert len(result) == 1 + assert result[0]["name"] == "github" + + def test_query_filter_by_description(self, sample_servers): + query = "messaging" + q = query.lower() + result = [ + s + for s in sample_servers + if q in s.get("name", "").lower() + or q in s.get("title", "").lower() + or q in s.get("description", "").lower() + ] + assert len(result) == 1 + assert result[0]["name"] == "slack" + + def test_category_filter(self, sample_servers): + category = "Databases" + result = [s for s in sample_servers if s.get("category") == category] + assert len(result) == 1 + assert result[0]["name"] == "postgresql" + + def test_no_filter_returns_all(self, sample_servers): + assert len(sample_servers) == 3 + + def test_query_filter_no_match(self, sample_servers): + query = "nonexistent" + q = query.lower() + result = [ + s + for s in sample_servers + if q in s.get("name", "").lower() + or q in s.get("title", "").lower() + or q in s.get("description", "").lower() + ] + assert len(result) == 0 + + def test_categories_extraction(self, sample_servers): + categories = sorted(set(s.get("category", "Other") for s in sample_servers)) + assert categories == ["Communication", "Databases", "Developer Tools"] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index b4b5811666b..1a50cacd308 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -112,7 +112,44 @@ class TestMCPServerManager: assert client.stdio_config is not None assert client.stdio_config["command"] == "node" assert client.stdio_config["args"] == ["server.js"] - assert client.stdio_config["env"] == {"NODE_ENV": "test"} + # NPM_CONFIG_CACHE is injected automatically for container compatibility + from litellm.constants import MCP_NPM_CACHE_DIR + + assert client.stdio_config["env"]["NODE_ENV"] == "test" + assert client.stdio_config["env"]["NPM_CONFIG_CACHE"] == MCP_NPM_CACHE_DIR + + async def test_create_mcp_client_stdio_injects_npm_config_cache(self): + """Test that _create_mcp_client injects NPM_CONFIG_CACHE when not already set, + and preserves user-provided NPM_CONFIG_CACHE when present.""" + from litellm.constants import MCP_NPM_CACHE_DIR + + manager = MCPServerManager() + + # Case 1: NPM_CONFIG_CACHE not set -> should be injected + server_no_cache = MCPServer( + server_id="stdio-npm-1", + name="test_npm_server", + url=None, + transport=MCPTransport.stdio, + command="npx", + args=["-y", "@modelcontextprotocol/server-everything"], + env={}, + ) + client = await manager._create_mcp_client(server_no_cache) + assert client.stdio_config["env"]["NPM_CONFIG_CACHE"] == MCP_NPM_CACHE_DIR + + # Case 2: NPM_CONFIG_CACHE already set -> should NOT be overwritten + server_with_cache = MCPServer( + server_id="stdio-npm-2", + name="test_npm_server_custom", + url=None, + transport=MCPTransport.stdio, + command="npx", + args=["-y", "@modelcontextprotocol/server-everything"], + env={"NPM_CONFIG_CACHE": "/custom/cache"}, + ) + client2 = await manager._create_mcp_client(server_with_cache) + assert client2.stdio_config["env"]["NPM_CONFIG_CACHE"] == "/custom/cache" def test_build_stdio_env_only_accepts_x_prefixed_placeholders(self): """Ensure only ${X-*} placeholders are substituted from headers.""" diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py index 9993b25dfdd..0ed5940dd75 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py @@ -225,6 +225,39 @@ async def test_aggregate_queue_updates_accuracy(spend_queue): assert aggregated["team_list_transactions"]["team1"] == 5.0 +def test_get_aggregated_spend_update_queue_item_does_not_mutate_original_updates( + spend_queue, +): + original_update: SpendUpdateQueueItem = { + "entity_type": Litellm_EntityType.USER, + "entity_id": "user1", + "response_cost": 10.0, + } + duplicate_key_update: SpendUpdateQueueItem = { + "entity_type": Litellm_EntityType.USER, + "entity_id": "user1", + "response_cost": 20.0, + } + + aggregated_updates = spend_queue._get_aggregated_spend_update_queue_item( + [original_update, duplicate_key_update] + ) + user1_aggregated_update = next( + ( + update + for update in aggregated_updates + if update.get("entity_type") == Litellm_EntityType.USER + and update.get("entity_id") == "user1" + ), + None, + ) + + assert original_update["response_cost"] == 10.0 + assert user1_aggregated_update is not None + assert user1_aggregated_update["response_cost"] == 30.0 + assert user1_aggregated_update is not original_update + + @pytest.mark.asyncio async def test_queue_size_reduction_with_large_volume(monkeypatch, spend_queue): """Test that queue size is actually reduced when dealing with many items""" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index 5c039141928..7d2b6e84de7 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -14,10 +14,14 @@ import pytest import litellm from litellm import ModelResponse from litellm.exceptions import GuardrailRaisedException +from litellm._version import version as litellm_version from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( GenericGuardrailAPI, ) +from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api.generic_guardrail_api import ( + _HEADER_PRESENT_PLACEHOLDER, +) from litellm.types.utils import Choices, Message @@ -351,6 +355,58 @@ class TestMetadataExtraction: # Should be empty dict assert request_metadata == {} + @pytest.mark.asyncio + async def test_inbound_headers_and_litellm_version_forwarded_and_sanitized( + self, generic_guardrail, mock_request_data_input + ): + """ + Ensure inbound proxy request headers are forwarded in JSON payload with allowlist: + allowed headers show their value; all other headers show presence only ([present]). + """ + # Add proxy_server_request headers as they exist in proxy request context + request_data = dict(mock_request_data_input) + request_data["proxy_server_request"] = { + "headers": { + "User-Agent": "OpenAI/Python 2.17.0", + "Authorization": "Bearer should-not-forward", + "Cookie": "session=should-not-forward", + "X-Request-Id": "req_123", + } + } + + mock_response = MagicMock() + mock_response.json.return_value = { + "action": "NONE", + "texts": ["test"], + } + mock_response.raise_for_status = MagicMock() + + with patch.object( + generic_guardrail.async_handler, "post", return_value=mock_response + ) as mock_post: + await generic_guardrail.apply_guardrail( + inputs={"texts": ["test"]}, + request_data=request_data, + input_type="request", + ) + + call_args = mock_post.call_args + json_payload = call_args.kwargs["json"] + + # New fields should exist + assert json_payload["litellm_version"] == litellm_version + assert "request_headers" in json_payload + assert isinstance(json_payload["request_headers"], dict) + req_headers = json_payload["request_headers"] + + # Allowed: value forwarded + assert req_headers.get("User-Agent") == "OpenAI/Python 2.17.0" + + # Not on allowlist: key present, value is placeholder only + assert req_headers.get("Authorization") == _HEADER_PRESENT_PLACEHOLDER + assert req_headers.get("Cookie") == _HEADER_PRESENT_PLACEHOLDER + assert req_headers.get("X-Request-Id") == _HEADER_PRESENT_PLACEHOLDER + class TestGuardrailActions: """Test different guardrail action responses""" diff --git a/tests/test_litellm/proxy/image_endpoints/test_endpoints.py b/tests/test_litellm/proxy/image_endpoints/test_endpoints.py index a3b6a9c6022..c35630176bc 100644 --- a/tests/test_litellm/proxy/image_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/image_endpoints/test_endpoints.py @@ -40,10 +40,14 @@ async def test_image_generation_prompt_rerouting(monkeypatch): async def fake_post_call_failure_hook(**_: Any) -> None: return None + async def fake_post_call_success_hook(*, data, user_api_key_dict, response): + return response + fake_proxy_logger = SimpleNamespace( pre_call_hook=fake_pre_call_hook, update_request_status=fake_update_request_status, post_call_failure_hook=fake_post_call_failure_hook, + post_call_success_hook=fake_post_call_success_hook, ) captured_route_request_data: Dict[str, Any] = {} diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py new file mode 100644 index 00000000000..5f204918d08 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -0,0 +1,583 @@ +""" +Tests for access group management endpoints. +""" + +import os +import sys +import types +from contextlib import asynccontextmanager +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi.testclient import TestClient +from prisma.errors import PrismaError + +import litellm.proxy.proxy_server as ps +from litellm.proxy.proxy_server import app +from litellm.proxy._types import ( + CommonProxyErrors, + LitellmUserRoles, + UserAPIKeyAuth, +) + +sys.path.insert(0, os.path.abspath("../../../")) + + +def _make_access_group_record( + access_group_id: str = "ag-123", + access_group_name: str = "test-group", + description: str | None = "Test description", + access_model_ids: list | None = None, + access_mcp_server_ids: list | None = None, + access_agent_ids: list | None = None, + assigned_team_ids: list | None = None, + assigned_key_ids: list | None = None, + created_by: str | None = "admin-user", + updated_by: str | None = "admin-user", + created_at: datetime | None = None, +): + record = MagicMock() + record.access_group_id = access_group_id + record.access_group_name = access_group_name + record.description = description + record.access_model_ids = access_model_ids or [] + record.access_mcp_server_ids = access_mcp_server_ids or [] + record.access_agent_ids = access_agent_ids or [] + record.assigned_team_ids = assigned_team_ids or [] + record.assigned_key_ids = assigned_key_ids or [] + record.created_at = created_at or datetime.now() + record.created_by = created_by + record.updated_at = datetime.now() + record.updated_by = updated_by + return record + + +@pytest.fixture +def client_and_mocks(monkeypatch): + """Setup mock prisma and admin auth for access group endpoints.""" + mock_access_group_table = MagicMock() + mock_prisma = MagicMock() + + def _create_side_effect(*, data): + return _make_access_group_record( + access_group_id="ag-new", + access_group_name=data.get("access_group_name", "new"), + description=data.get("description"), + access_model_ids=data.get("access_model_ids", []), + access_mcp_server_ids=data.get("access_mcp_server_ids", []), + access_agent_ids=data.get("access_agent_ids", []), + assigned_team_ids=data.get("assigned_team_ids", []), + assigned_key_ids=data.get("assigned_key_ids", []), + created_by=data.get("created_by"), + updated_by=data.get("updated_by"), + ) + + mock_access_group_table.create = AsyncMock(side_effect=_create_side_effect) + mock_access_group_table.find_unique = AsyncMock(return_value=None) + mock_access_group_table.find_many = AsyncMock(return_value=[]) + mock_access_group_table.update = AsyncMock(side_effect=lambda *, where, data: _make_access_group_record( + access_group_id=where.get("access_group_id", "ag-123"), + access_group_name=data.get("access_group_name", "updated"), + description=data.get("description"), + access_model_ids=data.get("access_model_ids", []), + access_mcp_server_ids=data.get("access_mcp_server_ids", []), + access_agent_ids=data.get("access_agent_ids", []), + assigned_team_ids=data.get("assigned_team_ids", []), + assigned_key_ids=data.get("assigned_key_ids", []), + updated_by=data.get("updated_by"), + )) + mock_access_group_table.delete = AsyncMock(return_value=None) + + mock_team_table = MagicMock() + mock_team_table.find_many = AsyncMock(return_value=[]) + mock_team_table.update = AsyncMock(return_value=None) + + mock_key_table = MagicMock() + mock_key_table.find_many = AsyncMock(return_value=[]) + mock_key_table.update = AsyncMock(return_value=None) + + @asynccontextmanager + async def mock_tx(): + tx = types.SimpleNamespace( + litellm_accessgrouptable=mock_access_group_table, + litellm_teamtable=mock_team_table, + litellm_verificationtoken=mock_key_table, + ) + yield tx + + mock_db = types.SimpleNamespace( + litellm_accessgrouptable=mock_access_group_table, + litellm_teamtable=mock_team_table, + litellm_verificationtoken=mock_key_table, + tx=mock_tx, + ) + mock_prisma.db = mock_db + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + admin_user = UserAPIKeyAuth( + user_id="admin_user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: admin_user + + client = TestClient(app) + + yield client, mock_prisma, mock_access_group_table + + app.dependency_overrides.clear() + monkeypatch.setattr(ps, "prisma_client", ps.prisma_client) + + +# Paths for primary and alias endpoints (alias: /v1/unified_access_group) +ACCESS_GROUP_PATHS = ["/v1/access_group", "/v1/unified_access_group"] + + +# --------------------------------------------------------------------------- +# CREATE +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize( + "payload", + [ + {"access_group_name": "group-a"}, + { + "access_group_name": "group-b", + "description": "Group B description", + "access_model_ids": ["model-1"], + "access_mcp_server_ids": ["mcp-1"], + "assigned_team_ids": ["team-1"], + }, + ], +) +def test_create_access_group_success(client_and_mocks, base_path, payload): + """Create access group with various payloads returns 201.""" + client, _, mock_table = client_and_mocks + + resp = client.post(base_path, json=payload) + assert resp.status_code == 201 + body = resp.json() + assert body["access_group_name"] == payload["access_group_name"] + assert body.get("access_group_id") is not None + mock_table.create.assert_awaited_once() + + +def test_create_access_group_duplicate_name_conflict(client_and_mocks): + """Create with duplicate name returns 409.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_name="existing-group") + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.post("/v1/access_group", json={"access_group_name": "existing-group"}) + assert resp.status_code == 409 + assert "already exists" in resp.json()["detail"] + + +@pytest.mark.parametrize( + "error_message", + [ + "Unique constraint failed on the fields: (`access_group_name`)", + "P2002: Unique constraint failed", + "unique constraint violation", + ], +) +def test_create_access_group_race_condition_returns_409(client_and_mocks, error_message): + """Create race condition: Prisma unique constraint surfaces as 409, not 500.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + mock_table.create = AsyncMock(side_effect=Exception(error_message)) + + resp = client.post("/v1/access_group", json={"access_group_name": "race-group"}) + assert resp.status_code == 409 + assert "already exists" in resp.json()["detail"] + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_create_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot create access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.post("/v1/access_group", json={"access_group_name": "forbidden"}) + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +def test_create_access_group_validation_missing_name(client_and_mocks): + """Create with missing access_group_name returns 422.""" + client, _, _ = client_and_mocks + + resp = client.post("/v1/access_group", json={}) + assert resp.status_code == 422 + + +def test_create_access_group_500_on_non_constraint_prisma_error(client_and_mocks): + """Create with non-unique-constraint Prisma error returns 500.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + mock_table.create = AsyncMock(side_effect=Exception("Some other database error")) + + # Use raise_server_exceptions=False so unhandled exceptions become 500 responses + test_client = TestClient(app, raise_server_exceptions=False) + resp = test_client.post("/v1/access_group", json={"access_group_name": "test-group"}) + assert resp.status_code == 500 + + +# --------------------------------------------------------------------------- +# LIST +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +def test_list_access_groups_success_empty(client_and_mocks, base_path): + """List access groups returns empty list when none exist.""" + client, _, mock_table = client_and_mocks + + resp = client.get(base_path) + assert resp.status_code == 200 + assert resp.json() == [] + mock_table.find_many.assert_awaited_once() + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +def test_list_access_groups_success_with_items(client_and_mocks, base_path): + """List access groups returns items when they exist.""" + client, _, mock_table = client_and_mocks + + records = [ + _make_access_group_record(access_group_id="ag-1", access_group_name="group-1"), + _make_access_group_record(access_group_id="ag-2", access_group_name="group-2"), + ] + mock_table.find_many = AsyncMock(return_value=records) + + resp = client.get(base_path) + assert resp.status_code == 200 + body = resp.json() + assert len(body) == 2 + assert body[0]["access_group_name"] == "group-1" + assert body[1]["access_group_name"] == "group-2" + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +def test_list_access_groups_ordered_by_created_at_desc(client_and_mocks, base_path): + """List access groups calls find_many with created_at desc order.""" + client, _, mock_table = client_and_mocks + + older = datetime(2025, 1, 1, 12, 0, 0) + newer = datetime(2025, 1, 2, 12, 0, 0) + records = [ + _make_access_group_record( + access_group_id="ag-newer", + access_group_name="newer-group", + created_at=newer, + ), + _make_access_group_record( + access_group_id="ag-older", + access_group_name="older-group", + created_at=older, + ), + ] + mock_table.find_many = AsyncMock(return_value=records) + + resp = client.get(base_path) + assert resp.status_code == 200 + body = resp.json() + assert len(body) == 2 + # Mock returns newest first (simulating Prisma order desc) + assert body[0]["access_group_name"] == "newer-group" + assert body[1]["access_group_name"] == "older-group" + mock_table.find_many.assert_awaited_once_with(order={"created_at": "desc"}) + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_list_access_groups_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot list access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.get("/v1/access_group") + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +# --------------------------------------------------------------------------- +# GET +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize("access_group_id", ["ag-123", "ag-other-id"]) +def test_get_access_group_success(client_and_mocks, base_path, access_group_id): + """Get access group by id returns record when found.""" + client, _, mock_table = client_and_mocks + + record = _make_access_group_record(access_group_id=access_group_id) + mock_table.find_unique = AsyncMock(return_value=record) + + resp = client.get(f"{base_path}/{access_group_id}") + assert resp.status_code == 200 + assert resp.json()["access_group_id"] == access_group_id + + +def test_get_access_group_not_found(client_and_mocks): + """Get access group returns 404 when not found.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + + resp = client.get("/v1/access_group/nonexistent-id") + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_get_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot get access group.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.get("/v1/access_group/ag-123") + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +# --------------------------------------------------------------------------- +# UPDATE +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize( + "update_payload", + [ + {"description": "Updated description"}, + {"access_model_ids": ["model-1", "model-2"]}, + {"assigned_team_ids": [], "assigned_key_ids": ["key-1"]}, + ], +) +def test_update_access_group_success(client_and_mocks, base_path, update_payload): + """Update access group with various payloads returns 200.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-update") + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.put(f"{base_path}/ag-update", json=update_payload) + assert resp.status_code == 200 + mock_table.update.assert_awaited_once() + + +def test_update_access_group_not_found(client_and_mocks): + """Update access group returns 404 when not found.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + + resp = client.put( + "/v1/access_group/nonexistent-id", + json={"description": "Updated"}, + ) + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + mock_table.update.assert_not_awaited() + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_update_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot update access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.put("/v1/access_group/ag-123", json={"description": "Updated"}) + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +def test_update_access_group_empty_body(client_and_mocks): + """Update with empty body succeeds; only updated_by is set.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-update", access_group_name="unchanged") + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.put("/v1/access_group/ag-update", json={}) + assert resp.status_code == 200 + mock_table.update.assert_awaited_once() + call_kwargs = mock_table.update.call_args.kwargs + assert call_kwargs["where"] == {"access_group_id": "ag-update"} + assert "updated_by" in call_kwargs["data"] + assert call_kwargs["data"]["updated_by"] == "admin_user" + + +# --------------------------------------------------------------------------- +# DELETE +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize("access_group_id", ["ag-123", "ag-delete-me"]) +def test_delete_access_group_success(client_and_mocks, base_path, access_group_id): + """Delete access group returns 204 when found.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id=access_group_id) + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.delete(f"{base_path}/{access_group_id}") + assert resp.status_code == 204 + mock_table.delete.assert_awaited_once() + + +def test_delete_access_group_not_found(client_and_mocks): + """Delete access group returns 404 when not found.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + + resp = client.delete("/v1/access_group/nonexistent-id") + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + mock_table.delete.assert_not_awaited() + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_delete_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot delete access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.delete("/v1/access_group/ag-123") + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +def test_delete_access_group_cleans_up_teams_and_keys(client_and_mocks): + """Delete removes access_group_id from teams and keys before deleting the group.""" + client, mock_prisma, mock_access_group_table = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + team_with_group = MagicMock() + team_with_group.team_id = "team-1" + team_with_group.access_group_ids = ["ag-to-delete", "ag-other"] + mock_team_table.find_many = AsyncMock(return_value=[team_with_group]) + + key_with_group = MagicMock() + key_with_group.token = "key-token-1" + key_with_group.access_group_ids = ["ag-to-delete"] + mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 204 + + mock_team_table.update.assert_awaited_once_with( + where={"team_id": "team-1"}, + data={"access_group_ids": ["ag-other"]}, + ) + mock_key_table.update.assert_awaited_once_with( + where={"token": "key-token-1"}, + data={"access_group_ids": []}, + ) + mock_access_group_table.delete.assert_awaited_once_with( + where={"access_group_id": "ag-to-delete"} + ) + + +def test_delete_access_group_503_on_db_connection_error(client_and_mocks): + """Delete returns 503 when DB connection error occurs during transaction.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_table.find_unique = AsyncMock(return_value=existing) + mock_table.delete = AsyncMock(side_effect=PrismaError()) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 503 + assert resp.json()["detail"] == CommonProxyErrors.db_not_connected_error.value + + +def test_delete_access_group_404_on_p2025_or_record_not_found(client_and_mocks): + """Delete returns 404 when Prisma raises P2025 or record-not-found error.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_table.find_unique = AsyncMock(return_value=existing) + mock_table.delete = AsyncMock(side_effect=Exception("P2025: Record to delete does not exist")) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + + +def test_delete_access_group_500_on_generic_exception(client_and_mocks): + """Delete returns 500 when generic exception occurs during transaction.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_table.find_unique = AsyncMock(return_value=existing) + mock_table.delete = AsyncMock(side_effect=RuntimeError("Unexpected error")) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 500 + assert "Failed to delete access group" in resp.json()["detail"] + + +# --------------------------------------------------------------------------- +# DB NOT CONNECTED +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "method,url,factory", + [ + ("post", "/v1/access_group", lambda: {"json": {"access_group_name": "test"}}), + ("get", "/v1/access_group", lambda: {}), + ("get", "/v1/access_group/ag-123", lambda: {}), + ("put", "/v1/access_group/ag-123", lambda: {"json": {"description": "x"}}), + ("delete", "/v1/access_group/ag-123", lambda: {}), + # Alias: /v1/unified_access_group + ("post", "/v1/unified_access_group", lambda: {"json": {"access_group_name": "test"}}), + ("get", "/v1/unified_access_group", lambda: {}), + ("get", "/v1/unified_access_group/ag-123", lambda: {}), + ("put", "/v1/unified_access_group/ag-123", lambda: {"json": {"description": "x"}}), + ("delete", "/v1/unified_access_group/ag-123", lambda: {}), + ], +) +def test_access_group_endpoints_db_not_connected(client_and_mocks, monkeypatch, method, url, factory): + """All endpoints return 500 when DB is not connected.""" + client, _, _ = client_and_mocks + + monkeypatch.setattr(ps, "prisma_client", None) + + resp = getattr(client, method)(url, **factory()) + assert resp.status_code == 500 + assert resp.json()["detail"]["error"] == CommonProxyErrors.db_not_connected_error.value diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index e7da4256182..d65df0087ad 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -2996,9 +2996,13 @@ async def test_get_image_non_root_uses_var_lib_assets_dir(monkeypatch): monkeypatch.setenv("LITELLM_NON_ROOT", "true") monkeypatch.delenv("UI_LOGO_PATH", raising=False) - # Mock os.path operations + # Mock os.path operations - exists=False for assets_dir so makedirs gets called + def exists_side_effect(path): + return False if path == "/var/lib/litellm/assets" else True + with patch("litellm.proxy.proxy_server.os.makedirs") as mock_makedirs, \ - patch("litellm.proxy.proxy_server.os.path.exists", return_value=True), \ + patch("litellm.proxy.proxy_server.os.path.exists", side_effect=exists_side_effect), \ + patch("litellm.proxy.proxy_server.os.access", return_value=True), \ patch("litellm.proxy.proxy_server.os.getenv") as mock_getenv, \ patch("litellm.proxy.proxy_server.FileResponse") as mock_file_response: @@ -3038,14 +3042,16 @@ async def test_get_image_non_root_fallback_to_default_logo(monkeypatch): def exists_side_effect(path): exists_calls.append(path) - # Return False for /var/lib/litellm/assets/logo.jpg to trigger fallback - if "/var/lib/litellm/assets/logo.jpg" in path: + # Return False for /var/lib/litellm/assets* so: makedirs is called, logo fallback + # triggers, and we don't return early with cached file + if "/var/lib/litellm/assets" in path: return False return True # Mock os.path operations with patch("litellm.proxy.proxy_server.os.makedirs") as mock_makedirs, \ patch("litellm.proxy.proxy_server.os.path.exists", side_effect=exists_side_effect), \ + patch("litellm.proxy.proxy_server.os.access", return_value=True), \ patch("litellm.proxy.proxy_server.os.getenv") as mock_getenv, \ patch("litellm.proxy.proxy_server.FileResponse") as mock_file_response: diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py new file mode 100644 index 00000000000..9c20d630a1b --- /dev/null +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -0,0 +1,103 @@ +""" +Test that litellm.responses() / litellm.aresponses() send the expected request body +over the wire. Expected JSON bodies are stored in expected_responses_api_request/. +""" +import json +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +import litellm + + +def _expected_dir() -> Path: + """Path to expected_responses_api_request folder (sibling of test_litellm/responses).""" + return Path(__file__).resolve().parent.parent / "expected_responses_api_request" + + +@pytest.mark.asyncio +async def test_aresponses_context_management_and_shell_request_body_matches_expected(): + """ + Call litellm.aresponses() with context_management and shell tool; + assert the httpx POST request body matches the expected JSON. + """ + expected_path = _expected_dir() / "context_management_and_shell.json" + assert expected_path.exists(), f"Expected file not found: {expected_path}" + with open(expected_path) as f: + expected_body = json.load(f) + + # Minimal Responses API response so parsing succeeds + mock_response = { + "id": "resp_ctx_shell_test", + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "gpt-4o", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "Done.", "annotations": []} + ], + } + ], + "parallel_tool_calls": True, + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "truncation": None, + "user": None, + } + + class MockResponse: + def __init__(self, json_data, status_code=200): + self._json_data = json_data + self.status_code = status_code + self.text = json.dumps(json_data) + self.headers = httpx.Headers({}) + + def json(self): + return self._json_data + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(mock_response, 200) + + await litellm.aresponses( + model="openai/gpt-4o", + input=expected_body["input"], + context_management=expected_body["context_management"], + tools=expected_body["tools"], + tool_choice=expected_body["tool_choice"], + max_output_tokens=expected_body["max_output_tokens"], + ) + + mock_post.assert_called_once() + request_body = mock_post.call_args.kwargs["json"] + + for key, expected_value in expected_body.items(): + assert key in request_body, f"Missing key in request body: {key}" + assert request_body[key] == expected_value, ( + f"Mismatch for key {key}: got {request_body[key]!r}, expected {expected_value!r}" + ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 794b3b87187..7374a605798 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -3305,3 +3305,73 @@ class TestIsStreamingRequest: def test_stream_true_overrides_non_streaming_call_type(self): assert _is_streaming_request(kwargs={"stream": True}, call_type=CallTypes.acompletion) is True + + +class TestMetadataNoneHandling: + """ + Test that metadata=None in kwargs doesn't cause TypeError. + + When metadata key exists with value None (e.g., from Azure OpenAI streaming), + dict.get("metadata", {}) returns None (key exists, so default is ignored). + The fix uses (kwargs.get("metadata") or {}) which handles both missing key + and explicit None value. + + Related: #20871 + """ + + def test_metadata_none_get_previous_models(self): + """kwargs.get("metadata") or {} should return {} when metadata is None.""" + kwargs = {"metadata": None} + previous_models = (kwargs.get("metadata") or {}).get( + "previous_models", None + ) + assert previous_models is None + + def test_metadata_none_model_group_check(self): + """'model_group' in (kwargs.get("metadata") or {}) should not raise TypeError.""" + kwargs = {"metadata": None} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} + ) + assert _is_litellm_router_call is False + + def test_metadata_missing_key(self): + """Should work when metadata key is completely absent.""" + kwargs = {} + previous_models = (kwargs.get("metadata") or {}).get( + "previous_models", None + ) + assert previous_models is None + + def test_metadata_present_with_values(self): + """Should work when metadata has actual values.""" + kwargs = {"metadata": {"previous_models": ["model1"], "model_group": "test"}} + previous_models = (kwargs.get("metadata") or {}).get( + "previous_models", None + ) + assert previous_models == ["model1"] + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} + ) + assert _is_litellm_router_call is True + + def test_metadata_none_causes_error_with_old_pattern(self): + """Demonstrate the bug: dict.get('metadata', {}) returns None when key exists with None value.""" + kwargs = {"metadata": None} + # Old pattern: kwargs.get("metadata", {}) returns None because key exists + result = kwargs.get("metadata", {}) + assert result is None # This is the root cause of the bug + + # Attempting to use .get() on None raises AttributeError or TypeError + with pytest.raises((TypeError, AttributeError)): + kwargs.get("metadata", {}).get("previous_models", None) + + # Attempting 'in' on None raises TypeError + with pytest.raises(TypeError): + "model_group" in kwargs.get("metadata", {}) + + def test_litellm_params_metadata_none(self): + """litellm_params.get("metadata") or {} should handle None value.""" + litellm_params = {"metadata": None} + metadata = litellm_params.get("metadata") or {} + assert metadata == {} diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx index 8a7119aa4a3..94bfbd54426 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx @@ -14,6 +14,10 @@ vi.mock("../../../playground/llm_calls/fetch_models", () => ({ fetchAvailableModels: vi.fn(), })); +vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ + useModelCostMap: vi.fn().mockReturnValue({ data: null }), +})); + vi.mock("openai", () => ({ default: { OpenAI: vi.fn().mockImplementation(() => ({ @@ -97,20 +101,9 @@ describe("Fallbacks", () => { modelData: mockModelData, }; - const findDeleteButton = (container: HTMLElement) => { - const tableRows = container.querySelectorAll("tbody tr"); - if (tableRows.length === 0) return null; - const firstRow = tableRows[0]; - const actionCells = firstRow.querySelectorAll("td"); - const lastCell = actionCells[actionCells.length - 1]; - const buttons = lastCell.querySelectorAll("button"); - if (buttons.length >= 2) { - return buttons[buttons.length - 1]; - } - const clickableElements = lastCell.querySelectorAll("[class*='cursor-pointer'], button"); - return Array.from(clickableElements).find((el) => - el.className.includes("red") || el.className.includes("hover:text-red") - ) || clickableElements[clickableElements.length - 1]; + const getFirstRowDeleteButton = () => { + const deleteButtons = screen.getAllByTestId("delete-fallback-button"); + return deleteButtons.length > 0 ? deleteButtons[0] : null; }; beforeEach(() => { @@ -156,20 +149,31 @@ describe("Fallbacks", () => { await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); - expect(screen.getByText("gpt-3.5-turbo, claude-3-opus")).toBeInTheDocument(); - expect(screen.getByText("claude-3-opus")).toBeInTheDocument(); + expect(screen.getAllByText(/gpt-3\.5-turbo/).length).toBeGreaterThan(0); + expect(screen.getAllByText(/claude-3-opus/).length).toBeGreaterThan(0); }); }); - it("should open delete modal when delete icon is clicked", async () => { - const user = userEvent.setup(); - const { container } = render(); + it("should show delete button for each fallback row when fallbacks exist", async () => { + render(); await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); }); - const deleteButton = findDeleteButton(container); + const deleteButtons = screen.getAllByTestId("delete-fallback-button"); + expect(deleteButtons.length).toBe(2); + }); + + it("should open delete modal when delete icon is clicked", async () => { + const user = userEvent.setup(); + render(); + + await waitFor(() => { + expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); + }); + + const deleteButton = getFirstRowDeleteButton(); expect(deleteButton).not.toBeNull(); await user.click(deleteButton as HTMLElement); @@ -182,13 +186,13 @@ describe("Fallbacks", () => { it("should delete fallback when confirmed", async () => { const user = userEvent.setup(); - const { container } = render(); + render(); await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); }); - const deleteButton = findDeleteButton(container); + const deleteButton = getFirstRowDeleteButton(); expect(deleteButton).not.toBeNull(); await user.click(deleteButton as HTMLElement); @@ -210,13 +214,13 @@ describe("Fallbacks", () => { it("should close delete modal when cancel is clicked", async () => { const user = userEvent.setup(); - const { container } = render(); + render(); await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); }); - const deleteButton = findDeleteButton(container); + const deleteButton = getFirstRowDeleteButton(); expect(deleteButton).not.toBeNull(); await user.click(deleteButton as HTMLElement); @@ -237,13 +241,13 @@ describe("Fallbacks", () => { const user = userEvent.setup(); const error = new Error("Delete failed"); vi.mocked(networkingModule.setCallbacksCall).mockRejectedValueOnce(error); - const { container } = render(); + render(); await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); }); - const deleteButton = findDeleteButton(container); + const deleteButton = getFirstRowDeleteButton(); expect(deleteButton).not.toBeNull(); await user.click(deleteButton as HTMLElement); @@ -264,13 +268,13 @@ describe("Fallbacks", () => { const user = userEvent.setup(); const error = new Error("Delete failed"); vi.mocked(networkingModule.setCallbacksCall).mockRejectedValueOnce(error); - const { container } = render(); + render(); await waitFor(() => { expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); }); - const deleteButton = findDeleteButton(container); + const deleteButton = getFirstRowDeleteButton(); expect(deleteButton).not.toBeNull(); await user.click(deleteButton as HTMLElement); @@ -296,6 +300,9 @@ describe("Fallbacks", () => { await waitFor(() => { expect(screen.getByTestId("add-fallbacks-button")).toBeInTheDocument(); + expect( + screen.getByText(/No fallbacks configured. Add fallbacks to automatically try another model/), + ).toBeInTheDocument(); }); expect(screen.queryByText("gpt-4")).not.toBeInTheDocument(); @@ -309,6 +316,9 @@ describe("Fallbacks", () => { await waitFor(() => { expect(screen.getByTestId("add-fallbacks-button")).toBeInTheDocument(); + expect( + screen.getByText(/No fallbacks configured. Add fallbacks to automatically try another model/), + ).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx index f493cc51323..9c5933aba3a 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx @@ -1,9 +1,11 @@ -import { PlayIcon, TrashIcon } from "@heroicons/react/outline"; +import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; +import { ArrowRightIcon, PlayIcon, TrashIcon } from "@heroicons/react/outline"; import { Icon, Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow } from "@tremor/react"; -import { Tooltip } from "antd"; +import { Tooltip, Typography } from "antd"; import openai from "openai"; import React, { useEffect, useState } from "react"; import DeleteResourceModal from "../../../common_components/DeleteResourceModal"; +import { ProviderLogo } from "../../../molecules/models/ProviderLogo"; import NotificationsManager from "../../../molecules/notifications_manager"; import { getCallbacksCall, setCallbacksCall } from "../../../networking"; import AddFallbacks from "./AddFallbacks"; @@ -11,6 +13,61 @@ import AddFallbacks from "./AddFallbacks"; type FallbackEntry = { [modelName: string]: string[] }; type Fallbacks = FallbackEntry[]; +const modelCardClass = + "inline-flex items-center gap-2 px-2.5 py-1 rounded-md border border-gray-200 bg-gray-50 text-sm font-medium text-gray-800 shrink-0"; + +function renderModelNameCell( + modelName: string, + getProviderFromModel?: (modelName: string) => string, +): React.ReactNode { + const provider = getProviderFromModel?.(modelName) ?? modelName; + return ( + + + {modelName} + + ); +} + +function renderFallbacksChain( + _primaryModel: string, + fallbackModels: string[], + getProviderFromModel?: (modelName: string) => string, +): React.ReactNode { + const list = Array.isArray(fallbackModels) ? fallbackModels : []; + if (list.length === 0) return null; + + const ChainCard = ({ modelName }: { modelName: string }) => { + const provider = getProviderFromModel?.(modelName) ?? modelName; + return ( + + + {modelName} + + ); + }; + return ( + + + + + + {list.map((model, i) => ( + + {i > 0 && ( + + )} + + + ))} + + + ); +} + interface FallbacksProps { accessToken: string | null; userRole: string | null; @@ -71,6 +128,14 @@ const Fallbacks: React.FC = ({ accessToken, userRole, userID, mo const [fallbackToDelete, setFallbackToDelete] = useState(null); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); + const { data: modelCostMapData } = useModelCostMap(); + const getProviderFromModel = (model: string): string => { + if (modelCostMapData != null && typeof modelCostMapData === "object" && model in modelCostMapData) { + return modelCostMapData[model]["litellm_provider"] ?? ""; + } + return ""; + }; + useEffect(() => { if (!accessToken || !userRole || !userID) { return; @@ -177,6 +242,8 @@ const Fallbacks: React.FC = ({ accessToken, userRole, userID, mo } }; + const hasFallbacks = Array.isArray(routerSettings.fallbacks) && routerSettings.fallbacks.length > 0; + return ( <> = ({ accessToken, userRole, userID, mo value={routerSettings.fallbacks || []} onChange={handleFallbacksChange} /> - - - - Model Name - Fallbacks - Actions - - + {!hasFallbacks ? ( +
+ + No fallbacks configured. Add fallbacks to automatically try another model when the primary + fails. + +
+ ) : ( +
+ + + Model Name + Fallbacks + Actions + + - - {routerSettings["fallbacks"] && - routerSettings["fallbacks"].map((item: FallbackEntry, index: number) => + + {routerSettings["fallbacks"].map((item: FallbackEntry, index: number) => Object.entries(item).map(([key, value]) => ( - {key} - {Array.isArray(value) ? value.join(", ") : value} - + + {renderModelNameCell(key, getProviderFromModel)} + + + {renderFallbacksChain(key, Array.isArray(value) ? value : [], getProviderFromModel)} + + = ({ accessToken, userRole, userID, mo /> - handleDeleteClick(item)} - className="cursor-pointer hover:text-red-600" - /> + onKeyDown={(e) => e.key === "Enter" && handleDeleteClick(item)} + className="cursor-pointer inline-flex" + > + + )), )} - -
+ + + )} { + it("should render", () => { + render(); + + expect(screen.getByText("Routes Configuration")).toBeInTheDocument(); + }); + + it("should display Add Route button", () => { + render(); + + expect(screen.getByRole("button", { name: /add route/i })).toBeInTheDocument(); + }); + + it("should show empty state when no routes are configured", () => { + render(); + + expect(screen.getByText(/no routes configured/i)).toBeInTheDocument(); + }); + + it("should add a route when Add Route is clicked", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: /add route/i })); + + expect(screen.getByText("Route 1: Unnamed")).toBeInTheDocument(); + }); + + it("should call onChange when a route is added", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: /add route/i })); + + expect(onChange).toHaveBeenCalledWith({ + routes: [ + expect.objectContaining({ + name: "", + utterances: [], + description: "", + score_threshold: 0.5, + }), + ], + }); + }); + + it("should initialize routes from value prop", async () => { + const value = { + routes: [ + { + name: "gpt-4", + utterances: ["hello", "hi"], + description: "For greetings", + score_threshold: 0.7, + }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + }); + + it("should support both name and model fields in value prop", async () => { + const value = { + routes: [{ model: "gpt-3.5-turbo", utterances: [], description: "", score_threshold: 0.5 }], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-3.5-turbo")).toBeInTheDocument(); + }); + }); + + it("should remove a route when delete button is clicked", async () => { + const user = userEvent.setup(); + const value = { + routes: [ + { + name: "gpt-4", + utterances: [], + description: "", + score_threshold: 0.5, + }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + const deleteButton = screen.getByRole("button", { name: "delete" }); + await user.click(deleteButton); + + await waitFor(() => { + expect(screen.queryByText("Route 1: gpt-4")).not.toBeInTheDocument(); + expect(screen.getByText(/no routes configured/i)).toBeInTheDocument(); + }); + }); + + it("should call onChange when route is removed", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + const value = { + routes: [ + { + name: "gpt-4", + utterances: [], + description: "", + score_threshold: 0.5, + }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + const deleteButton = screen.getByRole("button", { name: "delete" }); + await user.click(deleteButton); + + await waitFor(() => { + expect(onChange).toHaveBeenCalledWith({ routes: [] }); + }); + }); + + + + + it("should update route when description is changed", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + const value = { + routes: [ + { + name: "gpt-4", + utterances: [], + description: "", + score_threshold: 0.5, + }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + const descriptionInput = screen.getByPlaceholderText("Describe when this route should be used..."); + await user.type(descriptionInput, "For code generation"); + + await waitFor(() => { + const lastCall = onChange.mock.calls[onChange.mock.calls.length - 1]; + expect(lastCall[0].routes[0].description).toBe("For code generation"); + }); + }); + + it("should update route when score threshold is changed", async () => { + const onChange = vi.fn(); + const value = { + routes: [ + { + name: "gpt-4", + utterances: [], + description: "", + score_threshold: 0.5, + }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + const scoreInput = screen.getByRole("spinbutton"); + fireEvent.change(scoreInput, { target: { value: "0.9" } }); + + await waitFor(() => { + const lastCall = onChange.mock.calls[onChange.mock.calls.length - 1]; + expect(lastCall[0].routes[0].score_threshold).toBe(0.9); + }); + }); + + it("should add multiple routes", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: /add route/i })); + await user.click(screen.getByRole("button", { name: /add route/i })); + + expect(screen.getByText("Route 1: Unnamed")).toBeInTheDocument(); + expect(screen.getByText("Route 2: Unnamed")).toBeInTheDocument(); + }); + + it("should toggle JSON preview visibility", async () => { + const user = userEvent.setup(); + const { container } = render(); + + expect(screen.getByText("JSON Preview")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Show" })).toBeInTheDocument(); + expect(container.querySelector("pre")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Show" })); + + expect(screen.getByRole("button", { name: "Hide" })).toBeInTheDocument(); + expect(container.querySelector("pre")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Hide" })); + + expect(screen.getByRole("button", { name: "Show" })).toBeInTheDocument(); + expect(container.querySelector("pre")).not.toBeInTheDocument(); + }); + + it("should display JSON preview with route data when routes exist", async () => { + const user = userEvent.setup(); + const { container } = render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: "Show" })); + + const preElement = container.querySelector("pre"); + expect(preElement).toBeInTheDocument(); + expect(preElement?.textContent).toContain("gpt-4"); + expect(preElement?.textContent).toContain("hello"); + expect(preElement?.textContent).toContain("0.8"); + }); + + it("should display model selector with options from modelInfo", async () => { + const value = { + routes: [ + { name: "", utterances: [], description: "", score_threshold: 0.5 }, + ], + }; + render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: Unnamed")).toBeInTheDocument(); + }); + + expect(screen.getByText("Model")).toBeInTheDocument(); + const comboboxes = screen.getAllByRole("combobox"); + expect(comboboxes.length).toBeGreaterThan(0); + }); + + it("should clear routes when value prop changes to empty", async () => { + const value = { + routes: [ + { + name: "gpt-4", + utterances: [], + description: "", + score_threshold: 0.5, + }, + ], + }; + const { rerender } = render(); + + await waitFor(() => { + expect(screen.getByText("Route 1: gpt-4")).toBeInTheDocument(); + }); + + rerender(); + + await waitFor(() => { + expect(screen.getByText(/no routes configured/i)).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.tsx b/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.tsx new file mode 100644 index 00000000000..9b93287b36d --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/RouterConfigBuilder.tsx @@ -0,0 +1,277 @@ +import { DeleteOutlined, InfoCircleOutlined, PlusOutlined } from "@ant-design/icons"; +import { Select as AntdSelect, Button, Card, Collapse, Divider, Empty, Flex, Input, InputNumber, Space, Tooltip, Typography } from "antd"; +import React, { useEffect, useState } from "react"; +import { ModelGroup } from "../playground/llm_calls/fetch_models"; + +const { Text } = Typography; + +const { TextArea } = Input; + +interface Route { + id: string; + model: string; + utterances: string[]; + description: string; + score_threshold: number; +} + +interface SavedRoute { + id?: string; + name?: string; + model?: string; + utterances?: string[]; + description?: string; + score_threshold?: number; +} + +interface RouterConfig { + routes?: SavedRoute[]; +} + +interface RouterConfigBuilderProps { + modelInfo: ModelGroup[]; + value?: RouterConfig; + onChange?: (config: any) => void; +} + +const RouterConfigBuilder: React.FC = ({ modelInfo, value, onChange }) => { + const [routes, setRoutes] = useState([]); + const [showJsonPreview, setShowJsonPreview] = useState(false); + const [expandedRoutes, setExpandedRoutes] = useState([]); + + // Initialize routes from value prop - preserve existing route IDs to avoid focus loss when parent re-renders + useEffect(() => { + const routesFromValue = value?.routes; + if (routesFromValue) { + const routeIds: string[] = []; + setRoutes((prevRoutes) => { + const initializedRoutes = routesFromValue.map((route: SavedRoute, index: number) => { + const existingRoute = prevRoutes[index]; + const id = existingRoute?.id || route.id || `route-${index}-${Date.now()}`; + routeIds.push(id); + return { + id, + model: route.name || route.model || "", // handle both 'name' and 'model' fields + utterances: route.utterances || [], + description: route.description || "", + score_threshold: route.score_threshold ?? 0.5, + }; + }); + return initializedRoutes; + }); + setExpandedRoutes(routeIds); + } else { + setRoutes([]); + setExpandedRoutes([]); + } + }, [value]); + + // Handle adding a new route + const addRoute = () => { + const newRouteId = `route-${Date.now()}`; + const newRoute: Route = { + id: newRouteId, + model: "", + utterances: [], + description: "", + score_threshold: 0.5, + }; + const updatedRoutes = [...routes, newRoute]; + setRoutes(updatedRoutes); + updateConfig(updatedRoutes); + // Automatically expand the new route + setExpandedRoutes((prev) => [...prev, newRouteId]); + }; + + // Handle removing a route + const removeRoute = (routeId: string) => { + const updatedRoutes = routes.filter((route) => route.id !== routeId); + setRoutes(updatedRoutes); + updateConfig(updatedRoutes); + // Remove from expanded routes as well + setExpandedRoutes((prev) => prev.filter((id) => id !== routeId)); + }; + + // Handle updating a route + const updateRoute = (routeId: string, field: keyof Route, value: any) => { + const updatedRoutes = routes.map((route) => (route.id === routeId ? { ...route, [field]: value } : route)); + setRoutes(updatedRoutes); + updateConfig(updatedRoutes); + }; + + // Update the overall configuration + const updateConfig = (updatedRoutes: Route[]) => { + const config = { + routes: updatedRoutes.map((route) => ({ + name: route.model, + utterances: route.utterances, + description: route.description, + score_threshold: route.score_threshold, + })), + }; + onChange?.(config); + }; + + // Handle utterances change (convert textarea string to array) + const handleUtterancesChange = (routeId: string, utterancesText: string) => { + const utterancesArray = utterancesText + .split("\n") + .map((line) => line.trim()) // Only trims leading/trailing whitespace, preserves internal spaces + .filter((line) => line.length > 0); + updateRoute(routeId, "utterances", utterancesArray); + }; + + // Prepare model options for dropdowns + const modelOptions = modelInfo.map((model) => ({ + value: model.model_group, + label: model.model_group, + })); + + const generateConfig = () => { + return { + routes: routes.map((route) => ({ + name: route.model, + utterances: route.utterances, + description: route.description, + score_threshold: route.score_threshold, + })), + }; + }; + + return ( +
+ + + Routes Configuration + + + + + + + + {/* Routes */} + {routes.length === 0 ? ( + + + + ) : ( + setExpandedRoutes(Array.isArray(keys) ? keys : [keys].filter(Boolean))} + style={{ width: "100%" }} + items={routes.map((route, index) => ({ + key: route.id, + label: ( + + Route {index + 1}: {route.model || "Unnamed"} + + ), + extra: ( +