diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml new file mode 100644 index 00000000000..b442c7dd5f5 --- /dev/null +++ b/.github/workflows/test-litellm-matrix.yml @@ -0,0 +1,109 @@ +name: LiteLLM Unit Tests (Matrix) + +on: + pull_request: + branches: [main] + +# Cancel in-progress runs for the same PR +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + test-group: + # tests/test_litellm split by subdirectory (~560 files total) + - name: "llms" + path: "tests/test_litellm/llms" + workers: 4 + # tests/test_litellm/proxy split by subdirectory (~180 files total) + - name: "proxy-guardrails" + path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers" + workers: 4 + - name: "proxy-core" + path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine" + workers: 4 + - name: "proxy-misc" + path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py" + workers: 4 + - name: "integrations" + path: "tests/test_litellm/integrations" + workers: 4 + - name: "core-utils" + path: "tests/test_litellm/litellm_core_utils" + workers: 2 + - name: "other" + path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types" + workers: 4 + - name: "root" + path: "tests/test_litellm/test_*.py" + workers: 4 + # tests/proxy_unit_tests split alphabetically (~48 files total) + - name: "proxy-unit-a" + path: "tests/proxy_unit_tests/test_[a-o]*.py" + workers: 2 + - name: "proxy-unit-b" + path: "tests/proxy_unit_tests/test_[p-z]*.py" + workers: 2 + + name: test (${{ matrix.test-group.name }}) + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Poetry + uses: snok/install-poetry@v1 + + - name: Cache Poetry dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install pytest-retry==1.6.3 pytest-xdist google-genai==1.22.0 \ + google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core + + - name: Setup litellm-enterprise + run: | + cd enterprise && poetry run pip install -e . && cd .. + + - name: Run tests - ${{ matrix.test-group.name }} + run: | + poetry run pytest ${{ matrix.test-group.path }} \ + --tb=short -vv \ + --maxfail=10 \ + -n ${{ matrix.test-group.workers }} \ + --durations=20 + + # Aggregate job to require all matrix jobs pass + test-complete: + needs: test + runs-on: ubuntu-latest + if: always() + steps: + - name: Check test results + run: | + if [ "${{ needs.test.result }}" != "success" ]; then + echo "Some test groups failed" + exit 1 + fi + echo "All test groups passed!" diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index d9cf2e74a11..dc9b48c28f6 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -1,8 +1,12 @@ name: LiteLLM Mock Tests (folder - tests/test_litellm) +# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs +# the same tests in parallel across 10 jobs for faster CI times. +# Kept for manual debugging only. on: - pull_request: - branches: [ main ] + workflow_dispatch: # Manual trigger only + # pull_request: + # branches: [ main ] jobs: test: diff --git a/.semgrep/rules/README.md b/.semgrep/rules/README.md index 6cffcc32963..0dbb77cdd48 100644 --- a/.semgrep/rules/README.md +++ b/.semgrep/rules/README.md @@ -1,52 +1,22 @@ -# Custom Semgrep Rules +# Custom Semgrep rules for LiteLLM -All `.yml` files under `.semgrep/rules/` run in CI (CircleCI `semgrep` job). +Add custom rule YAML files here. Semgrep loads all `.yml`/`.yaml` files under this directory. -## Add a Rule - -* Add a `.yml` file under `.semgrep/rules///` - - -[Rule syntax →](https://semgrep.dev/docs/writing-rules/rule-syntax/) - -## Organizing Rules - -### Structure: language → domain - -``` -.semgrep/rules///.yml -``` - -Examples: - -- `python/security/unsafe-yaml-load.yml` -- `python/reliability/missing-timeout-http.yml` -- `python/performance/blocking-io-in-async.yml` - -### Rule metadata - -Match tags to the folder for consistent filtering: - -```yaml -metadata: - tags: [python, security] -``` - -### Severity expectations - -All rules must fail CI on findings. No warn-only rules. - -- Use `severity: ERROR` in rule metadata -- If a rule is noisy → refine until low false positives before adding - -## Run Locally +**Run only custom rules (CI / fail on findings):** ```bash semgrep scan --config .semgrep/rules . --error ``` -With Semgrep registry: +**Run with registry + custom rules:** ```bash semgrep scan --config auto --config .semgrep/rules . ``` + +**Layout:** + +- `python/` – Python-specific rules (security, patterns) +- Add more subdirs as needed (e.g. `generic/` for language-agnostic rules) + +See [Semgrep rule syntax](https://semgrep.dev/docs/writing-rules/rule-syntax/). diff --git a/.semgrep/rules/python/unbounded-memory.yml b/.semgrep/rules/python/unbounded-memory.yml new file mode 100644 index 00000000000..811ef689344 --- /dev/null +++ b/.semgrep/rules/python/unbounded-memory.yml @@ -0,0 +1,14 @@ +# Unbounded memory growth – data structures without a clear max limit +# Can lead to OOM under load. + +rules: + - id: unbounded-asyncio-queue + message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues). + severity: ERROR + languages: [python] + pattern-either: + - pattern: asyncio.Queue() + - pattern: asyncio.Queue(maxsize=0) + metadata: + category: correctness + cwe: "CWE-400: Uncontrolled Resource Consumption" \ No newline at end of file diff --git a/Makefile b/Makefile index b867d7ea35e..74031f418d6 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ # LiteLLM Makefile # Simple Makefile for running tests and basic development tasks -.PHONY: help test test-unit test-integration test-unit-helm \ +.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \ + test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \ + test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ info lint lint-dev format \ install-dev install-proxy-dev install-test-deps \ install-helm-unittest check-circular-imports check-import-safety @@ -25,6 +27,16 @@ help: @echo " make check-import-safety - Check import safety" @echo " make test - Run all tests" @echo " make test-unit - Run unit tests (tests/test_litellm)" + @echo " make test-unit-llms - Run LLM provider tests (~225 files)" + @echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)" + @echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)" + @echo " make test-unit-proxy-misc - Run proxy misc tests (~77 files)" + @echo " make test-unit-integrations - Run integration tests (~60 files)" + @echo " make test-unit-core-utils - Run core utils tests (~32 files)" + @echo " make test-unit-other - Run other tests (caching, responses, etc., ~69 files)" + @echo " make test-unit-root - Run root-level tests (~34 files)" + @echo " make test-proxy-unit-a - Run proxy_unit_tests (a-o, ~20 files)" + @echo " make test-proxy-unit-b - Run proxy_unit_tests (p-z, ~28 files)" @echo " make test-integration - Run integration tests" @echo " make test-unit-helm - Run helm unit tests" @@ -129,6 +141,38 @@ test: test-unit: install-test-deps poetry run pytest tests/test_litellm -x -vv -n 4 +# Matrix test targets (matching CI workflow groups) +test-unit-llms: install-test-deps + poetry run pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-guardrails: install-test-deps + poetry run pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-core: install-test-deps + poetry run pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20 + +test-unit-proxy-misc: install-test-deps + poetry run pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 + +test-unit-integrations: install-test-deps + poetry run pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20 + +test-unit-core-utils: install-test-deps + poetry run pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20 + +test-unit-other: install-test-deps + poetry run pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20 + +test-unit-root: install-test-deps + poetry run pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20 + +# Proxy unit tests (tests/proxy_unit_tests split alphabetically) +test-proxy-unit-a: install-test-deps + poetry run pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20 + +test-proxy-unit-b: install-test-deps + poetry run pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20 + test-integration: poetry run pytest tests/ -k "not test_litellm" diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 64126bb0292..004377e19b3 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -59,7 +59,8 @@ RUN mkdir -p /var/lib/litellm/ui && \ mkdir -p "$folder_name" && \ mv "$html_file" "$folder_name/index.html"; \ fi; \ - done ) && \ + done && \ + touch .litellm_ui_ready ) && \ cd /app/ui/litellm-dashboard && rm -rf ./out # Build litellm wheel and place it in wheels dir (replace any PyPI wheels) diff --git a/docker/README.md b/docker/README.md index 6d81276bb4b..7027a30fdd7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -70,9 +70,12 @@ docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d This setup: - Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image. -- Runs the proxy as a non-root user with a read-only rootfs and only two writable tmpfs mounts: +- Runs the proxy as a non-root user with a read-only rootfs and only writable tmpfs mounts: - `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`) - `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`) +- Pre-builds and serves the admin UI from read-only paths: + - `/var/lib/litellm/ui` (pre-restructured Next.js UI with `.litellm_ui_ready` marker) + - `/var/lib/litellm/assets` (UI logos and assets) - Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines. You should also verify offline Prisma behaviour with: diff --git a/docs/my-website/blog/minimax_m2_5/index.md b/docs/my-website/blog/minimax_m2_5/index.md new file mode 100644 index 00000000000..50084fcc1e5 --- /dev/null +++ b/docs/my-website/blog/minimax_m2_5/index.md @@ -0,0 +1,394 @@ +--- +slug: minimax_m2_5 +title: "Day 0 Support: MiniMax-M2.5" +date: 2026-02-12T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +description: "Day 0 support for MiniMax-M2.5 on LiteLLM" +tags: [minimax, M2.5, llm] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway. + +## Supported Models + +LiteLLM supports the following MiniMax models: + +| Model | Description | Input Cost | Output Cost | Context Window | +|-------|-------------|------------|-------------|----------------| +| **MiniMax-M2.5** | Advanced reasoning, Agentic capabilities | $0.3/M tokens | $1.2/M tokens | 1M tokens | +| **MiniMax-M2.5-lightning** | Faster and More Agile (~100 tps) | $0.3/M tokens | $2.4/M tokens | 1M tokens | + +## Features Supported + +- **Prompt Caching**: Reduce costs with cached prompts ($0.03/M tokens for cache read, $0.375/M tokens for cache write) +- **Function Calling**: Built-in tool calling support +- **Reasoning**: Advanced reasoning capabilities with thinking support +- **System Messages**: Full system message support +- **Cost Tracking**: Automatic cost calculation for all requests + +## Docker Image + +```bash +docker pull litellm/litellm:v1.81.3-stable +``` + +## Usage - OpenAI Compatible API (/v1/chat/completions) + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: minimax-m2-5 + litellm_params: + model: minimax/MiniMax-M2.5 + api_key: os.environ/MINIMAX_API_KEY + api_base: https://api.minimax.io/v1 +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e MINIMAX_API_KEY=$MINIMAX_API_KEY \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:v1.81.3-stable \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +### With Reasoning Split + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "messages": [ + { + "role": "user", + "content": "Solve: 2+2=?" + } + ], + "extra_body": { + "reasoning_split": true + } +}' +``` + +## Usage - Anthropic Compatible API (/v1/messages) + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: minimax-m2-5 + litellm_params: + model: minimax/MiniMax-M2.5 + api_key: os.environ/MINIMAX_API_KEY + api_base: https://api.minimax.io/anthropic/v1/messages +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e MINIMAX_API_KEY=$MINIMAX_API_KEY \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:v1.81.3-stable \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "max_tokens": 1000, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + +### With Thinking + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "minimax-m2-5", + "max_tokens": 1000, + "thinking": { + "type": "enabled", + "budget_tokens": 1000 + }, + "messages": [ + { + "role": "user", + "content": "Solve: 2+2=?" + } + ] +}' +``` + +## Usage - LiteLLM SDK + +### OpenAI-compatible API + +```python +import litellm + +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[ + {"role": "user", "content": "Hello, how are you?"} + ], + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +print(response.choices[0].message.content) +``` + +### Anthropic-compatible API + +```python +import litellm + +response = litellm.anthropic.messages.acreate( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Hello, how are you?"}], + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/anthropic/v1/messages", + max_tokens=1000 +) + +print(response.choices[0].message.content) +``` + +### With Thinking + +```python +response = litellm.anthropic.messages.acreate( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Solve: 2+2=?"}], + thinking={"type": "enabled", "budget_tokens": 1000}, + api_key="your-minimax-api-key" +) + +# Access thinking content +for block in response.choices[0].message.content: + if hasattr(block, 'type') and block.type == 'thinking': + print(f"Thinking: {block.thinking}") +``` + +### With Reasoning Split (OpenAI API) + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[ + {"role": "user", "content": "Solve: 2+2=?"} + ], + extra_body={"reasoning_split": True}, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +# Access thinking and response +if hasattr(response.choices[0].message, 'reasoning_details'): + print(f"Thinking: {response.choices[0].message.reasoning_details}") +print(f"Response: {response.choices[0].message.content}") +``` + +## Cost Tracking + +LiteLLM automatically tracks costs for MiniMax-M2.5 requests. The pricing is: + +- **Input**: $0.3 per 1M tokens +- **Output**: $1.2 per 1M tokens +- **Cache Read**: $0.03 per 1M tokens +- **Cache Write**: $0.375 per 1M tokens + +### Accessing Cost Information + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Hello!"}], + api_key="your-minimax-api-key" +) + +# Access cost information +print(f"Cost: ${response._hidden_params.get('response_cost', 0)}") +``` + +## Streaming Support + +### OpenAI API + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[{"role": "user", "content": "Tell me a story"}], + stream=True, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + +### Streaming with Reasoning Split + +```python +stream = litellm.completion( + model="minimax/MiniMax-M2.5", + messages=[ + {"role": "user", "content": "Tell me a story"}, + ], + extra_body={"reasoning_split": True}, + stream=True, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +reasoning_buffer = "" +text_buffer = "" + +for chunk in stream: + if hasattr(chunk.choices[0].delta, "reasoning_details") and chunk.choices[0].delta.reasoning_details: + for detail in chunk.choices[0].delta.reasoning_details: + if "text" in detail: + reasoning_text = detail["text"] + new_reasoning = reasoning_text[len(reasoning_buffer):] + if new_reasoning: + print(new_reasoning, end="", flush=True) + reasoning_buffer = reasoning_text + + if chunk.choices[0].delta.content: + content_text = chunk.choices[0].delta.content + new_text = content_text[len(text_buffer):] if text_buffer else content_text + if new_text: + print(new_text, end="", flush=True) + text_buffer = content_text +``` + +## Using with Native SDKs + +### Anthropic SDK via LiteLLM Proxy + +```python +import os +os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000" +os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM proxy key + +import anthropic + +client = anthropic.Anthropic() + +message = client.messages.create( + model="minimax-m2-5", + max_tokens=1000, + system="You are a helpful assistant.", + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hi, how are you?" + } + ] + } + ] +) + +for block in message.content: + if block.type == "thinking": + print(f"Thinking:\n{block.thinking}\n") + elif block.type == "text": + print(f"Text:\n{block.text}\n") +``` + +### OpenAI SDK via LiteLLM Proxy + +```python +import os +os.environ["OPENAI_BASE_URL"] = "http://localhost:4000" +os.environ["OPENAI_API_KEY"] = "sk-1234" # Your LiteLLM proxy key + +from openai import OpenAI + +client = OpenAI() + +response = client.chat.completions.create( + model="minimax-m2-5", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hi, how are you?"}, + ], + extra_body={"reasoning_split": True}, +) + +# Access thinking and response +if hasattr(response.choices[0].message, 'reasoning_details'): + print(f"Thinking:\n{response.choices[0].message.reasoning_details[0]['text']}\n") +print(f"Text:\n{response.choices[0].message.content}\n") +``` diff --git a/docs/my-website/docs/adding_provider/generic_guardrail_api.md b/docs/my-website/docs/adding_provider/generic_guardrail_api.md index 482dedaa8a9..0931c349e48 100644 --- a/docs/my-website/docs/adding_provider/generic_guardrail_api.md +++ b/docs/my-website/docs/adding_provider/generic_guardrail_api.md @@ -93,6 +93,12 @@ Implement `POST /beta/litellm_basic_guardrail_api` "user_api_key_end_user_id": "end user id associated with the litellm virtual key used", "user_api_key_org_id": "org id associated with the litellm virtual key used" }, + "request_headers": { // optional: inbound request headers (allowlist). Allowed headers show their value; all others show "[present]" to indicate the header existed. + "User-Agent": "OpenAI/Python 2.17.0", + "Content-Type": "application/json", + "X-Request-Id": "[present]" + }, + "litellm_version": "1.x.y", // optional: LiteLLM library version running this proxy "input_type": "request", // "request" or "response" "litellm_call_id": "unique_call_id", // the call id of the individual LLM call "litellm_trace_id": "trace_id", // the trace id of the LLM call - useful if there are multiple LLM calls for the same conversation diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 38ad9bdd0ee..05e490a8b46 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -746,6 +746,7 @@ router_settings: | LITERAL_API_URL | API URL for Literal service | LITERAL_BATCH_SIZE | Batch size for Literal operations | LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints +| LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker. | LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours | LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API | LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518 @@ -760,6 +761,7 @@ router_settings: | LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems. | LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM | LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset. +| LITELLM_UI_PATH | Path to directory for Admin UI files. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/ui` in Docker. | LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval. | LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false. | LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours). diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index a5674bf2bc5..56a8b9566db 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -469,6 +469,7 @@ credential_list: api_version: "2023-05-15" credential_info: description: "Production credentials for EU region" + custom_llm_provider: "azure" ``` #### Key Parameters diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index a42d91a7d5f..994788a3ad9 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -250,11 +250,133 @@ The migrate deploy command: ### Read-only File System -If you see a `Permission denied` error, it means the LiteLLM pod is running with a read-only file system. +Running LiteLLM with `readOnlyRootFilesystem: true` is a Kubernetes security best practice that prevents container processes from writing to the root filesystem. LiteLLM fully supports this configuration. -To fix this, just set `LITELLM_MIGRATION_DIR="/path/to/writeable/directory"` in your environment. +#### Quick Fix for Permission Errors -LiteLLM will use this directory to write migration files. +If you see a `Permission denied` error, it means the LiteLLM pod is running with a read-only file system. LiteLLM needs writable directories for: +- **Database migrations**: Set `LITELLM_MIGRATION_DIR="/path/to/writable/directory"` +- **Admin UI**: Set `LITELLM_UI_PATH="/path/to/writable/directory"` +- **UI assets/logos**: Set `LITELLM_ASSETS_PATH="/path/to/writable/directory"` + +#### Complete Read-Only Filesystem Setup (Kubernetes) + +For production deployments with enhanced security, use this configuration: + +**Option 1: Using EmptyDir Volumes with InitContainer (Recommended)** + +This approach copies the pre-built UI from the Docker image to writable emptyDir volumes at pod startup. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm-proxy +spec: + template: + spec: + initContainers: + - name: setup-ui + image: ghcr.io/berriai/litellm:main-stable + command: + - sh + - -c + - | + cp -r /var/lib/litellm/ui/* /app/var/litellm/ui/ && \ + cp -r /var/lib/litellm/assets/* /app/var/litellm/assets/ + volumeMounts: + - name: ui-volume + mountPath: /app/var/litellm/ui + - name: assets-volume + mountPath: /app/var/litellm/assets + + containers: + - name: litellm + image: ghcr.io/berriai/litellm:main-stable + env: + - name: LITELLM_NON_ROOT + value: "true" + - name: LITELLM_UI_PATH + value: "/app/var/litellm/ui" + - name: LITELLM_ASSETS_PATH + value: "/app/var/litellm/assets" + - name: LITELLM_MIGRATION_DIR + value: "/app/migrations" + - name: PRISMA_BINARY_CACHE_DIR + value: "/app/cache/prisma-python/binaries" + - name: XDG_CACHE_HOME + value: "/app/cache" + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 101 + capabilities: + drop: + - ALL + volumeMounts: + - name: config + mountPath: /app/config.yaml + subPath: config.yaml + readOnly: true + - name: ui-volume + mountPath: /app/var/litellm/ui + - name: assets-volume + mountPath: /app/var/litellm/assets + - name: cache + mountPath: /app/cache + - name: migrations + mountPath: /app/migrations + + volumes: + - name: config + configMap: + name: litellm-config + - name: ui-volume + emptyDir: + sizeLimit: 100Mi + - name: assets-volume + emptyDir: + sizeLimit: 10Mi + - name: cache + emptyDir: + sizeLimit: 500Mi + - name: migrations + emptyDir: + sizeLimit: 64Mi +``` + +**Option 2: Without UI (API-only deployment)** + +If you don't need the admin UI, you can run with minimal configuration: + +```yaml +env: + - name: LITELLM_NON_ROOT + value: "true" + - name: LITELLM_MIGRATION_DIR + value: "/app/migrations" +securityContext: + readOnlyRootFilesystem: true +``` + +The proxy will log a warning about the UI but API endpoints will work normally. + +#### Environment Variables for Read-Only Filesystems + +| Variable | Purpose | Default | +|----------|---------|---------| +| `LITELLM_UI_PATH` | Admin UI directory | `/var/lib/litellm/ui` (Docker) | +| `LITELLM_ASSETS_PATH` | UI assets/logos | `/var/lib/litellm/assets` (Docker) | +| `LITELLM_MIGRATION_DIR` | Database migrations | Package directory | +| `PRISMA_BINARY_CACHE_DIR` | Prisma binary cache | System default | +| `XDG_CACHE_HOME` | General cache directory | System default | + +#### Important Notes + +1. **Migrations**: Always set `LITELLM_MIGRATION_DIR` to a writable emptyDir path +2. **Prisma Cache**: Set `PRISMA_BINARY_CACHE_DIR` and `XDG_CACHE_HOME` to writable paths +3. **Server Root Path**: If using a custom `server_root_path`, you must pre-process UI files in your Dockerfile as the proxy cannot modify files at runtime with read-only filesystem +4. **Automatic Detection**: The UI is automatically detected as pre-restructured if it contains a `.litellm_ui_ready` marker file (created by the official Docker images) ## 10. Use a Separate Health Check App :::info diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md index 140dfd4faf8..dd2b77712c4 100644 --- a/docs/my-website/docs/response_api.md +++ b/docs/my-website/docs/response_api.md @@ -1023,6 +1023,134 @@ curl http://localhost:4000/v1/responses \ +## Server-side compaction + +For long-running conversations, you can enable **server-side compaction** so that when the rendered context size crosses a threshold, the server automatically runs compaction in-stream and emits a compaction item—no separate `POST /v1/responses/compact` call is required. + +Supported on the OpenAI Responses API when using the `openai` or `azure` provider. Pass `context_management` with a compaction entry and `compact_threshold` (token count; minimum 1000). When the context crosses the threshold, the server compacts in-stream and continues. Chain turns with `previous_response_id` or by appending output items to your next input array. See [OpenAI Compaction guide](https://developers.openai.com/api/docs/guides/compaction) for details. + +For explicit control over when compaction runs, use the standalone compact endpoint (`POST /v1/responses/compact`) instead. + +### Python SDK + +```python showLineNumbers title="Server-side compaction with LiteLLM Python SDK" +import litellm + +# Non-streaming: enable compaction when context exceeds 200k tokens +response = litellm.responses( + model="openai/gpt-4o", + input="Your conversation input...", + context_management=[{"type": "compaction", "compact_threshold": 200000}], + max_output_tokens=1024, +) +print(response) + +# Streaming: same context_management, compaction runs in-stream if threshold is crossed +stream = litellm.responses( + model="openai/gpt-4o", + input="Your conversation input...", + context_management=[{"type": "compaction", "compact_threshold": 200000}], + stream=True, +) +for event in stream: + print(event) +``` + +### LiteLLM Proxy (AI Gateway) + +Use the OpenAI SDK with your proxy as `base_url`, or call the proxy with curl. The proxy forwards `context_management` to the provider. + +**OpenAI Python SDK (proxy as base_url):** + +```python showLineNumbers title="Server-side compaction via LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", # LiteLLM Proxy (AI Gateway) + api_key="your-proxy-api-key", +) + +response = client.responses.create( + model="openai/gpt-4o", + input="Your conversation input...", + context_management=[{"type": "compaction", "compact_threshold": 200000}], + max_output_tokens=1024, +) +print(response) +``` + +**curl (proxy):** + +```bash title="Server-side compaction via curl to LiteLLM Proxy" +curl -X POST "http://localhost:4000/v1/responses" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "openai/gpt-4o", + "input": "Your conversation input...", + "context_management": [{"type": "compaction", "compact_threshold": 200000}], + "max_output_tokens": 1024 + }' +``` + +## Shell tool + +The **Shell tool** lets the model run commands in a hosted container or local runtime (OpenAI Responses API). You pass `tools=[{"type": "shell", "environment": {...}}]`; the `environment` object configures the runtime (e.g. `type: "container_auto"` for auto-provisioned containers). See [OpenAI Shell tool guide](https://developers.openai.com/api/docs/guides/tools-shell) for full options. + +Supported when using the `openai` or `azure` provider with a model that supports the Shell tool. + +### Python SDK + +```python showLineNumbers title="Shell tool with LiteLLM Python SDK" +import litellm + +response = litellm.responses( + model="openai/gpt-5.2", + input="List files in /mnt/data and run python --version.", + tools=[{"type": "shell", "environment": {"type": "container_auto"}}], + tool_choice="auto", + max_output_tokens=1024, +) +``` + +### LiteLLM Proxy (AI Gateway) + +Use the OpenAI SDK with your proxy as `base_url`, or call the proxy with curl. The proxy forwards `tools` (including `type: "shell"`) to the provider. + +**OpenAI Python SDK (proxy as base_url):** + +```python showLineNumbers title="Shell tool via LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-proxy-api-key", +) + +response = client.responses.create( + model="openai/gpt-5.2", + input="List files in /mnt/data.", + tools=[{"type": "shell", "environment": {"type": "container_auto"}}], + tool_choice="auto", + max_output_tokens=1024, +) +``` + +**curl:** + +```bash title="Shell tool via curl to LiteLLM Proxy" +curl -X POST "http://localhost:4000/v1/responses" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-proxy-api-key" \ + -d '{ + "model": "openai/gpt-5.2", + "input": "List files in /mnt/data.", + "tools": [{"type": "shell", "environment": {"type": "container_auto"}}], + "tool_choice": "auto", + "max_output_tokens": 1024 + }' +``` + ## Session Management LiteLLM Proxy supports session management for all supported models. This allows you to store and fetch conversation history (state) in LiteLLM Proxy. diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35-py3-none-any.whl new file mode 100644 index 00000000000..8a443f38ef5 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35.tar.gz new file mode 100644 index 00000000000..4dde13b32e2 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35.tar.gz differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl new file mode 100644 index 00000000000..c98d9cfcfac Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz new file mode 100644 index 00000000000..c8c33404620 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz differ diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql new file mode 100644 index 00000000000..f3a0821d37f --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "LiteLLM_PolicyAttachmentTable" ADD COLUMN "tags" TEXT[] DEFAULT ARRAY[]::TEXT[]; + diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql new file mode 100644 index 00000000000..67e75e84c4a --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql @@ -0,0 +1,33 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- AlterTable +ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- AlterTable +ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- AlterTable +ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; + +-- CreateTable +CREATE TABLE "LiteLLM_AccessGroupTable" ( + "access_group_id" TEXT NOT NULL, + "access_group_name" TEXT NOT NULL, + "description" TEXT, + "access_model_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "access_mcp_server_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "access_agent_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "assigned_team_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "assigned_key_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_by" TEXT, + + CONSTRAINT "LiteLLM_AccessGroupTable_pkey" PRIMARY KEY ("access_group_id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_AccessGroupTable_access_group_name_key" ON "LiteLLM_AccessGroupTable"("access_group_name"); + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 558dfcc9517..2a0faee2808 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -128,6 +128,7 @@ model LiteLLM_TeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team @@ -161,6 +162,7 @@ model LiteLLM_DeletedTeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) @@ -293,6 +295,7 @@ model LiteLLM_VerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") budget_id String? @@ -348,6 +351,7 @@ model LiteLLM_DeletedVerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") router_settings Json? @default("{}") @@ -920,3 +924,23 @@ model LiteLLM_PolicyAttachmentTable { updated_at DateTime @default(now()) @updatedAt updated_by String? } + +//Unified Access Groups table for storing unified access groups +model LiteLLM_AccessGroupTable { + access_group_id String @id @default(uuid()) + access_group_name String @unique + description String? + + // Resource memberships - explicit arrays per type + access_model_ids String[] @default([]) + access_mcp_server_ids String[] @default([]) + access_agent_ids String[] @default([]) + + assigned_team_ids String[] @default([]) + assigned_key_ids String[] @default([]) + + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? +} \ No newline at end of file diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index e0a769a5edf..eda49bfb9fa 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.34" +version = "0.4.36" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.34" +version = "0.4.36" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index 538ee727612..0fdbac63feb 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -175,6 +175,7 @@ _async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # Custo pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False +standard_logging_payload_excluded_fields: Optional[List[str]] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 753a94295b3..e546a0dbb02 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -227,6 +227,84 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return input_items, instructions + def _map_optional_params_to_responses_api_request( + self, + optional_params: dict, + responses_api_request: "ResponsesAPIOptionalRequestParams", + ) -> None: + """Map optional_params into responses_api_request (mutates in place).""" + for key, value in optional_params.items(): + if value is None: + continue + if key in ("max_tokens", "max_completion_tokens"): + responses_api_request["max_output_tokens"] = value + elif key == "tools" and value is not None: + responses_api_request["tools"] = ( + self._convert_tools_to_responses_format( + cast(List[Dict[str, Any]], value) + ) + ) + elif key == "response_format": + text_format = self._transform_response_format_to_text_format(value) + if text_format: + responses_api_request["text"] = text_format # type: ignore + elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): + responses_api_request[key] = value # type: ignore + elif key == "previous_response_id": + responses_api_request["previous_response_id"] = value + elif key == "reasoning_effort": + responses_api_request["reasoning"] = self._map_reasoning_effort(value) + elif key == "web_search_options": + self._add_web_search_tool(responses_api_request, value) + + def _build_sanitized_litellm_params( + self, litellm_params: dict + ) -> Dict[str, Any]: + """Build sanitized litellm_params with merged metadata.""" + responses_optional_param_keys = set( + ResponsesAPIOptionalRequestParams.__annotations__.keys() + ) + sanitized: Dict[str, Any] = { + key: value + for key, value in litellm_params.items() + if key not in responses_optional_param_keys + } + legacy_metadata = litellm_params.get("metadata") + existing_litellm_metadata = litellm_params.get("litellm_metadata") + merged_litellm_metadata: Dict[str, Any] = {} + if isinstance(legacy_metadata, dict): + merged_litellm_metadata.update(legacy_metadata) + if isinstance(existing_litellm_metadata, dict): + merged_litellm_metadata.update(existing_litellm_metadata) + if merged_litellm_metadata: + sanitized["litellm_metadata"] = merged_litellm_metadata + else: + sanitized.pop("litellm_metadata", None) + return sanitized + + def _merge_responses_api_request_into_request_data( + self, + request_data: Dict[str, Any], + responses_api_request: "ResponsesAPIOptionalRequestParams", + instructions: Optional[str], + ) -> None: + """Add non-None values from responses_api_request into request_data.""" + for key, value in responses_api_request.items(): + if value is None: + continue + if key == "instructions" and instructions: + request_data["instructions"] = instructions + elif key == "stream_options" and isinstance(value, dict): + request_data["stream_options"] = value.get("include_obfuscation") + elif key == "user" and isinstance(value, str): + # OpenAI API requires user param to be max 64 chars - truncate if longer + if len(value) <= 64: + request_data["user"] = value + else: + request_data["user"] = value[:64] + else: + request_data[key] = value + def transform_request( self, model: str, @@ -251,36 +329,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if instructions: responses_api_request["instructions"] = instructions - # Map optional parameters - for key, value in optional_params.items(): - if value is None: - continue - if key in ("max_tokens", "max_completion_tokens"): - responses_api_request["max_output_tokens"] = value - elif key == "tools" and value is not None: - # Convert chat completion tools to responses API tools format - responses_api_request["tools"] = ( - self._convert_tools_to_responses_format( - cast(List[Dict[str, Any]], value) - ) - ) - elif key == "response_format": - # Convert response_format to text.format - text_format = self._transform_response_format_to_text_format(value) - if text_format: - responses_api_request["text"] = text_format # type: ignore - elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): - responses_api_request[key] = value # type: ignore - elif key == "metadata": - responses_api_request["metadata"] = value - elif key == "previous_response_id": - responses_api_request["previous_response_id"] = value - elif key == "reasoning_effort": - responses_api_request["reasoning"] = self._map_reasoning_effort(value) - elif key == "web_search_options": - self._add_web_search_tool(responses_api_request, value) + self._map_optional_params_to_responses_api_request( + optional_params, responses_api_request + ) - # Get stream parameter from litellm_params if not in optional_params stream = optional_params.get("stream") or litellm_params.get("stream", False) verbose_logger.debug(f"Chat provider: Stream parameter: {stream}") @@ -304,11 +356,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): setattr(litellm_logging_obj, "call_type", CallTypes.responses.value) + sanitized_litellm_params = self._build_sanitized_litellm_params( + litellm_params + ) + request_data = { "model": api_model, "input": input_items, "litellm_logging_obj": litellm_logging_obj, - **litellm_params, + **sanitized_litellm_params, "client": client, } @@ -316,18 +372,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): f"Chat provider: Final request model={api_model}, input_items={len(input_items)}" ) - # Add non-None values from responses_api_request - for key, value in responses_api_request.items(): - if value is not None: - if key == "instructions" and instructions: - request_data["instructions"] = instructions - elif key == "stream_options" and isinstance(value, dict): - request_data["stream_options"] = value.get("include_obfuscation") - elif key == "user": # string can't be longer than 64 characters - if isinstance(value, str) and len(value) <= 64: - request_data["user"] = value - else: - request_data[key] = value + self._merge_responses_api_request_into_request_data( + request_data, responses_api_request, instructions + ) if headers: request_data["extra_headers"] = headers diff --git a/litellm/constants.py b/litellm/constants.py index 88c57d3ce4c..4e89ddd7bd9 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -101,6 +101,11 @@ MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE = int( MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL = int( os.getenv("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", "3600") ) + +# Default npm cache directory for STDIO MCP servers. +# npm/npx needs a writable cache dir; in containers the default (~/.npm) +# may not exist or be read-only. /tmp is always writable. +MCP_NPM_CACHE_DIR = os.getenv("MCP_NPM_CACHE_DIR", "/tmp/.npm_mcp_cache") MCP_OAUTH2_TOKEN_CACHE_MIN_TTL = int( os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10") ) diff --git a/litellm/integrations/cloudzero/transform.py b/litellm/integrations/cloudzero/transform.py index c36833a6dbf..b40a71da1c6 100644 --- a/litellm/integrations/cloudzero/transform.py +++ b/litellm/integrations/cloudzero/transform.py @@ -141,7 +141,7 @@ class CBFTransformer: # Required CBF fields 'time/usage_start': usage_date.isoformat() if usage_date else None, # Required: ISO-formatted UTC datetime 'cost/cost': float(row.get('spend', 0.0)), # Required: billed cost - 'resource/id': model, # Send model name + 'resource/id': resource_id, # CZRN (CloudZero Resource Name) # Usage metrics for token consumption 'usage/amount': total_tokens, # Numeric value of tokens consumed diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 407bc581f71..fb395366004 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -624,7 +624,9 @@ class CustomGuardrail(CustomLogger): This gets logged on downsteam Langfuse, DataDog, etc. """ # Convert None to empty dict to satisfy type requirements - guardrail_response = {} if response is None else response + guardrail_response: Union[Dict[str, Any], str] = ( + {} if response is None else response + ) # For apply_guardrail functions in custom_code_guardrail scenario, # simplify the logged response to "allow", "deny", or "mask" diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 4a341863d4b..c244363e389 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -774,15 +774,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac self, model_call_details: Dict ) -> Dict: """ - Only redacts messages and responses when self.turn_off_message_logging is True + Redacts or excludes fields from StandardLoggingPayload before callbacks receive it. + This method handles two features: + 1. turn_off_message_logging: When True, redacts messages and responses + 2. standard_logging_payload_excluded_fields: Removes specified fields entirely - By default, self.turn_off_message_logging is False and this does nothing. - - Return a redacted deepcopy of the provided logging payload. + Return a modified copy of the provided logging payload. This is useful for logging payloads that contain sensitive information. """ + import litellm from copy import copy from litellm import Choices, Message, ModelResponse @@ -790,14 +792,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac turn_off_message_logging: bool = getattr( self, "turn_off_message_logging", False ) + excluded_fields: Optional[List[str]] = getattr( + litellm, "standard_logging_payload_excluded_fields", None + ) - if turn_off_message_logging is False: + # Early return if no processing needed + if turn_off_message_logging is False and not excluded_fields: return model_call_details # Only make a shallow copy of the top-level dict to avoid deepcopy issues # with complex objects like AuthenticationError that may be present model_call_details_copy = copy(model_call_details) - redacted_str = "redacted-by-litellm" standard_logging_object = model_call_details.get("standard_logging_object") if standard_logging_object is None: return model_call_details_copy @@ -805,39 +810,58 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac # Make a copy of just the standard_logging_object to avoid modifying the original standard_logging_object_copy = copy(standard_logging_object) - if standard_logging_object_copy.get("messages") is not None: - standard_logging_object_copy["messages"] = [ - Message(content=redacted_str).model_dump() - ] + # Handle excluded fields - remove them entirely from the payload + if excluded_fields: + for field in excluded_fields: + if field in standard_logging_object_copy: + del standard_logging_object_copy[field] - if standard_logging_object_copy.get("response") is not None: - response = standard_logging_object_copy["response"] - # Check if this is a ResponsesAPIResponse (has "output" field) - if isinstance(response, dict) and "output" in response: - # Make a copy to avoid modifying the original - from copy import deepcopy + # Handle turn_off_message_logging - redact messages and responses (if not already excluded) + if turn_off_message_logging: + redacted_str = "redacted-by-litellm" - response_copy = deepcopy(response) - # Redact content in output array - if isinstance(response_copy.get("output"), list): - for output_item in response_copy["output"]: - if isinstance(output_item, dict) and "content" in output_item: - if isinstance(output_item["content"], list): - # Redact text in content items - for content_item in output_item["content"]: - if ( - isinstance(content_item, dict) - and "text" in content_item - ): - content_item["text"] = redacted_str - standard_logging_object_copy["response"] = response_copy - else: - # Standard ModelResponse format - model_response = ModelResponse( - choices=[Choices(message=Message(content=redacted_str))] - ) - model_response_dict = model_response.model_dump() - standard_logging_object_copy["response"] = model_response_dict + if ( + "messages" not in (excluded_fields or []) + and standard_logging_object_copy.get("messages") is not None + ): + standard_logging_object_copy["messages"] = [ + Message(content=redacted_str).model_dump() + ] + + if ( + "response" not in (excluded_fields or []) + and standard_logging_object_copy.get("response") is not None + ): + response = standard_logging_object_copy["response"] + # Check if this is a ResponsesAPIResponse (has "output" field) + if isinstance(response, dict) and "output" in response: + # Make a copy to avoid modifying the original + from copy import deepcopy + + response_copy = deepcopy(response) + # Redact content in output array + if isinstance(response_copy.get("output"), list): + for output_item in response_copy["output"]: + if ( + isinstance(output_item, dict) + and "content" in output_item + ): + if isinstance(output_item["content"], list): + # Redact text in content items + for content_item in output_item["content"]: + if ( + isinstance(content_item, dict) + and "text" in content_item + ): + content_item["text"] = redacted_str + standard_logging_object_copy["response"] = response_copy + else: + # Standard ModelResponse format + model_response = ModelResponse( + choices=[Choices(message=Message(content=redacted_str))] + ) + model_response_dict = model_response.model_dump() + standard_logging_object_copy["response"] = model_response_dict model_call_details_copy["standard_logging_object"] = ( standard_logging_object_copy diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 3ddcae69315..dde44cced36 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -70,6 +70,11 @@ class ExceptionCheckers: Check if an error string indicates a context window exceeded error. """ _error_str_lowercase = error_str.lower() + # Exclude param validation errors (e.g. OpenAI "user" param max 64 chars) + if "string_above_max_length" in _error_str_lowercase: + return False + if "invalid 'user'" in _error_str_lowercase and "string too long" in _error_str_lowercase: + return False known_exception_substrings = [ "exceed context limit", "this model's maximum context length is", @@ -98,16 +103,18 @@ class ExceptionCheckers: """ Check if an error string indicates a content policy violation error. """ + _lower = error_str.lower() known_exception_substrings = [ - "invalid_request_error", "content_policy_violation", + "responsibleaipolicyviolation", "the response was filtered due to the prompt triggering azure openai's content management", "your task failed as a result of our safety system", "the model produced invalid content", "content_filter_policy", + "your request was rejected as a result of our safety system", ] for substring in known_exception_substrings: - if substring in error_str.lower(): + if substring in _lower: return True return False @@ -2060,6 +2067,19 @@ def exception_type( # type: ignore # noqa: PLR0915 if isinstance(body_dict, dict): if isinstance(body_dict.get("error"), dict): azure_error_code = body_dict["error"].get("code") # type: ignore[index] + # Also check inner_error for + # ResponsibleAIPolicyViolation which indicates a + # content policy violation even when the top-level + # code is generic (e.g. "invalid_request_error"). + if azure_error_code != "content_policy_violation": + _inner = ( + body_dict["error"].get("inner_error") # type: ignore[index] + or body_dict["error"].get("innererror") # type: ignore[index] + ) + if isinstance(_inner, dict) and _inner.get( + "code" + ) == "ResponsibleAIPolicyViolation": + azure_error_code = "content_policy_violation" else: azure_error_code = body_dict.get("code") except Exception: diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 718773a1b16..8ab4ec15b07 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -51,7 +51,7 @@ def handle_cohere_chat_model_custom_llm_provider( if custom_llm_provider == "cohere" and model in litellm.cohere_chat_models: return model, "cohere_chat" - if "/" in model: + if model and "/" in model: _custom_llm_provider, _model = model.split("/", 1) if ( _custom_llm_provider @@ -84,7 +84,7 @@ def handle_anthropic_text_model_custom_llm_provider( ): return model, "anthropic_text" - if "/" in model: + if model and "/" in model: _custom_llm_provider, _model = model.split("/", 1) if ( _custom_llm_provider @@ -113,6 +113,12 @@ def get_llm_provider( # noqa: PLR0915 Return model, custom_llm_provider, dynamic_api_key, api_base """ try: + # Early validation - model is required + if model is None: + raise ValueError( + "model parameter is required but was None. Please provide a valid model name." + ) + if litellm.LiteLLMProxyChatConfig._should_use_litellm_proxy_by_default( litellm_params=litellm_params ): diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index acd08f0a569..9938cd7979b 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -664,35 +664,34 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): reasoning_effort: Optional[Union[REASONING_EFFORT, str]], model: str, ) -> Optional[AnthropicThinkingParam]: + if reasoning_effort is None or reasoning_effort == "none": + return None if AnthropicConfig._is_claude_opus_4_6(model): return AnthropicThinkingParam( type="adaptive", ) + elif reasoning_effort == "low": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, + ) + elif reasoning_effort == "medium": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + ) + elif reasoning_effort == "high": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, + ) + elif reasoning_effort == "minimal": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, + ) else: - if reasoning_effort is None: - return None - elif reasoning_effort == "low": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, - ) - elif reasoning_effort == "medium": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - ) - elif reasoning_effort == "high": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, - ) - elif reasoning_effort == "minimal": - return AnthropicThinkingParam( - type="enabled", - budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, - ) - else: - raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}") + raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}") def _extract_json_schema_from_response_format( self, value: Optional[dict] diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 76fa713ca8c..44ee51d14ab 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -901,7 +901,20 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): if response.json()["status"] == "failed": error_data = response.json() - raise AzureOpenAIError(status_code=400, message=json.dumps(error_data)) + # Preserve Azure error details (e.g. content_policy_violation, + # inner_error, content_filter_results) as structured body so + # exception_type() can route them correctly. + _error_body = error_data.get("error", error_data) + _error_msg = ( + _error_body.get("message", "Image generation failed") + if isinstance(_error_body, dict) + else json.dumps(error_data) + ) + raise AzureOpenAIError( + status_code=400, + message=_error_msg, + body=error_data, + ) result = response.json()["result"] return httpx.Response( @@ -999,7 +1012,20 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): if response.json()["status"] == "failed": error_data = response.json() - raise AzureOpenAIError(status_code=400, message=json.dumps(error_data)) + # Preserve Azure error details (e.g. content_policy_violation, + # inner_error, content_filter_results) as structured body so + # exception_type() can route them correctly. + _error_body = error_data.get("error", error_data) + _error_msg = ( + _error_body.get("message", "Image generation failed") + if isinstance(_error_body, dict) + else json.dumps(error_data) + ) + raise AzureOpenAIError( + status_code=400, + message=_error_msg, + body=error_data, + ) result = response.json()["result"] return httpx.Response( diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 2c041adba7a..477fa3316d1 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -246,6 +246,11 @@ class AmazonAnthropicClaudeMessagesConfig( "sonnet_4.5", "sonnet-4-5", "sonnet_4_5", + # Opus 4.6 + "opus-4.6", + "opus_4.6", + "opus-4-6", + "opus_4_6", ] return any(pattern in model_lower for pattern in supported_patterns) diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index 1b03ec47643..fb98006c7e4 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -4,7 +4,7 @@ import os import ssl import typing import urllib.request -from typing import Callable, Dict, Optional, Union +from typing import Any, Callable, Dict, Optional, Union import aiohttp import aiohttp.client_exceptions @@ -248,26 +248,25 @@ class LiteLLMAiohttpTransport(AiohttpTransport): # Only pass ssl kwarg when explicitly configured, to avoid # overriding the session/connector defaults with None (which is # not a valid value for aiohttp's ssl parameter). - ssl_kwargs: Dict[str, Union[bool, ssl.SSLContext]] = {} - if ssl_verify is not None: - ssl_kwargs["ssl"] = ssl_verify - - response = await client_session.request( - method=request.method, - url=YarlURL(str(request.url), encoded=True), - headers=request.headers, - data=data, - allow_redirects=False, - auto_decompress=False, - timeout=ClientTimeout( + request_kwargs: Dict[str, Any] = { + "method": request.method, + "url": YarlURL(str(request.url), encoded=True), + "headers": request.headers, + "data": data, + "allow_redirects": False, + "auto_decompress": False, + "timeout": ClientTimeout( sock_connect=timeout.get("connect"), sock_read=timeout.get("read"), connect=timeout.get("pool"), ), - proxy=proxy, - server_hostname=sni_hostname, - **ssl_kwargs, - ).__aenter__() + "proxy": proxy, + "server_hostname": sni_hostname, + } + if ssl_verify is not None: + request_kwargs["ssl"] = ssl_verify + + response = await client_session.request(**request_kwargs).__aenter__() return response diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index cc2439b431a..5c870711062 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -240,24 +240,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): event_pydantic_model = OpenAIResponsesAPIConfig.get_event_model_class( event_type=event_type ) - # Defensive: Some OpenAI-compatible providers may send `error.code: null`. - # Pydantic will raise a ValidationError when it expects a string but gets None. - # Coalesce a None `error.code` to a stable default string so streaming - # iteration does not crash (see issue report). This keeps behavior similar - # to previous fixes (coalesce before validation) and lets higher-level - # handlers still receive an `ErrorEvent` object. + # Some OpenAI-compatible providers send error.code: null; coalesce so validation succeeds. try: error_obj = parsed_chunk.get("error") if isinstance(error_obj, dict) and error_obj.get("code") is None: - # Preserve other fields, but ensure `code` is a non-null string parsed_chunk = dict(parsed_chunk) parsed_chunk["error"] = dict(error_obj) parsed_chunk["error"]["code"] = "unknown_error" except Exception: - # If anything unexpected happens here, fall back to attempting - # instantiation and let higher-level handlers manage errors. verbose_logger.debug("Failed to coalesce error.code in parsed_chunk") - return event_pydantic_model(**parsed_chunk) @staticmethod @@ -307,6 +298,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ResponsesAPIStreamEvents.MCP_CALL_FAILED: MCPCallFailedEvent, ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE: ImageGenerationPartialImageEvent, ResponsesAPIStreamEvents.ERROR: ErrorEvent, + # Shell tool events: passthrough as GenericEvent so payload is preserved + ResponsesAPIStreamEvents.SHELL_CALL_IN_PROGRESS: GenericEvent, + ResponsesAPIStreamEvents.SHELL_CALL_COMPLETED: GenericEvent, + ResponsesAPIStreamEvents.SHELL_CALL_OUTPUT: GenericEvent, } model_class = event_models.get(cast(ResponsesAPIStreamEvents, event_type)) diff --git a/litellm/llms/sagemaker/embedding/transformation.py b/litellm/llms/sagemaker/embedding/transformation.py index bd8abc5e01a..04b201380fc 100644 --- a/litellm/llms/sagemaker/embedding/transformation.py +++ b/litellm/llms/sagemaker/embedding/transformation.py @@ -102,11 +102,18 @@ class SagemakerEmbeddingConfig(BaseEmbeddingConfig): status_code=raw_response.status_code ) - if "embedding" not in response_data: + # Handle both raw array format (TEI) and wrapped format (standard HF) + if isinstance(response_data, list): + # TEI and some HF models return raw embedding arrays directly + embeddings = response_data + elif isinstance(response_data, dict) and "embedding" in response_data: + # Standard HF format with "embedding" key + embeddings = response_data["embedding"] + else: raise SagemakerError( - status_code=500, message="HF response missing 'embedding' field" + status_code=500, + message=f"Unexpected response format. Expected list or dict with 'embedding' key, got: {type(response_data).__name__}", ) - embeddings = response_data["embedding"] if not isinstance(embeddings, list): raise SagemakerError( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f6edcf7efd0..d0556fc4e89 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -21432,6 +21432,36 @@ "max_input_tokens": 1000000, "max_output_tokens": 8192 }, + "minimax/MiniMax-M2.5": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, + "minimax/MiniMax-M2.5-lightning": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, "minimax/MiniMax-M2": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, @@ -30799,6 +30829,21 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/zai-org/glm-5-maas": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "vertex_ai/mistral-medium-3": { "input_cost_per_token": 4e-07, "litellm_provider": "vertex_ai-mistral_models", diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 548e3bc3dbf..ed4fb133478 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -536,10 +536,25 @@ class MCPRequestHandler: user_api_key_auth: Optional[UserAPIKeyAuth] = None, ) -> List[str]: try: - # Get key object permission (already loaded in main auth flow) + # Get key object permission (already loaded in main auth flow, or fetch from DB) key_object_permission = MCPRequestHandler._get_key_object_permission( user_api_key_auth ) + if key_object_permission is None and user_api_key_auth and user_api_key_auth.object_permission_id: + from litellm.proxy.auth.auth_checks import get_object_permission + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + if prisma_client is not None: + key_object_permission = await get_object_permission( + object_permission_id=user_api_key_auth.object_permission_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) if key_object_permission is None: return [] diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index fd251488db4..fbb0603100e 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -14,6 +14,7 @@ import re from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Union, cast from urllib.parse import urlparse +import anyio from fastapi import HTTPException from httpx import HTTPStatusError from mcp import ReadResourceResult, Resource @@ -30,6 +31,7 @@ from pydantic import AnyUrl import litellm from litellm._logging import verbose_logger +from litellm.constants import MCP_NPM_CACHE_DIR from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.experimental_mcp_client.client import MCPClient from litellm.llms.custom_httpx.http_handler import get_async_httpx_client @@ -70,7 +72,9 @@ try: from mcp.shared.tool_name_validation import ( validate_tool_name, # pyright: ignore[reportAssignmentType] ) - from mcp.shared.tool_name_validation import SEP_986_URL + from mcp.shared.tool_name_validation import ( + SEP_986_URL, + ) except ImportError: from pydantic import BaseModel @@ -887,8 +891,15 @@ class MCPServerManager: # Handle stdio transport if transport == MCPTransport.stdio: - # For stdio, we need to get the stdio config from the server - resolved_env = stdio_env if stdio_env is not None else server.env or {} + resolved_env = stdio_env if stdio_env is not None else dict(server.env or {}) + + # Ensure npm-based STDIO MCP servers have a writable cache dir. + # In containers the default (~/.npm or /app/.npm) may not exist + # or be read-only, causing npx to fail with ENOENT. + if "NPM_CONFIG_CACHE" not in resolved_env: + from litellm.constants import MCP_NPM_CACHE_DIR + + resolved_env["NPM_CONFIG_CACHE"] = MCP_NPM_CACHE_DIR stdio_config: Optional[MCPStdioConfig] = None if server.command and server.args is not None: stdio_config = MCPStdioConfig( @@ -1437,6 +1448,9 @@ class MCPServerManager: """ Fetch tools from MCP client with timeout and error handling. + Uses anyio.fail_after() instead of asyncio.wait_for() to avoid conflicts + with the MCP SDK's anyio TaskGroup. See GitHub issue #20715 for details. + Args: client: MCP client instance server_name: Name of the server for logging @@ -1444,24 +1458,12 @@ class MCPServerManager: Returns: List of tools from the server """ - - async def _list_tools_task(): - try: + try: + with anyio.fail_after(30.0): tools = await client.list_tools() verbose_logger.debug(f"Tools from {server_name}: {tools}") return tools - except asyncio.CancelledError: - verbose_logger.warning(f"Client operation cancelled for {server_name}") - return [] - except Exception as e: - verbose_logger.warning( - f"Client operation failed for {server_name}: {str(e)}" - ) - return [] - - try: - return await asyncio.wait_for(_list_tools_task(), timeout=30.0) - except asyncio.TimeoutError: + except TimeoutError: verbose_logger.warning(f"Timeout while listing tools from {server_name}") return [] except asyncio.CancelledError: @@ -2481,6 +2483,9 @@ class MCPServerManager: except asyncio.TimeoutError: health_check_error = "Health check timed out after 10 seconds" status = "unhealthy" + except asyncio.CancelledError: + health_check_error = "Health check was cancelled" + status = "unknown" except Exception as e: health_check_error = str(e) status = "unhealthy" diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 58cd8c99e7b..fa28b6070b3 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -41,6 +41,9 @@ from litellm.proxy._experimental.mcp_server.utils import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, +) from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall @@ -842,6 +845,7 @@ if MCP_AVAILABLE: raw_headers: Optional[Dict[str, str]] = None, log_list_tools_to_spendlogs: bool = False, list_tools_log_source: Optional[str] = None, + litellm_trace_id: Optional[str] = None, ) -> List[MCPTool]: """ Helper method to fetch tools from MCP servers based on server filtering criteria. @@ -879,6 +883,7 @@ if MCP_AVAILABLE: "model": "MCP: list_tools", "call_type": CallTypes.list_mcp_tools.value, "litellm_call_id": list_tools_call_id, + "litellm_trace_id": litellm_trace_id, "metadata": { "spend_logs_metadata": spend_logs_metadata, }, @@ -894,13 +899,14 @@ if MCP_AVAILABLE: ], } - # Attach user identifiers when available (matches call_mcp_tool style) + # Attach user identifiers using the standard helper if user_api_key_auth is not None: - user_api_key = getattr(user_api_key_auth, "api_key", None) - if user_api_key: - cast(dict, list_tools_request_data["metadata"])[ - "user_api_key" - ] = user_api_key + + LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( + data=list_tools_request_data, + user_api_key_dict=user_api_key_auth, + _metadata_variable_name="metadata", + ) user_identifier = getattr( user_api_key_auth, "end_user_id", None diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 87ff4a66e08..de785b221bc 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -514,6 +514,8 @@ class LiteLLMRoutes(enum.Enum): "/user/delete", "/user/info", "/user/list", + "/user/daily/activity", + "/user/daily/activity/aggregated", # team "/team/new", "/team/update", @@ -526,6 +528,7 @@ class LiteLLMRoutes(enum.Enum): "/team/available", "/team/permissions_list", "/team/permissions_update", + "/team/daily/activity", # model "/model/new", "/model/update", @@ -893,6 +896,7 @@ class KeyRequestBase(GenerateRequestBase): Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] ] = None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm router_settings: Optional[UpdateRouterConfig] = None + access_group_ids: Optional[List[str]] = None class LiteLLMKeyType(str, enum.Enum): @@ -1502,6 +1506,7 @@ class TeamBase(LiteLLMPydanticObjectBase): models: list = [] blocked: bool = False router_settings: Optional[dict] = None + access_group_ids: Optional[List[str]] = None class NewTeamRequest(TeamBase): @@ -1589,6 +1594,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): model_tpm_limit: Optional[Dict[str, int]] = None allowed_vector_store_indexes: Optional[List[AllowedVectorStoreIndexItem]] = None router_settings: Optional[dict] = None + access_group_ids: Optional[List[str]] = None class ResetTeamBudgetRequest(LiteLLMPydanticObjectBase): @@ -2177,6 +2183,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase): updated_by: Optional[str] = None object_permission_id: Optional[str] = None object_permission: Optional[LiteLLM_ObjectPermissionTable] = None + access_group_ids: Optional[List[str]] = None rotation_count: Optional[int] = 0 # Number of times key has been rotated auto_rotate: Optional[bool] = False # Whether this key should be auto-rotated rotation_interval: Optional[str] = None # How often to rotate (e.g., "30d", "90d") diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f33b2412260..a02bc7f9e51 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -282,7 +282,7 @@ def _override_openai_response_model( if isinstance(response_obj, dict): downstream_model = response_obj.get("model") if downstream_model != requested_model: - verbose_proxy_logger.warning( + verbose_proxy_logger.debug( "%s: response model mismatch - requested=%r downstream=%r. Overriding response['model'] to requested model.", log_context, requested_model, @@ -301,7 +301,7 @@ def _override_openai_response_model( downstream_model = getattr(response_obj, "model", None) if downstream_model != requested_model: - verbose_proxy_logger.warning( + verbose_proxy_logger.debug( "%s: response model mismatch - requested=%r downstream=%r. Overriding response.model to requested model.", log_context, requested_model, diff --git a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py index c96564252d0..b41ff121622 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py @@ -112,7 +112,8 @@ class SpendUpdateQueue(BaseUpdateQueue): for update in updates: _key = f"{update.get('entity_type')}:{update.get('entity_id')}" if _key not in _in_memory_map: - _in_memory_map[_key] = update + # avoid mutating caller-owned dicts while aggregating queue entries + _in_memory_map[_key] = update.copy() else: current_cost = _in_memory_map[_key].get("response_cost", 0) or 0 update_cost = update.get("response_cost", 0) or 0 diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index 9018675d7a5..9cded6f0ac2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -5,10 +5,12 @@ # +-------------------------------------------------------------+ # Thank you users! We ❤️ you! - Krrish & Ishaan +import fnmatch import os from typing import TYPE_CHECKING, Any, Dict, Literal, Optional from litellm._logging import verbose_proxy_logger +from litellm._version import version as litellm_version from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_guardrail import ( CustomGuardrail, @@ -31,6 +33,110 @@ if TYPE_CHECKING: GUARDRAIL_NAME = "generic_guardrail_api" +# Headers whose values are forwarded as-is (case-insensitive). Glob patterns supported (e.g. x-stainless-*, x-litellm*). +_HEADER_VALUE_ALLOWLIST = frozenset({ + "host", + "accept-encoding", + "connection", + "accept", + "content-type", + "user-agent", + "x-stainless-*", + "x-litellm-*", + "content-length", +}) + +# Placeholder for headers that exist but are not on the allowlist (we don't expose their value). +_HEADER_PRESENT_PLACEHOLDER = "[present]" + + +def _header_value_allowed(header_name: str) -> bool: + """Return True if this header's value may be forwarded (allowlist, including globs).""" + lower = header_name.lower() + if lower in _HEADER_VALUE_ALLOWLIST: + return True + for pattern in _HEADER_VALUE_ALLOWLIST: + if "*" in pattern and fnmatch.fnmatch(lower, pattern): + return True + return False + + +def _sanitize_inbound_headers(headers: Any) -> Optional[Dict[str, str]]: + """ + Sanitize inbound headers before passing them to a 3rd party guardrail service. + + - Allowlist: only headers in the allowlist have their values forwarded (exact + glob: x-stainless-*, x-litellm-*). + - All other headers are included with value "[present]" so the guardrail knows the header existed. + - Coerces values to str (for JSON serialization). + """ + if not headers or not isinstance(headers, dict): + return None + + sanitized: Dict[str, str] = {} + for k, v in headers.items(): + if k is None: + continue + key = str(k) + if _header_value_allowed(key): + try: + sanitized[key] = str(v) + except Exception: + continue + else: + sanitized[key] = _HEADER_PRESENT_PLACEHOLDER + + return sanitized or None + + +def _extract_inbound_headers( + request_data: dict, logging_obj: Optional["LiteLLMLoggingObj"] +) -> Optional[Dict[str, str]]: + """ + Extract inbound headers from available request context. + + We try multiple locations to support different call paths: + - proxy endpoints: request_data["proxy_server_request"]["headers"] + - if the guardrail is passed the proxy_server_request object directly + - metadata headers captured in litellm_pre_call_utils + - response hooks: fallback to logging_obj.model_call_details + """ + # 1) Most common path (proxy): full request context in proxy_server_request + headers = request_data.get("proxy_server_request", {}).get("headers") + if headers: + return _sanitize_inbound_headers(headers) + + # 2) Some guardrails pass proxy_server_request as request_data itself + headers = request_data.get("headers") + if headers: + return _sanitize_inbound_headers(headers) + + # 3) Pre-call: headers stored in request metadata + metadata_headers = (request_data.get("metadata") or {}).get("headers") + if metadata_headers: + return _sanitize_inbound_headers(metadata_headers) + + litellm_metadata_headers = (request_data.get("litellm_metadata") or {}).get( + "headers" + ) + if litellm_metadata_headers: + return _sanitize_inbound_headers(litellm_metadata_headers) + + # 4) Post-call: headers not present on response; fallback to logging object + if logging_obj and getattr(logging_obj, "model_call_details", None): + try: + details = logging_obj.model_call_details or {} + headers = ( + details.get("litellm_params", {}) + .get("metadata", {}) + .get("headers", None) + ) + if headers: + return _sanitize_inbound_headers(headers) + except Exception: + pass + + return None + class GenericGuardrailAPI(CustomGuardrail): """ @@ -207,6 +313,7 @@ class GenericGuardrailAPI(CustomGuardrail): # Extract user API key metadata user_metadata = self._extract_user_api_key_metadata(request_data) + inbound_headers = _extract_inbound_headers(request_data=request_data, logging_obj=logging_obj) # Create request payload guardrail_request = GenericGuardrailAPIRequest( @@ -214,6 +321,8 @@ class GenericGuardrailAPI(CustomGuardrail): litellm_trace_id=logging_obj.litellm_trace_id if logging_obj else None, texts=texts, request_data=user_metadata, + request_headers=inbound_headers, + litellm_version=litellm_version, images=images, tools=tools, structured_messages=structured_messages, diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index 9d64aea8910..9b2b43c6fcb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -29,10 +29,7 @@ from fastapi import HTTPException from litellm import Router from litellm._logging import verbose_proxy_logger -from litellm.integrations.custom_guardrail import ( - CustomGuardrail, - log_guardrail_information, -) +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ModelResponseStream @@ -1056,7 +1053,6 @@ class ContentFilterGuardrail(CustomGuardrail): masked_entity_count=masked_entity_count, ) - @log_guardrail_information async def apply_guardrail( self, inputs: "GenericGuardrailAPIInputs", diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 38462094b11..2d3f048f81b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -330,6 +330,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): end_time: Optional[float] = None, duration: Optional[float] = None, event_type: Optional[GuardrailEventHooks] = None, + original_inputs: Optional[dict] = None, ): """ Override to store only the Model Armor API response, not the entire data dict. diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py new file mode 100644 index 00000000000..33b81e85654 --- /dev/null +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -0,0 +1,264 @@ +from typing import List + +from fastapi import APIRouter, Depends, HTTPException, status + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.utils import get_prisma_client_or_throw +from litellm.types.access_group import ( + AccessGroupCreateRequest, + AccessGroupResponse, + AccessGroupUpdateRequest, +) + +router = APIRouter( + tags=["access group management"], +) + + +def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={"error": CommonProxyErrors.not_allowed_access.value}, + ) + + +def _record_to_response(record) -> AccessGroupResponse: + return AccessGroupResponse( + access_group_id=record.access_group_id, + access_group_name=record.access_group_name, + description=record.description, + access_model_ids=record.access_model_ids, + access_mcp_server_ids=record.access_mcp_server_ids, + access_agent_ids=record.access_agent_ids, + assigned_team_ids=record.assigned_team_ids, + assigned_key_ids=record.assigned_key_ids, + created_at=record.created_at, + created_by=record.created_by, + updated_at=record.updated_at, + updated_by=record.updated_by, + ) + + +@router.post( + "/v1/access_group", + response_model=AccessGroupResponse, + status_code=status.HTTP_201_CREATED, +) +async def create_access_group( + data: AccessGroupCreateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> AccessGroupResponse: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + existing = await prisma_client.db.litellm_accessgrouptable.find_unique( + where={"access_group_name": data.access_group_name} + ) + if existing is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Access group '{data.access_group_name}' already exists", + ) + + try: + record = await prisma_client.db.litellm_accessgrouptable.create( + data={ + "access_group_name": data.access_group_name, + "description": data.description, + "access_model_ids": data.access_model_ids or [], + "access_mcp_server_ids": data.access_mcp_server_ids or [], + "access_agent_ids": data.access_agent_ids or [], + "assigned_team_ids": data.assigned_team_ids or [], + "assigned_key_ids": data.assigned_key_ids or [], + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, + } + ) + except Exception as e: + # Race condition: another request created the same name between find_unique and create. + # Prisma raises UniqueViolationError (P2002) or similar for unique constraint. + if "unique constraint" in str(e).lower() or "P2002" in str(e): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Access group '{data.access_group_name}' already exists", + ) + raise + return _record_to_response(record) + + +@router.get( + "/v1/access_group", + response_model=List[AccessGroupResponse], +) +async def list_access_groups( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> List[AccessGroupResponse]: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + records = await prisma_client.db.litellm_accessgrouptable.find_many( + order={"created_at": "desc"} + ) + return [_record_to_response(r) for r in records] + + +@router.get( + "/v1/access_group/{access_group_id}", + response_model=AccessGroupResponse, +) +async def get_access_group( + access_group_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> AccessGroupResponse: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + record = await prisma_client.db.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if record is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + return _record_to_response(record) + + +@router.put( + "/v1/access_group/{access_group_id}", + response_model=AccessGroupResponse, +) +async def update_access_group( + access_group_id: str, + data: AccessGroupUpdateRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> AccessGroupResponse: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + existing = await prisma_client.db.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if existing is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + + update_data: dict = {"updated_by": user_api_key_dict.user_id} + for field, value in data.model_dump(exclude_unset=True).items(): + update_data[field] = value + + record = await prisma_client.db.litellm_accessgrouptable.update( + where={"access_group_id": access_group_id}, + data=update_data, + ) + return _record_to_response(record) + + +@router.delete( + "/v1/access_group/{access_group_id}", + status_code=status.HTTP_204_NO_CONTENT, +) +async def delete_access_group( + access_group_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> None: + _require_proxy_admin(user_api_key_dict) + prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) + + try: + async with prisma_client.db.tx() as tx: + existing = await tx.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if existing is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + + # Remove access_group_id from teams and keys that reference it + teams_with_group = await tx.litellm_teamtable.find_many( + where={"access_group_ids": {"hasSome": [access_group_id]}} + ) + for team in teams_with_group: + updated_ids = [tid for tid in (team.access_group_ids or []) if tid != access_group_id] + await tx.litellm_teamtable.update( + where={"team_id": team.team_id}, + data={"access_group_ids": updated_ids}, + ) + + keys_with_group = await tx.litellm_verificationtoken.find_many( + where={"access_group_ids": {"hasSome": [access_group_id]}} + ) + for key in keys_with_group: + updated_ids = [kid for kid in (key.access_group_ids or []) if kid != access_group_id] + await tx.litellm_verificationtoken.update( + where={"token": key.token}, + data={"access_group_ids": updated_ids}, + ) + + await tx.litellm_accessgrouptable.delete( + where={"access_group_id": access_group_id} + ) + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception( + "delete_access_group failed: access_group_id=%s error=%s", + access_group_id, + e, + ) + if PrismaDBExceptionHandler.is_database_connection_error(e): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=CommonProxyErrors.db_not_connected_error.value, + ) + if "P2025" in str(e) or ("record" in str(e).lower() and "not found" in str(e).lower()): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to delete access group. Please try again.", + ) + + +# Alias routes for /v1/unified_access_group +router.add_api_route( + "/v1/unified_access_group", + create_access_group, + methods=["POST"], + response_model=AccessGroupResponse, + status_code=status.HTTP_201_CREATED, +) +router.add_api_route( + "/v1/unified_access_group", + list_access_groups, + methods=["GET"], + response_model=List[AccessGroupResponse], +) +router.add_api_route( + "/v1/unified_access_group/{access_group_id}", + get_access_group, + methods=["GET"], + response_model=AccessGroupResponse, +) +router.add_api_route( + "/v1/unified_access_group/{access_group_id}", + update_access_group, + methods=["PUT"], + response_model=AccessGroupResponse, +) +router.add_api_route( + "/v1/unified_access_group/{access_group_id}", + delete_access_group, + methods=["DELETE"], + status_code=status.HTTP_204_NO_CONTENT, +) diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 942758e3bab..d6f6d1337a3 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -383,6 +383,17 @@ def _update_metadata_field(updated_kv: dict, field_name: str) -> None: updated_kv["metadata"] = {field_name: _value} +def _has_non_empty_value(value: Any) -> bool: + """Check if a value has real content (not None, not empty list, not blank string).""" + if value is None: + return False + if isinstance(value, list) and len(value) == 0: + return False + if isinstance(value, str) and value.strip() == "": + return False + return True + + def _update_metadata_fields(updated_kv: dict) -> None: """ Helper function to update all metadata fields (both premium and standard). @@ -391,7 +402,7 @@ def _update_metadata_fields(updated_kv: dict) -> None: updated_kv: The key-value dict being used for the update """ for field in LiteLLM_ManagementEndpoint_MetadataFields_Premium: - if field in updated_kv and updated_kv[field] is not None: + if field in updated_kv and _has_non_empty_value(updated_kv[field]): _update_metadata_field(updated_kv=updated_kv, field_name=field) for field in LiteLLM_ManagementEndpoint_MetadataFields: diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 597521ae773..8c4d4e7937e 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -10,10 +10,13 @@ Endpoints here: - DELETE `/v1/mcp/server/{server_id}` - Deletes the mcp server given `server_id`. - GET `/v1/mcp/tools - lists all the tools available for a key - GET `/v1/mcp/access_groups` - lists all available MCP access groups +- GET `/v1/mcp/discover` - Returns curated list of well-known MCP servers for discovery UI """ import importlib +import json +import os from dataclasses import dataclass from datetime import datetime, timedelta from typing import Any, Dict, Iterable, List, Literal, Optional @@ -1176,3 +1179,88 @@ if MCP_AVAILABLE: except Exception as e: verbose_proxy_logger.exception(f"Error making agent public: {e}") raise HTTPException(status_code=500, detail=str(e)) + + # --- MCP Discovery --- + + _MCP_REGISTRY_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "mcp_registry.json", + ) + + _mcp_registry_cache: Optional[Dict[str, Any]] = None + + def _load_mcp_registry() -> Dict[str, Any]: + """Load the curated MCP registry from disk. Cached after first read.""" + global _mcp_registry_cache + if _mcp_registry_cache is not None: + return _mcp_registry_cache + try: + with open(_MCP_REGISTRY_PATH, "r") as f: + data: Dict[str, Any] = json.load(f) + except Exception as e: + verbose_proxy_logger.warning( + f"Failed to load MCP registry from {_MCP_REGISTRY_PATH}: {e}" + ) + data = {"servers": []} + _mcp_registry_cache = data + return data + + @router.get( + "/discover", + description="Returns a curated list of well-known MCP servers for discovery UI", + dependencies=[Depends(user_api_key_auth)], + ) + async def discover_mcp_servers( + query: Optional[str] = Query( + None, description="Search filter for server names and descriptions" + ), + category: Optional[str] = Query( + None, description="Filter by category" + ), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + """ + Returns a curated list of well-known MCP servers that can be added to the proxy. + + Used by the UI to show a discovery grid when adding new MCP servers. + """ + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail={ + "error": "Only proxy admins can access MCP discovery. Your role={}".format( + user_api_key_dict.user_role + ) + }, + ) + + registry = _load_mcp_registry() + servers = registry.get("servers", []) + + # Apply query filter + if query: + query_lower = query.lower() + servers = [ + s + for s in servers + if query_lower in s.get("name", "").lower() + or query_lower in s.get("title", "").lower() + or query_lower in s.get("description", "").lower() + ] + + # Apply category filter + if category: + servers = [ + s for s in servers if s.get("category", "") == category + ] + + # Extract unique categories from the full list (before filtering) + all_servers = registry.get("servers", []) + categories = sorted( + set(s.get("category", "Other") for s in all_servers) + ) + + return { + "servers": servers, + "categories": categories, + } diff --git a/litellm/proxy/mcp_registry.json b/litellm/proxy/mcp_registry.json new file mode 100644 index 00000000000..2e1e8f64eae --- /dev/null +++ b/litellm/proxy/mcp_registry.json @@ -0,0 +1,426 @@ +{ + "servers": [ + { + "name": "github", + "title": "GitHub", + "description": "Manage repos, issues, PRs, and workflows through natural language", + "icon_url": "https://cdn.simpleicons.org/github", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.github%2Fgithub-mcp-server", + "transport": "http", + "url": "https://api.githubcopilot.com/mcp/", + "env_vars": [ + {"name": "GITHUB_PERSONAL_ACCESS_TOKEN", "description": "GitHub Personal Access Token", "secret": true} + ] + }, + { + "name": "gitlab", + "title": "GitLab", + "description": "Official GitLab MCP Server for project and repository management", + "icon_url": "https://cdn.simpleicons.org/gitlab", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.gitlab%2Fmcp", + "transport": "http", + "url": "https://gitlab.com/api/v4/mcp", + "env_vars": [ + {"name": "GITLAB_PERSONAL_ACCESS_TOKEN", "description": "GitLab Personal Access Token", "secret": true} + ] + }, + { + "name": "atlassian", + "title": "Atlassian (Jira & Confluence)", + "description": "Jira issues, Confluence pages, and Atlassian product integration", + "icon_url": "https://cdn.simpleicons.org/atlassian", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.atlassian%2Fatlassian-mcp-server", + "transport": "sse", + "url": "https://mcp.atlassian.com/v1/sse", + "env_vars": [] + }, + { + "name": "linear", + "title": "Linear", + "description": "Issue tracking, project management, and team workflow automation", + "icon_url": "https://cdn.simpleicons.org/linear", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/app.linear%2Flinear", + "transport": "sse", + "url": "https://mcp.linear.app/sse", + "env_vars": [] + }, + { + "name": "sentry", + "title": "Sentry", + "description": "Error monitoring, issue tracking, and debugging for AI assistants", + "icon_url": "https://cdn.simpleicons.org/sentry", + "category": "Developer Tools", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.getsentry%2Fsentry-mcp", + "transport": "stdio", + "command": "npx", + "args": ["-y", "@sentry/mcp-server"], + "env_vars": [ + {"name": "SENTRY_ACCESS_TOKEN", "description": "Sentry Access Token", "secret": true} + ] + }, + { + "name": "slack", + "title": "Slack", + "description": "Channel management, messaging, and Slack workspace integration", + "icon_url": "https://cdn.simpleicons.org/slack", + "category": "Communication", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-slack"], + "env_vars": [ + {"name": "SLACK_BOT_TOKEN", "description": "Slack Bot User OAuth Token", "secret": true}, + {"name": "SLACK_TEAM_ID", "description": "Slack Team/Workspace ID", "secret": false} + ] + }, + { + "name": "discord", + "title": "Discord", + "description": "Discord server management, messaging, and bot integration", + "icon_url": "https://cdn.simpleicons.org/discord", + "category": "Communication", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-discord"], + "env_vars": [ + {"name": "DISCORD_BOT_TOKEN", "description": "Discord Bot Token", "secret": true} + ] + }, + { + "name": "postgresql", + "title": "PostgreSQL", + "description": "Query and manage PostgreSQL databases with read-only access", + "icon_url": "https://cdn.simpleicons.org/postgresql", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-postgres"], + "env_vars": [ + {"name": "POSTGRES_CONNECTION_STRING", "description": "PostgreSQL connection string (e.g., postgresql://user:pass@host:5432/db)", "secret": true} + ] + }, + { + "name": "sqlite", + "title": "SQLite", + "description": "Query and manage SQLite databases", + "icon_url": "https://cdn.simpleicons.org/sqlite", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-sqlite"], + "env_vars": [ + {"name": "SQLITE_DB_PATH", "description": "Path to SQLite database file", "secret": false} + ] + }, + { + "name": "mysql", + "title": "MySQL", + "description": "Query and manage MySQL databases", + "icon_url": "https://cdn.simpleicons.org/mysql", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-mysql"], + "env_vars": [ + {"name": "MYSQL_HOST", "description": "MySQL host", "secret": false}, + {"name": "MYSQL_USER", "description": "MySQL username", "secret": false}, + {"name": "MYSQL_PASSWORD", "description": "MySQL password", "secret": true}, + {"name": "MYSQL_DATABASE", "description": "MySQL database name", "secret": false} + ] + }, + { + "name": "mongodb", + "title": "MongoDB", + "description": "Query and manage MongoDB databases and collections", + "icon_url": "https://cdn.simpleicons.org/mongodb", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-mongodb"], + "env_vars": [ + {"name": "MONGODB_CONNECTION_STRING", "description": "MongoDB connection string", "secret": true} + ] + }, + { + "name": "redis", + "title": "Redis", + "description": "Interact with Redis key-value stores", + "icon_url": "https://cdn.simpleicons.org/redis", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-redis"], + "env_vars": [ + {"name": "REDIS_URL", "description": "Redis connection URL (e.g., redis://localhost:6379)", "secret": true} + ] + }, + { + "name": "snowflake", + "title": "Snowflake", + "description": "MCP Server for Snowflake from Snowflake Labs", + "icon_url": "https://cdn.simpleicons.org/snowflake", + "category": "Databases", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.Snowflake-Labs%2Fmcp", + "transport": "stdio", + "command": "uvx", + "args": ["snowflake-labs-mcp"], + "env_vars": [ + {"name": "SNOWFLAKE_ACCOUNT", "description": "Snowflake account identifier (e.g., xy12345.us-east-1)", "secret": false}, + {"name": "SNOWFLAKE_USER", "description": "Snowflake username", "secret": false}, + {"name": "SNOWFLAKE_PASSWORD", "description": "Snowflake password", "secret": true} + ] + }, + { + "name": "notion", + "title": "Notion", + "description": "Official Notion MCP server for pages and databases", + "icon_url": "https://cdn.simpleicons.org/notion", + "category": "Productivity", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.notion%2Fmcp", + "transport": "sse", + "url": "https://mcp.notion.com/sse", + "env_vars": [] + }, + { + "name": "google_drive", + "title": "Google Drive", + "description": "Search and access files in Google Drive", + "icon_url": "https://cdn.simpleicons.org/googledrive", + "category": "Productivity", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-gdrive"], + "env_vars": [ + {"name": "GOOGLE_CLIENT_ID", "description": "Google OAuth Client ID", "secret": false}, + {"name": "GOOGLE_CLIENT_SECRET", "description": "Google OAuth Client Secret", "secret": true} + ] + }, + { + "name": "google_calendar", + "title": "Google Calendar", + "description": "Manage events and calendars in Google Calendar", + "icon_url": "https://cdn.simpleicons.org/googlecalendar", + "category": "Productivity", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-google-calendar"], + "env_vars": [ + {"name": "GOOGLE_CLIENT_ID", "description": "Google OAuth Client ID", "secret": false}, + {"name": "GOOGLE_CLIENT_SECRET", "description": "Google OAuth Client Secret", "secret": true} + ] + }, + { + "name": "obsidian", + "title": "Obsidian", + "description": "Read, search, and manage Obsidian vault notes and files", + "icon_url": "https://cdn.simpleicons.org/obsidian", + "category": "Productivity", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-obsidian"], + "env_vars": [ + {"name": "OBSIDIAN_VAULT_PATH", "description": "Path to Obsidian vault directory", "secret": false} + ] + }, + { + "name": "brave_search", + "title": "Brave Search", + "description": "Web results, images, videos, and AI summaries via Brave Search API", + "icon_url": "https://cdn.simpleicons.org/brave", + "category": "Search", + "registry_url": "https://registry.modelcontextprotocol.io/servers/io.github.brave%2Fbrave-search-mcp-server", + "transport": "stdio", + "command": "npx", + "args": ["-y", "@brave/brave-search-mcp-server"], + "env_vars": [ + {"name": "BRAVE_API_KEY", "description": "Brave Search API Key", "secret": true} + ] + }, + { + "name": "exa", + "title": "Exa", + "description": "Fast, intelligent web search and web crawling", + "icon_url": "https://cdn.simpleicons.org/exa", + "category": "Search", + "registry_url": "https://registry.modelcontextprotocol.io/servers/ai.exa%2Fexa", + "transport": "http", + "url": "https://mcp.exa.ai/mcp", + "env_vars": [ + {"name": "EXA_API_KEY", "description": "Exa API Key", "secret": true} + ] + }, + { + "name": "tavily", + "title": "Tavily", + "description": "AI-optimized search engine for research and retrieval", + "icon_url": "https://cdn.simpleicons.org/tavily", + "category": "Search", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-tavily"], + "env_vars": [ + {"name": "TAVILY_API_KEY", "description": "Tavily API Key", "secret": true} + ] + }, + { + "name": "puppeteer", + "title": "Puppeteer", + "description": "Browser automation, web scraping, and screenshot capture", + "icon_url": "https://cdn.simpleicons.org/puppeteer", + "category": "Web & Browser", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-puppeteer"], + "env_vars": [] + }, + { + "name": "playwright", + "title": "Playwright", + "description": "Browser automation and testing with Playwright", + "icon_url": "https://cdn.simpleicons.org/playwright", + "category": "Web & Browser", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-playwright"], + "env_vars": [] + }, + { + "name": "browserbase", + "title": "Browserbase", + "description": "Cloud browser automation and session management", + "icon_url": "https://cdn.simpleicons.org/browserbase", + "category": "Web & Browser", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-browserbase"], + "env_vars": [ + {"name": "BROWSERBASE_API_KEY", "description": "Browserbase API Key", "secret": true}, + {"name": "BROWSERBASE_PROJECT_ID", "description": "Browserbase Project ID", "secret": false} + ] + }, + { + "name": "aws", + "title": "AWS", + "description": "Interact with Amazon Web Services resources and APIs", + "icon_url": "https://cdn.simpleicons.org/amazonaws", + "category": "Cloud", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-aws"], + "env_vars": [ + {"name": "AWS_ACCESS_KEY_ID", "description": "AWS Access Key ID", "secret": true}, + {"name": "AWS_SECRET_ACCESS_KEY", "description": "AWS Secret Access Key", "secret": true}, + {"name": "AWS_REGION", "description": "AWS Region (e.g., us-east-1)", "secret": false} + ] + }, + { + "name": "cloudflare", + "title": "Cloudflare", + "description": "Manage Cloudflare Workers, KV, R2, D1, and more", + "icon_url": "https://cdn.simpleicons.org/cloudflare", + "category": "Cloud", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.cloudflare.mcp%2Fmcp", + "transport": "sse", + "url": "https://bindings.mcp.cloudflare.com/sse", + "env_vars": [] + }, + { + "name": "filesystem", + "title": "Filesystem", + "description": "Read, write, and manage files and directories on disk", + "icon_url": "https://cdn.simpleicons.org/files", + "category": "System", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem"], + "env_vars": [] + }, + { + "name": "docker", + "title": "Docker", + "description": "Manage Docker containers, images, and networks", + "icon_url": "https://cdn.simpleicons.org/docker", + "category": "System", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-docker"], + "env_vars": [] + }, + { + "name": "stripe", + "title": "Stripe", + "description": "Manage payments, customers, and subscriptions via Stripe", + "icon_url": "https://cdn.simpleicons.org/stripe", + "category": "Finance", + "registry_url": "https://registry.modelcontextprotocol.io/servers/com.stripe%2Fmcp", + "transport": "http", + "url": "https://mcp.stripe.com", + "env_vars": [] + }, + { + "name": "shopify", + "title": "Shopify", + "description": "Manage Shopify stores, products, orders, and customers", + "icon_url": "https://cdn.simpleicons.org/shopify", + "category": "E-Commerce", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-shopify"], + "env_vars": [ + {"name": "SHOPIFY_ACCESS_TOKEN", "description": "Shopify Admin API Access Token", "secret": true}, + {"name": "SHOPIFY_STORE_URL", "description": "Shopify Store URL (e.g., mystore.myshopify.com)", "secret": false} + ] + }, + { + "name": "twilio", + "title": "Twilio", + "description": "Send SMS, make calls, and manage communication via Twilio", + "icon_url": "https://cdn.simpleicons.org/twilio", + "category": "Communication", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-twilio"], + "env_vars": [ + {"name": "TWILIO_ACCOUNT_SID", "description": "Twilio Account SID", "secret": false}, + {"name": "TWILIO_AUTH_TOKEN", "description": "Twilio Auth Token", "secret": true} + ] + }, + { + "name": "supabase", + "title": "Supabase", + "description": "Manage Supabase projects, databases, and storage", + "icon_url": "https://cdn.simpleicons.org/supabase", + "category": "Databases", + "registry_url": null, + "transport": "stdio", + "command": "npx", + "args": ["-y", "@anthropic/mcp-server-supabase"], + "env_vars": [ + {"name": "SUPABASE_URL", "description": "Supabase Project URL", "secret": false}, + {"name": "SUPABASE_SERVICE_ROLE_KEY", "description": "Supabase Service Role Key", "secret": true} + ] + } + ] +} diff --git a/litellm/proxy/policy_engine/policy_resolve_endpoints.py b/litellm/proxy/policy_engine/policy_resolve_endpoints.py index eb4d3fc5845..318e990ff12 100644 --- a/litellm/proxy/policy_engine/policy_resolve_endpoints.py +++ b/litellm/proxy/policy_engine/policy_resolve_endpoints.py @@ -12,6 +12,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query from litellm._logging import verbose_proxy_logger from litellm.constants import MAX_POLICY_ESTIMATE_IMPACT_ROWS from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry from litellm.proxy.policy_engine.policy_registry import get_policy_registry @@ -85,7 +86,6 @@ def _filter_keys_by_tags(keys: list, tag_patterns: list) -> tuple: Returns (named_aliases, unnamed_count). """ - from litellm.proxy.auth.route_checks import RouteChecks affected: list = [] unnamed_count = 0 @@ -111,7 +111,6 @@ def _filter_teams_by_tags(teams: list, tag_patterns: list) -> tuple: Returns (named_aliases, unnamed_count). """ - from litellm.proxy.auth.route_checks import RouteChecks affected: list = [] unnamed_count = 0 @@ -141,7 +140,6 @@ async def _find_affected_by_team_patterns( Returns (new_teams, new_keys, unnamed_keys_count). """ - from litellm.proxy.auth.route_checks import RouteChecks new_teams: list = [] matched_team_ids: list = [] @@ -178,7 +176,6 @@ async def _find_affected_keys_by_alias( prisma_client: object, key_patterns: list, existing_keys: list ) -> list: """Find keys whose alias matches the given patterns.""" - from litellm.proxy.auth.route_checks import RouteChecks affected: list = [] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6286d6dd1ca..63109916ab1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -393,6 +393,9 @@ from litellm.proxy.management_endpoints.tag_management_endpoints import ( from litellm.proxy.management_endpoints.team_callback_endpoints import ( router as team_callback_router, ) +from litellm.proxy.management_endpoints.access_group_endpoints import ( + router as access_group_router, +) from litellm.proxy.management_endpoints.team_endpoints import router as team_router from litellm.proxy.management_endpoints.team_endpoints import ( update_team, @@ -1051,98 +1054,236 @@ try: except FileNotFoundError: return False + def _validate_ui_directory(ui_path: str) -> bool: + """ + Verify UI directory has minimum required structure. + + Checks for: + - Directory exists + - Has index.html (main entry point) + - Has _next directory (Next.js assets) + + Returns True if UI directory appears valid and servable. + """ + if not os.path.isdir(ui_path): + return False + + # Must have main index.html + if not os.path.exists(os.path.join(ui_path, "index.html")): + return False + + # Must have _next directory with Next.js assets + next_dir = os.path.join(ui_path, "_next") + if not os.path.isdir(next_dir): + return False + + return True + + def _is_ui_pre_restructured(ui_dir: str) -> bool: + """ + Detect if UI directory is already pre-restructured and ready to serve. + + Returns True if: + 1. Marker file .litellm_ui_ready exists (created by Dockerfile), OR + 2. Restructuring pattern detected (subdirectories with index.html inside) + + This allows skipping copy/restructure operations on read-only filesystems. + """ + if not os.path.isdir(ui_dir): + return False + + # Primary signal: marker file created by Dockerfile + marker_file = os.path.join(ui_dir, ".litellm_ui_ready") + if os.path.exists(marker_file): + verbose_proxy_logger.debug(f"Found UI ready marker: {marker_file}") + return True + + # Fallback signal: Detect restructuring pattern + # After restructuring, routes exist as directories with index.html inside + # (e.g., login/index.html instead of login.html) + # Check for main index.html first (basic UI structure requirement) + if not os.path.exists(os.path.join(ui_dir, "index.html")): + return False + + # Look for ANY subdirectory with index.html (proves restructuring happened) + # Ignore directories starting with _ (Next.js internals like _next) + try: + for entry in os.scandir(ui_dir): + if entry.is_dir() and not entry.name.startswith("_"): + index_path = os.path.join(entry.path, "index.html") + if os.path.exists(index_path): + # Found at least one restructured route - this proves the pattern + verbose_proxy_logger.debug( + f"Detected restructured UI via pattern: found {entry.name}/index.html" + ) + return True + except (PermissionError, OSError) as e: + verbose_proxy_logger.debug( + f"Could not scan {ui_dir} for restructuring detection: {e}" + ) + return False + + # No restructured routes found + return False + + def _try_populate_ui_directory( + source_path: str, target_path: str + ) -> tuple[bool, str]: + """ + Attempt to populate target UI directory from source. + + Returns: (success: bool, error_message: str) + """ + try: + os.makedirs(target_path, exist_ok=True) + if not _dir_has_content(target_path) and _dir_has_content(source_path): + shutil.copytree( + source_path, + target_path, + dirs_exist_ok=True, + ) + verbose_proxy_logger.info(f"Successfully populated UI at {target_path}") + return True, "" + else: + return False, "Source or target directory state invalid" + except (PermissionError, OSError) as e: + return False, str(e) + # Use a writable runtime UI directory whenever possible. # This prevents mutating the packaged UI directory (e.g. site-packages or the repo checkout) # and ensures extensionless routes like /ui/login work via /index.html. is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - # Only use runtime UI path in Docker/non-root environments - # In local development, use the packaged UI directly + # Determine runtime UI path + # Priority: LITELLM_UI_PATH env var > default path based on is_non_root if is_non_root: - # Use /var/lib/litellm/ui for Docker (more secure than /tmp) - runtime_ui_path = "/var/lib/litellm/ui" + default_runtime_ui_path = "/var/lib/litellm/ui" + else: + default_runtime_ui_path = packaged_ui_path - if _dir_has_content(runtime_ui_path): + runtime_ui_path = os.getenv("LITELLM_UI_PATH", default_runtime_ui_path) + + # Validate packaged UI before proceeding + if not _validate_ui_directory(packaged_ui_path): + verbose_proxy_logger.error( + f"Packaged UI at {packaged_ui_path} is invalid or incomplete. " + f"UI may not function correctly." + ) + + # Decision tree for UI path selection: + # 1. If runtime path == packaged path: use packaged UI directly + # 2. If runtime UI exists and is pre-restructured: use it + # 3. If runtime UI exists but not restructured: use it (will restructure later) + # 4. If runtime UI missing: try to populate from packaged UI + # 4a. If population succeeds: use runtime UI + # 4b. If population fails: fall back to packaged UI + + should_use_runtime_path = runtime_ui_path != packaged_ui_path + + if should_use_runtime_path: + is_pre_restructured = _is_ui_pre_restructured(runtime_ui_path) + has_content = _dir_has_content(runtime_ui_path) + + # Case 2: Runtime UI exists and is ready + if has_content and is_pre_restructured: verbose_proxy_logger.info( - f"Using pre-built UI for non-root Docker: {runtime_ui_path}" + f"Using pre-restructured UI at {runtime_ui_path}" ) ui_path = runtime_ui_path + + # Case 3: Runtime UI exists but needs restructuring + elif has_content and not is_pre_restructured: + verbose_proxy_logger.warning( + f"UI at {runtime_ui_path} has content but is not properly restructured. " + f"Will attempt to restructure in place." + ) + ui_path = runtime_ui_path + + # Case 4: Runtime UI missing - try to populate else: - verbose_proxy_logger.error( - f"UI not found at {runtime_ui_path}. Attempting to populate it from packaged UI." - ) - verbose_proxy_logger.error( - f"Path exists: {os.path.exists(runtime_ui_path)}, Has content: {_dir_has_content(runtime_ui_path)}" + verbose_proxy_logger.info( + f"UI not found at {runtime_ui_path}. Attempting to populate from packaged UI." ) - try: - os.makedirs(runtime_ui_path, exist_ok=True) - if not _dir_has_content(runtime_ui_path) and _dir_has_content( - packaged_ui_path - ): - shutil.copytree( - packaged_ui_path, - runtime_ui_path, - dirs_exist_ok=True, - ) - except Exception as e: - verbose_proxy_logger.exception( - f"Failed to populate runtime UI directory {runtime_ui_path} from {packaged_ui_path}: {e}" - ) + success, error = _try_populate_ui_directory( + packaged_ui_path, runtime_ui_path + ) + + if success: + # Case 4a: Population succeeded + ui_path = runtime_ui_path else: - if _dir_has_content(runtime_ui_path): - verbose_proxy_logger.info( - f"Using populated UI for non-root Docker: {runtime_ui_path}" - ) - ui_path = runtime_ui_path + # Case 4b: Population failed - fall back to packaged UI + verbose_proxy_logger.warning( + f"Failed to populate UI at {runtime_ui_path}: {error}. " + f"Falling back to packaged UI at {packaged_ui_path}. " + f"For read-only deployments, pre-build UI in Dockerfile " + f"or set LITELLM_UI_PATH to a writable emptyDir volume." + ) + ui_path = packaged_ui_path else: - # Local development: use packaged UI directly, no runtime copy needed - verbose_proxy_logger.info( - f"Using packaged UI directory for local development: {packaged_ui_path}" - ) + # Case 1: Using packaged UI directly (local development) + verbose_proxy_logger.info(f"Using packaged UI directory: {packaged_ui_path}") ui_path = packaged_ui_path - # Only modify files if a custom server root path is set + + # Validate final UI path + if not _validate_ui_directory(ui_path): + verbose_proxy_logger.error( + f"Selected UI path {ui_path} is invalid or incomplete. UI may not work correctly." + ) + + # Only modify files if a custom server root path is set AND filesystem is writable if server_root_path and server_root_path != "/": - # Iterate through files in the UI directory - for root, dirs, files in os.walk(ui_path): - for filename in files: - file_path = os.path.join(root, filename) - # Skip binary files and files that don't need path replacement - if filename.endswith( - ( - ".png", - ".jpg", - ".jpeg", - ".gif", - ".ico", - ".woff", - ".woff2", - ".ttf", - ".eot", - ) - ): - continue - try: - with open(file_path, "r", encoding="utf-8") as f: - content = f.read() + # Check if UI path is writable + is_writable = os.access(ui_path, os.W_OK) - # Replace the asset prefix with the server root path - modified_content = content.replace( - f"{litellm_asset_prefix}", - f"{server_root_path}", - ) + if not is_writable: + verbose_proxy_logger.warning( + f"Cannot apply server_root_path replacements to UI at {ui_path}: " + f"path is not writable. Ensure server_root_path is '/' or pre-process " + f"UI files in Dockerfile with custom server_root_path." + ) + else: + # Iterate through files in the UI directory + for root, dirs, files in os.walk(ui_path): + for filename in files: + file_path = os.path.join(root, filename) + # Skip binary files and files that don't need path replacement + if filename.endswith( + ( + ".png", + ".jpg", + ".jpeg", + ".gif", + ".ico", + ".woff", + ".woff2", + ".ttf", + ".eot", + ) + ): + continue + try: + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() - # Replace the /.well-known/litellm-ui-config with the server root path - modified_content = modified_content.replace( - "/litellm/.well-known/litellm-ui-config", - f"{server_root_path}/.well-known/litellm-ui-config", - ) + # Replace the asset prefix with the server root path + modified_content = content.replace( + f"{litellm_asset_prefix}", + f"{server_root_path}", + ) - with open(file_path, "w", encoding="utf-8") as f: - f.write(modified_content) - except UnicodeDecodeError: - # Skip binary files that can't be decoded - continue + # Replace the /.well-known/litellm-ui-config with the server root path + modified_content = modified_content.replace( + "/litellm/.well-known/litellm-ui-config", + f"{server_root_path}/.well-known/litellm-ui-config", + ) + + with open(file_path, "w", encoding="utf-8") as f: + f.write(modified_content) + except (UnicodeDecodeError, PermissionError, OSError): + # Skip binary files or files we can't write to + continue # # Mount the _next directory at the root level app.mount( @@ -1186,14 +1327,22 @@ try: continue # Handle HTML file restructuring - # Always restructure the directory we actually serve. - # This is critical for extensionless routes like /ui/login (expects login/index.html). - # In development, we restructure directly in _experimental/out. - # In non-root Docker, we restructure in /var/lib/litellm/ui. + # Only restructure if: + # 1. UI is not already pre-restructured + # 2. Filesystem is writable try: - if is_non_root and ui_path == "/var/lib/litellm/ui": + is_pre_restructured = _is_ui_pre_restructured(ui_path) + is_writable = os.access(ui_path, os.W_OK) + + if is_pre_restructured: verbose_proxy_logger.info( - f"Skipping runtime UI restructuring for non-root Docker. UI at {ui_path} is pre-restructured." + f"Skipping UI restructuring: {ui_path} is already pre-restructured" + ) + elif not is_writable: + verbose_proxy_logger.warning( + f"Cannot restructure UI at {ui_path}: path is not writable. " + f"UI may not work correctly for extensionless routes. " + f"Pre-build and restructure UI in Dockerfile for read-only deployments." ) else: _restructure_ui_html_files(ui_path) @@ -4716,8 +4865,10 @@ async def async_assistants_data_generator( if isinstance(e, HTTPException): raise e else: - error_traceback = traceback.format_exc() - error_msg = f"{str(e)}\n\n{error_traceback}" + # Only include the error message, not the traceback. + # The traceback is already logged above via verbose_proxy_logger.exception(). + # Including it in the SSE response leaks internal details to clients. + error_msg = str(e) proxy_exception = ProxyException( message=getattr(e, "message", error_msg), @@ -4764,7 +4915,7 @@ def _restamp_streaming_chunk_model( chunk.get("model") if isinstance(chunk, dict) else getattr(chunk, "model", None) ) if not model_mismatch_logged and downstream_model != requested_model_from_client: - verbose_proxy_logger.warning( + verbose_proxy_logger.debug( "litellm_call_id=%s: streaming chunk model mismatch - requested=%r downstream=%r. Overriding model to requested.", request_data.get("litellm_call_id"), requested_model_from_client, @@ -4867,8 +5018,10 @@ async def async_data_generator( elif isinstance(e, StreamingCallbackError): error_msg = str(e) else: - error_traceback = traceback.format_exc() - error_msg = f"{str(e)}\n\n{error_traceback}" + # Only include the error message, not the traceback. + # The traceback is already logged above via verbose_proxy_logger.exception(). + # Including it in the SSE response leaks internal details to clients. + error_msg = str(e) proxy_exception = ProxyException( message=getattr(e, "message", error_msg), @@ -10294,18 +10447,34 @@ async def get_image(): default_site_logo = os.path.join(current_dir, "logo.jpg") is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - assets_dir = "/var/lib/litellm/assets" if is_non_root else current_dir - if is_non_root: - os.makedirs(assets_dir, exist_ok=True) + # Determine assets directory + # Priority: LITELLM_ASSETS_PATH env var > default based on is_non_root + default_assets_dir = "/var/lib/litellm/assets" if is_non_root else current_dir + assets_dir = os.getenv("LITELLM_ASSETS_PATH", default_assets_dir) + # Try to create assets_dir if it doesn't exist (simple try/except approach) + if not os.path.exists(assets_dir): + try: + os.makedirs(assets_dir, exist_ok=True) + verbose_proxy_logger.debug(f"Created assets directory at {assets_dir}") + except (PermissionError, OSError) as e: + verbose_proxy_logger.warning( + f"Cannot create assets directory at {assets_dir}: {e}. " + f"Logo caching may not work. Using current directory for assets." + ) + assets_dir = current_dir + + # Determine default logo path default_logo = ( - os.path.join(assets_dir, "logo.jpg") if is_non_root else default_site_logo + os.path.join(assets_dir, "logo.jpg") + if assets_dir != current_dir + else default_site_logo ) - if is_non_root and not os.path.exists(default_logo): + if assets_dir != current_dir and not os.path.exists(default_logo): default_logo = default_site_logo - cache_dir = assets_dir if is_non_root else current_dir + cache_dir = assets_dir if os.access(assets_dir, os.W_OK) else current_dir cache_path = os.path.join(cache_dir, "cached_logo.jpg") # [OPTIMIZATION] Check if the cached image exists first @@ -11832,6 +12001,7 @@ app.include_router(enterprise_router) app.include_router(ui_discovery_endpoints_router) app.include_router(agent_endpoints_router) app.include_router(a2a_router) +app.include_router(access_group_router) ######################################################## # MCP Server ######################################################## diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 37ed0182663..b3caa46147b 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -128,6 +128,7 @@ model LiteLLM_TeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) @@ -160,6 +161,7 @@ model LiteLLM_DeletedTeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases @@ -291,6 +293,7 @@ model LiteLLM_VerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") budget_id String? @@ -346,6 +349,7 @@ model LiteLLM_DeletedVerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") router_settings Json? @default("{}") @@ -917,3 +921,23 @@ model LiteLLM_PolicyAttachmentTable { updated_at DateTime @default(now()) @updatedAt updated_by String? } + +//Unified Access Groups table for storing unified access groups +model LiteLLM_AccessGroupTable { + access_group_id String @id @default(uuid()) + access_group_name String @unique + description String? + + // Resource memberships - explicit arrays per type + access_model_ids String[] @default([]) + access_mcp_server_ids String[] @default([]) + access_agent_ids String[] @default([]) + + assigned_team_ids String[] @default([]) + assigned_key_ids String[] @default([]) + + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? +} \ No newline at end of file diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 4076e13bcce..0c91fd00b38 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1951,13 +1951,15 @@ async def ui_view_spend_logs( # noqa: PLR0915 verbose_proxy_logger.debug("data= %s", json.dumps(data, indent=4, default=str)) - return { - "data": data, - "total": total_records, - "page": page, - "page_size": page_size, - "total_pages": total_pages, - } + return await _build_ui_spend_logs_response( + prisma_client, + data, + total_records, + page, + page_size, + total_pages, + enrich_session_counts=not is_v2, + ) except Exception as e: verbose_proxy_logger.exception(f"Error in ui_view_spend_logs: {e}") raise handle_exception_on_proxy(e) @@ -3233,6 +3235,91 @@ async def ui_view_session_spend_logs( ) +async def _build_ui_spend_logs_response( + prisma_client: "PrismaClient", + data: list, + total_records: int, + page: int, + page_size: int, + total_pages: int, + enrich_session_counts: bool = True, +) -> dict: + """ + Build the paginated response for the UI spend-logs endpoint. + + When ``enrich_session_counts`` is ``True`` (the default for the v1/UI + endpoint), each row is enriched with ``session_total_count`` so the + frontend knows which sessions are expandable (multi-call sessions). + For every row that carries a ``session_id``, a single ``GROUP BY`` query + fetches the total number of logs in each referenced session. Rows without + a ``session_id`` default to ``1``. + + When ``enrich_session_counts`` is ``False`` (v2 endpoint), rows are + serialised without the extra query. + + Args: + prisma_client: The connected Prisma client instance. + data: A list of Prisma model instances (must support ``.model_dump()`` + and have a ``session_id`` attribute). + total_records: Total number of matching records (for pagination). + page: Current page number. + page_size: Number of items per page. + total_pages: Total number of pages. + enrich_session_counts: Whether to add ``session_total_count`` to each + row. Defaults to ``True``. + + Returns: + A dict with ``data`` (enriched rows), ``total``, ``page``, + ``page_size``, and ``total_pages``. + """ + count_map: dict[str, int] = {} + if enrich_session_counts: + session_ids = list( + {row.session_id for row in data if getattr(row, "session_id", None)} + ) + if session_ids: + # NOTE: This GROUP BY runs on every v1/UI page load. The IN clause + # is bounded by page_size (typically 25-50 distinct session IDs). + # If performance degrades at scale, consider short-lived caching or + # folding the count into the main query via a window function. + counts = await prisma_client.db.litellm_spendlogs.group_by( + by=["session_id"], + where={"session_id": {"in": session_ids}}, + count={"session_id": True}, + ) + count_map = { + r["session_id"]: r["_count"]["session_id"] + for r in counts + if r.get("session_id") + } + + if enrich_session_counts: + enriched: List[dict] = [] + for row in data: + row_dict = ( + dict(row) + if isinstance(row, dict) + else row.model_dump() + ) + sid = row_dict.get("session_id") + row_dict["session_total_count"] = count_map.get(sid, 1) if sid else 1 + enriched.append(row_dict) + response_data: list = enriched + else: + # v2 path: return raw Prisma model instances so FastAPI applies its + # own Pydantic-aware serialisation (preserves alias handling, custom + # serializers, etc.). + response_data = data # type: ignore[assignment] + + return { + "data": response_data, + "total": total_records, + "page": page, + "page_size": page_size, + "total_pages": total_pages, + } + + def _build_status_filter_condition(status_filter: Optional[str]) -> Dict[str, Any]: """ Helper function to build the status filter condition for database queries. diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 8f524690be1..7105f1ae6fb 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -181,6 +181,7 @@ async def aresponses_api_with_mcp( ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + litellm_trace_id=kwargs.get("litellm_trace_id"), ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( original_mcp_tools diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index 4a640a61064..377ce396457 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -127,6 +127,7 @@ async def acompletion_with_mcp( # noqa: PLR0915 ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + litellm_trace_id=kwargs.get("litellm_trace_id"), ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( @@ -235,7 +236,10 @@ async def acompletion_with_mcp( # noqa: PLR0915 def _add_mcp_list_tools_to_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: """Add mcp_list_tools to the first chunk.""" - from litellm.types.utils import StreamingChoices, add_provider_specific_fields + from litellm.types.utils import ( + StreamingChoices, + add_provider_specific_fields, + ) if not self.openai_tools: return chunk @@ -258,7 +262,10 @@ async def acompletion_with_mcp( # noqa: PLR0915 def _add_mcp_tool_metadata_to_final_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: """Add mcp_tool_calls and mcp_call_results to the final chunk.""" - from litellm.types.utils import StreamingChoices, add_provider_specific_fields + from litellm.types.utils import ( + StreamingChoices, + add_provider_specific_fields, + ) if hasattr(chunk, "choices") and chunk.choices: for choice in chunk.choices: diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 297ccf4355e..805a1958552 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -6,10 +6,10 @@ from typing import ( Dict, Iterable, List, + Literal, Optional, Tuple, Union, - Literal, ) from litellm._logging import verbose_logger @@ -29,6 +29,7 @@ from litellm.utils import Rules, function_setup if TYPE_CHECKING: from mcp.types import Tool as MCPTool + from litellm.proxy.utils import ProxyLogging else: MCPTool = Any @@ -97,6 +98,7 @@ class LiteLLM_Proxy_MCP_Handler: async def _get_mcp_tools_from_manager( user_api_key_auth: Any, mcp_tools_with_litellm_proxy: Optional[Iterable[ToolParam]], + litellm_trace_id: Optional[str] = None, ) -> tuple[List[MCPTool], List[str]]: """ Get available tools from the MCP server manager. @@ -109,13 +111,13 @@ class LiteLLM_Proxy_MCP_Handler: List of MCP tools List names of allowed MCP servers """ - from litellm.proxy._experimental.mcp_server.server import ( - _get_tools_from_mcp_servers, - _get_allowed_mcp_servers_from_mcp_server_names, - ) from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) + from litellm.proxy._experimental.mcp_server.server import ( + _get_allowed_mcp_servers_from_mcp_server_names, + _get_tools_from_mcp_servers, + ) mcp_servers: List[str] = [] if mcp_tools_with_litellm_proxy: @@ -136,6 +138,7 @@ class LiteLLM_Proxy_MCP_Handler: mcp_server_auth_headers=None, log_list_tools_to_spendlogs=True, list_tools_log_source="responses", + litellm_trace_id=litellm_trace_id, ) allowed_mcp_server_ids = ( await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) @@ -239,7 +242,9 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _process_mcp_tools_to_openai_format( - user_api_key_auth: Any, mcp_tools_with_litellm_proxy: List[ToolParam] + user_api_key_auth: Any, + mcp_tools_with_litellm_proxy: List[ToolParam], + litellm_trace_id: Optional[str] = None, ) -> tuple[List[Any], dict[str, str]]: """ Centralized method to process MCP tools through the complete pipeline. @@ -247,6 +252,7 @@ class LiteLLM_Proxy_MCP_Handler: Args: user_api_key_auth: User authentication info for access control mcp_tools_with_litellm_proxy: ToolParam objects with server_url starting with "litellm_proxy" + litellm_trace_id: Optional trace ID for linking list_mcp_tools spend logs to parent request Returns: List of tools in OpenAI format ready to be sent to the LLM @@ -258,6 +264,7 @@ class LiteLLM_Proxy_MCP_Handler: ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform( user_api_key_auth, mcp_tools_with_litellm_proxy, + litellm_trace_id=litellm_trace_id, ) openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( @@ -268,7 +275,9 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod async def _process_mcp_tools_without_openai_transform( - user_api_key_auth: Any, mcp_tools_with_litellm_proxy: List[ToolParam] + user_api_key_auth: Any, + mcp_tools_with_litellm_proxy: List[ToolParam], + litellm_trace_id: Optional[str] = None, ) -> tuple[List[Any], dict[str, str]]: """ Process MCP tools through filtering and deduplication pipeline without OpenAI transformation. @@ -291,6 +300,7 @@ class LiteLLM_Proxy_MCP_Handler: ) = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager( user_api_key_auth=user_api_key_auth, mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy, + litellm_trace_id=litellm_trace_id, ) # Step 2: Filter tools based on allowed_tools parameter @@ -495,14 +505,13 @@ class LiteLLM_Proxy_MCP_Handler: """Execute tool calls and return results.""" from fastapi import HTTPException + from litellm._uuid import uuid from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) from litellm.proxy.proxy_server import proxy_logging_obj - from litellm._uuid import uuid - tool_results = [] tool_call_id: Optional[str] = None rules_obj = Rules() @@ -1025,7 +1034,6 @@ class LiteLLM_Proxy_MCP_Handler: List of MCP tool execution events for streaming """ from litellm._uuid import uuid - from litellm.responses.mcp.mcp_streaming_iterator import create_mcp_call_events tool_execution_events: List[Any] = [] @@ -1108,8 +1116,8 @@ class LiteLLM_Proxy_MCP_Handler: """Add custom output elements to the final response for MCP tool execution.""" # Import the required classes for creating output items import json - from litellm._uuid import uuid + from litellm._uuid import uuid from litellm.types.responses.main import GenericResponseOutputItem, OutputText # Create output element for initial MCP tools diff --git a/litellm/router.py b/litellm/router.py index 37fa3926b4d..7bba0902a5e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2283,7 +2283,7 @@ class Router: item = FlowItem( priority=priority, # 👈 SET PRIORITY FOR REQUEST request_id=_request_id, # 👈 SET REQUEST ID - model_name="gpt-3.5-turbo", # 👈 SAME as 'Router' + model_name=model, # 👈 SAME as 'Router' ) ### [fin] ### @@ -2325,6 +2325,10 @@ class Router: setattr(e, "priority", priority) raise e else: + # Clean up the request from the scheduler queue also before raising the timeout exception + await self.scheduler.remove_request( + request_id=item.request_id, model_name=item.model_name + ) raise litellm.Timeout( message="Request timed out while polling queue", model=model, @@ -2386,6 +2390,10 @@ class Router: setattr(e, "priority", priority) raise e else: + # Clean up the request from the scheduler queue also before raising the timeout exception + await self.scheduler.remove_request( + request_id=item.request_id, model_name=item.model_name + ) raise litellm.Timeout( message="Request timed out while polling queue", model=model, @@ -5039,7 +5047,7 @@ class Router: else: _healthy_deployments = [] _timeout = self._time_to_sleep_before_retry( - e=original_exception, + e=e, remaining_retries=remaining_retries, num_retries=num_retries, healthy_deployments=_healthy_deployments, diff --git a/litellm/scheduler.py b/litellm/scheduler.py index 5f3dd4cbf61..0221e249848 100644 --- a/litellm/scheduler.py +++ b/litellm/scheduler.py @@ -92,6 +92,17 @@ class Scheduler: return True + async def remove_request(self, request_id: str, model_name: str) -> None: + """ + Remove a specific request from the priority queue for a model. + Used when a request times out while waiting in the queue. + """ + queue = await self.get_queue(model_name=model_name) + filtered_queue = [item for item in queue if item[1] != request_id] + heapq.heapify(filtered_queue) # restore heap invariant after filtering + await self.save_queue(queue=filtered_queue, model_name=model_name) + print_verbose(f"Removed request_id: {request_id} from queue for model: {model_name}") + async def peek(self, id: str, model_name: str, health_deployments: list) -> bool: """Return if the id is at the top of the queue. Don't pop the value from heap.""" queue = await self.get_queue(model_name=model_name) diff --git a/litellm/types/access_group.py b/litellm/types/access_group.py new file mode 100644 index 00000000000..3a6b75768ef --- /dev/null +++ b/litellm/types/access_group.py @@ -0,0 +1,38 @@ +from datetime import datetime +from typing import List, Optional + +from pydantic import BaseModel + + +class AccessGroupCreateRequest(BaseModel): + access_group_name: str + description: Optional[str] = None + access_model_ids: Optional[List[str]] = None + access_mcp_server_ids: Optional[List[str]] = None + access_agent_ids: Optional[List[str]] = None + assigned_team_ids: Optional[List[str]] = None + assigned_key_ids: Optional[List[str]] = None + + +class AccessGroupUpdateRequest(BaseModel): + description: Optional[str] = None + access_model_ids: Optional[List[str]] = None + access_mcp_server_ids: Optional[List[str]] = None + access_agent_ids: Optional[List[str]] = None + assigned_team_ids: Optional[List[str]] = None + assigned_key_ids: Optional[List[str]] = None + + +class AccessGroupResponse(BaseModel): + access_group_id: str + access_group_name: str + description: Optional[str] = None + access_model_ids: List[str] + access_mcp_server_ids: List[str] + access_agent_ids: List[str] + assigned_team_ids: List[str] + assigned_key_ids: List[str] + created_at: datetime + created_by: Optional[str] = None + updated_at: datetime + updated_by: Optional[str] = None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 299b47199ed..4ab81f8fd57 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1058,7 +1058,20 @@ class ComputerToolParam(TypedDict, total=False): type: Required[Union[Literal["computer_use_preview"], str]] -ALL_RESPONSES_API_TOOL_PARAMS = Union[ToolParam, ComputerToolParam] +class ShellToolParam(TypedDict, total=False): + """ + Shell tool for Responses API: run commands in hosted containers or local runtime. + See https://developers.openai.com/api/docs/guides/tools-shell. + """ + + type: Required[Union[Literal["shell"], str]] + """The type of tool. Use ``\"shell\"``.""" + + environment: Required[Dict[str, Any]] + """Environment config: ``type`` (e.g. ``\"container_auto\"``, ``\"container_reference\"``, ``\"local\"``), optional ``container_id``, ``network_policy``, ``domain_secrets``, ``skills``.""" + + +ALL_RESPONSES_API_TOOL_PARAMS = Union[ToolParam, ComputerToolParam, ShellToolParam] class PromptObject(TypedDict, total=False): @@ -1074,6 +1087,19 @@ class PromptObject(TypedDict, total=False): """Optional version of the prompt template.""" +class ContextManagementEntry(TypedDict, total=False): + """ + Context management configuration entry for a request. + See https://developers.openai.com/api/docs/guides/compaction. + """ + + type: str + """The context management entry type. Currently only ``'compaction'`` is supported.""" + + compact_threshold: int + """Token threshold at which compaction is triggered for this entry. Minimum 1000.""" + + class ResponsesAPIOptionalRequestParams(TypedDict, total=False): """TypedDict for Optional parameters supported by the responses API.""" @@ -1104,6 +1130,8 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): partial_images: Optional[ int ] # Number of partial images to generate (1-3) for streaming image generation + context_management: Optional[List[ContextManagementEntry]] + """Context management configuration. E.g. [{\"type\": \"compaction\", \"compact_threshold\": 200000}] for server-side compaction (minimum 1000).""" class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): @@ -1189,7 +1217,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): top_p: Optional[float] = None max_output_tokens: Optional[int] = None previous_response_id: Optional[str] = None - reasoning: Optional[Reasoning] = None + reasoning: Optional[Dict[str, Any]] = None status: Optional[str] = None text: Optional[Union["ResponseText", Dict[str, Any]]] = None truncation: Optional[Literal["auto", "disabled"]] = None @@ -1199,6 +1227,18 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): # Define private attributes using PrivateAttr _hidden_params: dict = PrivateAttr(default_factory=dict) + @field_validator("reasoning", mode="before") + @classmethod + def validate_reasoning_to_dict(cls, value: Any) -> Optional[Dict[str, Any]]: + """Accept API reasoning dict (including effort 'none'/'xhigh'); always store as dict.""" + if value is None: + return None + if isinstance(value, dict): + return value + if hasattr(value, "model_dump"): + return value.model_dump() + return value + @field_validator("usage", mode="before") @classmethod def validate_usage(cls, value): @@ -1307,6 +1347,11 @@ class ResponsesAPIStreamEvents(str, Enum): # Image generation events IMAGE_GENERATION_PARTIAL_IMAGE = "image_generation.partial_image" + # Shell tool events (Responses API; passthrough via GenericEvent) + SHELL_CALL_IN_PROGRESS = "response.shell_call.in_progress" + SHELL_CALL_COMPLETED = "response.shell_call.completed" + SHELL_CALL_OUTPUT = "response.shell_call_output.done" + # Error event ERROR = "error" @@ -1593,12 +1638,12 @@ class ImageGenerationPartialImageEvent(BaseLiteLLMOpenAIResponseObject): class ErrorEventError(BaseLiteLLMOpenAIResponseObject): - """Nested error object within ErrorEvent""" + """Nested error object within ErrorEvent.""" type: str # e.g., 'invalid_request_error' code: str # e.g., 'context_length_exceeded' message: str - param: Optional[str] + param: Optional[str] = None class ErrorEvent(BaseLiteLLMOpenAIResponseObject): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py index 96d78cf8827..21f6f5b3b4e 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py @@ -60,6 +60,14 @@ class GenericGuardrailAPIRequest(BaseModel): tools: Optional[List[ChatCompletionToolParam]] = None texts: Optional[List[str]] = None request_data: GenericGuardrailAPIMetadata + request_headers: Optional[Dict[str, str]] = Field( + default=None, + description="Sanitized inbound request headers from the original proxy request.", + ) + litellm_version: Optional[str] = Field( + default=None, + description="LiteLLM library version running this proxy.", + ) additional_provider_specific_params: Optional[Dict[str, Any]] = None tool_calls: Optional[ Union[List[ChatCompletionToolCallChunk], List[ChatCompletionMessageToolCall]] diff --git a/litellm/utils.py b/litellm/utils.py index 0fa5436d98d..0fd21f09919 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1406,7 +1406,7 @@ def client(original_function): # noqa: PLR0915 # [OPTIONAL] CHECK MAX RETRIES / REQUEST if litellm.num_retries_per_request is not None: # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] - previous_models = kwargs.get("metadata", {}).get( + previous_models = (kwargs.get("metadata") or {}).get( "previous_models", None ) if previous_models is not None: @@ -1483,7 +1483,7 @@ def client(original_function): # noqa: PLR0915 # [OPTIONAL] CHECK MAX RETRIES / REQUEST if litellm.num_retries_per_request is not None: # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] - previous_models = kwargs.get("metadata", {}).get( + previous_models = (kwargs.get("metadata") or {}).get( "previous_models", None ) if previous_models is not None: @@ -1678,8 +1678,8 @@ def client(original_function): # noqa: PLR0915 "context_window_fallback_dict", {} ) - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( num_retries and not _is_litellm_router_call @@ -1724,8 +1724,8 @@ def client(original_function): # noqa: PLR0915 None # set retries to None to prevent infinite loops ) - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( num_retries and not _is_litellm_router_call @@ -1974,8 +1974,8 @@ def client(original_function): # noqa: PLR0915 "context_window_fallback_dict", {} ) - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( @@ -2008,8 +2008,8 @@ def client(original_function): # noqa: PLR0915 kwargs["model"] = context_window_fallback_dict[model] return await original_function(*args, **kwargs) elif call_type == CallTypes.aresponses.value: - _is_litellm_router_call = "model_group" in kwargs.get( - "metadata", {} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} ) # check if call from litellm.router/proxy if ( @@ -7337,7 +7337,7 @@ def _get_base_model_from_metadata(model_call_details=None): _base_model = litellm_params.get("base_model", None) if _base_model is not None: return _base_model - metadata = litellm_params.get("metadata", {}) + metadata = litellm_params.get("metadata") or {} _get_base_model_from_litellm_call_metadata = getattr( sys.modules[__name__], "_get_base_model_from_litellm_call_metadata" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f6edcf7efd0..d0556fc4e89 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -21432,6 +21432,36 @@ "max_input_tokens": 1000000, "max_output_tokens": 8192 }, + "minimax/MiniMax-M2.5": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, + "minimax/MiniMax-M2.5-lightning": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, "minimax/MiniMax-M2": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, @@ -30799,6 +30829,21 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/zai-org/glm-5-maas": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-zai_models", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "vertex_ai/mistral-medium-3": { "input_cost_per_token": 4e-07, "litellm_provider": "vertex_ai-mistral_models", diff --git a/pyproject.toml b/pyproject.toml index acb8bc2ada3..6ed7618dd26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.81.10" +version = "1.81.11" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -61,7 +61,7 @@ boto3 = { version = "1.40.76", optional = true } redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"} a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.4.34", optional = true} +litellm-proxy-extras = {version = "0.4.36", optional = true} rich = {version = "13.7.1", optional = true} litellm-enterprise = {version = "0.1.31", optional = true} diskcache = {version = "^5.6.1", optional = true} @@ -175,7 +175,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.81.10" +version = "1.81.11" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index bd313b105e5..f31730e20f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,8 @@ urllib3>=2.6.0 # CVE-2025-66471, CVE-2025-66418, CVE-2026-21441 tornado>=6.5.3 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724 filelock>=3.20.1 # CVE-2025-68146 +Pillow==12.1.1 #GHSA-cfh3-3jmp-rvhc +cryptography==46.0.5 #GHSA-r6ph-v2qm-q3c2 anyio==4.8.0 # openai + http req. httpx==0.28.1 @@ -38,7 +40,6 @@ apscheduler==3.10.4 # for resetting budget in background fastapi-sso==0.19.0 # admin UI, SSO pyjwt[crypto]==2.10.1 ; python_version >= "3.9" python-multipart==0.0.22 # admin UI -Pillow==11.0.0 jaraco.context>=6.1.0 azure-ai-contentsafety==1.0.0 # for azure content safety azure-identity==1.16.1 ; python_version >= "3.9" # for azure content safety @@ -53,9 +54,8 @@ grpcio>=1.62.3,!=1.68.*,!=1.69.*,!=1.70.*,!=1.71.0,!=1.71.1,!=1.72.0,!=1.72.1,!= grpcio>=1.75.0; python_version >= "3.14" sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests -cryptography==44.0.1 tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.34 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.36 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env diff --git a/schema.prisma b/schema.prisma index 4329f939a7b..2a11d0028fb 100644 --- a/schema.prisma +++ b/schema.prisma @@ -128,6 +128,7 @@ model LiteLLM_TeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team @@ -161,6 +162,7 @@ model LiteLLM_DeletedTeamTable { model_max_budget Json @default("{}") router_settings Json? @default("{}") team_member_permissions String[] @default([]) + access_group_ids String[] @default([]) policies String[] @default([]) model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) @@ -293,6 +295,7 @@ model LiteLLM_VerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") budget_id String? @@ -348,6 +351,7 @@ model LiteLLM_DeletedVerificationToken { allowed_cache_controls String[] @default([]) allowed_routes String[] @default([]) policies String[] @default([]) + access_group_ids String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") router_settings Json? @default("{}") @@ -363,7 +367,6 @@ model LiteLLM_DeletedVerificationToken { rotation_interval String? last_rotation_at DateTime? key_rotation_at DateTime? - // Deletion metadata deleted_at DateTime @default(now()) @map("deleted_at") deleted_by String? @map("deleted_by") // User who deleted the key @@ -919,3 +922,23 @@ model LiteLLM_PolicyAttachmentTable { updated_at DateTime @default(now()) @updatedAt updated_by String? } + +//Unified Access Groups table for storing unified access groups +model LiteLLM_AccessGroupTable { + access_group_id String @id @default(uuid()) + access_group_name String @unique + description String? + + // Resource memberships - explicit arrays per type + access_model_ids String[] @default([]) + access_mcp_server_ids String[] @default([]) + access_agent_ids String[] @default([]) + + assigned_team_ids String[] @default([]) + assigned_key_ids String[] @default([]) + + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @default(now()) @updatedAt + updated_by String? +} \ No newline at end of file diff --git a/tests/code_coverage_tests/check_guardrail_apply_decorator.py b/tests/code_coverage_tests/check_guardrail_apply_decorator.py index 18a86277aa9..523b9f5b35b 100644 --- a/tests/code_coverage_tests/check_guardrail_apply_decorator.py +++ b/tests/code_coverage_tests/check_guardrail_apply_decorator.py @@ -83,6 +83,12 @@ def test_guardrail_apply_decorator(): if python_file.name == "bedrock_guardrails.py": continue + # Skip content_filter.py - it implements its own detailed logging via + # _log_guardrail_information with detections, masked_entity_count, etc. + # Using the decorator would cause duplicate entries. + if python_file.name == "content_filter.py": + continue + results = find_apply_guardrail_methods(python_file) for class_name, line_num, has_decorator in results: diff --git a/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py b/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py new file mode 100644 index 00000000000..89da8d87e63 --- /dev/null +++ b/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py @@ -0,0 +1,64 @@ +""" +Tests for _map_reasoning_effort in AnthropicConfig. + +Verifies that reasoning_effort=None returns None for all models, +including Claude Opus 4.6. +""" + +from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + +class TestMapReasoningEffort: + def test_none_returns_none_for_opus_4_6(self): + """reasoning_effort=None should return None for Opus 4.6, not adaptive.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort=None, model="claude-opus-4-6" + ) + assert result is None + + def test_none_returns_none_for_other_models(self): + """reasoning_effort=None should return None for non-Opus models.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort=None, model="claude-3-7-sonnet-20250219" + ) + assert result is None + + def test_opus_4_6_returns_adaptive_for_low(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="low", model="claude-opus-4-6" + ) + assert result["type"] == "adaptive" + + def test_opus_4_6_returns_adaptive_for_high(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="high", model="claude-opus-4-6" + ) + assert result["type"] == "adaptive" + + def test_other_model_low_returns_enabled_with_budget(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="low", model="claude-3-7-sonnet-20250219" + ) + assert result["type"] == "enabled" + assert "budget_tokens" in result + + def test_other_model_high_returns_enabled_with_budget(self): + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="high", model="claude-3-7-sonnet-20250219" + ) + assert result["type"] == "enabled" + assert "budget_tokens" in result + + def test_none_string_returns_none_for_opus_4_6(self): + """reasoning_effort='none' should return None for Opus 4.6.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="none", model="claude-opus-4-6" + ) + assert result is None + + def test_none_string_returns_none_for_other_models(self): + """reasoning_effort='none' should return None for non-Opus models.""" + result = AnthropicConfig._map_reasoning_effort( + reasoning_effort="none", model="claude-3-7-sonnet-20250219" + ) + assert result is None diff --git a/tests/litellm/proxy/management_endpoints/test_common_utils.py b/tests/litellm/proxy/management_endpoints/test_common_utils.py new file mode 100644 index 00000000000..f857db770d0 --- /dev/null +++ b/tests/litellm/proxy/management_endpoints/test_common_utils.py @@ -0,0 +1,159 @@ +""" +Tests for litellm/proxy/management_endpoints/common_utils.py + +Specifically tests that _update_metadata_fields does not trigger premium +user checks when premium fields are present but empty. + +Related: https://github.com/BerriAI/litellm/issues/20534 +""" + +from unittest.mock import patch + +import pytest + +from litellm.proxy.management_endpoints.common_utils import ( + _has_non_empty_value, + _update_metadata_fields, +) + + +class TestHasNonEmptyValue: + """Tests for the _has_non_empty_value helper.""" + + def test_none_is_empty(self): + assert _has_non_empty_value(None) is False + + def test_empty_list_is_empty(self): + assert _has_non_empty_value([]) is False + + def test_empty_string_is_empty(self): + assert _has_non_empty_value("") is False + + def test_blank_string_is_empty(self): + assert _has_non_empty_value(" ") is False + + def test_non_empty_list_has_value(self): + assert _has_non_empty_value(["policy-a"]) is True + + def test_non_empty_string_has_value(self): + assert _has_non_empty_value("30d") is True + + def test_dict_has_value(self): + assert _has_non_empty_value({"key": "val"}) is True + + def test_empty_dict_has_value(self): + # empty dict is not None/list/str, so it counts as non-empty + assert _has_non_empty_value({}) is True + + +class TestUpdateMetadataFieldsPremiumCheck: + """ + Tests that _update_metadata_fields skips premium user checks for empty + values but still enforces them for real values. + + Issue: The UI sends the full form on every team update, including premium + fields like `policies: []`. The backend was treating these empty values + as premium feature usage and returning 403. + """ + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_empty_policies_skips_premium_check(self, mock_check): + """policies: [] should NOT trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "team_alias": "my-team", + "policies": [], + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_empty_guardrails_skips_premium_check(self, mock_check): + """guardrails: [] should NOT trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "guardrails": [], + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_empty_string_team_member_key_duration_skips_premium_check( + self, mock_check + ): + """team_member_key_duration: '' should NOT trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "team_member_key_duration": "", + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + side_effect=Exception("Should not be called"), + ) + def test_full_ui_payload_with_empty_premium_fields_skips_premium_check( + self, mock_check + ): + """A realistic UI payload with all empty premium fields should not 403.""" + updated_kv = { + "team_id": "team-123", + "team_alias": "renamed-team", + "models": ["gpt-4o"], + "max_budget": 200, + "policies": [], + "guardrails": [], + "logging": [], + "team_member_key_duration": "", + "prompts": [], + } + _update_metadata_fields(updated_kv) + mock_check.assert_not_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + ) + def test_non_empty_policies_triggers_premium_check(self, mock_check): + """policies: ['real-policy'] SHOULD trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "policies": ["real-policy"], + } + _update_metadata_fields(updated_kv) + mock_check.assert_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + ) + def test_non_empty_guardrails_triggers_premium_check(self, mock_check): + """guardrails: ['my-guardrail'] SHOULD trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "guardrails": ["my-guardrail"], + } + _update_metadata_fields(updated_kv) + mock_check.assert_called() + + @patch( + "litellm.proxy.management_endpoints.common_utils._premium_user_check", + ) + def test_non_empty_team_member_key_duration_triggers_premium_check( + self, mock_check + ): + """team_member_key_duration: '30d' SHOULD trigger premium user check.""" + updated_kv = { + "team_id": "team-123", + "team_member_key_duration": "30d", + } + _update_metadata_fields(updated_kv) + mock_check.assert_called() diff --git a/tests/litellm/test_router_retry_backoff_headers.py b/tests/litellm/test_router_retry_backoff_headers.py new file mode 100644 index 00000000000..03c3af692ce --- /dev/null +++ b/tests/litellm/test_router_retry_backoff_headers.py @@ -0,0 +1,88 @@ +""" +Tests for router retry backoff behavior. +""" + +from unittest.mock import patch + +import httpx +import pytest + +import litellm +from litellm import Router + + +@pytest.mark.asyncio +async def test_retry_backoff_uses_current_exception_headers(): + """ + Ensure retry backoff uses the current retry exception, not the initial one. + """ + router = Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "sk-test", + }, + } + ], + num_retries=2, + ) + + first_error = litellm.RateLimitError( + message="Rate limited on first attempt", + model="gpt-3.5-turbo", + llm_provider="openai", + ) + first_error.litellm_response_headers = httpx.Headers({"retry-after": "1"}) + + second_error = litellm.RateLimitError( + message="Rate limited on second attempt", + model="gpt-3.5-turbo", + llm_provider="openai", + ) + second_error.litellm_response_headers = httpx.Headers({"retry-after": "15"}) + + third_error = litellm.RateLimitError( + message="Rate limited on third attempt", + model="gpt-3.5-turbo", + llm_provider="openai", + ) + third_error.litellm_response_headers = httpx.Headers({"retry-after": "30"}) + + raised_errors = [first_error, second_error, third_error] + captured_backoff_errors = [] + + async def mock_make_call(*args, **kwargs): + raise raised_errors.pop(0) + + def mock_time_to_sleep_before_retry(*args, **kwargs): + captured_backoff_errors.append(kwargs["e"]) + return 0.01 + + with patch.object(router, "make_call", side_effect=mock_make_call): + with patch.object( + router, + "_async_get_healthy_deployments", + return_value=( + [{"model_info": {"id": "test-id"}}], + [{"model_info": {"id": "test-id"}}], + ), + ): + with patch.object( + router, + "_time_to_sleep_before_retry", + side_effect=mock_time_to_sleep_before_retry, + ): + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello"}], + ) + + # Router computes backoff once after the initial failure, then once per failed retry. + # With num_retries=2 and all attempts failing, that's 1 + 2 = 3 invocations. + assert len(captured_backoff_errors) == router.num_retries + 1 + assert captured_backoff_errors[0] is first_error + assert captured_backoff_errors[1] is second_error + assert captured_backoff_errors[2] is third_error diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 37ed1a9b08c..7ac83753d17 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -2,7 +2,7 @@ import httpx import json import pytest import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from unittest.mock import MagicMock, Mock, patch import os from litellm._uuid import uuid @@ -114,6 +114,10 @@ class BaseResponsesAPITest(ABC): """Must return the base completion reasoning call args""" return None + def get_advanced_model_for_shell_tool(self) -> Optional[str]: + """If specified, overrides the model used by test_responses_api_shell_tool_streaming_sees_shell_output (e.g. openai/gpt-5.2 for shell support).""" + return None + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_basic_openai_responses_api(self, sync_mode): @@ -669,7 +673,7 @@ class BaseResponsesAPITest(ABC): async def test_cancel_responses_invalid_response_id(self, sync_mode): """Test cancel_responses with invalid response ID should raise appropriate error""" base_completion_call_args = self.get_base_completion_call_args() - + if sync_mode: with pytest.raises(Exception): litellm.cancel_responses( @@ -679,4 +683,128 @@ class BaseResponsesAPITest(ABC): with pytest.raises(Exception): await litellm.acancel_responses( response_id="invalid_response_id_12345", **base_completion_call_args - ) \ No newline at end of file + ) + + @pytest.mark.asyncio + async def test_responses_api_context_management_server_side_compaction(self): + """ + E2E test for server-side compaction (context_management) on OpenAI Responses API. + Passes context_management with compact_threshold; validates that the request is + accepted and returns a valid response. Compaction may not run for short inputs. + """ + base_completion_call_args = self.get_base_completion_call_args() + model = base_completion_call_args.get("model") or "" + # Only run with context_management for OpenAI (OAI) for now + if "openai/" not in str(model) and "azure/" not in str(model): + pytest.skip( + "context_management server-side compaction e2e is only run for OpenAI/Azure" + ) + context_management = [{"type": "compaction", "compact_threshold": 200000}] + try: + response = await litellm.aresponses( + input="Short ping to verify context_management is accepted.", + max_output_tokens=20, + context_management=context_management, + **base_completion_call_args, + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + validate_responses_api_response(response, final_chunk=True) + assert response.get("id") is not None + assert response.get("status") is not None + + @pytest.mark.asyncio + async def test_responses_api_shell_tool(self): + """ + E2E test for Shell tool on OpenAI Responses API. + Passes tools=[{"type": "shell", "environment": {"type": "container_auto"}}]; + validates that the request is accepted and returns a valid response. + Only runs for OpenAI/Azure (Responses API with shell support). + """ + base_completion_call_args = self.get_base_completion_call_args() + model = self.get_advanced_model_for_shell_tool() or base_completion_call_args.get( + "model" + ) or "" + if "openai/" not in str(model) and "azure/" not in str(model): + pytest.skip( + "Shell tool e2e is only run for OpenAI/Azure Responses API" + ) + tools = [{"type": "shell", "environment": {"type": "container_auto"}}] + input_msg = "List files in /mnt/data and show python --version." + try: + response = await litellm.aresponses( + **{**base_completion_call_args, "model": model}, + input=input_msg, + max_output_tokens=256, + tools=tools, + tool_choice="auto", + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + except litellm.BadRequestError as e: + if "shell" in str(e).lower() and "not supported" in str(e).lower(): + pytest.skip( + "Shell tool is not supported for this model (e.g. gpt-4o); use a model that supports shell" + ) + raise + validate_responses_api_response(response, final_chunk=True) + assert response.get("id") is not None + assert response.get("status") is not None + + @pytest.mark.asyncio + async def test_responses_api_shell_tool_streaming_sees_shell_output(self): + """ + E2E streaming call with Shell tool; validate we can see shell output in the stream. + + Calls aresponses(..., tools=[shell], stream=True), then iterates the stream and + asserts at least one event is shell-related or response output contains shell_call. + Skips when model does not support shell (e.g. gpt-4o). + """ + base_completion_call_args = self.get_base_completion_call_args() + model = self.get_advanced_model_for_shell_tool() or base_completion_call_args.get( + "model" + ) or "openai/gpt-5.2" + tools = [{"type": "shell", "environment": {"type": "container_auto"}}] + input_msg = "List files in /mnt/data and run python --version." + + stream = await litellm.aresponses( + **{**base_completion_call_args, "model": model}, + input=input_msg, + max_output_tokens=512, + tools=tools, + tool_choice="auto", + stream=True, + ) + + + event_types_seen = [] + output_items_with_shell = [] + + async for event in stream: + print("event=", json.dumps(event, indent=4, default=str)) + event_type = getattr(event, "type", None) or ( + event.get("type") if isinstance(event, dict) else None + ) + if event_type is not None: + event_types_seen.append(str(event_type)) + if "shell" in str(event_type or "").lower(): + output_items_with_shell.append(event_type) + response_obj = getattr(event, "response", None) or ( + event.get("response") if isinstance(event, dict) else None + ) + if response_obj is not None: + output = getattr(response_obj, "output", None) or ( + response_obj.get("output") if isinstance(response_obj, dict) else None + ) + if isinstance(output, list): + for item in output: + item_type = getattr(item, "type", None) or ( + item.get("type") if isinstance(item, dict) else None + ) + if item_type and "shell" in str(item_type).lower(): + output_items_with_shell.append(item_type) + + assert len(event_types_seen) > 0, "Expected at least one stream event" + assert len(output_items_with_shell) > 0, ( + f"Expected to see shell output in stream; event types seen: {event_types_seen!r}" + ) diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 5f35d6837c0..4972aa385ce 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -36,6 +36,9 @@ class TestOpenAIResponsesAPITest(BaseResponsesAPITest): "model": "openai/gpt-5-mini", } + def get_advanced_model_for_shell_tool(self): + return "openai/gpt-5.2" + class TestCustomLogger(CustomLogger): def __init__( diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 6a2ad406788..eff11c9cee3 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -519,7 +519,7 @@ async def test_openai_codex_stream(sync_mode): from litellm.main import stream_chunk_builder kwargs = { - "model": "openai/codex-mini-latest", + "model": "openai/gpt-5-codex-mini", "messages": [{"role": "user", "content": "Hey!"}], "stream": True, } @@ -549,16 +549,16 @@ async def test_openai_codex(sync_mode): router = Router( model_list=[ { - "model_name": "openai-codex-mini-latest", + "model_name": "openai-gpt-5-codex-mini", "litellm_params": { - "model": "openai/codex-mini-latest", + "model": "openai/gpt-5-codex-mini", }, } ] ) kwargs = { - "model": "openai-codex-mini-latest", + "model": "openai-gpt-5-codex-mini", "messages": [{"role": "user", "content": "Hey!"}], } diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index c322db157e9..26a93343caf 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -2189,7 +2189,7 @@ def test_completion_openrouter1(): try: litellm.set_verbose = True response = completion( - model="openrouter/mistralai/mistral-tiny", + model="openrouter/mistralai/ministral-8b", messages=messages, max_tokens=5, ) diff --git a/tests/local_testing/test_scheduler.py b/tests/local_testing/test_scheduler.py index f5b44224853..f198e572b21 100644 --- a/tests/local_testing/test_scheduler.py +++ b/tests/local_testing/test_scheduler.py @@ -117,3 +117,41 @@ async def test_scheduler_prioritized_requests(p0, p1, healthy_deployments): ) == False ) + + +@pytest.mark.asyncio +async def test_scheduler_queue_cleanup_on_timeout(): + """ + Test that a timed-out request is properly removed from the queue. + This prevents memory leaks from accumulating timed-out requests. + """ + scheduler = Scheduler() + + # Add multiple requests with different priorities + item1 = FlowItem(priority=0, request_id="req-0", model_name="gpt-3.5-turbo") + item2 = FlowItem(priority=1, request_id="req-1", model_name="gpt-3.5-turbo") + item3 = FlowItem(priority=2, request_id="req-2", model_name="gpt-3.5-turbo") + + await scheduler.add_request(item1) + await scheduler.add_request(item2) + await scheduler.add_request(item3) + + # Verify initial queue size + queue_before = await scheduler.get_queue(model_name="gpt-3.5-turbo") + assert len(queue_before) == 3, f"Expected 3 items in queue, got {len(queue_before)}" + + # Simulate timeout cleanup - remove a non-front request (item2) + await scheduler.remove_request(request_id="req-1", model_name="gpt-3.5-turbo") + + # Verify queue was cleaned up + queue_after = await scheduler.get_queue(model_name="gpt-3.5-turbo") + assert len(queue_after) == 2, f"Expected 2 items after cleanup, got {len(queue_after)}" + + # Verify the correct request was removed + remaining_ids = [item[1] for item in queue_after] + assert "req-1" not in remaining_ids, "Expected req-1 to be removed" + assert "req-0" in remaining_ids, "Expected req-0 to remain" + assert "req-2" in remaining_ids, "Expected req-2 to remain" + + # Verify remaining items are in correct priority order (0 should be first) + assert queue_after[0][1] == "req-0", "Expected req-0 (priority 0) to be at front" diff --git a/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py new file mode 100644 index 00000000000..d3c4ac80565 --- /dev/null +++ b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py @@ -0,0 +1,415 @@ +""" +Tests for standard_logging_payload_excluded_fields feature. + +This feature allows users to exclude specific fields from StandardLoggingPayload +before any callback receives it. This is useful for: +- Reducing log sizes (excluding large fields like 'response' or 'messages') +- Privacy compliance (excluding sensitive fields) +- Cost management (less data stored/transmitted) + +Example config: + litellm_settings: + success_callback: ["s3"] + standard_logging_payload_excluded_fields: ["response", "messages"] +""" + +import os +import sys +from copy import deepcopy +from typing import Dict, List, Optional +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import StandardLoggingPayload + + +def create_sample_standard_logging_payload() -> Dict: + """Create a sample StandardLoggingPayload for testing.""" + return { + "id": "test-id-123", + "trace_id": "trace-123", + "call_type": "completion", + "stream": False, + "response_cost": 0.001, + "cost_breakdown": None, + "response_cost_failure_debug_info": None, + "status": "success", + "status_fields": {}, + "custom_llm_provider": "openai", + "total_tokens": 100, + "prompt_tokens": 50, + "completion_tokens": 50, + "startTime": 1234567890.0, + "endTime": 1234567891.0, + "completionStartTime": 1234567890.5, + "response_time": 1.0, + "model_map_information": {}, + "model": "gpt-4", + "model_id": "model-123", + "model_group": None, + "api_base": "https://api.openai.com/v1", + "metadata": {}, + "cache_hit": False, + "cache_key": None, + "saved_cache_cost": 0.0, + "request_tags": [], + "end_user": None, + "requester_ip_address": None, + "user_agent": None, + "messages": [{"role": "user", "content": "Hello, this is sensitive data!"}], + "response": { + "choices": [ + {"message": {"content": "This is a sensitive response!"}} + ] + }, + "error_str": None, + "error_information": None, + "model_parameters": {}, + "hidden_params": {}, + "guardrail_information": None, + "standard_built_in_tools_params": None, + } + + +def create_model_call_details( + standard_logging_payload: Optional[Dict] = None, +) -> Dict: + """Create model_call_details dict with standard_logging_object.""" + if standard_logging_payload is None: + standard_logging_payload = create_sample_standard_logging_payload() + return { + "standard_logging_object": standard_logging_payload, + "other_key": "other_value", + } + + +class TestStandardLoggingPayloadExcludedFields: + """Test suite for standard_logging_payload_excluded_fields feature.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def test_no_excluded_fields_no_change(self): + """Test that payload is unchanged when no fields are excluded.""" + logger = CustomLogger() + model_call_details = create_model_call_details() + original_keys = set(model_call_details["standard_logging_object"].keys()) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + result_keys = set(result["standard_logging_object"].keys()) + assert result_keys == original_keys + + def test_exclude_single_field(self): + """Test excluding a single field (response).""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" in result["standard_logging_object"] + assert "model" in result["standard_logging_object"] + + def test_exclude_multiple_fields(self): + """Test excluding multiple fields (response, messages).""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" not in result["standard_logging_object"] + assert "model" in result["standard_logging_object"] + assert "model_parameters" in result["standard_logging_object"] + + def test_exclude_metadata_field(self): + """Test excluding the metadata field.""" + litellm.standard_logging_payload_excluded_fields = ["metadata"] + + logger = CustomLogger() + payload = create_sample_standard_logging_payload() + payload["metadata"] = {"sensitive_key": "sensitive_value"} + model_call_details = create_model_call_details(payload) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "metadata" not in result["standard_logging_object"] + + def test_exclude_hidden_params(self): + """Test excluding hidden_params field.""" + litellm.standard_logging_payload_excluded_fields = ["hidden_params"] + + logger = CustomLogger() + payload = create_sample_standard_logging_payload() + payload["hidden_params"] = {"api_key": "sk-secret-key"} + model_call_details = create_model_call_details(payload) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "hidden_params" not in result["standard_logging_object"] + + def test_exclude_nonexistent_field_no_error(self): + """Test that excluding a non-existent field doesn't cause an error.""" + litellm.standard_logging_payload_excluded_fields = [ + "nonexistent_field", + "response", + ] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + # Should not raise an exception + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" in result["standard_logging_object"] + + def test_original_payload_not_modified(self): + """Test that the original model_call_details is not modified.""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + logger = CustomLogger() + model_call_details = create_model_call_details() + original_payload = deepcopy(model_call_details) + + logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # Original should still have the fields + assert "response" in model_call_details["standard_logging_object"] + assert "messages" in model_call_details["standard_logging_object"] + assert model_call_details == original_payload + + def test_combined_with_turn_off_message_logging(self): + """Test that excluded_fields works together with turn_off_message_logging.""" + litellm.standard_logging_payload_excluded_fields = ["metadata", "hidden_params"] + + logger = CustomLogger(turn_off_message_logging=True) + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # excluded_fields should remove these + assert "metadata" not in result["standard_logging_object"] + assert "hidden_params" not in result["standard_logging_object"] + + # turn_off_message_logging should redact these + redacted_str = "redacted-by-litellm" + assert ( + result["standard_logging_object"]["messages"][0]["content"] == redacted_str + ) + assert ( + result["standard_logging_object"]["response"]["choices"][0]["message"][ + "content" + ] + == redacted_str + ) + + def test_excluded_fields_takes_precedence_over_redaction(self): + """Test that if a field is both excluded and would be redacted, it's excluded.""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger(turn_off_message_logging=True) + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # response should be excluded (not redacted) + assert "response" not in result["standard_logging_object"] + + # messages should still be redacted + redacted_str = "redacted-by-litellm" + assert ( + result["standard_logging_object"]["messages"][0]["content"] == redacted_str + ) + + def test_exclude_all_sensitive_fields(self): + """Test excluding all potentially sensitive fields.""" + litellm.standard_logging_payload_excluded_fields = [ + "messages", + "response", + "metadata", + "hidden_params", + "model_parameters", + "error_str", + "error_information", + ] + + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + standard_obj = result["standard_logging_object"] + + # All sensitive fields should be removed + assert "messages" not in standard_obj + assert "response" not in standard_obj + assert "metadata" not in standard_obj + assert "hidden_params" not in standard_obj + assert "model_parameters" not in standard_obj + assert "error_str" not in standard_obj + assert "error_information" not in standard_obj + + # Non-sensitive fields should remain + assert "id" in standard_obj + assert "model" in standard_obj + assert "response_cost" in standard_obj + assert "total_tokens" in standard_obj + + def test_empty_excluded_fields_list(self): + """Test that an empty list doesn't affect the payload.""" + litellm.standard_logging_payload_excluded_fields = [] + + logger = CustomLogger() + model_call_details = create_model_call_details() + original_keys = set(model_call_details["standard_logging_object"].keys()) + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + result_keys = set(result["standard_logging_object"].keys()) + assert result_keys == original_keys + + def test_none_standard_logging_object(self): + """Test handling when standard_logging_object is None.""" + litellm.standard_logging_payload_excluded_fields = ["response"] + + logger = CustomLogger() + model_call_details = {"other_key": "other_value"} + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + # Should return unchanged when no standard_logging_object + assert result == model_call_details + + +class TestExcludedFieldsIntegration: + """Integration tests for excluded fields with actual callbacks.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + litellm.callbacks = [] + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + litellm.callbacks = [] + + def test_custom_callback_receives_filtered_payload(self): + """Test that a custom callback receives the filtered payload.""" + captured_payloads = [] + + class TestCallback(CustomLogger): + def log_success_event(self, kwargs, response_obj, start_time, end_time): + captured_payloads.append(kwargs.get("standard_logging_object", {})) + + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + + callback = TestCallback() + model_call_details = create_model_call_details() + + # Simulate what litellm_logging.py does + filtered_details = callback.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + callback.log_success_event( + kwargs=filtered_details, + response_obj=None, + start_time=None, + end_time=None, + ) + + assert len(captured_payloads) == 1 + assert "response" not in captured_payloads[0] + assert "messages" not in captured_payloads[0] + assert "model" in captured_payloads[0] + + +class TestExcludedFieldsConfigLoading: + """Test that the config is properly loaded from litellm_settings.""" + + def setup_method(self): + """Reset litellm settings before each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def teardown_method(self): + """Clean up after each test.""" + litellm.standard_logging_payload_excluded_fields = None + + def test_config_attribute_exists(self): + """Test that the config attribute exists on litellm module.""" + assert hasattr(litellm, "standard_logging_payload_excluded_fields") + + def test_config_default_is_none(self): + """Test that the default value is None.""" + # Reset to ensure we're testing the default + litellm.standard_logging_payload_excluded_fields = None + assert litellm.standard_logging_payload_excluded_fields is None + + def test_config_can_be_set_to_list(self): + """Test that the config can be set to a list.""" + litellm.standard_logging_payload_excluded_fields = ["response", "messages"] + assert litellm.standard_logging_payload_excluded_fields == [ + "response", + "messages", + ] + + def test_config_setattr_simulates_proxy_loading(self): + """Test that setattr works as the proxy would use it.""" + # Simulating how proxy_server.py sets litellm_settings + config_value = ["response", "messages", "metadata"] + setattr(litellm, "standard_logging_payload_excluded_fields", config_value) + + assert litellm.standard_logging_payload_excluded_fields == config_value + + # Test it actually works in the logger + logger = CustomLogger() + model_call_details = create_model_call_details() + + result = logger.redact_standard_logging_payload_from_model_call_details( + model_call_details + ) + + assert "response" not in result["standard_logging_object"] + assert "messages" not in result["standard_logging_object"] + assert "metadata" not in result["standard_logging_object"] diff --git a/tests/mcp_tests/test_aresponses_api_with_mcp.py b/tests/mcp_tests/test_aresponses_api_with_mcp.py index a7bbfef14af..bae0b15dfec 100644 --- a/tests/mcp_tests/test_aresponses_api_with_mcp.py +++ b/tests/mcp_tests/test_aresponses_api_with_mcp.py @@ -683,8 +683,8 @@ async def test_streaming_responses_api_with_mcp_tools( Return the user the result of request 2 """ - # Skip test if required API keys are not set - if ("anthropic" in model.lower() or "claude" in model.lower()) and not os.getenv("ANTHROPIC_API_KEY"): + # Skip test if API keys are not set for the respective models + if ("claude" in model.lower() or "anthropic" in model.lower()) and not os.getenv("ANTHROPIC_API_KEY"): pytest.skip("ANTHROPIC_API_KEY not set, skipping anthropic model test") if ("gpt" in model.lower() or "openai" in model.lower()) and not os.getenv("OPENAI_API_KEY"): pytest.skip("OPENAI_API_KEY not set, skipping openai model test") diff --git a/tests/mcp_tests/test_mcp_chat_completions.py b/tests/mcp_tests/test_mcp_chat_completions.py index 0617dcb2e42..9010e8c0d29 100644 --- a/tests/mcp_tests/test_mcp_chat_completions.py +++ b/tests/mcp_tests/test_mcp_chat_completions.py @@ -19,7 +19,7 @@ async def test_acompletion_mcp_auto_exec(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -95,7 +95,7 @@ async def test_acompletion_mcp_respects_manual_approval(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -170,7 +170,7 @@ async def test_completion_mcp_with_streaming_no_timeout_error(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -470,7 +470,7 @@ async def test_mcp_metadata_in_streaming_final_chunk(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): @@ -793,7 +793,7 @@ async def test_mcp_streaming_metadata_ordering(monkeypatch): inputSchema={"type": "object", "properties": {}}, ) - async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy): + async def fake_process(user_api_key_auth, mcp_tools_with_litellm_proxy, **kwargs): return [dummy_tool], {"local_search": "local"} async def fake_execute(**kwargs): diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index ea823df1fb2..9718d714cfe 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -1053,6 +1053,7 @@ async def test_mcp_server_manager_access_groups_from_config(): mcp_server_manager_mod.global_mcp_server_manager = original_manager +@pytest.mark.asyncio async def test_mcp_server_manager_config_integration_with_database(): """ Test that config-based servers properly integrate with database servers, diff --git a/tests/proxy_unit_tests/test_ui_path_detection.py b/tests/proxy_unit_tests/test_ui_path_detection.py new file mode 100644 index 00000000000..72ee7770f94 --- /dev/null +++ b/tests/proxy_unit_tests/test_ui_path_detection.py @@ -0,0 +1,157 @@ +""" +Unit tests for UI path detection and configuration. + +Tests the new LITELLM_UI_PATH and LITELLM_ASSETS_PATH functionality +for read-only filesystem support. + +Note: Tests involving proxy_server imports are intentionally minimal +to avoid long module load times during testing. +""" + +import os +import tempfile +from pathlib import Path +from unittest import mock + +import pytest + + +class TestUIPathEnvironmentVariable: + """Test LITELLM_UI_PATH environment variable handling.""" + + def test_custom_ui_path_env_var(self): + """Test that LITELLM_UI_PATH overrides default.""" + custom_path = "/custom/ui/path" + + with mock.patch.dict( + os.environ, {"LITELLM_UI_PATH": custom_path, "LITELLM_NON_ROOT": "true"} + ): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_runtime_ui_path = ( + "/var/lib/litellm/ui" if is_non_root else "/default/packaged/path" + ) + runtime_ui_path = os.getenv("LITELLM_UI_PATH", default_runtime_ui_path) + + assert runtime_ui_path == custom_path + + def test_default_ui_path_non_root(self): + """Test default UI path in non-root mode.""" + with mock.patch.dict( + os.environ, {"LITELLM_NON_ROOT": "true"}, clear=False + ): + # Clear LITELLM_UI_PATH if it exists + env_copy = os.environ.copy() + if "LITELLM_UI_PATH" in env_copy: + del env_copy["LITELLM_UI_PATH"] + + with mock.patch.dict(os.environ, env_copy, clear=True): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_runtime_ui_path = ( + "/var/lib/litellm/ui" + if is_non_root + else "/default/packaged/path" + ) + runtime_ui_path = os.getenv( + "LITELLM_UI_PATH", default_runtime_ui_path + ) + + assert runtime_ui_path == "/var/lib/litellm/ui" + + +class TestAssetsPathEnvironmentVariable: + """Test LITELLM_ASSETS_PATH environment variable handling.""" + + def test_custom_assets_path_env_var(self): + """Test that LITELLM_ASSETS_PATH overrides default.""" + custom_path = "/custom/assets/path" + + with mock.patch.dict( + os.environ, + {"LITELLM_ASSETS_PATH": custom_path, "LITELLM_NON_ROOT": "true"}, + ): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_assets_dir = ( + "/var/lib/litellm/assets" if is_non_root else "/default/current/dir" + ) + assets_dir = os.getenv("LITELLM_ASSETS_PATH", default_assets_dir) + + assert assets_dir == custom_path + + def test_default_assets_path_non_root(self): + """Test default assets path in non-root mode.""" + env_copy = os.environ.copy() + env_copy["LITELLM_NON_ROOT"] = "true" + if "LITELLM_ASSETS_PATH" in env_copy: + del env_copy["LITELLM_ASSETS_PATH"] + + with mock.patch.dict(os.environ, env_copy, clear=True): + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + default_assets_dir = ( + "/var/lib/litellm/assets" if is_non_root else "/default/current/dir" + ) + assets_dir = os.getenv("LITELLM_ASSETS_PATH", default_assets_dir) + + assert assets_dir == "/var/lib/litellm/assets" + + +class TestUIDetectionLogic: + """Test UI pre-restructured detection logic without importing proxy_server.""" + + def setup_method(self): + """Create temporary directory for testing.""" + self.temp_dir = tempfile.mkdtemp() + + def teardown_method(self): + """Clean up temporary directory.""" + import shutil + + if os.path.exists(self.temp_dir): + shutil.rmtree(self.temp_dir) + + def test_marker_file_exists(self): + """Test marker file detection logic.""" + marker_path = os.path.join(self.temp_dir, ".litellm_ui_ready") + Path(marker_path).touch() + + # Verify marker file exists + assert os.path.exists(marker_path) + + def test_structural_routes_exist(self): + """Test structural detection logic.""" + routes = ["login", "guardrails", "logs"] + for route in routes: + route_dir = os.path.join(self.temp_dir, route) + os.makedirs(route_dir, exist_ok=True) + index_html = os.path.join(route_dir, "index.html") + Path(index_html).touch() + + # Verify routes exist + found_routes = 0 + expected_routes = ["login", "guardrails", "logs", "api-reference"] + for route in expected_routes: + route_index = os.path.join(self.temp_dir, route, "index.html") + if os.path.exists(route_index): + found_routes += 1 + + assert found_routes >= 3 + + def test_writability_check(self): + """Test that os.access() correctly detects writable directories.""" + # Should be writable + assert os.access(self.temp_dir, os.W_OK) is True + + # Create a directory we can't write to (platform-dependent) + if os.name != "nt": # Skip on Windows + readonly_dir = os.path.join(self.temp_dir, "readonly") + os.makedirs(readonly_dir) + os.chmod(readonly_dir, 0o444) # Read-only + + # Should not be writable + assert os.access(readonly_dir, os.W_OK) is False + + # Restore permissions for cleanup + os.chmod(readonly_dir, 0o755) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py index 57352eafaf1..5458b466f68 100644 --- a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py @@ -156,3 +156,83 @@ def test_transform_request_includes_extra_headers(): litellm_logging_obj=MockLoggingObj(), ) assert result.get("extra_headers") == headers + + +def test_transform_request_strips_internal_metadata_to_litellm_metadata(): + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = { + "metadata": {"user_api_key_auth": {"id": "abc"}}, + "litellm_metadata": {"trace_id": "trace-1"}, + "api_key": "sk-test", + } + + class MockLoggingObj: + pass + + result = handler.transform_request( + model="gpt-5-pro", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + litellm_logging_obj=MockLoggingObj(), + ) + + assert "metadata" not in result + assert result["litellm_metadata"]["user_api_key_auth"]["id"] == "abc" + assert result["litellm_metadata"]["trace_id"] == "trace-1" + + +def test_transform_request_preserves_user_metadata(): + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {"metadata": {"customer_id": "cust-123"}} + litellm_params = {"metadata": {"internal_key": "secret"}} + + class MockLoggingObj: + pass + + result = handler.transform_request( + model="gpt-5-pro", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + litellm_logging_obj=MockLoggingObj(), + ) + + assert result["metadata"] == {"customer_id": "cust-123"} + assert "internal_key" not in result["metadata"] + assert result["litellm_metadata"]["internal_key"] == "secret" + + +def test_transform_request_drops_user_metadata_with_additional_drop_params(): + from litellm.utils import get_optional_params + + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": "Hello"}] + optional_params = get_optional_params( + model="gpt-4o", + messages=messages, + metadata={"customer_id": "cust-123"}, + additional_drop_params=["metadata"], + custom_llm_provider="openai", + ) + litellm_params = {"metadata": {"internal_key": "secret"}} + + class MockLoggingObj: + pass + + result = handler.transform_request( + model="gpt-4o", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + litellm_logging_obj=MockLoggingObj(), + ) + + assert "metadata" not in result + assert result["litellm_metadata"]["internal_key"] == "secret" diff --git a/tests/test_litellm/expected_responses_api_request/context_management_and_shell.json b/tests/test_litellm/expected_responses_api_request/context_management_and_shell.json new file mode 100644 index 00000000000..1e34b230182 --- /dev/null +++ b/tests/test_litellm/expected_responses_api_request/context_management_and_shell.json @@ -0,0 +1,20 @@ +{ + "model": "gpt-4o", + "input": "List files in /mnt/data and run python --version.", + "context_management": [ + { + "type": "compaction", + "compact_threshold": 200000 + } + ], + "tools": [ + { + "type": "shell", + "environment": { + "type": "container_auto" + } + } + ], + "tool_choice": "auto", + "max_output_tokens": 256 +} diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index c7ad18cfb0c..beb978584cb 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -77,6 +77,15 @@ context_window_test_cases = [ ("Rate limit reached for requests.", False), ("The context is large, but acceptable.", False), ("", False), # Empty string + # OpenAI user param length validation - not a context window error + ( + "Invalid 'user': string too long. Expected a string with maximum length 64, but got a string with length 123 instead.", + False, + ), + ( + '{"error": {"message": "Invalid \'user\': string too long.", "code": "string_above_max_length"}}', + False, + ), ] diff --git a/tests/test_litellm/llms/azure/test_azure_exception_mapping.py b/tests/test_litellm/llms/azure/test_azure_exception_mapping.py index f4abe7f2b9a..495ca958cf5 100644 --- a/tests/test_litellm/llms/azure/test_azure_exception_mapping.py +++ b/tests/test_litellm/llms/azure/test_azure_exception_mapping.py @@ -239,4 +239,149 @@ class TestAzureExceptionMapping: assert e.provider_specific_fields is not None assert e.provider_specific_fields["inner_error"]["code"] == "ResponsibleAIPolicyViolation" assert e.provider_specific_fields["inner_error"]["revised_prompt"] == "revised" - assert e.provider_specific_fields["inner_error"]["content_filter_results"]["violence"]["filtered"] is True \ No newline at end of file + assert e.provider_specific_fields["inner_error"]["content_filter_results"]["violence"]["filtered"] is True + + def test_azure_content_policy_violation_detected_via_inner_error_code(self): + """Regression test for #20811: Azure returns inner_error with + ResponsibleAIPolicyViolation but the top-level error message is + generic. Previously this fell through to the generic + BadRequestError handler and all error details were lost.""" + + mock_exception = Exception("Bad request") + # This body structure mirrors what Azure OpenAI Images API returns + # for DALL-E 3 content policy violations (issue #20811). + mock_exception.body = { + "error": { + "code": "content_policy_violation", + "inner_error": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "profanity": {"detected": False, "filtered": False}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": True, "severity": "low"}, + }, + "revised_prompt": ( + "A dark and intense illustration of a man " + "in a dramatic action scene." + ), + }, + "message": ( + "Your request was rejected as a result of our safety system." + ), + "type": "invalid_request_error", + } + } + + mock_response = MagicMock() + mock_response.status_code = 400 + mock_exception.response = mock_response + + with pytest.raises(ContentPolicyViolationError) as exc_info: + exception_type( + model="azure/dall-e-3", + original_exception=mock_exception, + custom_llm_provider="azure", + ) + + e = exc_info.value + # Must surface as ContentPolicyViolationError, not generic BadRequestError + assert "safety system" in str(e) + assert e.provider_specific_fields is not None + inner = e.provider_specific_fields["inner_error"] + assert inner["code"] == "ResponsibleAIPolicyViolation" + assert inner["content_filter_results"]["violence"]["filtered"] is True + assert inner["revised_prompt"] is not None + + def test_azure_policy_violation_detected_via_inner_error_without_top_code(self): + """When the top-level code is NOT 'content_policy_violation' but + inner_error.code IS 'ResponsibleAIPolicyViolation', the error + should still be recognized as a content policy violation.""" + + mock_exception = Exception("Some error") + mock_exception.body = { + "error": { + "code": "BadRequest", + "inner_error": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_results": { + "violence": {"filtered": True, "severity": "medium"}, + }, + }, + "message": "The request was rejected.", + "type": "invalid_request_error", + } + } + + mock_response = MagicMock() + mock_response.status_code = 400 + mock_exception.response = mock_response + + with pytest.raises(ContentPolicyViolationError) as exc_info: + exception_type( + model="azure/dall-e-3", + original_exception=mock_exception, + custom_llm_provider="azure", + ) + + e = exc_info.value + assert e.provider_specific_fields is not None + assert e.provider_specific_fields["inner_error"]["code"] == "ResponsibleAIPolicyViolation" + + def test_azure_image_polling_error_preserves_body(self): + """Verify that AzureOpenAIError raised from the DALL-E polling path + carries the structured body so exception_type() can inspect it.""" + from litellm.llms.azure.common_utils import AzureOpenAIError + + error_payload = { + "status": "failed", + "error": { + "code": "content_policy_violation", + "message": "Your request was rejected.", + "inner_error": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_results": { + "violence": {"filtered": True, "severity": "low"}, + }, + }, + }, + } + + # Simulate what the fixed polling path now does + _error_body = error_payload.get("error", error_payload) + _error_msg = ( + _error_body.get("message", "Image generation failed") + if isinstance(_error_body, dict) + else json.dumps(error_payload) + ) + exc = AzureOpenAIError( + status_code=400, + message=_error_msg, + body=error_payload, + ) + + assert exc.body is not None + assert isinstance(exc.body, dict) + assert exc.body["error"]["code"] == "content_policy_violation" + assert "Your request was rejected" in exc.message + + def test_azure_safety_system_message_detected_as_policy_violation(self): + """Azure's rejection message 'Your request was rejected as a result + of our safety system' should be detected by string matching even + when the structured body is unavailable.""" + + mock_exception = Exception( + "Your request was rejected as a result of our safety system. " + "The revised prompt may contain text that is not allowed." + ) + mock_response = MagicMock() + mock_response.status_code = 400 + mock_exception.response = mock_response + + with pytest.raises(ContentPolicyViolationError): + exception_type( + model="azure/dall-e-3", + original_exception=mock_exception, + custom_llm_provider="azure", + ) \ No newline at end of file diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py index 989a06f80b4..42e62c75d63 100644 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py @@ -352,17 +352,17 @@ class TestErrorHandling: def test_hf_response_missing_embedding(self): """Test handling of HF response missing embedding field""" config = SagemakerEmbeddingConfig() - + # Mock response without embedding field mock_response = httpx.Response( status_code=200, content=json.dumps({"object": "list"}).encode('utf-8'), headers={"content-type": "application/json"} ) - + model_response = EmbeddingResponse() - - with pytest.raises(Exception, match="HF response missing 'embedding' field"): + + with pytest.raises(Exception, match="Unexpected response format"): config.transform_embedding_response( model="sentence-transformers-model", raw_response=mock_response, @@ -372,5 +372,99 @@ class TestErrorHandling: ) +class TestTEIEmbeddingResponse: + """Test HuggingFace Text Embeddings Inference (TEI) response format support""" + + def setup_method(self): + self.config = SagemakerEmbeddingConfig() + + def test_transform_embedding_response_tei_raw_array(self): + """Test TEI response transformation - raw array format [[...]]""" + # TEI returns raw embedding arrays without wrapper + tei_response = [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6] + ] + + mock_response = httpx.Response( + status_code=200, + content=json.dumps(tei_response).encode('utf-8'), + headers={"content-type": "application/json"} + ) + + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="tei-qwen-embedding", + raw_response=mock_response, + model_response=model_response, + logging_obj=None, + request_data={"inputs": ["Hello", "World"]} + ) + + # Verify response structure + assert result.object == "list" + assert result.model == "tei-qwen-embedding" + assert len(result.data) == 2 + assert result.data[0]["object"] == "embedding" + assert result.data[0]["index"] == 0 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert result.data[1]["object"] == "embedding" + assert result.data[1]["index"] == 1 + assert result.data[1]["embedding"] == [0.4, 0.5, 0.6] + assert isinstance(result.usage, Usage) + + def test_transform_embedding_response_tei_single_input(self): + """Test TEI response with single input""" + tei_response = [ + [0.1, 0.2, 0.3, 0.4, 0.5] + ] + + mock_response = httpx.Response( + status_code=200, + content=json.dumps(tei_response).encode('utf-8'), + headers={"content-type": "application/json"} + ) + + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="tei-model", + raw_response=mock_response, + model_response=model_response, + logging_obj=None, + request_data={"inputs": ["Hello"]} + ) + + assert len(result.data) == 1 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3, 0.4, 0.5] + + def test_transform_embedding_response_wrapped_format_still_works(self): + """Test that wrapped format {"embedding": [...]} still works""" + hf_response = { + "embedding": [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6] + ] + } + + mock_response = httpx.Response( + status_code=200, + content=json.dumps(hf_response).encode('utf-8'), + headers={"content-type": "application/json"} + ) + + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="hf-model", + raw_response=mock_response, + model_response=model_response, + logging_obj=None, + request_data={"inputs": ["Hello", "World"]} + ) + + assert len(result.data) == 2 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert result.data[1]["embedding"] == [0.4, 0.5, 0.6] + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py new file mode 100644 index 00000000000..dde73016271 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_discovery.py @@ -0,0 +1,182 @@ +import json +import os +import sys + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../..") +) # Adds the parent directory to the system path + + +class TestMCPRegistryFile: + """Tests for the curated MCP registry JSON file.""" + + @pytest.fixture + def registry_path(self): + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "..", + "..", + "..", + "..", + "litellm", + "proxy", + "mcp_registry.json", + ) + + def test_registry_file_exists(self, registry_path): + assert os.path.exists(registry_path), f"Registry file not found at {registry_path}" + + def test_registry_file_is_valid_json(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + assert isinstance(data, dict) + assert "servers" in data + + def test_registry_servers_have_required_fields(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + servers = data["servers"] + assert len(servers) > 0, "Registry should have at least one server" + + required_fields = ["name", "title", "description", "category", "transport"] + for server in servers: + for field in required_fields: + assert field in server, f"Server {server.get('name', '?')} missing field '{field}'" + + def test_registry_server_names_are_unique(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + names = [s["name"] for s in data["servers"]] + assert len(names) == len(set(names)), f"Duplicate server names found: {[n for n in names if names.count(n) > 1]}" + + def test_registry_transport_values_are_valid(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + valid_transports = {"stdio", "http", "sse"} + for server in data["servers"]: + assert server["transport"] in valid_transports, ( + f"Server {server['name']} has invalid transport '{server['transport']}'" + ) + + def test_stdio_servers_have_command(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + for server in data["servers"]: + if server["transport"] == "stdio": + assert "command" in server and server["command"], ( + f"stdio server {server['name']} missing 'command'" + ) + + def test_http_servers_have_url(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + for server in data["servers"]: + if server["transport"] in ("http", "sse"): + assert "url" in server and server["url"], ( + f"HTTP/SSE server {server['name']} missing 'url'" + ) + + def test_well_known_servers_present(self, registry_path): + """Ensure key well-known MCPs are in the registry.""" + with open(registry_path, "r") as f: + data = json.load(f) + names = {s["name"] for s in data["servers"]} + expected = {"github", "slack", "postgresql", "snowflake", "atlassian"} + missing = expected - names + assert not missing, f"Missing well-known servers: {missing}" + + def test_env_vars_structure(self, registry_path): + with open(registry_path, "r") as f: + data = json.load(f) + for server in data["servers"]: + if "env_vars" in server: + assert isinstance(server["env_vars"], list) + for var in server["env_vars"]: + assert "name" in var, f"env_var in {server['name']} missing 'name'" + + +class TestDiscoverEndpointFiltering: + """Tests for the discover endpoint filtering logic (unit-level).""" + + @pytest.fixture + def sample_servers(self): + return [ + { + "name": "github", + "title": "GitHub", + "description": "Repository management", + "category": "Developer Tools", + "transport": "http", + "url": "https://mcp.github.com/sse", + }, + { + "name": "slack", + "title": "Slack", + "description": "Channel management and messaging", + "category": "Communication", + "transport": "stdio", + "command": "npx", + }, + { + "name": "postgresql", + "title": "PostgreSQL", + "description": "Query and manage databases", + "category": "Databases", + "transport": "stdio", + "command": "npx", + }, + ] + + def test_query_filter_by_name(self, sample_servers): + query = "github" + q = query.lower() + result = [ + s + for s in sample_servers + if q in s.get("name", "").lower() + or q in s.get("title", "").lower() + or q in s.get("description", "").lower() + ] + assert len(result) == 1 + assert result[0]["name"] == "github" + + def test_query_filter_by_description(self, sample_servers): + query = "messaging" + q = query.lower() + result = [ + s + for s in sample_servers + if q in s.get("name", "").lower() + or q in s.get("title", "").lower() + or q in s.get("description", "").lower() + ] + assert len(result) == 1 + assert result[0]["name"] == "slack" + + def test_category_filter(self, sample_servers): + category = "Databases" + result = [s for s in sample_servers if s.get("category") == category] + assert len(result) == 1 + assert result[0]["name"] == "postgresql" + + def test_no_filter_returns_all(self, sample_servers): + assert len(sample_servers) == 3 + + def test_query_filter_no_match(self, sample_servers): + query = "nonexistent" + q = query.lower() + result = [ + s + for s in sample_servers + if q in s.get("name", "").lower() + or q in s.get("title", "").lower() + or q in s.get("description", "").lower() + ] + assert len(result) == 0 + + def test_categories_extraction(self, sample_servers): + categories = sorted(set(s.get("category", "Other") for s in sample_servers)) + assert categories == ["Communication", "Databases", "Developer Tools"] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index b4b5811666b..1a50cacd308 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -112,7 +112,44 @@ class TestMCPServerManager: assert client.stdio_config is not None assert client.stdio_config["command"] == "node" assert client.stdio_config["args"] == ["server.js"] - assert client.stdio_config["env"] == {"NODE_ENV": "test"} + # NPM_CONFIG_CACHE is injected automatically for container compatibility + from litellm.constants import MCP_NPM_CACHE_DIR + + assert client.stdio_config["env"]["NODE_ENV"] == "test" + assert client.stdio_config["env"]["NPM_CONFIG_CACHE"] == MCP_NPM_CACHE_DIR + + async def test_create_mcp_client_stdio_injects_npm_config_cache(self): + """Test that _create_mcp_client injects NPM_CONFIG_CACHE when not already set, + and preserves user-provided NPM_CONFIG_CACHE when present.""" + from litellm.constants import MCP_NPM_CACHE_DIR + + manager = MCPServerManager() + + # Case 1: NPM_CONFIG_CACHE not set -> should be injected + server_no_cache = MCPServer( + server_id="stdio-npm-1", + name="test_npm_server", + url=None, + transport=MCPTransport.stdio, + command="npx", + args=["-y", "@modelcontextprotocol/server-everything"], + env={}, + ) + client = await manager._create_mcp_client(server_no_cache) + assert client.stdio_config["env"]["NPM_CONFIG_CACHE"] == MCP_NPM_CACHE_DIR + + # Case 2: NPM_CONFIG_CACHE already set -> should NOT be overwritten + server_with_cache = MCPServer( + server_id="stdio-npm-2", + name="test_npm_server_custom", + url=None, + transport=MCPTransport.stdio, + command="npx", + args=["-y", "@modelcontextprotocol/server-everything"], + env={"NPM_CONFIG_CACHE": "/custom/cache"}, + ) + client2 = await manager._create_mcp_client(server_with_cache) + assert client2.stdio_config["env"]["NPM_CONFIG_CACHE"] == "/custom/cache" def test_build_stdio_env_only_accepts_x_prefixed_placeholders(self): """Ensure only ${X-*} placeholders are substituted from headers.""" diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py index 9993b25dfdd..0ed5940dd75 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_update_queue.py @@ -225,6 +225,39 @@ async def test_aggregate_queue_updates_accuracy(spend_queue): assert aggregated["team_list_transactions"]["team1"] == 5.0 +def test_get_aggregated_spend_update_queue_item_does_not_mutate_original_updates( + spend_queue, +): + original_update: SpendUpdateQueueItem = { + "entity_type": Litellm_EntityType.USER, + "entity_id": "user1", + "response_cost": 10.0, + } + duplicate_key_update: SpendUpdateQueueItem = { + "entity_type": Litellm_EntityType.USER, + "entity_id": "user1", + "response_cost": 20.0, + } + + aggregated_updates = spend_queue._get_aggregated_spend_update_queue_item( + [original_update, duplicate_key_update] + ) + user1_aggregated_update = next( + ( + update + for update in aggregated_updates + if update.get("entity_type") == Litellm_EntityType.USER + and update.get("entity_id") == "user1" + ), + None, + ) + + assert original_update["response_cost"] == 10.0 + assert user1_aggregated_update is not None + assert user1_aggregated_update["response_cost"] == 30.0 + assert user1_aggregated_update is not original_update + + @pytest.mark.asyncio async def test_queue_size_reduction_with_large_volume(monkeypatch, spend_queue): """Test that queue size is actually reduced when dealing with many items""" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index 5c039141928..7d2b6e84de7 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -14,10 +14,14 @@ import pytest import litellm from litellm import ModelResponse from litellm.exceptions import GuardrailRaisedException +from litellm._version import version as litellm_version from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( GenericGuardrailAPI, ) +from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api.generic_guardrail_api import ( + _HEADER_PRESENT_PLACEHOLDER, +) from litellm.types.utils import Choices, Message @@ -351,6 +355,58 @@ class TestMetadataExtraction: # Should be empty dict assert request_metadata == {} + @pytest.mark.asyncio + async def test_inbound_headers_and_litellm_version_forwarded_and_sanitized( + self, generic_guardrail, mock_request_data_input + ): + """ + Ensure inbound proxy request headers are forwarded in JSON payload with allowlist: + allowed headers show their value; all other headers show presence only ([present]). + """ + # Add proxy_server_request headers as they exist in proxy request context + request_data = dict(mock_request_data_input) + request_data["proxy_server_request"] = { + "headers": { + "User-Agent": "OpenAI/Python 2.17.0", + "Authorization": "Bearer should-not-forward", + "Cookie": "session=should-not-forward", + "X-Request-Id": "req_123", + } + } + + mock_response = MagicMock() + mock_response.json.return_value = { + "action": "NONE", + "texts": ["test"], + } + mock_response.raise_for_status = MagicMock() + + with patch.object( + generic_guardrail.async_handler, "post", return_value=mock_response + ) as mock_post: + await generic_guardrail.apply_guardrail( + inputs={"texts": ["test"]}, + request_data=request_data, + input_type="request", + ) + + call_args = mock_post.call_args + json_payload = call_args.kwargs["json"] + + # New fields should exist + assert json_payload["litellm_version"] == litellm_version + assert "request_headers" in json_payload + assert isinstance(json_payload["request_headers"], dict) + req_headers = json_payload["request_headers"] + + # Allowed: value forwarded + assert req_headers.get("User-Agent") == "OpenAI/Python 2.17.0" + + # Not on allowlist: key present, value is placeholder only + assert req_headers.get("Authorization") == _HEADER_PRESENT_PLACEHOLDER + assert req_headers.get("Cookie") == _HEADER_PRESENT_PLACEHOLDER + assert req_headers.get("X-Request-Id") == _HEADER_PRESENT_PLACEHOLDER + class TestGuardrailActions: """Test different guardrail action responses""" diff --git a/tests/test_litellm/proxy/image_endpoints/test_endpoints.py b/tests/test_litellm/proxy/image_endpoints/test_endpoints.py index a3b6a9c6022..c35630176bc 100644 --- a/tests/test_litellm/proxy/image_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/image_endpoints/test_endpoints.py @@ -40,10 +40,14 @@ async def test_image_generation_prompt_rerouting(monkeypatch): async def fake_post_call_failure_hook(**_: Any) -> None: return None + async def fake_post_call_success_hook(*, data, user_api_key_dict, response): + return response + fake_proxy_logger = SimpleNamespace( pre_call_hook=fake_pre_call_hook, update_request_status=fake_update_request_status, post_call_failure_hook=fake_post_call_failure_hook, + post_call_success_hook=fake_post_call_success_hook, ) captured_route_request_data: Dict[str, Any] = {} diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py new file mode 100644 index 00000000000..5f204918d08 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -0,0 +1,583 @@ +""" +Tests for access group management endpoints. +""" + +import os +import sys +import types +from contextlib import asynccontextmanager +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi.testclient import TestClient +from prisma.errors import PrismaError + +import litellm.proxy.proxy_server as ps +from litellm.proxy.proxy_server import app +from litellm.proxy._types import ( + CommonProxyErrors, + LitellmUserRoles, + UserAPIKeyAuth, +) + +sys.path.insert(0, os.path.abspath("../../../")) + + +def _make_access_group_record( + access_group_id: str = "ag-123", + access_group_name: str = "test-group", + description: str | None = "Test description", + access_model_ids: list | None = None, + access_mcp_server_ids: list | None = None, + access_agent_ids: list | None = None, + assigned_team_ids: list | None = None, + assigned_key_ids: list | None = None, + created_by: str | None = "admin-user", + updated_by: str | None = "admin-user", + created_at: datetime | None = None, +): + record = MagicMock() + record.access_group_id = access_group_id + record.access_group_name = access_group_name + record.description = description + record.access_model_ids = access_model_ids or [] + record.access_mcp_server_ids = access_mcp_server_ids or [] + record.access_agent_ids = access_agent_ids or [] + record.assigned_team_ids = assigned_team_ids or [] + record.assigned_key_ids = assigned_key_ids or [] + record.created_at = created_at or datetime.now() + record.created_by = created_by + record.updated_at = datetime.now() + record.updated_by = updated_by + return record + + +@pytest.fixture +def client_and_mocks(monkeypatch): + """Setup mock prisma and admin auth for access group endpoints.""" + mock_access_group_table = MagicMock() + mock_prisma = MagicMock() + + def _create_side_effect(*, data): + return _make_access_group_record( + access_group_id="ag-new", + access_group_name=data.get("access_group_name", "new"), + description=data.get("description"), + access_model_ids=data.get("access_model_ids", []), + access_mcp_server_ids=data.get("access_mcp_server_ids", []), + access_agent_ids=data.get("access_agent_ids", []), + assigned_team_ids=data.get("assigned_team_ids", []), + assigned_key_ids=data.get("assigned_key_ids", []), + created_by=data.get("created_by"), + updated_by=data.get("updated_by"), + ) + + mock_access_group_table.create = AsyncMock(side_effect=_create_side_effect) + mock_access_group_table.find_unique = AsyncMock(return_value=None) + mock_access_group_table.find_many = AsyncMock(return_value=[]) + mock_access_group_table.update = AsyncMock(side_effect=lambda *, where, data: _make_access_group_record( + access_group_id=where.get("access_group_id", "ag-123"), + access_group_name=data.get("access_group_name", "updated"), + description=data.get("description"), + access_model_ids=data.get("access_model_ids", []), + access_mcp_server_ids=data.get("access_mcp_server_ids", []), + access_agent_ids=data.get("access_agent_ids", []), + assigned_team_ids=data.get("assigned_team_ids", []), + assigned_key_ids=data.get("assigned_key_ids", []), + updated_by=data.get("updated_by"), + )) + mock_access_group_table.delete = AsyncMock(return_value=None) + + mock_team_table = MagicMock() + mock_team_table.find_many = AsyncMock(return_value=[]) + mock_team_table.update = AsyncMock(return_value=None) + + mock_key_table = MagicMock() + mock_key_table.find_many = AsyncMock(return_value=[]) + mock_key_table.update = AsyncMock(return_value=None) + + @asynccontextmanager + async def mock_tx(): + tx = types.SimpleNamespace( + litellm_accessgrouptable=mock_access_group_table, + litellm_teamtable=mock_team_table, + litellm_verificationtoken=mock_key_table, + ) + yield tx + + mock_db = types.SimpleNamespace( + litellm_accessgrouptable=mock_access_group_table, + litellm_teamtable=mock_team_table, + litellm_verificationtoken=mock_key_table, + tx=mock_tx, + ) + mock_prisma.db = mock_db + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + admin_user = UserAPIKeyAuth( + user_id="admin_user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: admin_user + + client = TestClient(app) + + yield client, mock_prisma, mock_access_group_table + + app.dependency_overrides.clear() + monkeypatch.setattr(ps, "prisma_client", ps.prisma_client) + + +# Paths for primary and alias endpoints (alias: /v1/unified_access_group) +ACCESS_GROUP_PATHS = ["/v1/access_group", "/v1/unified_access_group"] + + +# --------------------------------------------------------------------------- +# CREATE +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize( + "payload", + [ + {"access_group_name": "group-a"}, + { + "access_group_name": "group-b", + "description": "Group B description", + "access_model_ids": ["model-1"], + "access_mcp_server_ids": ["mcp-1"], + "assigned_team_ids": ["team-1"], + }, + ], +) +def test_create_access_group_success(client_and_mocks, base_path, payload): + """Create access group with various payloads returns 201.""" + client, _, mock_table = client_and_mocks + + resp = client.post(base_path, json=payload) + assert resp.status_code == 201 + body = resp.json() + assert body["access_group_name"] == payload["access_group_name"] + assert body.get("access_group_id") is not None + mock_table.create.assert_awaited_once() + + +def test_create_access_group_duplicate_name_conflict(client_and_mocks): + """Create with duplicate name returns 409.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_name="existing-group") + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.post("/v1/access_group", json={"access_group_name": "existing-group"}) + assert resp.status_code == 409 + assert "already exists" in resp.json()["detail"] + + +@pytest.mark.parametrize( + "error_message", + [ + "Unique constraint failed on the fields: (`access_group_name`)", + "P2002: Unique constraint failed", + "unique constraint violation", + ], +) +def test_create_access_group_race_condition_returns_409(client_and_mocks, error_message): + """Create race condition: Prisma unique constraint surfaces as 409, not 500.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + mock_table.create = AsyncMock(side_effect=Exception(error_message)) + + resp = client.post("/v1/access_group", json={"access_group_name": "race-group"}) + assert resp.status_code == 409 + assert "already exists" in resp.json()["detail"] + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_create_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot create access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.post("/v1/access_group", json={"access_group_name": "forbidden"}) + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +def test_create_access_group_validation_missing_name(client_and_mocks): + """Create with missing access_group_name returns 422.""" + client, _, _ = client_and_mocks + + resp = client.post("/v1/access_group", json={}) + assert resp.status_code == 422 + + +def test_create_access_group_500_on_non_constraint_prisma_error(client_and_mocks): + """Create with non-unique-constraint Prisma error returns 500.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + mock_table.create = AsyncMock(side_effect=Exception("Some other database error")) + + # Use raise_server_exceptions=False so unhandled exceptions become 500 responses + test_client = TestClient(app, raise_server_exceptions=False) + resp = test_client.post("/v1/access_group", json={"access_group_name": "test-group"}) + assert resp.status_code == 500 + + +# --------------------------------------------------------------------------- +# LIST +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +def test_list_access_groups_success_empty(client_and_mocks, base_path): + """List access groups returns empty list when none exist.""" + client, _, mock_table = client_and_mocks + + resp = client.get(base_path) + assert resp.status_code == 200 + assert resp.json() == [] + mock_table.find_many.assert_awaited_once() + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +def test_list_access_groups_success_with_items(client_and_mocks, base_path): + """List access groups returns items when they exist.""" + client, _, mock_table = client_and_mocks + + records = [ + _make_access_group_record(access_group_id="ag-1", access_group_name="group-1"), + _make_access_group_record(access_group_id="ag-2", access_group_name="group-2"), + ] + mock_table.find_many = AsyncMock(return_value=records) + + resp = client.get(base_path) + assert resp.status_code == 200 + body = resp.json() + assert len(body) == 2 + assert body[0]["access_group_name"] == "group-1" + assert body[1]["access_group_name"] == "group-2" + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +def test_list_access_groups_ordered_by_created_at_desc(client_and_mocks, base_path): + """List access groups calls find_many with created_at desc order.""" + client, _, mock_table = client_and_mocks + + older = datetime(2025, 1, 1, 12, 0, 0) + newer = datetime(2025, 1, 2, 12, 0, 0) + records = [ + _make_access_group_record( + access_group_id="ag-newer", + access_group_name="newer-group", + created_at=newer, + ), + _make_access_group_record( + access_group_id="ag-older", + access_group_name="older-group", + created_at=older, + ), + ] + mock_table.find_many = AsyncMock(return_value=records) + + resp = client.get(base_path) + assert resp.status_code == 200 + body = resp.json() + assert len(body) == 2 + # Mock returns newest first (simulating Prisma order desc) + assert body[0]["access_group_name"] == "newer-group" + assert body[1]["access_group_name"] == "older-group" + mock_table.find_many.assert_awaited_once_with(order={"created_at": "desc"}) + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_list_access_groups_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot list access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.get("/v1/access_group") + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +# --------------------------------------------------------------------------- +# GET +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize("access_group_id", ["ag-123", "ag-other-id"]) +def test_get_access_group_success(client_and_mocks, base_path, access_group_id): + """Get access group by id returns record when found.""" + client, _, mock_table = client_and_mocks + + record = _make_access_group_record(access_group_id=access_group_id) + mock_table.find_unique = AsyncMock(return_value=record) + + resp = client.get(f"{base_path}/{access_group_id}") + assert resp.status_code == 200 + assert resp.json()["access_group_id"] == access_group_id + + +def test_get_access_group_not_found(client_and_mocks): + """Get access group returns 404 when not found.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + + resp = client.get("/v1/access_group/nonexistent-id") + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_get_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot get access group.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.get("/v1/access_group/ag-123") + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +# --------------------------------------------------------------------------- +# UPDATE +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize( + "update_payload", + [ + {"description": "Updated description"}, + {"access_model_ids": ["model-1", "model-2"]}, + {"assigned_team_ids": [], "assigned_key_ids": ["key-1"]}, + ], +) +def test_update_access_group_success(client_and_mocks, base_path, update_payload): + """Update access group with various payloads returns 200.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-update") + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.put(f"{base_path}/ag-update", json=update_payload) + assert resp.status_code == 200 + mock_table.update.assert_awaited_once() + + +def test_update_access_group_not_found(client_and_mocks): + """Update access group returns 404 when not found.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + + resp = client.put( + "/v1/access_group/nonexistent-id", + json={"description": "Updated"}, + ) + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + mock_table.update.assert_not_awaited() + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_update_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot update access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.put("/v1/access_group/ag-123", json={"description": "Updated"}) + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +def test_update_access_group_empty_body(client_and_mocks): + """Update with empty body succeeds; only updated_by is set.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-update", access_group_name="unchanged") + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.put("/v1/access_group/ag-update", json={}) + assert resp.status_code == 200 + mock_table.update.assert_awaited_once() + call_kwargs = mock_table.update.call_args.kwargs + assert call_kwargs["where"] == {"access_group_id": "ag-update"} + assert "updated_by" in call_kwargs["data"] + assert call_kwargs["data"]["updated_by"] == "admin_user" + + +# --------------------------------------------------------------------------- +# DELETE +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +@pytest.mark.parametrize("access_group_id", ["ag-123", "ag-delete-me"]) +def test_delete_access_group_success(client_and_mocks, base_path, access_group_id): + """Delete access group returns 204 when found.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id=access_group_id) + mock_table.find_unique = AsyncMock(return_value=existing) + + resp = client.delete(f"{base_path}/{access_group_id}") + assert resp.status_code == 204 + mock_table.delete.assert_awaited_once() + + +def test_delete_access_group_not_found(client_and_mocks): + """Delete access group returns 404 when not found.""" + client, _, mock_table = client_and_mocks + + mock_table.find_unique = AsyncMock(return_value=None) + + resp = client.delete("/v1/access_group/nonexistent-id") + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + mock_table.delete.assert_not_awaited() + + +@pytest.mark.parametrize("user_role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY]) +def test_delete_access_group_forbidden_non_admin(client_and_mocks, user_role): + """Non-admin users cannot delete access groups.""" + client, _, _ = client_and_mocks + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="regular_user", + user_role=user_role, + ) + + resp = client.delete("/v1/access_group/ag-123") + assert resp.status_code == 403 + assert resp.json()["detail"]["error"] == CommonProxyErrors.not_allowed_access.value + + +def test_delete_access_group_cleans_up_teams_and_keys(client_and_mocks): + """Delete removes access_group_id from teams and keys before deleting the group.""" + client, mock_prisma, mock_access_group_table = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + team_with_group = MagicMock() + team_with_group.team_id = "team-1" + team_with_group.access_group_ids = ["ag-to-delete", "ag-other"] + mock_team_table.find_many = AsyncMock(return_value=[team_with_group]) + + key_with_group = MagicMock() + key_with_group.token = "key-token-1" + key_with_group.access_group_ids = ["ag-to-delete"] + mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 204 + + mock_team_table.update.assert_awaited_once_with( + where={"team_id": "team-1"}, + data={"access_group_ids": ["ag-other"]}, + ) + mock_key_table.update.assert_awaited_once_with( + where={"token": "key-token-1"}, + data={"access_group_ids": []}, + ) + mock_access_group_table.delete.assert_awaited_once_with( + where={"access_group_id": "ag-to-delete"} + ) + + +def test_delete_access_group_503_on_db_connection_error(client_and_mocks): + """Delete returns 503 when DB connection error occurs during transaction.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_table.find_unique = AsyncMock(return_value=existing) + mock_table.delete = AsyncMock(side_effect=PrismaError()) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 503 + assert resp.json()["detail"] == CommonProxyErrors.db_not_connected_error.value + + +def test_delete_access_group_404_on_p2025_or_record_not_found(client_and_mocks): + """Delete returns 404 when Prisma raises P2025 or record-not-found error.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_table.find_unique = AsyncMock(return_value=existing) + mock_table.delete = AsyncMock(side_effect=Exception("P2025: Record to delete does not exist")) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 404 + assert "not found" in resp.json()["detail"] + + +def test_delete_access_group_500_on_generic_exception(client_and_mocks): + """Delete returns 500 when generic exception occurs during transaction.""" + client, _, mock_table = client_and_mocks + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_table.find_unique = AsyncMock(return_value=existing) + mock_table.delete = AsyncMock(side_effect=RuntimeError("Unexpected error")) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 500 + assert "Failed to delete access group" in resp.json()["detail"] + + +# --------------------------------------------------------------------------- +# DB NOT CONNECTED +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "method,url,factory", + [ + ("post", "/v1/access_group", lambda: {"json": {"access_group_name": "test"}}), + ("get", "/v1/access_group", lambda: {}), + ("get", "/v1/access_group/ag-123", lambda: {}), + ("put", "/v1/access_group/ag-123", lambda: {"json": {"description": "x"}}), + ("delete", "/v1/access_group/ag-123", lambda: {}), + # Alias: /v1/unified_access_group + ("post", "/v1/unified_access_group", lambda: {"json": {"access_group_name": "test"}}), + ("get", "/v1/unified_access_group", lambda: {}), + ("get", "/v1/unified_access_group/ag-123", lambda: {}), + ("put", "/v1/unified_access_group/ag-123", lambda: {"json": {"description": "x"}}), + ("delete", "/v1/unified_access_group/ag-123", lambda: {}), + ], +) +def test_access_group_endpoints_db_not_connected(client_and_mocks, monkeypatch, method, url, factory): + """All endpoints return 500 when DB is not connected.""" + client, _, _ = client_and_mocks + + monkeypatch.setattr(ps, "prisma_client", None) + + resp = getattr(client, method)(url, **factory()) + assert resp.status_code == 500 + assert resp.json()["detail"]["error"] == CommonProxyErrors.db_not_connected_error.value diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index e7da4256182..d65df0087ad 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -2996,9 +2996,13 @@ async def test_get_image_non_root_uses_var_lib_assets_dir(monkeypatch): monkeypatch.setenv("LITELLM_NON_ROOT", "true") monkeypatch.delenv("UI_LOGO_PATH", raising=False) - # Mock os.path operations + # Mock os.path operations - exists=False for assets_dir so makedirs gets called + def exists_side_effect(path): + return False if path == "/var/lib/litellm/assets" else True + with patch("litellm.proxy.proxy_server.os.makedirs") as mock_makedirs, \ - patch("litellm.proxy.proxy_server.os.path.exists", return_value=True), \ + patch("litellm.proxy.proxy_server.os.path.exists", side_effect=exists_side_effect), \ + patch("litellm.proxy.proxy_server.os.access", return_value=True), \ patch("litellm.proxy.proxy_server.os.getenv") as mock_getenv, \ patch("litellm.proxy.proxy_server.FileResponse") as mock_file_response: @@ -3038,14 +3042,16 @@ async def test_get_image_non_root_fallback_to_default_logo(monkeypatch): def exists_side_effect(path): exists_calls.append(path) - # Return False for /var/lib/litellm/assets/logo.jpg to trigger fallback - if "/var/lib/litellm/assets/logo.jpg" in path: + # Return False for /var/lib/litellm/assets* so: makedirs is called, logo fallback + # triggers, and we don't return early with cached file + if "/var/lib/litellm/assets" in path: return False return True # Mock os.path operations with patch("litellm.proxy.proxy_server.os.makedirs") as mock_makedirs, \ patch("litellm.proxy.proxy_server.os.path.exists", side_effect=exists_side_effect), \ + patch("litellm.proxy.proxy_server.os.access", return_value=True), \ patch("litellm.proxy.proxy_server.os.getenv") as mock_getenv, \ patch("litellm.proxy.proxy_server.FileResponse") as mock_file_response: diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py new file mode 100644 index 00000000000..9c20d630a1b --- /dev/null +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -0,0 +1,103 @@ +""" +Test that litellm.responses() / litellm.aresponses() send the expected request body +over the wire. Expected JSON bodies are stored in expected_responses_api_request/. +""" +import json +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +import litellm + + +def _expected_dir() -> Path: + """Path to expected_responses_api_request folder (sibling of test_litellm/responses).""" + return Path(__file__).resolve().parent.parent / "expected_responses_api_request" + + +@pytest.mark.asyncio +async def test_aresponses_context_management_and_shell_request_body_matches_expected(): + """ + Call litellm.aresponses() with context_management and shell tool; + assert the httpx POST request body matches the expected JSON. + """ + expected_path = _expected_dir() / "context_management_and_shell.json" + assert expected_path.exists(), f"Expected file not found: {expected_path}" + with open(expected_path) as f: + expected_body = json.load(f) + + # Minimal Responses API response so parsing succeeds + mock_response = { + "id": "resp_ctx_shell_test", + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "gpt-4o", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "Done.", "annotations": []} + ], + } + ], + "parallel_tool_calls": True, + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "truncation": None, + "user": None, + } + + class MockResponse: + def __init__(self, json_data, status_code=200): + self._json_data = json_data + self.status_code = status_code + self.text = json.dumps(json_data) + self.headers = httpx.Headers({}) + + def json(self): + return self._json_data + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse(mock_response, 200) + + await litellm.aresponses( + model="openai/gpt-4o", + input=expected_body["input"], + context_management=expected_body["context_management"], + tools=expected_body["tools"], + tool_choice=expected_body["tool_choice"], + max_output_tokens=expected_body["max_output_tokens"], + ) + + mock_post.assert_called_once() + request_body = mock_post.call_args.kwargs["json"] + + for key, expected_value in expected_body.items(): + assert key in request_body, f"Missing key in request body: {key}" + assert request_body[key] == expected_value, ( + f"Mismatch for key {key}: got {request_body[key]!r}, expected {expected_value!r}" + ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 794b3b87187..7374a605798 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -3305,3 +3305,73 @@ class TestIsStreamingRequest: def test_stream_true_overrides_non_streaming_call_type(self): assert _is_streaming_request(kwargs={"stream": True}, call_type=CallTypes.acompletion) is True + + +class TestMetadataNoneHandling: + """ + Test that metadata=None in kwargs doesn't cause TypeError. + + When metadata key exists with value None (e.g., from Azure OpenAI streaming), + dict.get("metadata", {}) returns None (key exists, so default is ignored). + The fix uses (kwargs.get("metadata") or {}) which handles both missing key + and explicit None value. + + Related: #20871 + """ + + def test_metadata_none_get_previous_models(self): + """kwargs.get("metadata") or {} should return {} when metadata is None.""" + kwargs = {"metadata": None} + previous_models = (kwargs.get("metadata") or {}).get( + "previous_models", None + ) + assert previous_models is None + + def test_metadata_none_model_group_check(self): + """'model_group' in (kwargs.get("metadata") or {}) should not raise TypeError.""" + kwargs = {"metadata": None} + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} + ) + assert _is_litellm_router_call is False + + def test_metadata_missing_key(self): + """Should work when metadata key is completely absent.""" + kwargs = {} + previous_models = (kwargs.get("metadata") or {}).get( + "previous_models", None + ) + assert previous_models is None + + def test_metadata_present_with_values(self): + """Should work when metadata has actual values.""" + kwargs = {"metadata": {"previous_models": ["model1"], "model_group": "test"}} + previous_models = (kwargs.get("metadata") or {}).get( + "previous_models", None + ) + assert previous_models == ["model1"] + _is_litellm_router_call = "model_group" in ( + kwargs.get("metadata") or {} + ) + assert _is_litellm_router_call is True + + def test_metadata_none_causes_error_with_old_pattern(self): + """Demonstrate the bug: dict.get('metadata', {}) returns None when key exists with None value.""" + kwargs = {"metadata": None} + # Old pattern: kwargs.get("metadata", {}) returns None because key exists + result = kwargs.get("metadata", {}) + assert result is None # This is the root cause of the bug + + # Attempting to use .get() on None raises AttributeError or TypeError + with pytest.raises((TypeError, AttributeError)): + kwargs.get("metadata", {}).get("previous_models", None) + + # Attempting 'in' on None raises TypeError + with pytest.raises(TypeError): + "model_group" in kwargs.get("metadata", {}) + + def test_litellm_params_metadata_none(self): + """litellm_params.get("metadata") or {} should handle None value.""" + litellm_params = {"metadata": None} + metadata = litellm_params.get("metadata") or {} + assert metadata == {} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index bf7f1a67431..61cad4c437b 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -3,7 +3,7 @@ import { Modal, Tooltip, Form, Select, Input } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; import { Button, TextInput } from "@tremor/react"; import { createMCPServer } from "../networking"; -import { AUTH_TYPE, OAUTH_FLOW, MCPServer, MCPServerCostInfo } from "./types"; +import { AUTH_TYPE, DiscoverableMCPServer, OAUTH_FLOW, MCPServer, MCPServerCostInfo } from "./types"; import OAuthFormFields from "./OAuthFormFields"; import MCPServerCostConfig from "./mcp_server_cost_config"; import MCPConnectionStatus from "./mcp_connection_status"; @@ -25,6 +25,8 @@ interface CreateMCPServerProps { isModalVisible: boolean; setModalVisible: (visible: boolean) => void; availableAccessGroups: string[]; + prefillData?: DiscoverableMCPServer | null; + onBackToDiscovery?: () => void; } const AUTH_TYPES_REQUIRING_AUTH_VALUE = [AUTH_TYPE.API_KEY, AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.BASIC]; @@ -38,6 +40,8 @@ const CreateMCPServer: React.FC = ({ isModalVisible, setModalVisible, availableAccessGroups, + prefillData, + onBackToDiscovery, }) => { const [form] = Form.useForm(); const [isLoading, setIsLoading] = useState(false); @@ -183,6 +187,50 @@ const CreateMCPServer: React.FC = ({ setPendingRestoredValues(null); }, [pendingRestoredValues, form, transportType]); + // Pre-fill form from discovery selection + React.useEffect(() => { + if (!isModalVisible || !prefillData) { + return; + } + // Sanitize server name: strip vendor prefix, replace hyphens with underscores + const sanitizedName = (prefillData.name || "") + .replace(/[^a-zA-Z0-9_]/g, "_") + .replace(/_+/g, "_") + .replace(/^_|_$/g, ""); + + const transport = prefillData.transport || ""; + setTransportType(transport); + + const prefillValues: Record = { + server_name: sanitizedName, + alias: sanitizedName, + description: prefillData.description || "", + transport: transport, + }; + + if (transport === "stdio") { + const stdioObj: Record = {}; + if (prefillData.command) stdioObj.command = prefillData.command; + if (prefillData.args && prefillData.args.length > 0) stdioObj.args = prefillData.args; + if (prefillData.env_vars && prefillData.env_vars.length > 0) { + const envObj: Record = {}; + for (const v of prefillData.env_vars) { + envObj[v.name] = v.description ? `<${v.description}>` : ""; + } + stdioObj.env = envObj; + } + if (Object.keys(stdioObj).length > 0) { + prefillValues.stdio_config = JSON.stringify(stdioObj, null, 2); + } + } else if (prefillData.url) { + prefillValues.url = prefillData.url; + } + + form.setFieldsValue(prefillValues); + setFormValues(prefillValues); + setAliasManuallyEdited(false); + }, [isModalVisible, prefillData, form]); + const handleCreate = async (values: Record) => { setIsLoading(true); try { @@ -391,7 +439,16 @@ const CreateMCPServer: React.FC = ({ return ( +
+ {onBackToDiscovery && ( + + )} MCP Logo = ({ style={{ height: "20px", width: "20px", - marginRight: "8px", objectFit: "contain", }} /> @@ -429,7 +485,7 @@ const CreateMCPServer: React.FC = ({ label={ MCP Server Name - + @@ -450,7 +506,7 @@ const CreateMCPServer: React.FC = ({ label={ Alias - + @@ -458,12 +514,7 @@ const CreateMCPServer: React.FC = ({ name="alias" rules={[ { required: false }, - { - validator: (_, value) => - value && value.includes("-") - ? Promise.reject("Alias cannot contain '-' (hyphen). Please use '_' (underscore) instead.") - : Promise.resolve(), - }, + { validator: (_, value) => validateMCPServerName(value) }, ]} > void; + onSelectServer: (server: DiscoverableMCPServer) => void; + onCustomServer: () => void; + accessToken: string | null; +} + +const INITIAL_COLORS = [ + "#3B82F6", + "#10B981", + "#F59E0B", + "#EF4444", + "#8B5CF6", + "#EC4899", + "#06B6D4", + "#84CC16", +]; + +function getInitialAvatar(name: string) { + const initial = name.charAt(0).toUpperCase(); + const colorIndex = + name.split("").reduce((acc, ch) => acc + ch.charCodeAt(0), 0) % + INITIAL_COLORS.length; + return { initial, backgroundColor: INITIAL_COLORS[colorIndex] }; +} + +const MCPDiscovery: React.FC = ({ + isVisible, + onClose, + onSelectServer, + onCustomServer, + accessToken, +}) => { + const [servers, setServers] = useState([]); + const [categories, setCategories] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [searchQuery, setSearchQuery] = useState(""); + const [selectedCategory, setSelectedCategory] = useState("All"); + + useEffect(() => { + if (isVisible && accessToken) { + setLoading(true); + setError(null); + fetchDiscoverableMCPServers(accessToken) + .then((data: DiscoverMCPServersResponse) => { + setServers(data.servers || []); + setCategories(data.categories || []); + }) + .catch((err: Error) => { + setError(err.message || "Failed to load MCP servers"); + }) + .finally(() => { + setLoading(false); + }); + } + }, [isVisible, accessToken]); + + useEffect(() => { + if (isVisible) { + setSearchQuery(""); + setSelectedCategory("All"); + } + }, [isVisible]); + + const filteredServers = useMemo(() => { + let result = servers; + if (selectedCategory !== "All") { + result = result.filter((s) => s.category === selectedCategory); + } + if (searchQuery.trim()) { + const q = searchQuery.toLowerCase(); + result = result.filter( + (s) => + s.name.toLowerCase().includes(q) || + s.title.toLowerCase().includes(q) || + s.description.toLowerCase().includes(q), + ); + } + return result; + }, [servers, selectedCategory, searchQuery]); + + const groupedServers = useMemo(() => { + const groups: Record = {}; + for (const server of filteredServers) { + const cat = server.category || "Other"; + if (!groups[cat]) groups[cat] = []; + groups[cat].push(server); + } + return groups; + }, [filteredServers]); + + return ( + +
+ MCP Logo +

Add MCP Server

+
+ +
+ } + open={isVisible} + onCancel={onClose} + footer={null} + width={1000} + className="top-8" + styles={{ + body: { padding: "24px", maxHeight: "70vh", overflowY: "auto" }, + header: { padding: "24px 24px 0 24px", border: "none" }, + }} + > + {/* Filter pills */} +
+ {["All", ...categories].map((cat) => { + const isSelected = selectedCategory === cat; + return ( + + ); + })} +
+ + {/* Search */} + setSearchQuery(e.target.value)} + style={{ marginBottom: 16 }} + allowClear + /> + + {/* Loading skeleton */} + {loading && ( +
+ {Array.from({ length: 8 }).map((_, i) => ( +
+ ))} +
+ )} + + {error && ( +
+ Failed to load servers: {error} +
+ )} + + {!loading && !error && filteredServers.length === 0 && ( +
+ + No servers found.{" "} + + Add a custom server + + +
+ )} + + {/* Server list grouped by category — 2 columns */} + {!loading && + !error && + Object.entries(groupedServers).map(([category, categoryServers]) => ( +
+
+ {category} +
+
+ {categoryServers.map((server) => { + const avatar = getInitialAvatar(server.title || server.name); + return ( +
onSelectServer(server)} + style={{ + display: "flex", + alignItems: "center", + padding: "8px 10px", + borderRadius: 6, + cursor: "pointer", + transition: "background 0.1s ease", + }} + onMouseEnter={(e) => { + e.currentTarget.style.background = "#f9fafb"; + }} + onMouseLeave={(e) => { + e.currentTarget.style.background = "transparent"; + }} + > + {server.icon_url ? ( + {server.title} { + const target = e.currentTarget; + target.style.display = "none"; + const next = target.nextElementSibling as HTMLElement; + if (next) next.style.display = "flex"; + }} + /> + ) : null} +
+ {avatar.initial} +
+ + {server.title || server.name} + + + › + +
+ ); + })} +
+
+ ))} + + ); +}; + +export default MCPDiscovery; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx index 9ebf900cb92..1c851c43996 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx @@ -12,9 +12,10 @@ import CreateMCPServer from "./create_mcp_server"; import MCPConnect from "./mcp_connect"; import { mcpServerColumns } from "./mcp_server_columns"; import { MCPServerView } from "./mcp_server_view"; -import { MCPServer, MCPServerProps, Team } from "./types"; +import { DiscoverableMCPServer, MCPServer, MCPServerProps, Team } from "./types"; import MCPSemanticFilterSettings from "../Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings"; import MCPNetworkSettings from "./MCPNetworkSettings"; +import MCPDiscovery from "./mcp_discovery"; const { Text: AntdText, Title: AntdTitle } = Typography; const EDIT_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-edit-state"; @@ -66,6 +67,8 @@ const MCPServers: React.FC = ({ accessToken, userRole, userID }) const [selectedMcpAccessGroup, setSelectedMcpAccessGroup] = useState("all"); const [filteredServers, setFilteredServers] = useState([]); const [isModalVisible, setModalVisible] = useState(false); + const [isDiscoveryVisible, setDiscoveryVisible] = useState(false); + const [prefillData, setPrefillData] = useState(null); const [isDeletingServer, setIsDeletingServer] = useState(false); const isInternalUser = userRole === "Internal User"; @@ -291,14 +294,35 @@ const MCPServers: React.FC = ({ accessToken, userRole, userID }) isModalVisible={isModalVisible} setModalVisible={setModalVisible} availableAccessGroups={uniqueMcpAccessGroups} + prefillData={prefillData} + onBackToDiscovery={() => { + setModalVisible(false); + setPrefillData(null); + setDiscoveryVisible(true); + }} /> MCP Servers Configure and manage your MCP servers {isAdminRole(userRole) && ( - )} + setDiscoveryVisible(false)} + onSelectServer={(server: DiscoverableMCPServer) => { + setPrefillData(server); + setDiscoveryVisible(false); + setModalVisible(true); + }} + onCustomServer={() => { + setPrefillData(null); + setDiscoveryVisible(false); + setModalVisible(true); + }} + accessToken={accessToken} + />
diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index ecc4171a8ac..b5357e2b5b0 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -175,3 +175,23 @@ export interface MCPServerProps { userRole: string | null; userID: string | null; } + +// Discoverable MCP server from the curated registry +export interface DiscoverableMCPServer { + name: string; + title: string; + description: string; + icon_url?: string | null; + category: string; + registry_url?: string | null; + transport: string; + url?: string | null; + command?: string | null; + args?: string[] | null; + env_vars?: Array<{ name: string; description?: string; secret?: boolean }> | null; +} + +export interface DiscoverMCPServersResponse { + servers: DiscoverableMCPServer[]; + categories: string[]; +} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/utils.tsx b/ui/litellm-dashboard/src/components/mcp_tools/utils.tsx index 5dcb80107ca..44a06405615 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/utils.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/utils.tsx @@ -47,7 +47,7 @@ export const validateMCPServerUrl = (value: string) => { }; export const validateMCPServerName = (value: string) => { - return value && value.includes("-") - ? Promise.reject("Server name cannot contain '-' (hyphen). Please use '_' (underscore) instead.") + return value && (value.includes("-") || value.includes(" ")) + ? Promise.reject("Cannot contain '-' (hyphen) or spaces. Please use '_' (underscore) instead.") : Promise.resolve(); }; diff --git a/ui/litellm-dashboard/src/components/molecules/models/columns.test.tsx b/ui/litellm-dashboard/src/components/molecules/models/columns.test.tsx index 665023f1da4..0a8dacbadd2 100644 --- a/ui/litellm-dashboard/src/components/molecules/models/columns.test.tsx +++ b/ui/litellm-dashboard/src/components/molecules/models/columns.test.tsx @@ -200,7 +200,7 @@ describe("columns", () => { expect(screen.getByText("my-credential")).toBeInTheDocument(); }); - it("should display 'No credentials' when credential name is missing", () => { + it("should display 'Manual' when credential name is missing", () => { const cols = columns( defaultProps.userRole, defaultProps.userID, @@ -221,7 +221,132 @@ describe("columns", () => { }); render(); - expect(screen.getByText("No credentials")).toBeInTheDocument(); + expect(screen.getByText("Manual")).toBeInTheDocument(); + }); + + describe("credentials column", () => { + it("should display Credentials header with info icon", () => { + const cols = columns( + defaultProps.userRole, + defaultProps.userID, + defaultProps.premiumUser, + defaultProps.setSelectedModelId, + defaultProps.setSelectedTeamId, + defaultProps.getDisplayModelName, + defaultProps.handleEditClick, + defaultProps.handleRefreshClick, + defaultProps.expandedRows, + defaultProps.setExpandedRows, + ); + + const model = createMockModel(); + render(); + + expect(screen.getByText("Credentials")).toBeInTheDocument(); + // Info icon is in a flex container with Credentials - ant icons render as span with role="img" + const credentialsHeader = screen.getByText("Credentials").closest("span"); + expect(credentialsHeader?.parentElement?.querySelector('[role="img"]')).toBeInTheDocument(); + }); + + it("should display reusable credential with SyncOutlined icon and credential name", () => { + const cols = columns( + defaultProps.userRole, + defaultProps.userID, + defaultProps.premiumUser, + defaultProps.setSelectedModelId, + defaultProps.setSelectedTeamId, + defaultProps.getDisplayModelName, + defaultProps.handleEditClick, + defaultProps.handleRefreshClick, + defaultProps.expandedRows, + defaultProps.setExpandedRows, + ); + + const model = createMockModel({ + litellm_params: { + model: "gpt-4", + litellm_credential_name: "my-reusable-credential", + }, + }); + render(); + + expect(screen.getByText("my-reusable-credential")).toBeInTheDocument(); + const credentialCell = screen.getByText("my-reusable-credential").closest("div"); + expect(credentialCell).toHaveClass("flex"); + expect(screen.getByText("my-reusable-credential")).toHaveClass("text-blue-600"); + }); + + it("should display Manual with EditOutlined when no credential name", () => { + const cols = columns( + defaultProps.userRole, + defaultProps.userID, + defaultProps.premiumUser, + defaultProps.setSelectedModelId, + defaultProps.setSelectedTeamId, + defaultProps.getDisplayModelName, + defaultProps.handleEditClick, + defaultProps.handleRefreshClick, + defaultProps.expandedRows, + defaultProps.setExpandedRows, + ); + + const model = createMockModel({ + litellm_params: { + model: "gpt-4", + }, + }); + render(); + + expect(screen.getByText("Manual")).toBeInTheDocument(); + expect(screen.getByText("Manual")).toHaveClass("text-gray-500"); + }); + + it("should display Manual when litellm_params is undefined", () => { + const cols = columns( + defaultProps.userRole, + defaultProps.userID, + defaultProps.premiumUser, + defaultProps.setSelectedModelId, + defaultProps.setSelectedTeamId, + defaultProps.getDisplayModelName, + defaultProps.handleEditClick, + defaultProps.handleRefreshClick, + defaultProps.expandedRows, + defaultProps.setExpandedRows, + ); + + const model = createMockModel({ + litellm_params: undefined as any, + }); + render(); + + expect(screen.getByText("Manual")).toBeInTheDocument(); + }); + + it("should display Manual when litellm_credential_name is empty string", () => { + const cols = columns( + defaultProps.userRole, + defaultProps.userID, + defaultProps.premiumUser, + defaultProps.setSelectedModelId, + defaultProps.setSelectedTeamId, + defaultProps.getDisplayModelName, + defaultProps.handleEditClick, + defaultProps.handleRefreshClick, + defaultProps.expandedRows, + defaultProps.setExpandedRows, + ); + + const model = createMockModel({ + litellm_params: { + model: "gpt-4", + litellm_credential_name: "", + }, + }); + render(); + + expect(screen.getByText("Manual")).toBeInTheDocument(); + }); }); it("should display created by information for DB models", () => { diff --git a/ui/litellm-dashboard/src/components/molecules/models/columns.tsx b/ui/litellm-dashboard/src/components/molecules/models/columns.tsx index 8bb28e68b77..f6114bc9f3d 100644 --- a/ui/litellm-dashboard/src/components/molecules/models/columns.tsx +++ b/ui/litellm-dashboard/src/components/molecules/models/columns.tsx @@ -1,11 +1,45 @@ -import { KeyIcon, TrashIcon } from "@heroicons/react/outline"; +import { EditOutlined, InfoCircleOutlined, SyncOutlined } from "@ant-design/icons"; +import { TrashIcon } from "@heroicons/react/outline"; import { ColumnDef } from "@tanstack/react-table"; import { Badge, Button, Icon } from "@tremor/react"; -import { Popover, Tooltip, Typography, Space, Flex } from "antd"; +import { Divider, Flex, Popover, Space, Tooltip, Typography } from "antd"; import { ModelData } from "../../model_dashboard/types"; import { ProviderLogo } from "./ProviderLogo"; -const { Text } = Typography; +const { Text, Title } = Typography; + +const credentialsInfoPopoverContent = ( + + + Credential types + + + + + + + Reusable + + + Credentials saved in LiteLLM that can be added to models repeatedly. + + + + + + + + + Manual + + + Credentials added directly during model creation or defined in the config file. + + + + + +); export const columns = ( userRole: string, @@ -127,7 +161,21 @@ export const columns = ( }, }, { - header: () => Credentials, + header: () => ( + + Credentials + + + + + ), accessorKey: "litellm_credential_name", enableSorting: false, size: 180, @@ -135,20 +183,23 @@ export const columns = ( cell: ({ row }) => { const model = row.original; const credentialName = model.litellm_params?.litellm_credential_name; + const isReusable = !!credentialName; - return credentialName ? ( - -
- - - {credentialName} - -
- - ) : ( + return (
- - No credentials + {isReusable ? ( + <> + + + {credentialName} + + + ) : ( + <> + + Manual + + )}
); }, diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 162fdb9bc69..945ef02dd88 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -6097,6 +6097,34 @@ export const updateInternalUserSettings = async (accessToken: string, settings: } }; +export const fetchDiscoverableMCPServers = async (accessToken: string) => { + try { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/v1/mcp/discover` + : `/v1/mcp/discover`; + + const response = await fetch(url, { + method: HTTP_REQUEST.GET, + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.json(); + const errorMessage = deriveErrorMessage(errorData); + handleError(errorMessage); + throw new Error(errorMessage); + } + + return await response.json(); + } catch (error) { + console.error("Failed to fetch discoverable MCP servers:", error); + throw error; + } +}; + export const fetchMCPServers = async (accessToken: string) => { try { // Construct base URL diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx new file mode 100644 index 00000000000..9081219d5be --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -0,0 +1,452 @@ +import { useState } from "react"; +import { Typography, Descriptions, Card, Tag, Tabs, Alert, Collapse, Radio, Space, Spin } from "antd"; +import moment from "moment"; +import { LogEntry } from "../columns"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import GuardrailViewer from "../GuardrailViewer/GuardrailViewer"; +import { CostBreakdownViewer } from "../CostBreakdownViewer"; +import { ConfigInfoMessage } from "../ConfigInfoMessage"; +import { VectorStoreViewer } from "../VectorStoreViewer"; +import { TruncatedValue } from "./TruncatedValue"; +import { TokenFlow } from "./TokenFlow"; +import { JsonViewer } from "./JsonViewer"; +import { + formatData, + checkHasMessages, + checkHasResponse, + normalizeGuardrailEntries, + calculateTotalMaskedEntities, + getGuardrailLabel, + checkHasVectorStoreData, +} from "./utils"; +import { + DRAWER_CONTENT_PADDING, + API_BASE_MAX_WIDTH, + METADATA_MAX_HEIGHT, + TAB_REQUEST, + TAB_RESPONSE, + FONT_SIZE_SMALL, + FONT_FAMILY_MONO, + SPACING_XLARGE, + SPACING_MEDIUM, +} from "./constants"; +import { ToolsSection } from "../ToolsSection"; +import { PrettyMessagesView } from "./PrettyMessagesView"; + +const { Text } = Typography; + +export interface LogDetailContentProps { + logEntry: LogEntry; + onOpenSettings?: () => void; + /** When true, log details (messages/response) are still being lazy-loaded. */ + isLoadingDetails?: boolean; +} + +/** + * The scrollable detail content for a single log entry. + * Renders request details, metrics, cost breakdown, request/response, + * guardrails, vector store data, and metadata. + * + * Designed to be placed inside LogDetailsDrawer's right panel so it can + * be reused for both single-log and session-mode views. + */ +export function LogDetailContent({ logEntry, onOpenSettings, isLoadingDetails = false }: LogDetailContentProps) { + const metadata = logEntry.metadata || {}; + const hasError = metadata.status === "failure"; + const errorInfo = hasError ? metadata.error_information : null; + + const hasMessages = checkHasMessages(logEntry.messages); + const hasResponse = checkHasResponse(logEntry.response); + // Don't show "missing data" warning while details are still loading + const missingData = !hasMessages && !hasResponse && !hasError && !isLoadingDetails; + + // Guardrail data + const guardrailInfo = metadata?.guardrail_information; + const guardrailEntries = normalizeGuardrailEntries(guardrailInfo); + const hasGuardrailData = guardrailEntries.length > 0; + const totalMaskedEntities = calculateTotalMaskedEntities(guardrailEntries); + const primaryGuardrailLabel = getGuardrailLabel(guardrailEntries); + + // Vector store data + const hasVectorStoreData = checkHasVectorStoreData(metadata); + + const getRawRequest = () => { + return formatData(logEntry.proxy_server_request || logEntry.messages); + }; + + const getFormattedResponse = () => { + if (hasError && errorInfo) { + return { + error: { + message: errorInfo.error_message || "An error occurred", + type: errorInfo.error_class || "error", + code: errorInfo.error_code || "unknown", + param: null, + }, + }; + } + return formatData(logEntry.response); + }; + + return ( +
+ {/* Error Alert */} + {hasError && errorInfo && ( + } + className="mb-6" + /> + )} + + {/* Tags */} + {logEntry.request_tags && Object.keys(logEntry.request_tags).length > 0 && ( + + )} + + {/* Request Details */} +
+ + + {logEntry.model} + {logEntry.custom_llm_provider || "-"} + {logEntry.call_type} + + + + + + + {logEntry.requester_ip_address && ( + {logEntry.requester_ip_address} + )} + {hasGuardrailData && ( + + + + )} + + +
+ + {/* Metrics */} + + + {/* Cost Breakdown */} + + + {/* Tools */} + + + {/* Configuration Info Message */} + {missingData && ( +
+ +
+ )} + + {/* Request/Response JSON */} + {isLoadingDetails ? ( +
+ +
Loading request & response data...
+
+ ) : ( + + )} + + {/* Guardrail Data */} + {hasGuardrailData && } + + {/* Vector Store Data */} + {hasVectorStoreData && } + + {/* Metadata */} + {logEntry.metadata && Object.keys(logEntry.metadata).length > 0 && ( + + )} + + {/* Bottom spacing */} +
+
+ ); +} + +// ============================================================================ +// Helper Components +// ============================================================================ + +function ErrorDescription({ errorInfo }: { errorInfo: any }) { + return ( +
+ {errorInfo.error_code && ( +
+ Error Code: {errorInfo.error_code} +
+ )} + {errorInfo.error_message && ( +
+ Message: {errorInfo.error_message} +
+ )} +
+ ); +} + +function TagsSection({ tags }: { tags: Record }) { + return ( +
+ + Tags + + + {Object.entries(tags).map(([key, value]) => ( + + {key}: {String(value)} + + ))} + +
+ ); +} + +function GuardrailLabel({ label, maskedCount }: { label: string; maskedCount: number }) { + return ( + + {label} + {maskedCount > 0 && ( + + {maskedCount} masked + + )} + + ); +} + +function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record }) { + const hasCacheActivity = + logEntry.cache_hit || + (metadata?.additional_usage_values?.cache_read_input_tokens && + metadata.additional_usage_values.cache_read_input_tokens > 0); + + return ( +
+ + + + + + ${formatNumberWithCommas(logEntry.spend || 0, 8)} + {logEntry.duration?.toFixed(3)} s + + {hasCacheActivity && ( + <> + + {logEntry.cache_hit || "None"} + + {metadata?.additional_usage_values?.cache_read_input_tokens > 0 && ( + + {formatNumberWithCommas(metadata.additional_usage_values.cache_read_input_tokens)} + + )} + {metadata?.additional_usage_values?.cache_creation_input_tokens > 0 && ( + + {formatNumberWithCommas(metadata.additional_usage_values.cache_creation_input_tokens)} + + )} + + )} + + {metadata?.litellm_overhead_time_ms !== undefined && metadata.litellm_overhead_time_ms !== null && ( + + {metadata.litellm_overhead_time_ms.toFixed(2)} ms + + )} + + + {moment(logEntry.startTime).format("YYYY-MM-DDTHH:mm:ss.SSS[Z]")} + + + {moment(logEntry.endTime).format("YYYY-MM-DDTHH:mm:ss.SSS[Z]")} + + + +
+ ); +} + +interface RequestResponseSectionProps { + hasResponse: boolean; + hasError: boolean; + getRawRequest: () => any; + getFormattedResponse: () => any; + logEntry: LogEntry; +} + +function RequestResponseSection({ + hasResponse, + hasError, + getRawRequest, + getFormattedResponse, + logEntry, +}: RequestResponseSectionProps) { + const [activeTab, setActiveTab] = useState(TAB_REQUEST); + const [viewMode, setViewMode] = useState<'pretty' | 'json'>('pretty'); + + const getCopyText = () => { + const data = activeTab === TAB_REQUEST ? getRawRequest() : getFormattedResponse(); + return JSON.stringify(data, null, 2); + }; + + const totalSpend = logEntry.spend || 0; + const promptTokens = logEntry.prompt_tokens || 0; + const completionTokens = logEntry.completion_tokens || 0; + const totalTokens = promptTokens + completionTokens; + const inputCost = totalTokens > 0 ? (totalSpend * promptTokens) / totalTokens : 0; + const outputCost = totalTokens > 0 ? (totalSpend * completionTokens) / totalTokens : 0; + + return ( +
+ { + const target = e.target as HTMLElement; + if (target.closest('.ant-radio-group')) { + e.stopPropagation(); + } + }} + > +

Request & Response

+ setViewMode(e.target.value)} + > + Pretty + JSON + +
+ ), + children: ( +
+ {viewMode === 'pretty' ? ( + + ) : ( + setActiveTab(key as typeof TAB_REQUEST | typeof TAB_RESPONSE)} + tabBarExtraContent={ + + } + items={[ + { + key: TAB_REQUEST, + label: "Request", + children: ( +
+ +
+ ), + }, + { + key: TAB_RESPONSE, + label: "Response", + children: ( +
+ {hasResponse || hasError ? ( + + ) : ( +
+ Response data not available +
+ )} +
+ ), + }, + ]} + /> + )} +
+ ), + }, + ]} + /> +
+ ); +} + +function MetadataSection({ metadata }: { metadata: Record }) { + return ( +
+ Metadata, + children: ( +
+
+ +
+ 
+                  {JSON.stringify(metadata, null, 2)}
+
+
+ ), + }, + ]} + /> +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx index 6d7e3d75f8d..b6feff77a1c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx @@ -1,54 +1,94 @@ -import { useState } from "react"; -import { Drawer, Typography, Descriptions, Card, Tag, Tabs, Alert, Collapse, Radio, Space, Spin } from "antd"; -import moment from "moment"; +import { useEffect, useMemo, useState } from "react"; +import { Button, Drawer } from "antd"; +import { + CheckOutlined, + CopyOutlined, + LeftOutlined, + RightOutlined, +} from "@ant-design/icons"; +import { Sparkles, Wrench } from "lucide-react"; import { LogEntry } from "../columns"; -import { useLogDetails } from "@/app/(dashboard)/hooks/logDetails/useLogDetails"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; -import GuardrailViewer from "../GuardrailViewer/GuardrailViewer"; -import { CostBreakdownViewer } from "../CostBreakdownViewer"; -import { ConfigInfoMessage } from "../ConfigInfoMessage"; -import { VectorStoreViewer } from "../VectorStoreViewer"; -import { TruncatedValue } from "./TruncatedValue"; -import { TokenFlow } from "./TokenFlow"; -import { JsonViewer } from "./JsonViewer"; +import { MCP_CALL_TYPES } from "../constants"; +import { getEventDisplayName } from "../utils"; import { DrawerHeader } from "./DrawerHeader"; import { useKeyboardNavigation } from "./useKeyboardNavigation"; -import { - formatData, - checkHasMessages, - checkHasResponse, - normalizeGuardrailEntries, - calculateTotalMaskedEntities, - getGuardrailLabel, - checkHasVectorStoreData, -} from "./utils"; -import { - DRAWER_WIDTH, - DRAWER_CONTENT_PADDING, - API_BASE_MAX_WIDTH, - METADATA_MAX_HEIGHT, - TAB_REQUEST, - TAB_RESPONSE, - FONT_SIZE_SMALL, - FONT_FAMILY_MONO, - SPACING_XLARGE, - SPACING_MEDIUM, -} from "./constants"; -import { ToolsSection } from "../ToolsSection"; -import { PrettyMessagesView } from "./PrettyMessagesView"; - -const { Text } = Typography; +import { LogDetailContent } from "./LogDetailContent"; +import { sessionSpendLogsCall } from "../../networking"; +import { useQuery } from "@tanstack/react-query"; +import { getSpendString } from "@/utils/dataUtils"; +import { DRAWER_WIDTH } from "./constants"; +import { useLogDetails } from "@/app/(dashboard)/hooks/logDetails/useLogDetails"; export interface LogDetailsDrawerProps { open: boolean; onClose: () => void; logEntry: LogEntry | null; + sessionId?: string | null; + accessToken?: string | null; onOpenSettings?: () => void; allLogs?: LogEntry[]; onSelectLog?: (log: LogEntry) => void; startTime?: string; } +const SIDEBAR_WIDTH_PX = 224; + +/* ------------------------------------------------------------------ */ +/* TraceEventRow — compact event row used in both session & non- */ +/* session sidebar lists. Extracted to avoid JSX duplication. */ +/* ------------------------------------------------------------------ */ +interface TraceEventRowProps { + row: LogEntry; + isSelected: boolean; + onClick: () => void; +} + +function TraceEventRow({ row, isSelected, onClick }: TraceEventRowProps) { + const isMcp = MCP_CALL_TYPES.includes(row.call_type); + const durationValue = + row.duration != null + ? row.duration.toFixed(3) + : row.startTime && row.endTime + ? ((Date.parse(row.endTime) - Date.parse(row.startTime)) / 1000).toFixed(3) + : "-"; + + return ( + + ); +} + /** * Right-side drawer panel for displaying detailed log information. * Features: @@ -63,77 +103,138 @@ export function LogDetailsDrawer({ open, onClose, logEntry, + sessionId, + accessToken, onOpenSettings, allLogs = [], onSelectLog, startTime, }: LogDetailsDrawerProps) { - const [activeTab, setActiveTab] = useState(TAB_REQUEST); + const isSessionMode = Boolean(sessionId); + const [selectedSessionRequestId, setSelectedSessionRequestId] = useState(null); + const [isSidebarCollapsed, setIsSidebarCollapsed] = useState(false); + const [copiedLeftPanelId, setCopiedLeftPanelId] = useState(false); + + const { data: sessionLogs = [] } = useQuery({ + queryKey: ["sessionLogs", sessionId], + queryFn: async () => { + if (!sessionId || !accessToken) return []; + const response = await sessionSpendLogsCall(accessToken, sessionId); + const allSessionLogs: LogEntry[] = response.data || response || []; + return allSessionLogs + .map((row) => ({ + ...row, + duration: (Date.parse(row.endTime) - Date.parse(row.startTime)) / 1000, + })) + .sort((a, b) => { + const aIsMcp = MCP_CALL_TYPES.includes(a.call_type) ? 1 : 0; + const bIsMcp = MCP_CALL_TYPES.includes(b.call_type) ? 1 : 0; + if (aIsMcp !== bIsMcp) return aIsMcp - bIsMcp; + return new Date(a.startTime).getTime() - new Date(b.startTime).getTime(); + }); + }, + enabled: Boolean(open && isSessionMode && sessionId && accessToken), + }); + + const currentLog = useMemo(() => { + if (!isSessionMode) return logEntry; + if (!sessionLogs.length) return null; + if (selectedSessionRequestId) { + return sessionLogs.find((row) => row.request_id === selectedSessionRequestId) || sessionLogs[0]; + } + if (logEntry?.request_id) { + const clickedLog = sessionLogs.find((row) => row.request_id === logEntry.request_id); + return clickedLog || sessionLogs[0]; + } + return sessionLogs[0]; + }, [isSessionMode, logEntry, selectedSessionRequestId, sessionLogs]); + + useEffect(() => { + if (!isSessionMode || !sessionLogs.length) return; + if (!selectedSessionRequestId || !sessionLogs.some((row) => row.request_id === selectedSessionRequestId)) { + const fallbackRequestId = logEntry?.request_id && sessionLogs.some((row) => row.request_id === logEntry.request_id) + ? logEntry.request_id + : sessionLogs[0].request_id; + setSelectedSessionRequestId(fallbackRequestId); + } + }, [isSessionMode, logEntry, selectedSessionRequestId, sessionLogs]); + + // Reset transient UI state when the drawer opens or closes. + useEffect(() => { + if (open) { + setIsSidebarCollapsed(false); + } else { + if (isSessionMode) setSelectedSessionRequestId(null); + setCopiedLeftPanelId(false); + } + }, [open, isSessionMode]); // Keyboard navigation const { selectNextLog, selectPreviousLog } = useKeyboardNavigation({ isOpen: open, - currentLog: logEntry, - allLogs, + currentLog, + allLogs: isSessionMode ? sessionLogs : allLogs, onClose, - onSelectLog, + onSelectLog: (selected) => { + if (isSessionMode) { + setSelectedSessionRequestId(selected.request_id); + } + onSelectLog?.(selected); + }, }); - // Lazy-load log details (messages/response) only when drawer is open - const logDetails = useLogDetails(logEntry?.request_id, startTime, open && !!logEntry?.request_id); - - if (!logEntry) return null; - - // Use lazy-loaded details if available, fall back to list endpoint data. - // The list endpoint may already include messages/response when store_prompts_in_spend_logs is enabled, - // while the detail endpoint fetches from custom loggers (S3, GCS, etc.). + // Lazy-load log details (messages/response) only when drawer is open. + // This fetches data for a single log on-demand instead of prefetching all 50. + const logDetails = useLogDetails(currentLog?.request_id, startTime, open && !!currentLog?.request_id); const detailsData = logDetails.data as any; - const effectiveMessages = detailsData?.messages || logEntry.messages; - const effectiveResponse = detailsData?.response || logEntry.response; - const effectiveProxyServerRequest = detailsData?.proxy_server_request || logEntry.proxy_server_request; const isLoadingDetails = logDetails.isLoading; - const metadata = logEntry.metadata || {}; - const hasError = metadata.status === "failure"; - const errorInfo = hasError ? metadata.error_information : null; + // Build an enriched log entry that merges lazy-loaded details. + // The list endpoint may already include messages/response when store_prompts_in_spend_logs is enabled, + // while the detail endpoint fetches from custom loggers (S3, GCS, etc.) or DB fallback. + const enrichedLog = useMemo(() => { + if (!currentLog) return null; + return { + ...currentLog, + messages: detailsData?.messages || currentLog.messages, + response: detailsData?.response || currentLog.response, + proxy_server_request: detailsData?.proxy_server_request || currentLog.proxy_server_request, + }; + }, [currentLog, detailsData]); - // Check if request/response data is present (using lazy-loaded data) - const hasMessages = checkHasMessages(effectiveMessages); - const hasResponse = checkHasResponse(effectiveResponse); - const missingData = !hasMessages && !hasResponse && !isLoadingDetails; - - // Guardrail data - const guardrailInfo = metadata?.guardrail_information; - const guardrailEntries = normalizeGuardrailEntries(guardrailInfo); - const hasGuardrailData = guardrailEntries.length > 0; - const totalMaskedEntities = calculateTotalMaskedEntities(guardrailEntries); - const primaryGuardrailLabel = getGuardrailLabel(guardrailEntries); - - // Vector store data - const hasVectorStoreData = checkHasVectorStoreData(metadata); + const metadata = currentLog?.metadata || {}; // Status display values const statusLabel = metadata.status === "failure" ? "Failure" : "Success"; const statusColor = metadata.status === "failure" ? ("error" as const) : ("success" as const); const environment = metadata?.user_api_key_team_alias || "default"; - const getRawRequest = () => { - return formatData(effectiveProxyServerRequest || effectiveMessages); + const totalSessionCost = sessionLogs.reduce((sum, row) => sum + (row.spend || 0), 0); + const sessionStart = sessionLogs.length > 0 + ? new Date(Math.min(...sessionLogs.map((r) => new Date(r.startTime).getTime()))) + : null; + const sessionEnd = sessionLogs.length > 0 + ? new Date(Math.max(...sessionLogs.map((r) => new Date(r.endTime).getTime()))) + : null; + const sessionDurationSeconds = + sessionStart && sessionEnd ? ((sessionEnd.getTime() - sessionStart.getTime()) / 1000).toFixed(2) : "0.00"; + const llmCount = sessionLogs.filter((row) => !MCP_CALL_TYPES.includes(row.call_type)).length; + const mcpCount = sessionLogs.filter((row) => MCP_CALL_TYPES.includes(row.call_type)).length; + const logsForList = isSessionMode ? sessionLogs : currentLog ? [currentLog] : []; + const leftPanelId = isSessionMode ? sessionId || "" : currentLog?.request_id || ""; + const leftPanelDisplayId = + leftPanelId.length > 14 ? `${leftPanelId.slice(0, 11)}...` : leftPanelId; + + const handleCopyLeftPanelId = async () => { + if (!leftPanelId) return; + try { + await navigator.clipboard.writeText(leftPanelId); + setCopiedLeftPanelId(true); + setTimeout(() => setCopiedLeftPanelId(false), 1200); + } catch { /* clipboard unavailable in non-secure contexts */ } }; - const getFormattedResponse = () => { - if (hasError && errorInfo) { - return { - error: { - message: errorInfo.error_message || "An error occurred", - type: errorInfo.error_class || "error", - code: errorInfo.error_code || "unknown", - param: null, - }, - }; - } - return formatData(effectiveResponse); - }; + if (!currentLog || !enrichedLog) return null; return ( - +
+ {!isSidebarCollapsed ? ( + +
+
+
+
+ {logsForList.length} req + · + {isSessionMode + ? `${llmCount} LLM` + : `${logsForList.filter((row) => !MCP_CALL_TYPES.includes(row.call_type)).length} LLM`} + · + {isSessionMode + ? `${mcpCount} MCP` + : `${logsForList.filter((row) => MCP_CALL_TYPES.includes(row.call_type)).length} MCP`} + · + {isSessionMode + ? getSpendString(totalSessionCost) + : getSpendString(currentLog.spend || 0)} + {isSessionMode && ( + <> + · + {sessionDurationSeconds}s + + )} +
+ -
- {/* Error Alert - Show prominently at top for failures */} - {hasError && errorInfo && ( - } - className="mb-6" - /> - )} +
+ {isSessionMode ? ( +
+ {/* Child events — vertical tree line with horizontal connectors */} +
+
+ {logsForList.map((row, idx) => { + const isLast = idx === logsForList.length - 1; + return ( +
+
+ {isLast &&
} + { + setSelectedSessionRequestId(row.request_id); + onSelectLog?.(row); + }} + /> +
+ ); + })} +
+
+ ) : ( +
+ {logsForList.map((row) => ( + onSelectLog?.(row)} + /> + ))} +
+ )} +
+
+ )} - {/* Tags - Only show if present */} - {logEntry.request_tags && Object.keys(logEntry.request_tags).length > 0 && ( - - )} - - {/* Request Details Section */} -
- - - {logEntry.model} - {logEntry.custom_llm_provider || "-"} - {logEntry.call_type} - - - - - - - {logEntry.requester_ip_address && ( - {logEntry.requester_ip_address} - )} - {hasGuardrailData && ( - - - - )} - - +
+ +
+ +
+
- - {/* Metrics Section */} - - - {/* Cost Breakdown - Show if cost breakdown data is available */} - - - {/* Tools Section - Show if tools are present in request */} - - - {/* Configuration Info Message - Show when data is missing */} - {missingData && ( -
- -
- )} - - {/* Request/Response JSON - Collapsible */} - {isLoadingDetails ? ( -
- -
Loading request & response data...
-
- ) : ( - - )} - - {/* Guardrail Data - Show only if present */} - {hasGuardrailData && } - - {/* Vector Store Request Data - Show only if present */} - {hasVectorStoreData && } - - {/* Metadata Card - Only show if there's metadata */} - {logEntry.metadata && Object.keys(logEntry.metadata).length > 0 && ( - - )} - - {/* Bottom spacing for scroll area */} -
-
); } - -// ============================================================================ -// Helper Components -// ============================================================================ - -function ErrorDescription({ errorInfo }: { errorInfo: any }) { - return ( -
- {errorInfo.error_code && ( -
- Error Code: {errorInfo.error_code} -
- )} - {errorInfo.error_message && ( -
- Message: {errorInfo.error_message} -
- )} -
- ); -} - -function TagsSection({ tags }: { tags: Record }) { - return ( -
- - Tags - - - {Object.entries(tags).map(([key, value]) => ( - - {key}: {String(value)} - - ))} - -
- ); -} - -function GuardrailLabel({ label, maskedCount }: { label: string; maskedCount: number }) { - return ( - - {label} - {maskedCount > 0 && ( - - {maskedCount} masked - - )} - - ); -} - -function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record }) { - const hasCacheActivity = - logEntry.cache_hit || - (metadata?.additional_usage_values?.cache_read_input_tokens && - metadata.additional_usage_values.cache_read_input_tokens > 0); - - return ( -
- - - - - - ${formatNumberWithCommas(logEntry.spend || 0, 8)} - {logEntry.duration?.toFixed(3)} s - - {/* Only show cache fields if there's cache activity */} - {hasCacheActivity && ( - <> - - {logEntry.cache_hit || "None"} - - {metadata?.additional_usage_values?.cache_read_input_tokens > 0 && ( - - {formatNumberWithCommas(metadata.additional_usage_values.cache_read_input_tokens)} - - )} - {metadata?.additional_usage_values?.cache_creation_input_tokens > 0 && ( - - {formatNumberWithCommas(metadata.additional_usage_values.cache_creation_input_tokens)} - - )} - - )} - - {metadata?.litellm_overhead_time_ms !== undefined && metadata.litellm_overhead_time_ms !== null && ( - - {metadata.litellm_overhead_time_ms.toFixed(2)} ms - - )} - - - {moment(logEntry.startTime).format("YYYY-MM-DDTHH:mm:ss.SSS[Z]")} - - - {moment(logEntry.endTime).format("YYYY-MM-DDTHH:mm:ss.SSS[Z]")} - - - -
- ); -} - -interface RequestResponseSectionProps { - hasResponse: boolean; - hasError: boolean; - getRawRequest: () => any; - getFormattedResponse: () => any; - logEntry: LogEntry; -} - -function RequestResponseSection({ - hasResponse, - hasError, - getRawRequest, - getFormattedResponse, - logEntry, -}: RequestResponseSectionProps) { - const [activeTab, setActiveTab] = useState(TAB_REQUEST); - const [viewMode, setViewMode] = useState<'pretty' | 'json'>('pretty'); - - const getCopyText = () => { - const data = activeTab === TAB_REQUEST ? getRawRequest() : getFormattedResponse(); - return JSON.stringify(data, null, 2); - }; - - // Calculate input and output costs - // Assume average cost if not explicitly provided - const totalSpend = logEntry.spend || 0; - const promptTokens = logEntry.prompt_tokens || 0; - const completionTokens = logEntry.completion_tokens || 0; - const totalTokens = promptTokens + completionTokens; - - // Estimate input/output costs proportionally if not available - const inputCost = totalTokens > 0 ? (totalSpend * promptTokens) / totalTokens : 0; - const outputCost = totalTokens > 0 ? (totalSpend * completionTokens) / totalTokens : 0; - - return ( -
- { - // Only prevent if clicking on the Radio.Group area - const target = e.target as HTMLElement; - if (target.closest('.ant-radio-group')) { - e.stopPropagation(); - } - }} - > -

Request & Response

- {/* View Mode Toggle - In the header */} - setViewMode(e.target.value)} - > - Pretty - JSON - -
- ), - children: ( -
- {viewMode === 'pretty' ? ( - - ) : ( - setActiveTab(key as typeof TAB_REQUEST | typeof TAB_RESPONSE)} - tabBarExtraContent={ - - } - items={[ - { - key: TAB_REQUEST, - label: "Request", - children: ( -
- -
- ), - }, - { - key: TAB_RESPONSE, - label: "Response", - children: ( -
- {hasResponse || hasError ? ( - - ) : ( -
- Response data not available -
- )} -
- ), - }, - ]} - /> - )} -
- ), - }, - ]} - /> -
- ); -} - -function MetadataSection({ metadata }: { metadata: Record }) { - return ( -
- Metadata, - children: ( -
-
- -
- 
-                  {JSON.stringify(metadata, null, 2)}
-
-
- ), - }, - ]} - /> -
- ); -} - diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/index.ts b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/index.ts index e1fdd9d2d60..c2839df27ef 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/index.ts +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/index.ts @@ -1,2 +1,4 @@ export { LogDetailsDrawer } from "./LogDetailsDrawer"; export type { LogDetailsDrawerProps } from "./LogDetailsDrawer"; +export { LogDetailContent } from "./LogDetailContent"; +export type { LogDetailContentProps } from "./LogDetailContent"; diff --git a/ui/litellm-dashboard/src/components/view_logs/SessionView.tsx b/ui/litellm-dashboard/src/components/view_logs/SessionView.tsx deleted file mode 100644 index ce77ad14a1b..00000000000 --- a/ui/litellm-dashboard/src/components/view_logs/SessionView.tsx +++ /dev/null @@ -1,193 +0,0 @@ -import React, { useState } from "react"; -import { LogEntry } from "./columns"; -import { DataTable } from "./table"; -import { columns } from "./columns"; -import { Card, Title, Text, Metric, Button as TremorButton } from "@tremor/react"; -import { RequestViewer } from "./index"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { ArrowLeftIcon } from "@heroicons/react/outline"; -import { Button } from "antd"; -import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; -import { CheckIcon, CopyIcon } from "lucide-react"; -import { Tooltip } from "antd"; - -interface SessionViewProps { - sessionId: string; - logs: LogEntry[]; - onBack: () => void; -} - -export const SessionView: React.FC = ({ sessionId, logs, onBack }) => { - // Track which log row is expanded - const [expandedRequestId, setExpandedRequestId] = useState(null); - const [copiedStates, setCopiedStates] = useState>({}); - - // Calculate session metrics - const totalCost = logs.reduce((sum, log) => sum + (log.spend || 0), 0); - const totalTokens = logs.reduce((sum, log) => sum + (log.total_tokens || 0), 0); - - // Calculate cache token totals from metadata - const totalCacheReadTokens = logs.reduce((sum, log) => { - const cacheReadTokens = log.metadata?.additional_usage_values?.cache_read_input_tokens || 0; - return sum + cacheReadTokens; - }, 0); - - const totalCacheCreationTokens = logs.reduce((sum, log) => { - const cacheCreationTokens = log.metadata?.additional_usage_values?.cache_creation_input_tokens || 0; - return sum + cacheCreationTokens; - }, 0); - - // Calculate total tokens including cache tokens - const totalTokensWithCache = totalTokens + totalCacheReadTokens + totalCacheCreationTokens; - - const startTime = logs.length > 0 ? new Date(logs[0].startTime) : new Date(); - const endTime = logs.length > 0 ? new Date(logs[logs.length - 1].endTime) : new Date(); - const durationMs = endTime.getTime() - startTime.getTime(); - const durationSec = (durationMs / 1000).toFixed(2); - - // Prepare data for the timeline chart - const timelineData = logs.map((log) => ({ - time: new Date(log.startTime).toISOString(), - tokens: log.total_tokens || 0, - cost: log.spend || 0, - })); - - const copyToClipboard = async (text: string, key: string) => { - const success = await utilCopyToClipboard(text); - if (success) { - setCopiedStates((prev) => ({ ...prev, [key]: true })); - setTimeout(() => { - setCopiedStates((prev) => ({ ...prev, [key]: false })); - }, 2000); - } - }; - - return ( -
- {/* Header with back button */} -
- - Back to All Logs - -
-

Session Details

-
-
-

{sessionId}

-
- - Get started with session management here - - - - -
-
-
- - {/* Session Overview Cards */} -
- - Total Requests - {logs.length} - - - Total Cost - ${formatNumberWithCommas(totalCost, 6)} - - -
Usage breakdown
-
-
-
Input usage:
-
-
- input: - - {formatNumberWithCommas(logs.reduce((sum, log) => sum + (log.prompt_tokens || 0), 0))} - -
- {totalCacheReadTokens > 0 && ( -
- input_cached_tokens: - {formatNumberWithCommas(totalCacheReadTokens)} -
- )} - {totalCacheCreationTokens > 0 && ( -
- input_cache_creation_tokens: - {formatNumberWithCommas(totalCacheCreationTokens)} -
- )} -
-
-
-
Output usage:
-
-
- output: - - {formatNumberWithCommas(logs.reduce((sum, log) => sum + (log.completion_tokens || 0), 0))} - -
-
-
-
-
- Total usage: - {formatNumberWithCommas(totalTokensWithCache)} -
-
-
-
- } - placement="top" - overlayStyle={{ minWidth: "300px" }} - > - -
- Total Tokens - ⓘ -
- {formatNumberWithCommas(totalTokensWithCache)} - - -
- {/* Request Timeline */} - Session Logs -
- true} - loadingMessage="Loading logs..." - noDataMessage="No logs found" - /> -
-
- ); -}; diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx new file mode 100644 index 00000000000..e4195ece9ec --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx @@ -0,0 +1,30 @@ +/** + * Compact type-indicator badges for LLM and MCP log entries. + * Used in the request logs table and session type column. + */ + +export const SparkleIcon = ({ size = 12 }: { size?: number }) => ( + + + +); + +export const WrenchIcon = ({ size = 10 }: { size?: number }) => ( + + + +); + +export const LlmBadge = ({ count }: { count?: number }) => ( + + + {count != null ? count : "LLM"} + +); + +export const McpBadge = ({ count }: { count?: number }) => ( + + + {count != null ? count : "MCP"} + +); diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx index 3e72c8e13b8..452d31aed51 100644 --- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx @@ -5,6 +5,8 @@ import { Tooltip } from "antd"; import React, { useState } from "react"; import { getProviderLogoAndName } from "../provider_info_helpers"; import { TimeCell } from "./time_cell"; +import { MCP_CALL_TYPES } from "./constants"; +import { LlmBadge, McpBadge, SparkleIcon, WrenchIcon } from "./TypeBadges"; // Helper to get the appropriate logo URL const getLogoUrl = (row: LogEntry, provider: string) => { @@ -44,6 +46,12 @@ export type LogEntry = { session_id?: string; status?: string; duration?: number; + session_total_count?: number; + session_total_spend?: number; + mcp_tool_call_count?: number; + mcp_tool_call_spend?: number; + session_llm_count?: number; + session_mcp_count?: number; onKeyHashClick?: (keyHash: string) => void; onSessionClick?: (sessionId: string) => void; }; @@ -54,6 +62,40 @@ export const columns: ColumnDef[] = [ accessorKey: "startTime", cell: (info: any) => , }, + { + header: "Type", + id: "type", + cell: (info: any) => { + const row = info.row.original; + const sessionCount = row.session_total_count || 1; + const isMcp = MCP_CALL_TYPES.includes(row.call_type); + const sessionLlmCount = row.session_llm_count ?? (isMcp ? 0 : sessionCount); + const sessionMcpCount = row.session_mcp_count ?? (isMcp ? sessionCount : 0); + + if (isMcp) return ; + if (sessionCount <= 1) return ; + + // Multi-call session — show total count, plus MCP indicator when mixed. + const sessionTypeBadge = ( + + + {sessionCount} + {sessionMcpCount > 0 && ( + <> + · + + + )} + + ); + + return ( + + {sessionTypeBadge} + + ); + }, + }, { header: "Status", accessorKey: "metadata.status", @@ -105,11 +147,24 @@ export const columns: ColumnDef[] = [ { header: "Cost", accessorKey: "spend", - cell: (info: any) => ( - - {getSpendString(info.getValue() || 0)} - - ), + cell: (info: any) => { + const row = info.row.original; + const mcpCount = row.mcp_tool_call_count || 0; + const mcpSpend = row.mcp_tool_call_spend || 0; + + return ( +
+ + {getSpendString(info.getValue() || 0)} + + {mcpCount > 0 && mcpSpend > 0 && ( + + incl. {getSpendString(mcpSpend)} from {mcpCount} MCP + + )} +
+ ); + }, }, { header: "Duration (s)", diff --git a/ui/litellm-dashboard/src/components/view_logs/constants.ts b/ui/litellm-dashboard/src/components/view_logs/constants.ts index 84862fa4632..949dab275fe 100644 --- a/ui/litellm-dashboard/src/components/view_logs/constants.ts +++ b/ui/litellm-dashboard/src/components/view_logs/constants.ts @@ -12,6 +12,9 @@ export const ERROR_CODE_OPTIONS: { label: string; value: string }[] = [ { label: "529 - Overloaded", value: "529" }, ]; +/** Call types that represent MCP tool invocations (shared across columns, index, drawer). */ +export const MCP_CALL_TYPES = ["call_mcp_tool", "list_mcp_tools"]; + export const QUICK_SELECT_OPTIONS: { label: string; value: number; unit: string }[] = [ { label: "Last 15 Minutes", value: 15, unit: "minutes" }, { label: "Last Hour", value: 1, unit: "hours" }, diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index 2bc26e434d0..0214a6ecf87 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -17,19 +17,18 @@ import { fetchAllKeyAliases } from "../key_team_helpers/filter_helpers"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; import { PaginatedModelSelect } from "../ModelSelect/PaginatedModelSelect/PaginatedModelSelect"; import FilterComponent, { FilterOption } from "../molecules/filter"; -import { allEndUsersCall, keyInfoV1Call, keyListCall, sessionSpendLogsCall, uiSpendLogsCall } from "../networking"; +import { allEndUsersCall, keyInfoV1Call, keyListCall, uiSpendLogsCall } from "../networking"; import KeyInfoView from "../templates/key_info_view"; import AuditLogs from "./audit_logs"; import { columns, LogEntry } from "./columns"; import { ConfigInfoMessage } from "./ConfigInfoMessage"; +import { ERROR_CODE_OPTIONS, MCP_CALL_TYPES, QUICK_SELECT_OPTIONS } from "./constants"; import { CostBreakdownViewer } from "./CostBreakdownViewer"; import { ErrorViewer } from "./ErrorViewer"; import { useLogFilterLogic } from "./log_filter_logic"; import { LogDetailsDrawer } from "./LogDetailsDrawer"; import { getTimeRangeDisplay } from "./logs_utils"; -import { ERROR_CODE_OPTIONS, QUICK_SELECT_OPTIONS } from "./constants"; import { RequestResponsePanel } from "./RequestResponsePanel"; -import { SessionView } from "./SessionView"; import SpendLogsSettingsModal from "./SpendLogsSettingsModal/SpendLogsSettingsModal"; import { DataTable } from "./table"; import { VectorStoreViewer } from "./VectorStoreViewer"; @@ -51,11 +50,6 @@ export interface PaginatedResponse { total_pages: number; } -interface PrefetchedLog { - messages: any[]; - response: any; -} - export default function SpendLogsTable({ accessToken, token, @@ -286,57 +280,70 @@ export default function SpendLogsTable({ } }, [filters, accessToken, fetchKeyHashForAlias]); - // Fetch logs for a session if selected - const sessionLogs = useQuery({ - queryKey: ["sessionLogs", selectedSessionId], - queryFn: async () => { - if (!accessToken || !selectedSessionId) return { data: [], total: 0, page: 1, page_size: 50, total_pages: 1 }; - const response = await sessionSpendLogsCall(accessToken, selectedSessionId); - // If the API returns an array, wrap it in the same shape as PaginatedResponse - return { - data: response.data || response || [], - total: (response.data || response || []).length, - page: 1, - page_size: 1000, - total_pages: 1, - }; - }, - enabled: !!accessToken && !!selectedSessionId, - }); - - if (!accessToken || !token || !userRole || !userID) { return null; } + const searchedLogs = filteredLogs.data.filter((log) => { + const matchesSearch = + !searchTerm || + log.request_id.includes(searchTerm) || + log.model.includes(searchTerm) || + (log.user && log.user.includes(searchTerm)); + + // No need for additional filtering since we're now handling this in the API call + return matchesSearch; + }); + + const sessionCompositionById = searchedLogs.reduce>((acc, log) => { + if (!log.session_id) return acc; + if (!acc[log.session_id]) { + acc[log.session_id] = { llm: 0, mcp: 0 }; + } + if (MCP_CALL_TYPES.includes(log.call_type)) { + acc[log.session_id].mcp += 1; + } else { + acc[log.session_id].llm += 1; + } + return acc; + }, {}); + + // Build a single-pass map of session_id → representative request_id. + // Prefers an LLM row over an MCP row as the representative. + const sessionRepresentativeMap = new Map(); + for (const log of searchedLogs) { + if (!log.session_id || (log.session_total_count || 1) <= 1) continue; + const isMcp = MCP_CALL_TYPES.includes(log.call_type); + const existing = sessionRepresentativeMap.get(log.session_id); + if (!existing || (existing.isMcp && !isMcp)) { + sessionRepresentativeMap.set(log.session_id, { requestId: log.request_id, isMcp }); + } + } + const filteredData = - filteredLogs.data - .filter((log) => { - const matchesSearch = - !searchTerm || - log.request_id.includes(searchTerm) || - log.model.includes(searchTerm) || - (log.user && log.user.includes(searchTerm)); - - // No need for additional filtering since we're now handling this in the API call - return matchesSearch; + searchedLogs + .map((log) => { + const sessionComposition = log.session_id ? sessionCompositionById[log.session_id] : undefined; + return { + ...log, + duration: (Date.parse(log.endTime) - Date.parse(log.startTime)) / 1000, + session_llm_count: sessionComposition?.llm ?? undefined, + session_mcp_count: sessionComposition?.mcp ?? undefined, + onKeyHashClick: (keyHash: string) => setSelectedKeyIdInfoView(keyHash), + onSessionClick: (sessionId: string) => { + if (sessionId) { + setSelectedSessionId(sessionId); + setSelectedLog(log); + setIsDrawerOpen(true); + } + }, + }; }) - .map((log) => ({ - ...log, - duration: (Date.parse(log.endTime) - Date.parse(log.startTime)) / 1000, - onKeyHashClick: (keyHash: string) => setSelectedKeyIdInfoView(keyHash), - onSessionClick: (sessionId: string) => { - if (sessionId) setSelectedSessionId(sessionId); - }, - })) || []; - - // For session logs, add onKeyHashClick/onSessionClick as well - const sessionData = - sessionLogs.data?.data?.map((log) => ({ - ...log, - onKeyHashClick: (keyHash: string) => setSelectedKeyIdInfoView(keyHash), - onSessionClick: (sessionId: string) => { }, - })) || []; + // Deduplicate multi-call sessions using the pre-built map (O(1) per row). + .filter((log) => { + if (!log.session_id || (log.session_total_count || 1) <= 1) return true; + return sessionRepresentativeMap.get(log.session_id)?.requestId === log.request_id; + }) || []; // Add this function to handle manual refresh const handleRefresh = () => { @@ -344,13 +351,22 @@ export default function SpendLogsTable({ }; const handleRowClick = (log: LogEntry) => { + // Multi-call session row: open in the same right-side drawer (session mode) + if (log.session_id && (log.session_total_count || 1) > 1) { + setSelectedSessionId(log.session_id); + setSelectedLog(log); + setIsDrawerOpen(true); + return; + } + // Single-call row: open the detail drawer + setSelectedSessionId(null); setSelectedLog(log); setIsDrawerOpen(true); }; const handleCloseDrawer = () => { setIsDrawerOpen(false); - // Optionally keep selectedLog for animation purposes + setSelectedSessionId(null); }; const handleSelectLog = (log: LogEntry) => { @@ -444,19 +460,6 @@ export default function SpendLogsTable({ }, ]; - // When a session is selected, render the SessionView component - if (selectedSessionId && sessionLogs.data) { - return ( -
- setSelectedSessionId(null)} - /> -
- ); - } - const formatTimeUnit = (value: number, unit: string) => { if (value === 1) { if (unit === "minutes") return "minute"; @@ -484,29 +487,12 @@ export default function SpendLogsTable({
-

- {selectedSessionId ? ( - <> - Session: {selectedSessionId} - - - ) : ( - "Request Logs" - )} -

- {!selectedSessionId && ( -
{selectedKeyInfo && selectedKeyIdInfoView && selectedKeyInfo.api_key === selectedKeyIdInfoView ? ( setSelectedKeyIdInfoView(null)} backButtonText="Back to Logs" /> - ) : selectedSessionId ? ( -
- -
) : ( <> setIsSpendLogsSettingsModalVisible(true)} allLogs={filteredData} onSelectLog={handleSelectLog} diff --git a/ui/litellm-dashboard/src/components/view_logs/table.tsx b/ui/litellm-dashboard/src/components/view_logs/table.tsx index fb7706cba19..77cb273d4fe 100644 --- a/ui/litellm-dashboard/src/components/view_logs/table.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/table.tsx @@ -7,8 +7,10 @@ interface DataTableProps { data: TData[]; columns: ColumnDef[]; onRowClick?: (row: TData) => void; - // Legacy props for backward compatibility (audit logs) + /** Renders inside a single colspan cell (used by audit logs) */ renderSubComponent?: (props: { row: Row }) => React.ReactElement; + /** Renders directly in tbody as sibling table rows (used by MCP children) */ + renderChildRows?: (props: { row: Row }) => React.ReactNode; getRowCanExpand?: (row: Row) => boolean; isLoading?: boolean; loadingMessage?: string; @@ -20,24 +22,24 @@ export function DataTable({ columns, onRowClick, renderSubComponent, + renderChildRows, getRowCanExpand, isLoading = false, loadingMessage = "🚅 Loading logs...", noDataMessage = "No logs found", }: DataTableProps) { - // Determine if we're in legacy expansion mode or new drawer mode - const isLegacyMode = !!renderSubComponent && !!getRowCanExpand; + const supportsExpansion = !!(renderSubComponent || renderChildRows) && !!getRowCanExpand; const table = useReactTable({ data, columns, - ...(isLegacyMode && { getRowCanExpand }), + ...(supportsExpansion && { getRowCanExpand }), getRowId: (row: TData, index: number) => { const _row: any = row as any; return _row?.request_id ?? String(index); }, getCoreRowModel: getCoreRowModel(), - ...(isLegacyMode && { getExpandedRowModel: getExpandedRowModel() }), + ...(supportsExpansion && { getExpandedRowModel: getExpandedRowModel() }), }); return ( @@ -69,8 +71,8 @@ export function DataTable({ table.getRowModel().rows.map((row) => ( !isLegacyMode && onRowClick?.(row.original)} + className={`h-8 ${onRowClick ? "cursor-pointer hover:bg-gray-50" : ""}`} + onClick={() => onRowClick?.(row.original)} > {row.getVisibleCells().map((cell) => ( @@ -79,8 +81,13 @@ export function DataTable({ ))} - {/* Legacy expansion mode for audit logs */} - {isLegacyMode && row.getIsExpanded() && renderSubComponent && ( + {/* Child rows rendered as real table rows (MCP children) */} + {supportsExpansion && row.getIsExpanded() && renderChildRows && ( + renderChildRows({ row }) + )} + + {/* Legacy sub-component in colspan cell (audit logs) */} + {supportsExpansion && row.getIsExpanded() && renderSubComponent && !renderChildRows && (
{renderSubComponent({ row })}
diff --git a/ui/litellm-dashboard/src/components/view_logs/utils.ts b/ui/litellm-dashboard/src/components/view_logs/utils.ts new file mode 100644 index 00000000000..a46aeccc707 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/utils.ts @@ -0,0 +1,20 @@ +import { MCP_CALL_TYPES } from "./constants"; + +/** + * Derive a short, human-readable display name for a log entry. + * Strips provider prefixes, date suffixes, and version tags. + */ +export function getEventDisplayName(callType: string, model: string): string { + const raw = (model || "").trim(); + const isMcp = MCP_CALL_TYPES.includes(callType); + + if (isMcp) { + return raw.replace(/^mcp:\s*/i, "").split("/").pop() || raw || "mcp_tool"; + } + + const lastSegment = raw.split("/").pop() || raw; + const noSuffix = lastSegment.replace(/-20\d{6}.*$/i, "").replace(/:.*$/, ""); + const claudeMatch = noSuffix.match(/claude-[a-z0-9-]+/i); + if (claudeMatch) return claudeMatch[0]; + return noSuffix || "llm_call"; +}