Merge pull request #25860 from BerriAI/litellm_internal_staging

merge internal staging

.circleci/config.yml

@@ -2911,95 +2911,11 @@ jobs:
             rm -f /tmp/uv-install.sh
             echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
             export PATH="$HOME/.local/bin:$PATH"
-            uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage
+            uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage
             uv tool run --from 'coverage[toml]==7.10.6' coverage xml
       - codecov/upload:
           file: ./coverage.xml
 
-  publish_proxy_extras:
-    docker:
-      - image: cimg/python:3.12
-    working_directory: ~/project/litellm-proxy-extras
-    environment:
-      TWINE_USERNAME: __token__
-
-    steps:
-      - checkout:
-          path: ~/project
-
-      - run:
-          name: Check if litellm-proxy-extras dir or pyproject.toml was modified
-          command: |
-            curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
-            echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f  /tmp/uv-install.sh" | sha256sum -c -
-            env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
-            rm -f /tmp/uv-install.sh
-            echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
-            echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
-            export PATH="$HOME/.local/bin:$PATH"
-            # Get current version from pyproject.toml
-            CURRENT_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])')
-
-            # Get last published version from PyPI
-            LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])")
-
-            echo "Current version: $CURRENT_VERSION"
-            echo "Last published version: $LAST_VERSION"
-
-            # Compare versions using Python's packaging.version
-            VERSION_COMPARE=$(uv run --with 'packaging==25.0' python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)")
-
-            echo "Version compare: $VERSION_COMPARE"
-            if [ "$VERSION_COMPARE" = "1" ]; then
-              echo "Error: Current version ($CURRENT_VERSION) is less than last published version ($LAST_VERSION)"
-              exit 1
-            fi
-
-            # If versions are equal or current is greater, compare against the published package contents.
-            EXTRACTED_DIR=$(uv run --with "litellm-proxy-extras==$LAST_VERSION" python -c 'import importlib.util; from pathlib import Path; spec = importlib.util.find_spec("litellm_proxy_extras"); assert spec is not None and spec.origin is not None, "litellm_proxy_extras not found in uv-run environment"; print(Path(spec.origin).resolve().parent)')
-
-            # Compare contents
-            if ! diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras; then
-              if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then
-                echo "Error: Changes detected in litellm-proxy-extras but version was not bumped"
-                echo "Current version: $CURRENT_VERSION"
-                echo "Last published version: $LAST_VERSION"
-                echo "Changes:"
-                diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras
-                exit 1
-              fi
-            else
-              echo "No changes detected in litellm-proxy-extras. Skipping PyPI publish."
-              circleci step halt
-            fi
-
-      - run:
-          name: Get new version
-          command: |
-            NEW_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])')
-            echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV
-
-      - run:
-          name: Check if versions match
-          command: |
-            cd ~/project
-            # Check pyproject.toml
-            CURRENT_VERSION=$(uv run --with 'packaging==25.0' python -c 'import tomllib; from packaging.requirements import Requirement; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); matches = [spec.version for requirement in data["project"]["optional-dependencies"]["proxy"] for parsed in [Requirement(requirement)] if parsed.name == "litellm-proxy-extras" and parsed.specifier for spec in parsed.specifier if spec.operator == "=="]; print(matches[0] if matches else (_ for _ in ()).throw(SystemExit("Could not find exact litellm-proxy-extras pin in project.optional-dependencies.proxy")))')
-            if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
-              echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)"
-              exit 1
-            fi
-
-      - run:
-          name: Publish to PyPI
-          command: |
-            echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
-            echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
-            export PATH="$HOME/.local/bin:$PATH"
-            rm -rf build dist
-            uv build
-            uv tool run --from 'twine==6.2.0' twine upload --verbose dist/*
-
   ui_build:
     docker:
       - image: cimg/node:20.19
@@ -3214,60 +3130,6 @@ jobs:
             - litellm-docker-database.tar.zst
 
 
-  prisma_schema_sync:
-    machine:
-      image: ubuntu-2204:2023.10.1
-    resource_class: medium
-    working_directory: ~/project
-    steps:
-      - checkout
-      - setup_google_dns
-      - attach_workspace:
-          at: ~/project
-      - run:
-          name: Start PostgreSQL Database
-          command: |
-            docker run -d \
-              --name postgres-db \
-              -e POSTGRES_USER=postgres \
-              -e POSTGRES_PASSWORD=postgres \
-              -e POSTGRES_DB=litellm_schema_sync \
-              -p 5432:5432 \
-              postgres:14
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Load Docker Database Image
-          command: |
-            zstd -d litellm-docker-database.tar.zst --stdout | docker load
-            docker images | grep litellm-docker-database
-      - run:
-          name: Run schema sync via prisma db push
-          command: |
-            docker run -d \
-              -p 4000:4000 \
-              -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \
-              -e LITELLM_MASTER_KEY="sk-1234" \
-              --name schema-sync \
-              --add-host=host.docker.internal:host-gateway \
-              -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
-              litellm-docker-database:ci \
-              --config /app/config.yaml \
-              --port 4000 \
-              --use_prisma_db_push
-      - run:
-          name: Start outputting logs
-          command: docker logs -f schema-sync
-          background: true
-      - wait_for_service:
-          url: http://localhost:4000
-          timeout: "300"
-      - run:
-          name: Stop schema sync container
-          command: docker stop schema-sync
-
-
   test_bad_database_url:
     machine:
       image: ubuntu-2204:2023.10.1
@@ -3421,14 +3283,6 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
-      - prisma_schema_sync:
-          requires:
-            - build_docker_database_image
-          filters:
-            branches:
-              only:
-                - main
-                - /litellm_.*/
       - e2e_ui_testing:
           filters:
             branches:
@@ -3688,9 +3542,3 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
-      - publish_proxy_extras:
-          filters:
-            branches:
-              only:
-                - main
-                - /litellm_release_day_.*/

.github/workflows/guard-main-branch.yml

@@ -0,0 +1,42 @@
+name: Guard main branch
+
+on:
+  pull_request:
+    branches:
+      - main
+  merge_group:
+
+permissions: {}
+
+# DO NOT RENAME the job's `name:` — it is referenced by GitHub branch
+# protection as a required status check on `main`. Renaming silently
+# breaks the gate.
+jobs:
+  guard:
+    name: Verify PR source branch
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    steps:
+      - name: Reject merge_group events
+        if: github.event_name == 'merge_group'
+        run: |
+          echo "::error::Merge queue is not supported for main. Disable merge queue or update this guard."
+          exit 1
+      - name: Check head branch name
+        env:
+          HEAD_REF: ${{ github.head_ref }}
+          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+          BASE_REPO: ${{ github.repository }}
+        run: |
+          echo "PR head repo: $HEAD_REPO"
+          echo "PR head branch: $HEAD_REF"
+          if [ "$HEAD_REPO" != "$BASE_REPO" ]; then
+            echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the 'litellm_oss_branch' branch instead."
+            exit 1
+          fi
+          if [ "$HEAD_REF" = "litellm_internal_staging" ] || [[ "$HEAD_REF" == litellm_hotfix_?* ]]; then
+            echo "Allowed source branch."
+            exit 0
+          fi
+          echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_oss_branch' instead."
+          exit 1

.github/workflows/test-linting.yml

@@ -2,7 +2,11 @@ name: LiteLLM Linting
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-litellm-ui-build.yml

@@ -4,7 +4,11 @@ permissions:
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 jobs:
   build-ui:

.github/workflows/test-mcp.yml

@@ -2,7 +2,11 @@ name: LiteLLM MCP Tests (folder - tests/mcp_tests)
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-model-map.yaml

@@ -2,7 +2,11 @@ name: Validate model_prices_and_context_window.json
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-core-utils.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Core Utilities"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-documentation.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Documentation Validation"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-enterprise-routing.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Enterprise, Google GenAI & Routing"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-integrations.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Integrations (Callbacks & Logging)"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-llm-providers.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: LLM Provider Transformations"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-misc.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: MCP, Secrets, Containers & Misc"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-auth.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Auth & Key Management"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-db.yml

@@ -3,7 +3,7 @@ name: "Unit Tests: Proxy DB Operations"
 # Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
 on:
   push:
-    branches: [main, "litellm_*"]
+    branches: [main, "litellm_**"]
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-endpoints.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy API Endpoints"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-infra.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Infrastructure"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-legacy.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Legacy Tests"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-responses-caching-types.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Responses, Caching & Types"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-security.yml

@@ -3,7 +3,7 @@ name: "Unit Tests: Security"
 # Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
 on:
   push:
-    branches: [main, "litellm_*"]
+    branches: [main, "litellm_**"]
 
 permissions:
   contents: read

.github/workflows/test_server_root_path.yml

@@ -4,12 +4,16 @@ permissions:
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 jobs:
   test-server-root-path:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
 
     strategy:
       matrix:

docs/my-website/docs/proxy/config_settings.md

@@ -487,6 +487,7 @@ router_settings:
 | AZURE_STORAGE_CLIENT_ID | The Application Client ID to use for Authentication to Azure Blob Storage logging
 | AZURE_STORAGE_CLIENT_SECRET | The Application Client Secret to use for Authentication to Azure Blob Storage logging
 | AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY | Cost per GB per day for Azure Vector Store service
+| BACKGROUND_HEALTH_CHECK_MAX_TOKENS | Optional global default for `max_tokens` on proxy background health checks when a model has no `health_check_max_tokens`. If unset, non-wildcard models default to 1. Applies to wildcard routes when set. Default is unset
 | BATCH_STATUS_POLL_INTERVAL_SECONDS | Interval in seconds for polling batch status. Default is 3600 (1 hour)
 | BATCH_STATUS_POLL_MAX_ATTEMPTS | Maximum number of attempts for polling batch status. Default is 24 (for 24 hours)
 | BEDROCK_MAX_POLICY_SIZE | Maximum size for Bedrock policy. Default is 75
@@ -804,6 +805,8 @@ router_settings:
 | LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker.
 | LITELLM_BLOG_POSTS_URL | Custom URL for fetching LiteLLM blog posts JSON. Default is the GitHub main branch URL
 | LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
+| LITELLM_CORS_ALLOW_CREDENTIALS | Set to `true` to explicitly allow credentials in CORS responses. When not set, credentials are disabled automatically if `LITELLM_CORS_ORIGINS` is `*` (wildcard) to prevent the browser security misconfiguration of reflecting any origin with credentials
+| LITELLM_CORS_ORIGINS | Comma-separated list of allowed CORS origins (e.g. `https://app.example.com,https://admin.example.com`). Defaults to `*` (all origins) when not set
 | LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
 | LITELLM_DEPLOYMENT_ENVIRONMENT | Environment name for the deployment (e.g., "production", "staging"). Used as a fallback when OTEL_ENVIRONMENT_NAME is not set. Sets the `environment` tag in telemetry data
 | LITELLM_DETAILED_TIMING | When true, adds detailed per-phase timing headers to responses (`x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms`). Default is false. See [latency overhead docs](../troubleshoot/latency_overhead.md)
@@ -925,6 +928,7 @@ router_settings:
 | OPENAI_CHATGPT_API_BASE | Alternative to CHATGPT_API_BASE. Base URL for ChatGPT API
 | OPENAI_FILE_SEARCH_COST_PER_1K_CALLS | Cost per 1000 calls for OpenAI file search. Default is 0.0025
 | OPENAI_ORGANIZATION | Organization identifier for OpenAI
+| OPENAPI_URL | The path to the OpenAPI JSON endpoint. **By default this is "/openapi.json"**
 | OPENID_BASE_URL | Base URL for OpenID Connect services
 | OPENID_CLIENT_ID | Client ID for OpenID Connect authentication
 | OPENID_CLIENT_SECRET | Client secret for OpenID Connect authentication

docs/my-website/docs/proxy/cost_tracking.md

@@ -14,6 +14,10 @@ Provider-specific cost tracking (e.g., [Vertex AI PayGo / priority pricing](../p
 [Sync model pricing data from GitHub](./sync_models_github.md) to ensure accurate cost tracking.
 :::
 
+:::info Cost does not match your provider bill?
+Use the step-by-step workflow in [Debugging a cost discrepancy](../troubleshoot/cost_discrepancy): align time ranges, compare token categories (including cache), then decide whether the gap is ingestion, formula, or model-map pricing.
+:::
+
 ### How to Track Spend with LiteLLM
 
 **Step 1**

docs/my-website/docs/proxy/guardrails/policy_flow_builder.md

@@ -71,11 +71,105 @@ For each step you choose an action for **pass**, **fail**, and optionally **erro
 3. Select **Flow Builder** (instead of the simple form)
 4. Design your flow:
    - **Trigger** — Incoming LLM request (runs when the policy matches)
-   - **Steps** — Add guardrails, set **ON PASS**, **ON FAIL**, and **ON ERROR** actions per step (ON ERROR is optional; when unset, errors follow ON FAIL)
-   - **End** — Request proceeds to the LLM
-5. Use the **+** between steps to insert new steps
-6. Use the **Test** panel to run sample messages through the pipeline before saving
-7. Click **Save** to create or update the policy
+   - **Steps** — Add guardrails; set **ON PASS**, **ON FAIL**, and **ON API FAILURE** / **ON ERROR** per step (when **ON API FAILURE** is unset, technical errors follow **ON FAIL**)
+   - **End** — Request proceeds to the LLM when the pipeline allows it
+5. Use **+** between steps to insert another guardrail step (for fallbacks, retries, or stricter second checks)
+6. Use **Test Pipeline** to run sample messages before saving
+7. Click **Save Policy** (or **Save**) to create or update the policy
+
+### Configure guardrail fallbacks in the UI (walkthrough)
+
+1. Click **Policies**
+
+![Policies tab in the Admin UI](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/1333f4ae-d7df-4645-bd33-fee11c80cb96/ascreenshot_ce21e8bd79324c4685ad6c191e39d89e_text_export.jpeg)
+
+2. Click **+ Add New Policy**
+
+![Add new policy](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/353c08ab-cdb5-490f-b54f-734f77c87c45/ascreenshot_223033a61071485187e87cbb8c41081e_text_export.jpeg)
+
+3. Click **Flow Builder**
+
+![Choose Flow Builder](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/70e99d1b-fd76-4143-93f4-296b8b4c3904/ascreenshot_ef49b2e2c5dc40e39cf8da7a37f346ac_text_export.jpeg)
+
+4. Click **Continue to Builder**
+
+![Continue to Builder](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/3de1beaf-9c52-4f03-9100-ce4d47e41967/ascreenshot_a1d64e7e58c54b6cb8a311173ffe435a_text_export.jpeg)
+
+5. Click the **guardrail search** field on the first step
+
+![Select first guardrail — search field](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/640f699b-bdde-4e6d-a226-1fede9477b22/ascreenshot_27f14445b78b4e61872f3f95c1c9bacd_text_export.jpeg)
+
+6. Choose **Test Moderation** (or your primary guardrail)
+
+![Pick Test Moderation](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/d46f7ab6-4231-44fb-b377-59f817cdfbe5/ascreenshot_e3a9f8e25ffe46ad82a73641b81d157c_text_export.jpeg)
+
+7. For one branch (e.g. **ON API FAILURE**), set the action to **Next Step** so the pipeline can fall through to the next guardrail when the API errors
+
+![Set action to Next Step](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/3a7ddc2a-4317-417b-9341-ff6b0913e64b/ascreenshot_8878486dc12b4dddafe0c8ba4382a0fb_text_export.jpeg)
+
+8. For **ON PASS**, set **Allow** (or **Next Step** if you need more steps before allowing)
+
+![Set ON PASS to Allow](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/0e31cde8-3075-4e17-b771-b2b1696db98f/ascreenshot_b4b1d232459e4941904c9fbcf90c70ca_text_export.jpeg)
+
+9. Open the next outcome’s search/dropdown (e.g. **ON FAIL**)
+
+![Configure another branch — search field](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/715fc3ad-f245-4ee8-bb36-cc13400d635d/ascreenshot_395fece82c124d4d826fb5d84c9c0529_text_export.jpeg)
+
+10. Set that branch to **Next Step** if failed checks should continue to your backup guardrail
+
+![ON FAIL or branch — Next Step](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/83156e9b-fc3f-4cc2-a6cb-2a13a5e77b06/ascreenshot_c61429bf7b354063afc57c40a6b45c7a_text_export.jpeg)
+
+11. Click **+** between steps to add a second guardrail
+
+![Add step — plus control](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/e76cff13-af73-4775-90f6-4d29cb97d401/ascreenshot_52c478e7afd5410f9f63b616c753c851_text_export.jpeg)
+
+12. Open the guardrail search field on the new step
+
+![Second step — guardrail search](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/5c1c4eea-d7da-41e5-bebd-945e97562aa5/ascreenshot_cef70e9146b148b1936e721638de0783_text_export.jpeg)
+
+13. Select **Insults & Personal Attacks** (or your fallback / stricter guardrail)
+
+![Pick Insults and Personal Attacks](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/e796c733-351f-494f-9261-795c27f2b519/ascreenshot_f0f778d50c2146e48829ffb203c7de92_text_export.jpeg)
+
+14. Set **Next Step** or **Block** on the branches as needed for this step
+
+![Second step branch — Next Step](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/c5fad953-4f4b-47ec-ab6d-81d21b2fb7b8/ascreenshot_b515fadec0534c6a9b9d66091398d82d_text_export.jpeg)
+
+15. Set **ON PASS** to **Allow** when this guardrail should complete the pipeline successfully
+
+![Second step — Allow on pass](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/8210f32a-8704-41b1-97cc-7d183682a2a4/ascreenshot_23361af2b7da482a8d89025ab285a72e_text_export.jpeg)
+
+16. Open the branch where you want a **Custom Response** (e.g. **ON FAIL** on the last step)
+
+![Custom response — open branch selector](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/98ab3a2c-f22f-4478-a146-d5d26cae9b10/ascreenshot_6a3b673654e64ce29c8c93fbf30c52ed_text_export.jpeg)
+
+17. Choose **Custom Response**
+
+![Select Custom Response](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/a9e69e82-d517-4426-95da-034643a2388b/ascreenshot_f8ef581fbfb440cdbf145a2e9368c8e8_text_export.jpeg)
+
+18. Click **Enter custom response...** and type your message
+
+![Custom response text field](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/ef0f90ba-d0bc-4220-874f-4998b2dcc5f6/ascreenshot_f3e825b57fa0478a92f56840af266e03_text_export.jpeg)
+
+19. Confirm or edit the message in **Enter custom response...** as needed
+
+![Custom response — confirm message](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/f9a4711d-655c-4f15-b0ea-6b7d33fe6e60/ascreenshot_5df4b465bc484d8f86a4af5a45e9ab42_text_export.jpeg)
+
+20. Open **Test Pipeline**
+
+![Test Pipeline panel](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/3f9ac555-66fe-43e0-a8d8-2288a5966c73/ascreenshot_b2319dae363346ebb4da5d09180b56e8_text_export.jpeg)
+
+21. Click **Run Test**
+
+![Run Test](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/8e21e973-8193-404b-9d97-fd85be5f90b6/ascreenshot_619ca71e3be244449ca2ab01dde3cc45_text_export.jpeg)
+
+22. Expand **Step 1** (or the first guardrail row) in the results to see **ERROR** / **Next Step** vs **PASS** / **Allow**
+
+![Expand first step in test results](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/b8010e20-dd9a-4e59-b0ca-1f2ba4c7b6ac/ascreenshot_da99f5761bbf44a08af4f1e1175a95fc_text_export.jpeg)
+
+23. Expand **Step 2** (e.g. **Insults & Personal Attacks**) to confirm **PASS** and **Allow** after the fallback
+
+![Expand Step 2 — second guardrail outcome](https://colony-recorder.s3.amazonaws.com/files/2026-04-15/cac5273c-dd4f-48a0-af58-12c428d0f0d0/ascreenshot_f74da58e280a47319a7d2fa41519f4fb_text_export.jpeg)
 
 ## Config (YAML)
 

docs/my-website/docs/proxy/ui_team_soft_budget_alerts.md

@@ -2,6 +2,16 @@ import Image from '@theme/IdealImage';
 
 # Team Soft Budget Alerts
 
+:::info
+
+✨ This is an Enterprise feature. Email budget alerts require an enterprise license.
+
+[Enterprise Pricing](https://www.litellm.ai/#pricing)
+
+[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial)
+
+:::
+
 Set a soft budget on a team and get email alerts when spending crosses the threshold — without blocking any requests.
 
 ## Overview

docs/my-website/docs/troubleshoot/cost_discrepancy.md

@@ -0,0 +1,205 @@
+# Debugging a cost discrepancy
+
+Cost discrepancies between LiteLLM and your provider bill usually come from one of three areas: token ingestion, the cost formula LiteLLM applies, or stale or incorrect pricing in the model map. This page walks through how to tell which case you are in.
+
+## Step 1: Pick a time range
+
+Lock down a specific window where the discrepancy is visible.
+
+- Use at least 7 days of data when you can.
+- Prefer a window with stable usage so one-off spikes do not dominate the comparison.
+- Set the **same start and end time** on both your provider dashboard and the LiteLLM UI.
+
+![LiteLLM dashboard date range picker](/img/cost-discrepancy-debug/date-range-picker.png)
+
+## Step 2: Confirm traffic only goes through LiteLLM
+
+If any requests hit the provider directly (bypassing LiteLLM), the provider will show higher usage. That is expected, not a LiteLLM bug.
+
+Before continuing, confirm:
+
+- All clients use your LiteLLM proxy base URL.
+- No SDK or script uses provider API keys against the provider directly for the models you are comparing.
+- During the selected period, the models in question are only called via LiteLLM.
+
+If you are unsure, filter the provider dashboard by the API key or IAM principal LiteLLM uses, rather than comparing to your whole account.
+
+## Step 3: Compare token categories
+
+In the LiteLLM UI, open **Model activity** (under Usage analytics) so you can inspect spend and tokens per model.
+
+![Navigate to Model activity in the LiteLLM UI](/img/cost-discrepancy-debug/go-to-model-activity.png)
+
+Scroll the **Model** list and select the model you are reconciling with your provider bill.
+
+![Scroll to your model in the Model activity table](/img/cost-discrepancy-debug/scroll-to-model.png)
+
+With the same time range on both sides, fill in:
+
+| Category | LiteLLM | Provider | Delta |
+| --- | --- | --- | --- |
+| Total requests | — | — | — |
+| Input tokens | — | — | — |
+| Output tokens | — | — | — |
+| Cache read tokens | — | — | — |
+| Cache write tokens | — | — | — |
+
+LiteLLM surfaces per-category token usage for the selected model—for example prompt, completion, and cache-related tokens.
+
+![LiteLLM usage breakdown by token category](/img/cost-discrepancy-debug/token-categories.png)
+
+Compare these figures with your provider’s usage view (for example AWS billing tools, Azure Monitor, or the OpenAI usage dashboard) for the same period.
+
+### Cache token reporting
+
+- **OpenAI:** Cache read tokens are typically included inside the reported input token count.
+- **Anthropic:** Cache read tokens are often reported separately from non-cached input tokens.
+
+Compare the correct columns on each side so you are not treating “input” differently between dashboards.
+
+### Why use a 10% threshold?
+
+Provider dashboards and LiteLLM do not bucket requests on identical timestamps. A call at 11:59 PM can land in different daily totals on each side. Token counts can also differ slightly due to rounding across SDKs and APIs. A delta **under ~10%** is often explained by boundary effects and rounding. A delta **over ~10%** usually means something is miscounted, dropped, or categorized differently.
+
+## Step 4: Follow the right path
+
+<svg width="100%" viewBox="0 0 680 482" role="img" xmlns="http://www.w3.org/2000/svg" style={{ maxWidth: '100%', fontFamily: 'system-ui, sans-serif' }} aria-labelledby="cost-disc-flow-title">
+  <title id="cost-disc-flow-title">Cost discrepancy debugging flowchart</title>
+  <desc>Flowchart branching into Path A (token ingestion) or Path B which splits further into B1 (formula issue) and B2 (model map issue).</desc>
+  <defs>
+    <marker id="cd-arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto-start-reverse">
+      <path d="M2 1L8 5L2 9" fill="none" stroke="#888780" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" />
+    </marker>
+  </defs>
+
+  <rect x="215" y="24" width="250" height="44" rx="8" fill="#F1EFE8" stroke="#5F5E5A" strokeWidth="0.5" />
+  <text x="340" y="47" textAnchor="middle" dominantBaseline="central" fill="#444441" fontSize="14" fontWeight="500">Compare provider vs LiteLLM</text>
+
+  <line x1="340" y1="68" x2="340" y2="104" stroke="#888780" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
+
+  <rect x="175" y="104" width="330" height="56" rx="8" fill="#F1EFE8" stroke="#5F5E5A" strokeWidth="0.5" />
+  <text x="340" y="126" textAnchor="middle" dominantBaseline="central" fill="#444441" fontSize="14" fontWeight="500">Any category off by &gt; 10%?</text>
+  <text x="340" y="148" textAnchor="middle" dominantBaseline="central" fill="#5F5E5A" fontSize="12">requests, input, output, cache tokens</text>
+
+  <path d="M220 132 L100 132 L100 250" fill="none" stroke="#0F6E56" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
+  <text x="157" y="122" textAnchor="middle" fill="#0F6E56" fontSize="12">YES</text>
+
+  <path d="M505 132 L580 132 L580 250" fill="none" stroke="#993C1D" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
+  <text x="543" y="122" textAnchor="middle" fill="#993C1D" fontSize="12">NO</text>
+
+  <rect x="40" y="250" width="220" height="56" rx="8" fill="#E1F5EE" stroke="#0F6E56" strokeWidth="0.5" />
+  <text x="150" y="271" textAnchor="middle" dominantBaseline="central" fill="#085041" fontSize="14" fontWeight="500">Path A</text>
+  <text x="150" y="291" textAnchor="middle" dominantBaseline="central" fill="#0F6E56" fontSize="12">Token ingestion issue</text>
+
+  <rect x="420" y="250" width="220" height="56" rx="8" fill="#FAECE7" stroke="#993C1D" strokeWidth="0.5" />
+  <text x="530" y="271" textAnchor="middle" dominantBaseline="central" fill="#712B13" fontSize="14" fontWeight="500">Path B</text>
+  <text x="530" y="291" textAnchor="middle" dominantBaseline="central" fill="#993C1D" fontSize="12">Quantities match, cost differs</text>
+
+  <line x1="150" y1="306" x2="150" y2="370" stroke="#0F6E56" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
+
+  <line x1="530" y1="306" x2="530" y2="318" stroke="#854F0B" strokeWidth="1.5" />
+  <line x1="435" y1="318" x2="575" y2="318" stroke="#854F0B" strokeWidth="1.5" />
+  <line x1="435" y1="318" x2="435" y2="370" stroke="#854F0B" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
+  <line x1="575" y1="318" x2="575" y2="370" stroke="#854F0B" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
+  <text x="448" y="312" textAnchor="middle" fill="#854F0B" fontSize="11">B1</text>
+  <text x="562" y="312" textAnchor="middle" fill="#854F0B" fontSize="11">B2</text>
+
+  <rect x="40" y="370" width="220" height="56" rx="8" fill="#E1F5EE" stroke="#0F6E56" strokeWidth="0.5" />
+  <text x="150" y="391" textAnchor="middle" dominantBaseline="central" fill="#085041" fontSize="14" fontWeight="500">Report to LiteLLM team</text>
+  <text x="150" y="411" textAnchor="middle" dominantBaseline="central" fill="#0F6E56" fontSize="12">endpoints + model + screenshots</text>
+
+  <rect x="380" y="370" width="110" height="56" rx="8" fill="#FAEEDA" stroke="#854F0B" strokeWidth="0.5" />
+  <text x="435" y="391" textAnchor="middle" dominantBaseline="central" fill="#633806" fontSize="14" fontWeight="500">B1</text>
+  <text x="435" y="411" textAnchor="middle" dominantBaseline="central" fill="#854F0B" fontSize="12">Fix formula</text>
+
+  <rect x="510" y="370" width="130" height="56" rx="8" fill="#FAEEDA" stroke="#854F0B" strokeWidth="0.5" />
+  <text x="575" y="391" textAnchor="middle" dominantBaseline="central" fill="#633806" fontSize="14" fontWeight="500">B2</text>
+  <text x="575" y="411" textAnchor="middle" dominantBaseline="central" fill="#854F0B" fontSize="12">Fix model map</text>
+
+  <path d="M150 426 L150 442 L340 442" fill="none" stroke="#888780" strokeWidth="0.5" strokeDasharray="4 3" />
+  <path d="M340 442 L435 442 L435 428" fill="none" stroke="#888780" strokeWidth="0.5" strokeDasharray="4 3" />
+  <path d="M340 442 L575 442 L575 428" fill="none" stroke="#888780" strokeWidth="0.5" strokeDasharray="4 3" />
+  <text x="340" y="454" textAnchor="middle" fill="#5F5E5A" fontSize="11">if neither path resolves it,</text>
+  <text x="340" y="470" textAnchor="middle" fill="#5F5E5A" fontSize="11">Open a github issue backing up with all your data</text>
+</svg>
+
+## Path A: Token quantity mismatch
+
+If any category is off by more than about 10%, LiteLLM may not be ingesting that category correctly (or the provider dashboard is categorizing tokens differently—recheck Step 3 first).
+
+**What to send the LiteLLM team:**
+
+1. Screenshots of both dashboards with the date range visible.
+2. Which category is off (input, output, cache reads, cache writes, or request count).
+3. Endpoints used (for example `/chat/completions`, `/responses`, `/embeddings`).
+4. Model names as sent in the request (for example `anthropic.claude-opus-4-5`, `gpt-4o`).
+
+### For maintainers debugging ingestion
+
+1. Start the proxy with verbose logging, for example:
+   ```bash
+   litellm --config config.yaml --detailed_debug
+   ```
+2. Reproduce a single request with the reported endpoint and model.
+3. Inspect the raw `usage` object in each streamed chunk (if streaming) or in the final response body.
+4. Compare that to the standard logging object (or the UI request log for that call).
+5. Any gap between raw provider usage and what LiteLLM logs or aggregates is where ingestion may be wrong.
+
+## Path B: Quantities match but cost is wrong
+
+If token and request counts agree within ~10% but dollar amounts differ, focus on how cost is computed.
+
+### B1: Formula issue
+
+Manually compute expected cost using the provider’s token breakdown and published rates (per million tokens or per token).
+
+Add other billed dimensions your provider applies (for example cache creation, audio, or tier surcharges). If your hand calculation matches the provider bill but not LiteLLM, the implementation in LiteLLM for that provider or modality may be wrong.
+
+### B2: Model map issue
+
+If the formula structure matches how the provider bills, the values in LiteLLM’s model map may be stale or incorrect. Cross-check:
+
+- [`model_prices_and_context_window.json`](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
+- The provider’s current public pricing
+
+Inspect `input_cost_per_token`, `output_cost_per_token`, and any cache-related pricing fields for your exact model id (including provider prefix).
+
+### For maintainers
+
+1. Take authoritative token quantities from the user’s provider report.
+2. Derive the formula that reproduces the provider’s line item.
+3. Diff that against LiteLLM’s cost path for the same provider and response shape.
+4. If the formula matches but numbers differ, update pricing in `model_prices_and_context_window.json` (and follow the project’s sync / backup rules for that file).
+5. If the formula in code is wrong, fix the calculation and add a regression test using the user’s token breakdown.
+
+## Still stuck?
+
+1. Open a GitHub issue on [BerriAI/litellm](https://github.com/BerriAI/litellm) with your Step 3 comparison table, endpoints, and model names.
+
+
+On the issue, it helps to clarify:
+
+- Reproducible on demand or intermittent?
+- Single model or many?
+- Steady over time, or starting from a specific release date or config change?
+
+### For LiteLLM maintainers
+
+If Path A and Path B do not close the case after triage, **you** should reach out and **schedule a call with the customer** (support or engineering), with the Step 3 table and screenshots—before treating the issue.
+
+## Checklist
+
+```
+□ Same time range on both dashboards
+□ Confirmed no direct-to-provider traffic for those models
+□ Compared: requests, input tokens, output tokens, cache tokens
+□ Noted cache reporting differences (OpenAI vs Anthropic, and so on)
+□ If > ~10% delta on quantities → Path A: report with screenshots, endpoints, model names
+□ If quantities match → Path B: verify formula (B1) and model map pricing (B2)
+□ If neither path fits → open a GitHub issue.
+```
+
+## See also
+
+- [Spend tracking](../proxy/cost_tracking)
+- [Sync model pricing from GitHub](../proxy/sync_models_github)

docs/my-website/sidebars.js

@@ -1149,6 +1149,7 @@ const sidebars = {
       label: "Troubleshooting",
       items: [
         "troubleshoot/ui_issues",
+        "troubleshoot/cost_discrepancy",
         "mcp_troubleshoot",
         {
           type: "category",

litellm-proxy-extras/litellm_proxy_extras/migrations/20260414140000_add_mcp_server_instructions/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "instructions" TEXT;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260415120000_health_check_latest_per_model_index/migration.sql

@@ -0,0 +1,12 @@
+-- CreateIndex (CONCURRENTLY)
+--
+-- Disclaimer:
+-- - CREATE INDEX CONCURRENTLY cannot run inside a transaction. This migration must stay a
+--   single statement so Prisma Migrate on PostgreSQL can apply it outside a transaction.
+-- - Builds are slower and use more I/O than a blocking CREATE INDEX; if the build is
+--   interrupted, Postgres may leave an INVALID index that must be dropped and recreated.
+-- - Do not edit this file after it has been applied to any database: Prisma checksums
+--   migrations; add a new migration instead.
+-- - Requires PostgreSQL that supports CONCURRENTLY with IF NOT EXISTS (use a new migration
+--   without IF NOT EXISTS if you must support older versions).
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx" ON "LiteLLM_HealthCheckTable"("model_id", "model_name", "checked_at" DESC);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -289,6 +289,7 @@ model LiteLLM_MCPServerTable {
   server_name  String?
   alias        String?
   description  String?
+  instructions String?
   url          String?
   spec_path    String?
   transport    String         @default("sse")
@@ -1045,6 +1046,7 @@ model LiteLLM_HealthCheckTable {
   @@index([model_name])
   @@index([checked_at])
   @@index([status])
+  @@index([model_id, model_name, checked_at(sort: Desc)], map: "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx")
 }
 
 // Search Tools table for storing search tool configurations

litellm/constants.py

@@ -1,6 +1,6 @@
 import os
 import sys
-from typing import List, Literal
+from typing import List, Literal, Optional
 
 from litellm.litellm_core_utils.env_utils import get_env_int
 
@@ -413,7 +413,20 @@ MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
 )
 DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
 #### Networking settings ####
-request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000))  # time in seconds
+# Sentinel used when `REQUEST_TIMEOUT` is unset: `litellm.request_timeout` keeps this
+# value so longer-running surfaces (Router `timeout or litellm.request_timeout`,
+# speech/TTS, responses, vector stores, etc.) get a long HTTP deadline. Chat
+# `completion()` maps this sentinel down to 600s when the caller did not set a
+# per-request/model timeout—see ``CompletionTimeout.resolve`` in completion_timeout.py. MCP uses
+# dedicated timeouts (e.g. `MCP_CLIENT_TIMEOUT`), not `request_timeout`.
+DEFAULT_REQUEST_TIMEOUT_SECONDS: float = 6000.0
+# Pair used for default httpx clients when no custom timeout is passed: read/write
+# deadline and connect handshake (see ``http_handler`` cached handler paths).
+COMPLETION_HTTP_FALLBACK_SECONDS: float = 600.0
+HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS: float = 5.0
+request_timeout: float = float(
+    os.getenv("REQUEST_TIMEOUT", str(int(DEFAULT_REQUEST_TIMEOUT_SECONDS)))
+)
 DEFAULT_A2A_AGENT_TIMEOUT: float = float(
     os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)
 )  # 10 minutes
@@ -1330,6 +1343,22 @@ BATCH_STATUS_POLL_MAX_ATTEMPTS = int(
 HEALTH_CHECK_TIMEOUT_SECONDS = int(
     os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60)
 )  # 60 seconds
+_background_health_check_max_tokens_env = os.getenv(
+    "BACKGROUND_HEALTH_CHECK_MAX_TOKENS"
+)
+try:
+    _raw_background_health_check_max_tokens = (
+        _background_health_check_max_tokens_env.strip()
+        if _background_health_check_max_tokens_env is not None
+        else ""
+    )
+    BACKGROUND_HEALTH_CHECK_MAX_TOKENS: Optional[int] = (
+        int(_raw_background_health_check_max_tokens)
+        if _raw_background_health_check_max_tokens
+        else None
+    )
+except (ValueError, TypeError):
+    BACKGROUND_HEALTH_CHECK_MAX_TOKENS = None
 LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check"
 LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli"
 LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"

litellm/cost_calculator.py

@@ -966,6 +966,8 @@ def _store_cost_breakdown_in_logging_obj(
     margin_percent: Optional[float] = None,
     margin_fixed_amount: Optional[float] = None,
     margin_total_amount: Optional[float] = None,
+    cache_read_cost: Optional[float] = None,
+    cache_creation_cost: Optional[float] = None,
 ) -> None:
     """
     Helper function to store cost breakdown in the logging object.
@@ -1001,6 +1003,8 @@ def _store_cost_breakdown_in_logging_obj(
             margin_percent=margin_percent,
             margin_fixed_amount=margin_fixed_amount,
             margin_total_amount=margin_total_amount,
+            cache_read_cost=cache_read_cost,
+            cache_creation_cost=cache_creation_cost,
         )
 
     except Exception as breakdown_error:
@@ -1599,6 +1603,22 @@ def completion_cost(  # noqa: PLR0915
 
                 # Store cost breakdown in logging object if available
                 if litellm_logging_obj is not None:
+                    _cache_read_cost: Optional[float] = None
+                    _cache_creation_cost: Optional[float] = None
+                    if cost_per_token_usage_object is not None:
+                        _cr = getattr(cost_per_token_usage_object, "cache_read_input_tokens", None) or (cost_per_token_usage_object.model_extra or {}).get("cache_read_input_tokens")
+                        _cc = getattr(cost_per_token_usage_object, "cache_creation_input_tokens", None) or (cost_per_token_usage_object.model_extra or {}).get("cache_creation_input_tokens")
+                        if (_cr or _cc) and model:
+                            try:
+                                _mi = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
+                                _cr_rate = _mi.get("cache_read_input_token_cost")
+                                if _cr and _cr_rate is not None:
+                                    _cache_read_cost = float(_cr) * float(_cr_rate)
+                                _cc_rate = _mi.get("cache_creation_input_token_cost")
+                                if _cc and _cc_rate is not None:
+                                    _cache_creation_cost = float(_cc) * float(_cc_rate)
+                            except Exception:
+                                pass
                     _store_cost_breakdown_in_logging_obj(
                         litellm_logging_obj=litellm_logging_obj,
                         prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar,
@@ -1612,6 +1632,8 @@ def completion_cost(  # noqa: PLR0915
                         margin_percent=margin_percent,
                         margin_fixed_amount=margin_fixed_amount,
                         margin_total_amount=margin_total_amount,
+                        cache_read_cost=_cache_read_cost,
+                        cache_creation_cost=_cache_creation_cost,
                     )
 
                 return _final_cost

litellm/experimental_mcp_client/client.py

@@ -221,6 +221,7 @@ class MCPClient:
         self.extra_headers: Optional[Dict[str, str]] = extra_headers
         self.ssl_verify: Optional[VerifyTypes] = ssl_verify
         self._aws_auth: Optional[httpx.Auth] = aws_auth
+        self._last_initialize_instructions: Optional[str] = None
         # handle the basic auth value if provided
         if auth_value:
             self.update_auth_value(auth_value)
@@ -296,7 +297,12 @@ class MCPClient:
             session_ctx = ClientSession(read_stream, write_stream)
             session = await session_ctx.__aenter__()
             try:
-                await session.initialize()
+                init_result = await session.initialize()
+                self._last_initialize_instructions = None
+                if init_result is not None:
+                    ins = getattr(init_result, "instructions", None)
+                    if isinstance(ins, str) and ins.strip():
+                        self._last_initialize_instructions = ins.strip()
                 return await operation(session)
             finally:
                 try:
@@ -315,6 +321,7 @@ class MCPClient:
         """Open a session, run the provided coroutine, and clean up."""
         http_client: Optional[httpx.AsyncClient] = None
         try:
+            self._last_initialize_instructions = None
             transport_ctx, http_client = self._create_transport_context()
             return await self._execute_session_operation(transport_ctx, operation)
         except Exception:

litellm/litellm_core_utils/completion_timeout.py

@@ -0,0 +1,83 @@
+"""Completion HTTP timeout resolution (kept out of ``main.py`` to limit import cycles)."""
+
+from __future__ import annotations
+
+from typing import Callable, Optional, Union
+
+import httpx
+
+from litellm.constants import (
+    COMPLETION_HTTP_FALLBACK_SECONDS,
+    DEFAULT_REQUEST_TIMEOUT_SECONDS,
+)
+
+
+class CompletionTimeout:
+    """Resolves HTTP timeout for ``completion()`` from model vs global settings."""
+
+    @staticmethod
+    def _fallback_when_no_explicit_timeout(
+        global_timeout: Optional[Union[float, str]],
+    ) -> float:
+        """
+        Used when ``model_timeout`` and kwargs timeouts are all unset.
+
+        ``global_timeout`` is :attr:`litellm.request_timeout` (numeric / string), not
+        :class:`httpx.Timeout`.
+
+        If it equals :data:`~litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS` (6000),
+        return :data:`~litellm.constants.COMPLETION_HTTP_FALLBACK_SECONDS`. Same if
+        ``None``. Otherwise return ``float(global_timeout)``.
+        """
+        if global_timeout is None:
+            return COMPLETION_HTTP_FALLBACK_SECONDS
+        if float(global_timeout) == float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
+            return COMPLETION_HTTP_FALLBACK_SECONDS
+        return float(global_timeout)
+
+    @staticmethod
+    def resolve(
+        model_timeout: Optional[Union[float, str, httpx.Timeout]],
+        kwargs: dict,
+        custom_llm_provider: str,
+        *,
+        global_timeout: Optional[Union[float, str]],
+        supports_httpx_timeout: Callable[[str], bool],
+    ) -> Union[float, httpx.Timeout]:
+        """
+        Resolution order (first non-None wins):
+
+        1. ``model_timeout`` (call argument / merged ``litellm_params``)
+        2. ``kwargs["timeout"]``
+        3. ``kwargs["request_timeout"]``
+        4. Fallback from ``global_timeout`` (:attr:`litellm.request_timeout`) — if it is
+           the package default (6000), use 600 instead.
+
+        Coerce :class:`httpx.Timeout` when the provider does not support it.
+        Explicit ``6000`` on the model or in kwargs is kept as ``6000``.
+        """
+        resolved: Union[float, str, httpx.Timeout]
+        if model_timeout is not None:
+            resolved = model_timeout
+        elif kwargs.get("timeout") is not None:
+            resolved = kwargs["timeout"]
+        elif kwargs.get("request_timeout") is not None:
+            resolved = kwargs["request_timeout"]
+        else:
+            resolved = CompletionTimeout._fallback_when_no_explicit_timeout(
+                global_timeout
+            )
+
+        if isinstance(resolved, httpx.Timeout) and not supports_httpx_timeout(
+            custom_llm_provider
+        ):
+            read_timeout = resolved.read
+            resolved = (
+                float(read_timeout)
+                if read_timeout is not None
+                else COMPLETION_HTTP_FALLBACK_SECONDS
+            )  # default 10 min timeout
+        elif not isinstance(resolved, httpx.Timeout):
+            resolved = float(resolved)  # type: ignore
+
+        return resolved

litellm/litellm_core_utils/litellm_logging.py

@@ -354,9 +354,9 @@ class Logging(LiteLLMLoggingBaseClass):
         )
         self.function_id = function_id
         self.streaming_chunks: List[Any] = []  # for generating complete stream response
-        self.sync_streaming_chunks: List[
-            Any
-        ] = []  # for generating complete stream response
+        self.sync_streaming_chunks: List[Any] = (
+            []
+        )  # for generating complete stream response
         self.log_raw_request_response = log_raw_request_response
 
         # Initialize dynamic callbacks
@@ -811,9 +811,9 @@ class Logging(LiteLLMLoggingBaseClass):
                         prompt_spec=prompt_spec,
                         dynamic_callback_params=dynamic_callback_params,
                     ):
-                        self.model_call_details[
-                            "prompt_integration"
-                        ] = logger.__class__.__name__
+                        self.model_call_details["prompt_integration"] = (
+                            logger.__class__.__name__
+                        )
                         return logger
                 except Exception:
                     # If check fails, continue to next logger
@@ -881,9 +881,9 @@ class Logging(LiteLLMLoggingBaseClass):
         if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook(
             non_default_params
         ):
-            self.model_call_details[
-                "prompt_integration"
-            ] = anthropic_cache_control_logger.__class__.__name__
+            self.model_call_details["prompt_integration"] = (
+                anthropic_cache_control_logger.__class__.__name__
+            )
             return anthropic_cache_control_logger
 
         #########################################################
@@ -895,9 +895,9 @@ class Logging(LiteLLMLoggingBaseClass):
                 internal_usage_cache=None,
                 llm_router=None,
             )
-            self.model_call_details[
-                "prompt_integration"
-            ] = vector_store_custom_logger.__class__.__name__
+            self.model_call_details["prompt_integration"] = (
+                vector_store_custom_logger.__class__.__name__
+            )
             # Add to global callbacks so post-call hooks are invoked
             if (
                 vector_store_custom_logger
@@ -957,9 +957,9 @@ class Logging(LiteLLMLoggingBaseClass):
             model
         ):  # if model name was changes pre-call, overwrite the initial model call name with the new one
             self.model_call_details["model"] = model
-        self.model_call_details["litellm_params"][
-            "api_base"
-        ] = self._get_masked_api_base(additional_args.get("api_base", ""))
+        self.model_call_details["litellm_params"]["api_base"] = (
+            self._get_masked_api_base(additional_args.get("api_base", ""))
+        )
 
     def pre_call(self, input, api_key, model=None, additional_args={}):  # noqa: PLR0915
         # Log the exact input to the LLM API
@@ -988,10 +988,10 @@ class Logging(LiteLLMLoggingBaseClass):
                 try:
                     # [Non-blocking Extra Debug Information in metadata]
                     if turn_off_message_logging is True:
-                        _metadata[
-                            "raw_request"
-                        ] = "redacted by litellm. \
+                        _metadata["raw_request"] = (
+                            "redacted by litellm. \
                             'litellm.turn_off_message_logging=True'"
+                        )
                     else:
                         curl_command = self._get_request_curl_command(
                             api_base=additional_args.get("api_base", ""),
@@ -1002,34 +1002,34 @@ class Logging(LiteLLMLoggingBaseClass):
 
                         _metadata["raw_request"] = str(curl_command)
                         # split up, so it's easier to parse in the UI
-                        self.model_call_details[
-                            "raw_request_typed_dict"
-                        ] = RawRequestTypedDict(
-                            raw_request_api_base=str(
-                                additional_args.get("api_base") or ""
-                            ),
-                            raw_request_body=self._get_raw_request_body(
-                                additional_args.get("complete_input_dict", {})
-                            ),
-                            # NOTE: setting ignore_sensitive_headers to True will cause
-                            # the Authorization header to be leaked when calls to the health
-                            # endpoint are made and fail.
-                            raw_request_headers=self._get_masked_headers(
-                                additional_args.get("headers", {}) or {},
-                            ),
-                            error=None,
+                        self.model_call_details["raw_request_typed_dict"] = (
+                            RawRequestTypedDict(
+                                raw_request_api_base=str(
+                                    additional_args.get("api_base") or ""
+                                ),
+                                raw_request_body=self._get_raw_request_body(
+                                    additional_args.get("complete_input_dict", {})
+                                ),
+                                # NOTE: setting ignore_sensitive_headers to True will cause
+                                # the Authorization header to be leaked when calls to the health
+                                # endpoint are made and fail.
+                                raw_request_headers=self._get_masked_headers(
+                                    additional_args.get("headers", {}) or {},
+                                ),
+                                error=None,
+                            )
                         )
                 except Exception as e:
-                    self.model_call_details[
-                        "raw_request_typed_dict"
-                    ] = RawRequestTypedDict(
-                        error=str(e),
+                    self.model_call_details["raw_request_typed_dict"] = (
+                        RawRequestTypedDict(
+                            error=str(e),
+                        )
                     )
-                    _metadata[
-                        "raw_request"
-                    ] = "Unable to Log \
+                    _metadata["raw_request"] = (
+                        "Unable to Log \
                         raw request: {}".format(
-                        str(e)
+                            str(e)
+                        )
                     )
             if getattr(self, "logger_fn", None) and callable(self.logger_fn):
                 try:
@@ -1330,13 +1330,13 @@ class Logging(LiteLLMLoggingBaseClass):
         for callback in callbacks:
             try:
                 if isinstance(callback, CustomLogger):
-                    response: Optional[
-                        MCPPostCallResponseObject
-                    ] = await callback.async_post_mcp_tool_call_hook(
-                        kwargs=kwargs,
-                        response_obj=post_mcp_tool_call_response_obj,
-                        start_time=start_time,
-                        end_time=end_time,
+                    response: Optional[MCPPostCallResponseObject] = (
+                        await callback.async_post_mcp_tool_call_hook(
+                            kwargs=kwargs,
+                            response_obj=post_mcp_tool_call_response_obj,
+                            start_time=start_time,
+                            end_time=end_time,
+                        )
                     )
                     ######################################################################
                     # if any of the callbacks modify the response, use the modified response
@@ -1387,6 +1387,8 @@ class Logging(LiteLLMLoggingBaseClass):
         margin_percent: Optional[float] = None,
         margin_fixed_amount: Optional[float] = None,
         margin_total_amount: Optional[float] = None,
+        cache_read_cost: Optional[float] = None,
+        cache_creation_cost: Optional[float] = None,
     ) -> None:
         """
         Helper method to store cost breakdown in the logging object.
@@ -1411,6 +1413,10 @@ class Logging(LiteLLMLoggingBaseClass):
             total_cost=total_cost,
             tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar,
         )
+        if cache_read_cost is not None and cache_read_cost > 0:
+            self.cost_breakdown["cache_read_cost"] = cache_read_cost
+        if cache_creation_cost is not None and cache_creation_cost > 0:
+            self.cost_breakdown["cache_creation_cost"] = cache_creation_cost
 
         # Store additional costs if provided (free-form dict for extensibility)
         if (
@@ -1537,9 +1543,9 @@ class Logging(LiteLLMLoggingBaseClass):
             verbose_logger.debug(
                 f"response_cost_failure_debug_information: {debug_info}"
             )
-            self.model_call_details[
-                "response_cost_failure_debug_information"
-            ] = debug_info
+            self.model_call_details["response_cost_failure_debug_information"] = (
+                debug_info
+            )
             return None
 
         try:
@@ -1565,9 +1571,9 @@ class Logging(LiteLLMLoggingBaseClass):
             verbose_logger.debug(
                 f"response_cost_failure_debug_information: {debug_info}"
             )
-            self.model_call_details[
-                "response_cost_failure_debug_information"
-            ] = debug_info
+            self.model_call_details["response_cost_failure_debug_information"] = (
+                debug_info
+            )
 
         return None
 
@@ -1716,9 +1722,9 @@ class Logging(LiteLLMLoggingBaseClass):
         self.model_call_details["litellm_params"].setdefault("metadata", {})
         if self.model_call_details["litellm_params"]["metadata"] is None:
             self.model_call_details["litellm_params"]["metadata"] = {}
-        self.model_call_details["litellm_params"]["metadata"][
-            "hidden_params"
-        ] = getattr(logging_result, "_hidden_params", {})
+        self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = (
+            getattr(logging_result, "_hidden_params", {})
+        )
 
     def _process_hidden_params_and_response_cost(
         self,
@@ -1747,9 +1753,9 @@ class Logging(LiteLLMLoggingBaseClass):
                 result=logging_result
             )
 
-        self.model_call_details[
-            "standard_logging_object"
-        ] = self._build_standard_logging_payload(logging_result, start_time, end_time)
+        self.model_call_details["standard_logging_object"] = (
+            self._build_standard_logging_payload(logging_result, start_time, end_time)
+        )
 
         if (
             standard_logging_payload := self.model_call_details.get(
@@ -1827,9 +1833,9 @@ class Logging(LiteLLMLoggingBaseClass):
                 end_time = datetime.datetime.now()
             if self.completion_start_time is None:
                 self.completion_start_time = end_time
-                self.model_call_details[
-                    "completion_start_time"
-                ] = self.completion_start_time
+                self.model_call_details["completion_start_time"] = (
+                    self.completion_start_time
+                )
 
             self.model_call_details["log_event_type"] = "successful_api_call"
             self.model_call_details["end_time"] = end_time
@@ -1866,10 +1872,10 @@ class Logging(LiteLLMLoggingBaseClass):
                         end_time=end_time,
                     )
                 elif isinstance(result, dict) or isinstance(result, list):
-                    self.model_call_details[
-                        "standard_logging_object"
-                    ] = self._build_standard_logging_payload(
-                        result, start_time, end_time
+                    self.model_call_details["standard_logging_object"] = (
+                        self._build_standard_logging_payload(
+                            result, start_time, end_time
+                        )
                     )
                     if (
                         standard_logging_payload := self.model_call_details.get(
@@ -1878,9 +1884,9 @@ class Logging(LiteLLMLoggingBaseClass):
                     ) is not None:
                         emit_standard_logging_payload(standard_logging_payload)
             elif standard_logging_object is not None:
-                self.model_call_details[
-                    "standard_logging_object"
-                ] = standard_logging_object
+                self.model_call_details["standard_logging_object"] = (
+                    standard_logging_object
+                )
             else:
                 self.model_call_details["response_cost"] = None
 
@@ -2038,20 +2044,20 @@ class Logging(LiteLLMLoggingBaseClass):
                 verbose_logger.debug(
                     "Logging Details LiteLLM-Success Call streaming complete"
                 )
-                self.model_call_details[
-                    "complete_streaming_response"
-                ] = complete_streaming_response
-                self.model_call_details[
-                    "response_cost"
-                ] = self._response_cost_calculator(result=complete_streaming_response)
+                self.model_call_details["complete_streaming_response"] = (
+                    complete_streaming_response
+                )
+                self.model_call_details["response_cost"] = (
+                    self._response_cost_calculator(result=complete_streaming_response)
+                )
                 self._merge_hidden_params_from_response_into_metadata(
                     complete_streaming_response
                 )
                 ## STANDARDIZED LOGGING PAYLOAD
-                self.model_call_details[
-                    "standard_logging_object"
-                ] = self._build_standard_logging_payload(
-                    complete_streaming_response, start_time, end_time
+                self.model_call_details["standard_logging_object"] = (
+                    self._build_standard_logging_payload(
+                        complete_streaming_response, start_time, end_time
+                    )
                 )
                 if (
                     standard_logging_payload := self.model_call_details.get(
@@ -2385,10 +2391,10 @@ class Logging(LiteLLMLoggingBaseClass):
                             )
                         else:
                             if self.stream and complete_streaming_response:
-                                self.model_call_details[
-                                    "complete_response"
-                                ] = self.model_call_details.get(
-                                    "complete_streaming_response", {}
+                                self.model_call_details["complete_response"] = (
+                                    self.model_call_details.get(
+                                        "complete_streaming_response", {}
+                                    )
                                 )
                                 result = self.model_call_details["complete_response"]
                             openMeterLogger.log_success_event(
@@ -2412,10 +2418,10 @@ class Logging(LiteLLMLoggingBaseClass):
                             )
                         else:
                             if self.stream and complete_streaming_response:
-                                self.model_call_details[
-                                    "complete_response"
-                                ] = self.model_call_details.get(
-                                    "complete_streaming_response", {}
+                                self.model_call_details["complete_response"] = (
+                                    self.model_call_details.get(
+                                        "complete_streaming_response", {}
+                                    )
                                 )
                                 result = self.model_call_details["complete_response"]
 
@@ -2554,9 +2560,9 @@ class Logging(LiteLLMLoggingBaseClass):
         if complete_streaming_response is not None:
             print_verbose("Async success callbacks: Got a complete streaming response")
 
-            self.model_call_details[
-                "async_complete_streaming_response"
-            ] = complete_streaming_response
+            self.model_call_details["async_complete_streaming_response"] = (
+                complete_streaming_response
+            )
 
             try:
                 if self.model_call_details.get("cache_hit", False) is True:
@@ -2567,10 +2573,10 @@ class Logging(LiteLLMLoggingBaseClass):
                         model_call_details=self.model_call_details
                     )
                     # base_model defaults to None if not set on model_info
-                    self.model_call_details[
-                        "response_cost"
-                    ] = self._response_cost_calculator(
-                        result=complete_streaming_response
+                    self.model_call_details["response_cost"] = (
+                        self._response_cost_calculator(
+                            result=complete_streaming_response
+                        )
                     )
 
                 verbose_logger.debug(
@@ -2587,10 +2593,10 @@ class Logging(LiteLLMLoggingBaseClass):
             )
 
             ## STANDARDIZED LOGGING PAYLOAD
-            self.model_call_details[
-                "standard_logging_object"
-            ] = self._build_standard_logging_payload(
-                complete_streaming_response, start_time, end_time
+            self.model_call_details["standard_logging_object"] = (
+                self._build_standard_logging_payload(
+                    complete_streaming_response, start_time, end_time
+                )
             )
 
             # print standard logging payload
@@ -2617,9 +2623,9 @@ class Logging(LiteLLMLoggingBaseClass):
             # _success_handler_helper_fn
             if self.model_call_details.get("standard_logging_object") is None:
                 ## STANDARDIZED LOGGING PAYLOAD
-                self.model_call_details[
-                    "standard_logging_object"
-                ] = self._build_standard_logging_payload(result, start_time, end_time)
+                self.model_call_details["standard_logging_object"] = (
+                    self._build_standard_logging_payload(result, start_time, end_time)
+                )
 
                 # print standard logging payload
                 if (
@@ -2862,18 +2868,18 @@ class Logging(LiteLLMLoggingBaseClass):
 
         ## STANDARDIZED LOGGING PAYLOAD
 
-        self.model_call_details[
-            "standard_logging_object"
-        ] = get_standard_logging_object_payload(
-            kwargs=self.model_call_details,
-            init_response_obj={},
-            start_time=start_time,
-            end_time=end_time,
-            logging_obj=self,
-            status="failure",
-            error_str=str(exception),
-            original_exception=exception,
-            standard_built_in_tools_params=self.standard_built_in_tools_params,
+        self.model_call_details["standard_logging_object"] = (
+            get_standard_logging_object_payload(
+                kwargs=self.model_call_details,
+                init_response_obj={},
+                start_time=start_time,
+                end_time=end_time,
+                logging_obj=self,
+                status="failure",
+                error_str=str(exception),
+                original_exception=exception,
+                standard_built_in_tools_params=self.standard_built_in_tools_params,
+            )
         )
         return start_time, end_time
 
@@ -3843,9 +3849,9 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
                 service_name=arize_config.project_name,
             )
 
-            os.environ[
-                "OTEL_EXPORTER_OTLP_TRACES_HEADERS"
-            ] = f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
+            os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
+                f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
+            )
             for callback in _in_memory_loggers:
                 if (
                     isinstance(callback, ArizeLogger)
@@ -3871,13 +3877,13 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
                 existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
                 # Add openinference.project.name attribute
                 if existing_attrs:
-                    os.environ[
-                        "OTEL_RESOURCE_ATTRIBUTES"
-                    ] = f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
+                    os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
+                        f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
+                    )
                 else:
-                    os.environ[
-                        "OTEL_RESOURCE_ATTRIBUTES"
-                    ] = f"openinference.project.name={arize_phoenix_config.project_name}"
+                    os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
+                        f"openinference.project.name={arize_phoenix_config.project_name}"
+                    )
 
             # Set Phoenix project name from environment variable
             phoenix_project_name = os.environ.get("PHOENIX_PROJECT_NAME", None)
@@ -3885,19 +3891,19 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
                 existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
                 # Add openinference.project.name attribute
                 if existing_attrs:
-                    os.environ[
-                        "OTEL_RESOURCE_ATTRIBUTES"
-                    ] = f"{existing_attrs},openinference.project.name={phoenix_project_name}"
+                    os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
+                        f"{existing_attrs},openinference.project.name={phoenix_project_name}"
+                    )
                 else:
-                    os.environ[
-                        "OTEL_RESOURCE_ATTRIBUTES"
-                    ] = f"openinference.project.name={phoenix_project_name}"
+                    os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
+                        f"openinference.project.name={phoenix_project_name}"
+                    )
 
             # auth can be disabled on local deployments of arize phoenix
             if arize_phoenix_config.otlp_auth_headers is not None:
-                os.environ[
-                    "OTEL_EXPORTER_OTLP_TRACES_HEADERS"
-                ] = arize_phoenix_config.otlp_auth_headers
+                os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
+                    arize_phoenix_config.otlp_auth_headers
+                )
 
             for callback in _in_memory_loggers:
                 if (
@@ -4084,9 +4090,9 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
                 exporter="otlp_http",
                 endpoint="https://langtrace.ai/api/trace",
             )
-            os.environ[
-                "OTEL_EXPORTER_OTLP_TRACES_HEADERS"
-            ] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
+            os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
+                f"api_key={os.getenv('LANGTRACE_API_KEY')}"
+            )
             for callback in _in_memory_loggers:
                 if (
                     isinstance(callback, OpenTelemetry)
@@ -4981,16 +4987,22 @@ class StandardLoggingPayloadSetup:
 
         additional_logging_headers: StandardLoggingAdditionalHeaders = {}
 
+        # Populate well-known typed fields with int/str coercion where needed
+        typed_keys: dict = {}
         for key in StandardLoggingAdditionalHeaders.__annotations__.keys():
-            _key = key.lower()
-            _key = _key.replace("_", "-")
+            _key = key.lower().replace("_", "-")
+            typed_keys[_key] = key
             if _key in additiona_headers:
                 try:
                     additional_logging_headers[key] = int(additiona_headers[_key])  # type: ignore
                 except (ValueError, TypeError):
-                    verbose_logger.debug(
-                        f"Could not convert {additiona_headers[_key]} to int for key {key}."
-                    )
+                    additional_logging_headers[key] = additiona_headers[_key]  # type: ignore
+
+        # Preserve all remaining headers verbatim (e.g. llm_provider-x-request-id)
+        for k, v in additiona_headers.items():
+            if k.lower() not in typed_keys:
+                additional_logging_headers[k] = v  # type: ignore
+
         return additional_logging_headers
 
     @staticmethod
@@ -5012,10 +5024,10 @@ class StandardLoggingPayloadSetup:
             for key in StandardLoggingHiddenParams.__annotations__.keys():
                 if key in hidden_params:
                     if key == "additional_headers":
-                        clean_hidden_params[
-                            "additional_headers"
-                        ] = StandardLoggingPayloadSetup.get_additional_headers(
-                            hidden_params[key]
+                        clean_hidden_params["additional_headers"] = (
+                            StandardLoggingPayloadSetup.get_additional_headers(
+                                hidden_params[key]
+                            )
                         )
                     else:
                         clean_hidden_params[key] = hidden_params[key]  # type: ignore
@@ -5656,9 +5668,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
     ):
         for k, v in metadata["user_api_key_metadata"].items():
             if k == "logging":  # prevent logging user logging keys
-                cleaned_user_api_key_metadata[
-                    k
-                ] = "scrubbed_by_litellm_for_sensitive_keys"
+                cleaned_user_api_key_metadata[k] = (
+                    "scrubbed_by_litellm_for_sensitive_keys"
+                )
             else:
                 cleaned_user_api_key_metadata[k] = v
 

litellm/litellm_core_utils/model_param_helper.py

@@ -11,6 +11,10 @@ from openai.types.completion_create_params import (
     CompletionCreateParamsStreaming as TextCompletionCreateParamsStreaming,
 )
 from openai.types.embedding_create_params import EmbeddingCreateParams
+from openai.types.responses.response_create_params import (
+    ResponseCreateParamsNonStreaming,
+    ResponseCreateParamsStreaming,
+)
 
 from litellm._logging import verbose_logger
 from litellm.types.rerank import RerankRequest
@@ -65,6 +69,9 @@ class ModelParamHelper:
             ModelParamHelper._get_litellm_supported_transcription_kwargs()
         )
         rerank_kwargs = ModelParamHelper._get_litellm_supported_rerank_kwargs()
+        responses_api_kwargs = (
+            ModelParamHelper._get_litellm_supported_responses_api_kwargs()
+        )
         exclude_kwargs = ModelParamHelper._get_exclude_kwargs()
 
         combined_kwargs = chat_completion_kwargs.union(
@@ -72,6 +79,7 @@ class ModelParamHelper:
             embedding_kwargs,
             transcription_kwargs,
             rerank_kwargs,
+            responses_api_kwargs,
         )
         combined_kwargs = combined_kwargs.difference(exclude_kwargs)
         return combined_kwargs
@@ -93,9 +101,9 @@ class ModelParamHelper:
         streaming_params: Set[str] = set(
             getattr(CompletionCreateParamsStreaming, "__annotations__", {}).keys()
         )
-        litellm_provider_specific_params: Set[
-            str
-        ] = ModelParamHelper.get_litellm_provider_specific_params_for_chat_params()
+        litellm_provider_specific_params: Set[str] = (
+            ModelParamHelper.get_litellm_provider_specific_params_for_chat_params()
+        )
         all_chat_completion_kwargs: Set[str] = non_streaming_params.union(
             streaming_params
         ).union(litellm_provider_specific_params)
@@ -167,6 +175,21 @@ class ModelParamHelper:
             verbose_logger.debug("Error getting transcription kwargs %s", str(e))
             return set()
 
+    @staticmethod
+    def _get_litellm_supported_responses_api_kwargs() -> Set[str]:
+        """
+        Get the litellm supported responses API kwargs
+
+        This follows the OpenAI API Spec
+        """
+        non_streaming_params: Set[str] = set(
+            getattr(ResponseCreateParamsNonStreaming, "__annotations__", {}).keys()
+        )
+        streaming_params: Set[str] = set(
+            getattr(ResponseCreateParamsStreaming, "__annotations__", {}).keys()
+        )
+        return non_streaming_params.union(streaming_params)
+
     @staticmethod
     def _get_exclude_kwargs() -> Set[str]:
         """

litellm/llms/anthropic/chat/transformation.py

@@ -1702,10 +1702,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 ),
             )
 
+        raw_input_tokens = usage_object.get("input_tokens", 0) or 0
         prompt_tokens_details = PromptTokensDetailsWrapper(
             cached_tokens=cache_read_input_tokens,
             cache_creation_tokens=cache_creation_input_tokens,
             cache_creation_token_details=cache_creation_token_details,
+            text_tokens=raw_input_tokens,
         )
         # Always populate completion_token_details, not just when there's reasoning_content
         reasoning_tokens = (

litellm/llms/anthropic/common_utils.py

@@ -2,6 +2,7 @@
 This file contains common utils for anthropic calls.
 """
 
+import copy
 from typing import Any, Dict, List, Optional, Union
 
 import httpx
@@ -736,6 +737,69 @@ def strip_advisor_blocks_from_messages(
     return messages
 
 
+def is_anthropic_invalid_thinking_signature_error(error_text: str) -> bool:
+    """
+    Detect Anthropic 400 when encrypted thinking signatures in history do not match
+    the current deployment (e.g. user rotated API key or switched model endpoint).
+
+    Example API message:
+    messages.N.content.M: Invalid `signature` in `thinking` block
+    """
+    if not error_text:
+        return False
+    lower = error_text.lower()
+    return (
+        "invalid" in lower
+        and "signature" in lower
+        and "thinking" in lower
+        and "block" in lower
+    )
+
+
+def strip_thinking_blocks_from_anthropic_messages(messages: List[Any]) -> List[Any]:
+    """
+    Return a new message list with thinking / redacted_thinking content blocks removed
+    from each message. Used to recover from invalid thinking signatures on retry.
+
+    Messages whose content is a list and becomes empty after stripping are omitted,
+    since Anthropic rejects empty content arrays.
+    """
+    out: List[Any] = []
+    for m in messages:
+        if not isinstance(m, dict):
+            out.append(m)
+            continue
+        mm = copy.deepcopy(m)
+        content = mm.get("content")
+        if isinstance(content, list):
+            filtered = [
+                b
+                for b in content
+                if not (
+                    isinstance(b, dict)
+                    and b.get("type") in ("thinking", "redacted_thinking")
+                )
+            ]
+            if not filtered:
+                continue
+            mm["content"] = filtered
+        out.append(mm)
+    return out
+
+
+def strip_thinking_blocks_from_anthropic_messages_request_dict(
+    data: Dict[str, Any],
+) -> None:
+    """
+    Mutate an Anthropic Messages-style request dict: strip thinking blocks from
+    ``messages`` and remove the top-level ``thinking`` extended-thinking param.
+    """
+    msgs = data.get("messages")
+    if isinstance(msgs, list):
+        data["messages"] = strip_thinking_blocks_from_anthropic_messages(msgs)
+    data.pop("thinking", None)
+
+
 def process_anthropic_headers(headers: Union[httpx.Headers, dict]) -> dict:
     openai_headers = {}
     if "anthropic-ratelimit-requests-limit" in headers:

litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py

@@ -27,6 +27,7 @@ class BaseAnthropicMessagesStreamingIterator:
         self.litellm_logging_obj = litellm_logging_obj
         self.request_body = request_body
         self.start_time = datetime.now()
+        self.completion_start_time: datetime | None = None
 
     async def _handle_streaming_logging(self, collected_chunks: List[bytes]):
         """Handle the logging after all chunks have been collected."""
@@ -35,6 +36,15 @@ class BaseAnthropicMessagesStreamingIterator:
         )
 
         end_time = datetime.now()
+        # Set completion_start_time so TTFT is calculated from the first
+        # chunk rather than falling back to end_time in async_success_handler.
+        if self.completion_start_time is not None:
+            self.litellm_logging_obj.completion_start_time = (
+                self.completion_start_time
+            )
+            self.litellm_logging_obj.model_call_details[
+                "completion_start_time"
+            ] = self.completion_start_time
         asyncio.create_task(
             PassThroughStreamingHandler._route_streaming_logging_to_handler(
                 litellm_logging_obj=self.litellm_logging_obj,
@@ -100,6 +110,8 @@ class BaseAnthropicMessagesStreamingIterator:
         collected_chunks = []
 
         async for chunk in completion_stream:
+            if self.completion_start_time is None:
+                self.completion_start_time = datetime.now()
             encoded_chunk = self._convert_chunk_to_sse_format(chunk)
             collected_chunks.append(encoded_chunk)
             yield encoded_chunk

litellm/llms/azure/passthrough/transformation.py

@@ -1,7 +1,9 @@
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
 import httpx
+from httpx import Response
 
+from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.llms.azure.common_utils import BaseAzureLLM
 from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
 from litellm.secret_managers.main import get_secret_str
@@ -11,6 +13,8 @@ from litellm.types.router import GenericLiteLLMParams
 if TYPE_CHECKING:
     from httpx import URL
 
+    from litellm.types.utils import CostResponseTypes
+
 
 class AzurePassthroughConfig(BasePassthroughConfig):
     def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
@@ -83,3 +87,36 @@ class AzurePassthroughConfig(BasePassthroughConfig):
         self, api_key: Optional[str] = None, api_base: Optional[str] = None
     ) -> List[str]:
         return super().get_models(api_key, api_base)
+
+    def logging_non_streaming_response(
+        self,
+        model: str,
+        custom_llm_provider: str,
+        httpx_response: Response,
+        request_data: dict,
+        logging_obj: Logging,
+        endpoint: str,
+    ) -> Optional["CostResponseTypes"]:
+        from litellm import encoding
+        from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+        from litellm.types.utils import ModelResponse
+
+        if "chat/completions" not in endpoint:
+            return None
+
+        openai_chat_config = OpenAIGPTConfig()
+
+        litellm_model_response: ModelResponse = openai_chat_config.transform_response(
+            model=model,
+            messages=[{"role": "user", "content": "no-message-pass-through-endpoint"}],
+            raw_response=httpx_response,
+            model_response=ModelResponse(),
+            logging_obj=logging_obj,
+            optional_params={},
+            litellm_params={},
+            api_key="",
+            request_data=request_data,
+            encoding=encoding,
+        )
+
+        return litellm_model_response

litellm/llms/base_llm/anthropic_messages/transformation.py

@@ -120,3 +120,46 @@ class BaseAnthropicMessagesConfig(ABC):
         return BaseLLMException(
             message=error_message, status_code=status_code, headers=headers
         )
+
+    @property
+    def max_retry_on_anthropic_messages_http_error(self) -> int:
+        """
+        Max HTTP attempts for /v1/messages when the handler may mutate the body and
+        retry (e.g. strip invalid encrypted thinking signatures after a deployment or
+        credential change).
+        """
+        return 2
+
+    def should_retry_anthropic_messages_on_http_error(
+        self, e: httpx.HTTPStatusError, litellm_params: dict
+    ) -> bool:
+        """
+        When True, async_anthropic_messages_handler will transform the request body
+        and issue one more attempt (bounded by max_retry_on_anthropic_messages_http_error).
+        """
+        from litellm.llms.anthropic.common_utils import (
+            is_anthropic_invalid_thinking_signature_error,
+        )
+
+        return (
+            e.response.status_code == 400
+            and is_anthropic_invalid_thinking_signature_error(e.response.text)
+        )
+
+    def transform_anthropic_messages_request_on_http_error(
+        self, e: httpx.HTTPStatusError, request_data: dict
+    ) -> dict:
+        """
+        Mutates request_data in place when retrying after a recoverable HTTP error.
+        """
+        from litellm.llms.anthropic.common_utils import (
+            is_anthropic_invalid_thinking_signature_error,
+            strip_thinking_blocks_from_anthropic_messages_request_dict,
+        )
+
+        if (
+            e.response.status_code == 400
+            and is_anthropic_invalid_thinking_signature_error(e.response.text)
+        ):
+            strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
+        return request_data

litellm/llms/bedrock/chat/converse_transformation.py

@@ -1657,6 +1657,7 @@ class AmazonConverseConfig(BaseConfig):
         cache_creation_input_tokens: int = 0
         cache_read_input_tokens: int = 0
 
+        raw_input_tokens = input_tokens  # capture before inflation
         if "cacheReadInputTokens" in usage:
             cache_read_input_tokens = usage["cacheReadInputTokens"]
             input_tokens += cache_read_input_tokens
@@ -1665,7 +1666,9 @@ class AmazonConverseConfig(BaseConfig):
             input_tokens += cache_creation_input_tokens
 
         prompt_tokens_details = PromptTokensDetailsWrapper(
-            cached_tokens=cache_read_input_tokens
+            cached_tokens=cache_read_input_tokens,
+            cache_creation_tokens=cache_creation_input_tokens,
+            text_tokens=raw_input_tokens,
         )
         reasoning_tokens = (
             token_counter(text=reasoning_content, count_response_tokens=True)

litellm/llms/custom_httpx/http_handler.py

@@ -30,7 +30,9 @@ from litellm.constants import (
     AIOHTTP_KEEPALIVE_TIMEOUT,
     AIOHTTP_NEEDS_CLEANUP_CLOSED,
     AIOHTTP_TTL_DNS_CACHE,
+    COMPLETION_HTTP_FALLBACK_SECONDS,
     DEFAULT_SSL_CIPHERS,
+    HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
 )
 from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
 from litellm.types.llms.custom_http import *
@@ -70,7 +72,10 @@ def get_default_headers() -> dict:
 headers = get_default_headers()
 
 # https://www.python-httpx.org/advanced/timeouts
-_DEFAULT_TIMEOUT = httpx.Timeout(timeout=5.0, connect=5.0)
+_DEFAULT_TIMEOUT = httpx.Timeout(
+    timeout=COMPLETION_HTTP_FALLBACK_SECONDS,
+    connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
+)
 
 
 def _prepare_request_data_and_content(
@@ -1244,7 +1249,7 @@ def get_async_httpx_client(
         _new_client = AsyncHTTPHandler(**handler_params)
     else:
         _new_client = AsyncHTTPHandler(
-            timeout=httpx.Timeout(timeout=600.0, connect=5.0),
+            timeout=_DEFAULT_TIMEOUT,
             shared_session=shared_session,
         )
 
@@ -1293,7 +1298,7 @@ def _get_httpx_client(params: Optional[dict] = None) -> HTTPHandler:
         }
         _new_client = HTTPHandler(**handler_params)
     else:
-        _new_client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
+        _new_client = HTTPHandler(timeout=_DEFAULT_TIMEOUT)
 
     cache.set_cache(
         key=_cache_key_name,

litellm/llms/custom_httpx/llm_http_handler.py

@@ -1816,6 +1816,73 @@ class BaseLLMHTTPHandler:
             logging_obj=logging_obj,
         )
 
+    async def _async_post_anthropic_messages_with_http_error_retry(
+        self,
+        async_httpx_client: AsyncHTTPHandler,
+        request_url: str,
+        headers: dict,
+        signed_json_body: Optional[bytes],
+        request_body: dict,
+        stream: bool,
+        logging_obj: LiteLLMLoggingObj,
+        provider_config: BaseAnthropicMessagesConfig,
+        litellm_params: GenericLiteLLMParams,
+        api_key: Optional[str],
+        model: str,
+    ) -> httpx.Response:
+        max_attempts = max(
+            provider_config.max_retry_on_anthropic_messages_http_error, 1
+        )
+        litellm_params_dict = dict(litellm_params)
+        optional_params_dict = dict(litellm_params)
+        for attempt_idx in range(max_attempts):
+            try:
+                response = await async_httpx_client.post(
+                    url=request_url,
+                    headers=headers,
+                    data=signed_json_body or json.dumps(request_body),
+                    stream=stream or False,
+                    logging_obj=logging_obj,
+                )
+                response.raise_for_status()
+                return response
+            except httpx.HTTPStatusError as e:
+                hit_max_attempt = attempt_idx + 1 == max_attempts
+                should_retry = (
+                    provider_config.should_retry_anthropic_messages_on_http_error(
+                        e=e, litellm_params=litellm_params_dict
+                    )
+                )
+                if should_retry and not hit_max_attempt:
+                    verbose_logger.debug(
+                        "Anthropic /v1/messages: invalid thinking signature; "
+                        "stripping thinking blocks and retrying (attempt %s/%s).",
+                        attempt_idx + 2,
+                        max_attempts,
+                    )
+                    provider_config.transform_anthropic_messages_request_on_http_error(
+                        e=e, request_data=request_body
+                    )
+                    headers, signed_json_body = provider_config.sign_request(
+                        headers=headers,
+                        optional_params=optional_params_dict,
+                        request_data=request_body,
+                        api_base=request_url,
+                        api_key=api_key,
+                        stream=stream,
+                        fake_stream=False,
+                        model=model,
+                    )
+                    logging_obj.model_call_details.update(request_body)
+                    continue
+                raise self._handle_error(e=e, provider_config=provider_config)
+            except Exception as e:
+                raise self._handle_error(e=e, provider_config=provider_config)
+
+        raise RuntimeError(
+            "unreachable: anthropic messages HTTP retry loop exited without return"
+        )
+
     async def async_anthropic_messages_handler(
         self,
         model: str,
@@ -1955,19 +2022,19 @@ class BaseLLMHTTPHandler:
             },
         )
 
-        try:
-            response = await async_httpx_client.post(
-                url=request_url,
-                headers=headers,
-                data=signed_json_body or json.dumps(request_body),
-                stream=stream or False,
-                logging_obj=logging_obj,
-            )
-            response.raise_for_status()
-        except Exception as e:
-            raise self._handle_error(
-                e=e, provider_config=anthropic_messages_provider_config
-            )
+        response = await self._async_post_anthropic_messages_with_http_error_retry(
+            async_httpx_client=async_httpx_client,
+            request_url=request_url,
+            headers=headers,
+            signed_json_body=signed_json_body,
+            request_body=request_body,
+            stream=stream or False,
+            logging_obj=logging_obj,
+            provider_config=anthropic_messages_provider_config,
+            litellm_params=litellm_params,
+            api_key=api_key,
+            model=model,
+        )
 
         # used for logging + cost tracking
         logging_obj.model_call_details["httpx_response"] = response
@@ -4496,9 +4563,9 @@ class BaseLLMHTTPHandler:
                         # Second: Execute agentic loop
                         # Add custom_llm_provider to kwargs so the agentic loop can reconstruct the full model name
                         kwargs_with_provider = kwargs.copy() if kwargs else {}
-                        kwargs_with_provider[
-                            "custom_llm_provider"
-                        ] = custom_llm_provider
+                        kwargs_with_provider["custom_llm_provider"] = (
+                            custom_llm_provider
+                        )
                         agentic_response = await callback.async_run_agentic_loop(
                             tools=tool_calls,
                             model=model,
@@ -4614,9 +4681,9 @@ class BaseLLMHTTPHandler:
                         # Second: Execute agentic loop
                         # Add custom_llm_provider to kwargs so the agentic loop can reconstruct the full model name
                         kwargs_with_provider = kwargs.copy() if kwargs else {}
-                        kwargs_with_provider[
-                            "custom_llm_provider"
-                        ] = custom_llm_provider
+                        kwargs_with_provider["custom_llm_provider"] = (
+                            custom_llm_provider
+                        )
                         agentic_response = (
                             await callback.async_run_chat_completion_agentic_loop(
                                 tools=tool_calls,
@@ -5110,7 +5177,10 @@ class BaseLLMHTTPHandler:
         _is_async: bool = False,
         fake_stream: bool = False,
         litellm_metadata: Optional[Dict[str, Any]] = None,
-    ) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
+    ) -> Union[
+        ImageResponse,
+        Coroutine[Any, Any, ImageResponse],
+    ]:
         """
 
         Handles image edit requests.
@@ -5322,7 +5392,10 @@ class BaseLLMHTTPHandler:
         fake_stream: bool = False,
         litellm_metadata: Optional[Dict[str, Any]] = None,
         api_key: Optional[str] = None,
-    ) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
+    ) -> Union[
+        ImageResponse,
+        Coroutine[Any, Any, ImageResponse],
+    ]:
         """
         Handles image generation requests.
         When _is_async=True, returns a coroutine instead of making the call directly.
@@ -5562,7 +5635,10 @@ class BaseLLMHTTPHandler:
         fake_stream: bool = False,
         litellm_metadata: Optional[Dict[str, Any]] = None,
         api_key: Optional[str] = None,
-    ) -> Union[VideoObject, Coroutine[Any, Any, VideoObject],]:
+    ) -> Union[
+        VideoObject,
+        Coroutine[Any, Any, VideoObject],
+    ]:
         """
         Handles video generation requests.
         When _is_async=True, returns a coroutine instead of making the call directly.

litellm/llms/ollama/chat/transformation.py

@@ -16,6 +16,7 @@ from httpx._models import Headers, Response
 from pydantic import BaseModel
 
 import litellm
+from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     _extract_reasoning_content,
     convert_content_list_to_str,
@@ -349,7 +350,8 @@ class OllamaChatConfig(BaseConfig):
         response_json = raw_response.json()
 
         ## RESPONSE OBJECT
-        model_response.choices[0].finish_reason = "stop"
+        _done_reason = map_finish_reason(response_json.get("done_reason") or "stop")
+        model_response.choices[0].finish_reason = _done_reason
         response_json_message = response_json.get("message")
         if response_json_message is not None:
             if "thinking" in response_json_message:
@@ -535,7 +537,7 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
             )
 
             if chunk["done"] is True:
-                finish_reason = chunk.get("done_reason", "stop")
+                finish_reason = chunk.get("done_reason") or "stop"
                 # Override finish_reason when tool_calls are present
                 # Fixes: https://github.com/BerriAI/litellm/issues/18922
                 if tool_calls is not None:

litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py

@@ -78,6 +78,19 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
 
         return endpoint
 
+    @staticmethod
+    def _strip_version_suffix(model: str) -> str:
+        """
+        Strip version suffixes (e.g. @default, @20251001) from model names.
+
+        The Vertex AI count-tokens endpoint rejects model names that include
+        version suffixes — for example, "claude-sonnet-4-6@default" returns
+        "not supported for token counting" while "claude-sonnet-4-6" works.
+        """
+        if "@" in model:
+            return model.split("@")[0]
+        return model
+
     async def handle_count_tokens_request(
         self,
         model: str,
@@ -98,6 +111,15 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
         Raises:
             ValueError: If required parameters are missing or invalid
         """
+        # Strip version suffixes (@default, @20251001, etc.) — the Vertex AI
+        # count-tokens endpoint does not accept versioned model names.
+        model = self._strip_version_suffix(model)
+        if "model" in request_data:
+            request_data = {
+                **request_data,
+                "model": self._strip_version_suffix(request_data["model"]),
+            }
+
         # Validate request
         if "messages" not in request_data:
             raise ValueError("messages required for token counting")

litellm/main.py

@@ -76,6 +76,7 @@ from litellm.litellm_core_utils.audio_utils.utils import (
     calculate_request_duration,
     get_audio_file_for_health_check,
 )
+from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.get_provider_specific_headers import (
     ProviderSpecificHeaderUtils,
@@ -1400,14 +1401,13 @@ def completion(  # type: ignore # noqa: PLR0915
             )  # support region-based pricing for bedrock
 
         ### TIMEOUT LOGIC ###
-        timeout = timeout or kwargs.get("request_timeout", 600) or 600
-        # set timeout for 10 minutes by default
-        if isinstance(timeout, httpx.Timeout) and not supports_httpx_timeout(
-            custom_llm_provider
-        ):
-            timeout = timeout.read or 600  # default 10 min timeout
-        elif not isinstance(timeout, httpx.Timeout):
-            timeout = float(timeout)  # type: ignore
+        timeout = CompletionTimeout.resolve(
+            timeout,
+            kwargs,
+            custom_llm_provider,
+            global_timeout=getattr(litellm, "request_timeout", None),
+            supports_httpx_timeout=supports_httpx_timeout,
+        )
 
         ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
         if (

litellm/proxy/_experimental/mcp_server/mcp_context.py

@@ -14,3 +14,8 @@ from typing import Optional
 _mcp_active_toolset_id: ContextVar[Optional[str]] = ContextVar(
     "_mcp_active_toolset_id", default=None
 )
+
+# Per-request merged InitializeResult.instructions; set in MCP HTTP/SSE handlers.
+_mcp_gateway_initialize_instructions: ContextVar[Optional[str]] = ContextVar(
+    "_mcp_gateway_initialize_instructions", default=None
+)

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -184,6 +184,16 @@ class MCPServerManager:
             "gmail_send_email": "zapier_mcp_server",
         }
         """
+        self._upstream_initialize_instructions_by_server_id: Dict[str, str] = {}
+
+    def _remember_upstream_initialize_instructions(
+        self, server: MCPServer, client: MCPClient
+    ) -> None:
+        raw = getattr(client, "_last_initialize_instructions", None)
+        if raw and str(raw).strip():
+            self._upstream_initialize_instructions_by_server_id[server.server_id] = str(
+                raw
+            ).strip()
 
     def get_registry(self) -> Dict[str, MCPServer]:
         """
@@ -204,6 +214,7 @@ class MCPServerManager:
             mcp_aliases: Optional dictionary mapping aliases to server names from litellm_settings
         """
         verbose_logger.debug("Loading MCP Servers from config-----")
+        self._upstream_initialize_instructions_by_server_id.clear()
 
         # Track which aliases have been used to ensure only first occurrence is used
         used_aliases = set()
@@ -351,6 +362,7 @@ class MCPServerManager:
                 aws_service_name=server_config.get("aws_service_name", None),
                 aws_role_name=server_config.get("aws_role_name", None),
                 aws_session_name=server_config.get("aws_session_name", None),
+                instructions=server_config.get("instructions", None),
             )
             self.config_mcp_servers[server_id] = new_server
 
@@ -693,6 +705,7 @@ class MCPServerManager:
             aws_service_name=aws_creds.get("aws_service_name"),
             aws_role_name=aws_creds.get("aws_role_name"),
             aws_session_name=aws_creds.get("aws_session_name"),
+            instructions=mcp_server.instructions,
         )
         return new_server
 
@@ -1247,6 +1260,7 @@ class MCPServerManager:
                 return tools
             else:
                 tools = await self._fetch_tools_with_timeout(client, server.name)
+                self._remember_upstream_initialize_instructions(server, client)
 
             prefixed_or_original_tools = self._create_prefixed_tools(
                 tools, server, add_prefix=add_prefix
@@ -2383,6 +2397,7 @@ class MCPServerManager:
         # If proxy_logging_obj is not None, the tool call result is at index 1 (after the during hook task)
         result_index = 1 if proxy_logging_obj else 0
         result = mcp_responses[result_index]
+        self._remember_upstream_initialize_instructions(mcp_server, client)
 
         return cast(CallToolResult, result)
 
@@ -2627,6 +2642,7 @@ class MCPServerManager:
         )
 
         verbose_logger.debug("Loading MCP servers from database into registry...")
+        self._upstream_initialize_instructions_by_server_id.clear()
 
         # perform authz check to filter the mcp servers user has access to
         prisma_client = get_prisma_client_or_throw(
@@ -2910,6 +2926,7 @@ class MCPServerManager:
                 await asyncio.wait_for(
                     client.run_with_session(_noop), timeout=MCP_HEALTH_CHECK_TIMEOUT
                 )
+                self._remember_upstream_initialize_instructions(server, client)
                 status = "healthy"
             except asyncio.TimeoutError:
                 health_check_error = (
@@ -2951,6 +2968,7 @@ class MCPServerManager:
             token_url=server.token_url,
             registration_url=server.registration_url,
             allow_all_keys=server.allow_all_keys,
+            instructions=server.instructions,
         )
 
     async def get_all_mcp_servers_with_health_and_teams(
@@ -3046,6 +3064,7 @@ class MCPServerManager:
             is_byok=server.is_byok,
             byok_description=server.byok_description,
             byok_api_key_help_url=server.byok_api_key_help_url,
+            instructions=server.instructions,
         )
 
     async def get_all_mcp_servers_unfiltered(self) -> List[LiteLLM_MCPServerTable]:

litellm/proxy/_experimental/mcp_server/rest_endpoints.py

@@ -933,6 +933,7 @@ if MCP_AVAILABLE:
                 authorization_url=request.authorization_url,
                 registration_url=request.registration_url,
                 oauth2_flow=_oauth2_flow,
+                instructions=request.instructions,
             )
 
             stdio_env = global_mcp_server_manager._build_stdio_env(

litellm/proxy/_experimental/mcp_server/server.py

@@ -7,6 +7,7 @@ LiteLLM MCP Server Routes
 import asyncio
 import contextlib
 import time
+import types
 import traceback
 import uuid
 from datetime import datetime
@@ -37,7 +38,10 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
 from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
     get_request_base_url,
 )
-from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_active_toolset_id
+from litellm.proxy._experimental.mcp_server.mcp_context import (
+    _mcp_active_toolset_id,
+    _mcp_gateway_initialize_instructions,
+)
 from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug
 from litellm.proxy._experimental.mcp_server.utils import (
     LITELLM_MCP_SERVER_DESCRIPTION,
@@ -122,6 +126,8 @@ _INITIALIZATION_LOCK = asyncio.Lock()
 
 if MCP_AVAILABLE:
     from mcp.server import Server
+    from mcp.server.lowlevel.server import NotificationOptions
+    from mcp.server.models import InitializationOptions
 
     # Import auth context variables and middleware
     from mcp.server.auth.middleware.auth_context import (
@@ -200,6 +206,21 @@ if MCP_AVAILABLE:
                 )
         return normalized
 
+    def _gateway_create_initialization_options(
+        self,
+        notification_options: Optional[NotificationOptions] = None,
+        experimental_capabilities: Optional[Dict[str, Dict[str, Any]]] = None,
+    ) -> InitializationOptions:
+        opts = Server.create_initialization_options(
+            self,
+            notification_options=notification_options,
+            experimental_capabilities=experimental_capabilities or {},
+        )
+        merged = _mcp_gateway_initialize_instructions.get()
+        if merged is not None:
+            return opts.model_copy(update={"instructions": merged})
+        return opts
+
     ########################################################
     ############ Initialize the MCP Server #################
     ########################################################
@@ -207,6 +228,9 @@ if MCP_AVAILABLE:
         name=LITELLM_MCP_SERVER_NAME,
         version=LITELLM_MCP_SERVER_VERSION,
     )
+    server.create_initialization_options = types.MethodType(  # type: ignore[method-assign]
+        _gateway_create_initialization_options, server
+    )
     sse: SseServerTransport = SseServerTransport("/mcp/sse/messages")
 
     # Create session managers
@@ -1021,7 +1045,9 @@ if MCP_AVAILABLE:
                     except (ValueError, TypeError):
                         pass
                 ttl = _compute_per_user_token_ttl(server, raw_expires)
-                await mcp_per_user_token_cache.set(user_id, server_id, access_token, ttl)
+                await mcp_per_user_token_cache.set(
+                    user_id, server_id, access_token, ttl
+                )
 
             return {"Authorization": f"Bearer {access_token}"}
         except Exception as e:
@@ -1103,6 +1129,57 @@ if MCP_AVAILABLE:
 
         return server_auth_header, extra_headers
 
+    def _merge_gateway_initialize_instructions(
+        allowed_mcp_servers: List[MCPServer],
+    ) -> Optional[str]:
+        """YAML/DB override, else in-memory upstream text from list_tools / health_check / call_tool."""
+        if not allowed_mcp_servers:
+            return None
+
+        texts: List[Tuple[str, str]] = []
+        for server in allowed_mcp_servers:
+            label = (
+                server.alias
+                or server.server_name
+                or server.name
+                or server.server_id
+                or "mcp"
+            )
+            if server.instructions and server.instructions.strip():
+                texts.append((label, server.instructions.strip()))
+                continue
+            if server.spec_path:
+                continue
+            cached = global_mcp_server_manager._upstream_initialize_instructions_by_server_id.get(
+                server.server_id
+            )
+            if cached and cached.strip():
+                texts.append((label, cached.strip()))
+
+        if not texts:
+            return None
+        if len(texts) == 1:
+            return texts[0][1]
+        return "\n\n---\n\n".join(f"[{lbl}]\n{txt}" for lbl, txt in texts)
+
+    @contextlib.asynccontextmanager
+    async def _gateway_initialize_instructions_request_scope(
+        user_api_key_auth: Optional[UserAPIKeyAuth],
+        mcp_servers: Optional[List[str]],
+        client_ip: Optional[str],
+    ) -> AsyncIterator[None]:
+        allowed = await _get_allowed_mcp_servers(
+            user_api_key_auth=user_api_key_auth,
+            mcp_servers=mcp_servers,
+            client_ip=client_ip,
+        )
+        merged = _merge_gateway_initialize_instructions(allowed_mcp_servers=allowed)
+        tok = _mcp_gateway_initialize_instructions.set(merged)
+        try:
+            yield
+        finally:
+            _mcp_gateway_initialize_instructions.reset(tok)
+
     async def _get_tools_from_mcp_servers(  # noqa: PLR0915
         user_api_key_auth: Optional[UserAPIKeyAuth],
         mcp_auth_header: Optional[str],
@@ -2670,7 +2747,12 @@ if MCP_AVAILABLE:
                 # Request was fully handled (e.g., DELETE on non-existent session)
                 return
 
-            await session_manager.handle_request(scope, receive, send)
+            async with _gateway_initialize_instructions_request_scope(
+                user_api_key_auth,
+                mcp_servers,
+                _client_ip,
+            ):
+                await session_manager.handle_request(scope, receive, send)
         except HTTPException:
             # Re-raise HTTP exceptions to preserve status codes and details
             raise
@@ -2729,7 +2811,12 @@ if MCP_AVAILABLE:
                 await initialize_session_managers()
                 await asyncio.sleep(0.1)
 
-            await sse_session_manager.handle_request(scope, receive, send)
+            async with _gateway_initialize_instructions_request_scope(
+                user_api_key_auth,
+                mcp_servers,
+                _sse_client_ip,
+            ):
+                await sse_session_manager.handle_request(scope, receive, send)
         except Exception as e:
             verbose_logger.exception(f"Error handling MCP request: {e}")
             # Instead of re-raising, try to send a graceful error response

litellm/proxy/_experimental/out/__next.__PAGE__.txt

@@ -1,28 +1,27 @@
 1:"$Sreact.fragment"
 2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
-3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/df37a0019220a941.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","/litellm-asset-prefix/_next/static/chunks/7834a5efb7b5f959.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/a7113797b37526f0.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/db0ac43a898048e2.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","/litellm-asset-prefix/_next/static/chunks/60b0cadba57cd7f7.js","/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","/litellm-asset-prefix/_next/static/chunks/1fd9dbe73d002173.js","/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/ed4f62880278d987.js","/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","/litellm-asset-prefix/_next/static/chunks/eaa9f9b9bb3e054b.js","/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/a02f90f97248b9aa.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/a5774cdb9f28daa1.js","/litellm-asset-prefix/_next/static/chunks/1501e804b4d0f510.js","/litellm-asset-prefix/_next/static/chunks/86819b3a4f820602.js","/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","/litellm-asset-prefix/_next/static/chunks/be00dd25857a2fb3.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/7dd16a650b98a4c5.js","/litellm-asset-prefix/_next/static/chunks/bd5cc6a7a48eedc7.js","/litellm-asset-prefix/_next/static/chunks/169b34fe8aeee0c7.js","/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/53ac95bfa383e1b4.js","/litellm-asset-prefix/_next/static/chunks/0c6c65a34bcde140.js"],"default"]
-18:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
-19:"$Sreact.suspense"
+3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/c0b640cc12a2b90e.js","/litellm-asset-prefix/_next/static/chunks/d439b54d089ced2b.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","/litellm-asset-prefix/_next/static/chunks/67faedd7c3f4c2e7.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/747ea6dd4e127c37.js","/litellm-asset-prefix/_next/static/chunks/bcbc49655bbecdc3.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","/litellm-asset-prefix/_next/static/chunks/d077cc2c21a37474.js","/litellm-asset-prefix/_next/static/chunks/2aa5ca37f441cf6f.js","/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/bd29d39cc81d3dc6.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/9edb3e10a3bcd754.js","/litellm-asset-prefix/_next/static/chunks/eaa9f9b9bb3e054b.js","/litellm-asset-prefix/_next/static/chunks/5e4cbfe76f1ba150.js","/litellm-asset-prefix/_next/static/chunks/ca91b0fa4d619698.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/443dce180e4b120d.js","/litellm-asset-prefix/_next/static/chunks/a5ab01e86df55e55.js","/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","/litellm-asset-prefix/_next/static/chunks/082a01ae76d64ee1.js","/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","/litellm-asset-prefix/_next/static/chunks/ac92164b24de092b.js","/litellm-asset-prefix/_next/static/chunks/7dd16a650b98a4c5.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/8ad88d515b60dca7.js"],"default"]
+17:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
+18:"$Sreact.suspense"
 :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
-0:{"buildId":"lRBQFcrGOsyCYLFEalzGW","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/df37a0019220a941.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/7834a5efb7b5f959.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/a7113797b37526f0.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/db0ac43a898048e2.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/60b0cadba57cd7f7.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/1fd9dbe73d002173.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/ed4f62880278d987.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/eaa9f9b9bb3e054b.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/a02f90f97248b9aa.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/a5774cdb9f28daa1.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/1501e804b4d0f510.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/86819b3a4f820602.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16"],"$L17"]}],"loading":null,"isPartial":false}
+0:{"buildId":"ak_B7XGok3Ra_ZXFSQmNR","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/c0b640cc12a2b90e.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/d439b54d089ced2b.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/67faedd7c3f4c2e7.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/747ea6dd4e127c37.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/bcbc49655bbecdc3.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/d077cc2c21a37474.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/2aa5ca37f441cf6f.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/bd29d39cc81d3dc6.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/9edb3e10a3bcd754.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/eaa9f9b9bb3e054b.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15"],"$L16"]}],"loading":null,"isPartial":false}
 4:{}
 5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
-6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","async":true}]
-7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/be00dd25857a2fb3.js","async":true}]
+6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/5e4cbfe76f1ba150.js","async":true}]
+7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/ca91b0fa4d619698.js","async":true}]
 8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
-9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}]
-a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}]
-b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/7dd16a650b98a4c5.js","async":true}]
-c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/bd5cc6a7a48eedc7.js","async":true}]
-d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/169b34fe8aeee0c7.js","async":true}]
+9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","async":true}]
+a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","async":true}]
+b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
+c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/443dce180e4b120d.js","async":true}]
+d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/a5ab01e86df55e55.js","async":true}]
 e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","async":true}]
-f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
-10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}]
-11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","async":true}]
-12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","async":true}]
-13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
-14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}]
-15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/53ac95bfa383e1b4.js","async":true}]
-16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/0c6c65a34bcde140.js","async":true}]
-17:["$","$L18",null,{"children":["$","$19",null,{"name":"Next.MetadataOutlet","children":"$@1a"}]}]
-1a:null
+f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/082a01ae76d64ee1.js","async":true}]
+10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","async":true}]
+11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","async":true}]
+12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/ac92164b24de092b.js","async":true}]
+13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/7dd16a650b98a4c5.js","async":true}]
+14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}]
+15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/8ad88d515b60dca7.js","async":true}]
+16:["$","$L17",null,{"children":["$","$18",null,{"name":"Next.MetadataOutlet","children":"$@19"}]}]
+19:null

litellm/proxy/_experimental/out/__next._head.txt

@@ -3,4 +3,4 @@
 3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
 4:"$Sreact.suspense"
 5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
-0:{"buildId":"lRBQFcrGOsyCYLFEalzGW","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
+0:{"buildId":"ak_B7XGok3Ra_ZXFSQmNR","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}

litellm/proxy/_experimental/out/__next._index.txt

@@ -4,5 +4,5 @@
 4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
 5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
 :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","style"]
-0:{"buildId":"lRBQFcrGOsyCYLFEalzGW","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
+:HL["/litellm-asset-prefix/_next/static/chunks/2f5024e5325fd185.css","style"]
+0:{"buildId":"ak_B7XGok3Ra_ZXFSQmNR","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/2f5024e5325fd185.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}

litellm/proxy/_experimental/out/__next._tree.txt

@@ -1,5 +1,5 @@
 :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","style"]
+:HL["/litellm-asset-prefix/_next/static/chunks/2f5024e5325fd185.css","style"]
 :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
 :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
-0:{"buildId":"lRBQFcrGOsyCYLFEalzGW","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
+0:{"buildId":"ak_B7XGok3Ra_ZXFSQmNR","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}