diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index 2ca2654a207..7aa0c3544ee 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -1,15 +1,17 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: classify_changes.sh }" +category="${1:?usage: classify_changes.sh }" has_client=false has_backend=false +has_ci=false while IFS= read -r file || [ -n "$file" ]; do [ -n "$file" ] || continue case "$file" in ui/* | tests/e2e/ui/*) has_client=true ;; docs/* | *.md | *.mdx) : ;; + .github/* | .circleci/*) has_ci=true; has_backend=true ;; *) has_backend=true ;; esac done @@ -21,6 +23,9 @@ case "$category" in client) { [ "$has_client" = true ] || [ "$has_backend" = true ]; } && echo run || echo skip ;; + ui) + { [ "$has_client" = true ] || [ "$has_ci" = true ]; } && echo run || echo skip + ;; *) echo run ;; diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 51d489459d9..118e5491939 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,5 @@ /ui/ @yuneng-jiang @ryan-crabbe-berri /litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri /ui/litellm-dashboard/src/lib/http/schema.d.ts +/model_prices_and_context_window.json @mateo-berri +/litellm/model_prices_and_context_window_backup.json @mateo-berri diff --git a/.github/actions/detect-backend-changes/action.yml b/.github/actions/detect-backend-changes/action.yml deleted file mode 100644 index af01038f294..00000000000 --- a/.github/actions/detect-backend-changes/action.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: "Detect backend-relevant changes" -description: >- - Classify the pull request's changed files with .circleci/scripts/classify_changes.sh - and expose decision=run|skip. decision=skip means only ui/**, **.md or **.mdx files - changed, so callers can short-circuit expensive steps while the job still completes - successfully and satisfies its required status check. The decision defaults to run for - any non pull_request event or whenever the changed set cannot be resolved, so tests are - never skipped when the classification is uncertain. - -outputs: - decision: - description: "run when backend-relevant files changed, otherwise skip" - value: ${{ steps.classify.outputs.decision }} - -runs: - using: composite - steps: - - id: classify - shell: bash - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - set -uo pipefail - if [ -z "${BASE_SHA:-}" ]; then - echo "detect-backend-changes: not a pull_request event; running job" - echo "decision=run" >> "${GITHUB_OUTPUT}" - exit 0 - fi - if ! git fetch --no-tags --depth=1 origin "${BASE_SHA}" >/dev/null 2>&1; then - echo "detect-backend-changes: could not fetch base ${BASE_SHA}; running job" - echo "decision=run" >> "${GITHUB_OUTPUT}" - exit 0 - fi - changed="$(git diff --name-only "${BASE_SHA}" HEAD 2>/dev/null)" || { - echo "detect-backend-changes: git diff failed; running job" - echo "decision=run" >> "${GITHUB_OUTPUT}" - exit 0 - } - if [ -z "${changed}" ]; then - echo "detect-backend-changes: no changed files vs ${BASE_SHA}; skipping job" - echo "decision=skip" >> "${GITHUB_OUTPUT}" - exit 0 - fi - echo "detect-backend-changes: changed files vs ${BASE_SHA}:" - printf '%s\n' "${changed}" | sed 's/^/ /' - decision="$(printf '%s\n' "${changed}" | bash .circleci/scripts/classify_changes.sh backend)" || decision="run" - echo "detect-backend-changes: decision=${decision}" - echo "decision=${decision}" >> "${GITHUB_OUTPUT}" diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml new file mode 100644 index 00000000000..9b22d2c23a8 --- /dev/null +++ b/.github/actions/detect-changes/action.yml @@ -0,0 +1,41 @@ +name: "Detect relevant changes" +description: >- + Classify the pull request's changed files with .circleci/scripts/classify_changes.sh + and expose decision=run|skip for one category. backend means anything outside ui/, + docs/ and markdown; ui means the dashboard sources alone. decision=skip lets callers + short-circuit expensive steps while the job still completes successfully and satisfies + its required status check, which a paths: filter cannot do because a workflow that + never starts never reports. The file list comes from the pull request itself rather + than from a git diff, because the checked-out merge ref is recomputed as the base + branch advances and would otherwise attribute the base branch's own commits to the + pull request. The decision defaults to run for any non pull_request event or whenever + the changed set cannot be resolved, so jobs are never skipped when the classification + is uncertain. + +inputs: + category: + description: "Which classification to apply: backend, client or ui" + required: false + default: backend + github-token: + description: "Token used to list the pull request's files; needs pull-requests: read" + required: false + default: ${{ github.token }} + +outputs: + decision: + description: "run when category-relevant files changed, otherwise skip" + value: ${{ steps.classify.outputs.decision }} + +runs: + using: composite + steps: + - id: classify + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + CATEGORY: ${{ inputs.category }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + CHANGED_FILE_COUNT: ${{ github.event.pull_request.changed_files }} + run: bash "${GITHUB_ACTION_PATH}/../../scripts/detect_changes.sh" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 10266228b1f..4e428d8cebf 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -53,7 +53,8 @@ After: the same request comes back with real token counts, so the dashboard show **Please complete all items before asking a LiteLLM maintainer to review your PR** - [ ] I have added meaningful tests -- [ ] My PR passes all CI/CD checks (e.g., lint, format, unit tests) +- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/test_litellm/.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more +- [ ] My PR passes all required CI/CD checks (e.g., lint, schema.d.ts sync check, etc.) - [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem - [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes) diff --git a/.github/scripts/detect_changes.sh b/.github/scripts/detect_changes.sh new file mode 100755 index 00000000000..2d427c92fb5 --- /dev/null +++ b/.github/scripts/detect_changes.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -uo pipefail + +readonly API_FILE_CEILING=3000 +readonly CATEGORY="${CATEGORY:-backend}" + +decide() { + echo "detect-changes[${CATEGORY}]: decision=$1" + [ -z "${GITHUB_OUTPUT:-}" ] || echo "decision=$1" >>"${GITHUB_OUTPUT}" + exit 0 +} + +run_full() { + echo "detect-changes[${CATEGORY}]: $1; running job" + decide run +} + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +classify="${here}/../../.circleci/scripts/classify_changes.sh" + +[ -n "${PR_NUMBER:-}" ] || run_full "not a pull_request event" +[ -n "${REPO:-}" ] || run_full "no repository in the environment" + +case "${CHANGED_FILE_COUNT:-}" in +'' | *[!0-9]*) run_full "the event payload carries no changed_files count" ;; +esac +[ "${CHANGED_FILE_COUNT}" -le "${API_FILE_CEILING}" ] || + run_full "PR #${PR_NUMBER} changes ${CHANGED_FILE_COUNT} files, past the ${API_FILE_CEILING}-file listing ceiling" + +changed="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename')" || + run_full "could not list the files on PR #${PR_NUMBER}" +[ -n "${changed}" ] || run_full "the API listed no files on PR #${PR_NUMBER}" + +echo "detect-changes[${CATEGORY}]: files changed by PR #${PR_NUMBER}:" +printf '%s\n' "${changed}" | sed 's/^/ /' + +decision="$(printf '%s\n' "${changed}" | bash "${classify}" "${CATEGORY}")" || + run_full "classify_changes.sh failed" +case "${decision}" in +run | skip) decide "${decision}" ;; +*) run_full "classify_changes.sh printed an unexpected decision: ${decision}" ;; +esac diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 58208988fca..4f4339a360a 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -60,6 +60,9 @@ jobs: name: Run tests runs-on: ubuntu-latest timeout-minutes: ${{ inputs.job-timeout-minutes }} + permissions: + contents: read + pull-requests: read outputs: decision: ${{ steps.changes.outputs.decision }} @@ -69,24 +72,27 @@ jobs: with: persist-credentials: false - - name: Detect backend-relevant changes + - name: Detect relevant changes id: changes timeout-minutes: 2 - uses: ./.github/actions/detect-backend-changes + uses: ./.github/actions/detect-changes - name: Set up Python + if: steps.changes.outputs.decision != 'skip' timeout-minutes: 3 uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' timeout-minutes: 3 uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Cache uv dependencies + if: steps.changes.outputs.decision != 'skip' timeout-minutes: 5 uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 69495cff896..7a0ae0faaa0 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -24,6 +24,7 @@ jobs: # re-running basedpyright over the merge-base tree. permissions: contents: read + pull-requests: read actions: read steps: @@ -37,7 +38,12 @@ jobs: clean: true persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + - name: Fetch gate base (merge-base with target branch) + if: steps.changes.outputs.decision != 'skip' env: GH_TOKEN: ${{ github.token }} BASE_SHA: ${{ github.event.pull_request.base.sha }} @@ -50,39 +56,47 @@ jobs: echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV" - name: Set up Python + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Clean Python cache + if: steps.changes.outputs.decision != 'skip' run: | find . -type d -name "__pycache__" -exec rm -rf {} + || true find . -name "*.pyc" -delete || true - name: Check uv.lock is up to date + if: steps.changes.outputs.decision != 'skip' run: | uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1) - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: | uv sync --frozen --group proxy-dev --group e2e-dev - name: Cache Prisma binaries + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/cache-prisma-binaries # basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma) # only after `prisma generate` writes prisma/client.py et al. Without this the # DB wrappers typed against the generated client would degrade to Unknown. - name: Generate Prisma client + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Check ruff format + if: steps.changes.outputs.decision != 'skip' run: | git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then @@ -92,6 +106,7 @@ jobs: xargs uv run --no-sync ruff format --check --exclude '/enterprise/' < "$RUNNER_TEMP/ruff_format_files.txt" - name: Debug - Check file state + if: steps.changes.outputs.decision != 'skip' run: | echo "Current branch:" git branch --show-current @@ -101,30 +116,36 @@ jobs: head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 - name: Run Ruff linting + if: steps.changes.outputs.decision != 'skip' run: | cd litellm uv run --no-sync ruff check . cd .. - name: Check strict-rule budget (delta vs base) + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA" - name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base) + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA" - name: Print OpenAI version + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')" - name: Check basedpyright budget (delta vs base) + if: steps.changes.outputs.decision != 'skip' env: GH_TOKEN: ${{ github.token }} run: | uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA" - name: Check tests/e2e basedpyright (zero errors) + if: steps.changes.outputs.decision != 'skip' run: | if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then uv run --no-sync basedpyright tests/e2e @@ -133,12 +154,14 @@ jobs: fi - name: Check for circular imports + if: steps.changes.outputs.decision != 'skip' run: | cd litellm uv run --no-sync python ../tests/documentation_tests/test_circular_imports.py cd .. - name: Check import safety + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index 618b0195b5a..b3a07a6e0ff 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -1,6 +1,7 @@ name: UI Build Check permissions: contents: read + pull-requests: read on: pull_request: @@ -28,7 +29,14 @@ jobs: with: persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + with: + category: ui + - name: Setup Node.js + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version-file: ui/litellm-dashboard/.nvmrc @@ -36,7 +44,9 @@ jobs: cache-dependency-path: ui/litellm-dashboard/package-lock.json - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: npm ci - name: Build + if: steps.changes.outputs.decision != 'skip' run: npm run build diff --git a/.github/workflows/test-litellm-ui-unit.yml b/.github/workflows/test-litellm-ui-unit.yml index 69cbc082d98..a7329432be5 100644 --- a/.github/workflows/test-litellm-ui-unit.yml +++ b/.github/workflows/test-litellm-ui-unit.yml @@ -1,6 +1,7 @@ name: UI Unit Tests permissions: contents: read + pull-requests: read on: pull_request: @@ -32,7 +33,14 @@ jobs: fetch-depth: 1 persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + with: + category: ui + - name: Setup Node.js + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version-file: ui/litellm-dashboard/.nvmrc @@ -40,14 +48,17 @@ jobs: cache-dependency-path: ui/litellm-dashboard/package-lock.json - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: npm ci - name: Run UI type tests (Vitest) + if: steps.changes.outputs.decision != 'skip' env: CI: "true" run: npm run test:types - name: Run UI unit tests (Vitest) + if: steps.changes.outputs.decision != 'skip' env: CI: "true" GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index 05cc13d0af2..95187ef2835 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -10,6 +10,7 @@ on: permissions: contents: read + pull-requests: read concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -25,26 +26,34 @@ jobs: with: persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + - name: Thank You Message run: | echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - name: Set up Python + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: | uv lock --check .github/scripts/uv_sync_with_retries.sh --frozen --group proxy-dev --extra proxy --extra semantic-router - name: Run MCP tests + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5 diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml index c93779c177f..cb8035aafa1 100644 --- a/.github/workflows/test-unit-documentation.yml +++ b/.github/workflows/test-unit-documentation.yml @@ -23,34 +23,41 @@ jobs: documentation: runs-on: ubuntu-latest timeout-minutes: 10 + permissions: + contents: read + pull-requests: read steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + - name: Checkout litellm-docs into docs/my-website (for documentation_tests) + if: steps.changes.outputs.decision != 'skip' uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: repository: BerriAI/litellm-docs path: docs/my-website persist-credentials: false - - name: Detect backend-relevant changes - id: changes - uses: ./.github/actions/detect-backend-changes - - name: Set up Python + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Cache uv dependencies + if: steps.changes.outputs.decision != 'skip' uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | diff --git a/.gitignore b/.gitignore index 3329f39ca10..201e02f2189 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,11 @@ .python-version .venv +tests/e2e/.fixtures/ .venv-typecheck .venv_policy_test .env .claude +CLAUDE.local.md .newenv newenv/* litellm/proxy/myenv/* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d995ddcc87e..9ef1d5ae2b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,8 +13,8 @@ Here are the core requirements for any PR submitted to LiteLLM: - [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing) - [ ] **Ensure your PR passes all checks**: - - [ ] [Unit Tests](#running-unit-tests) - `make test-unit` - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint` + - [ ] [The tests covering your change](#running-unit-tests) pass, e.g. `uv run pytest tests/test_litellm/.py -v`. CI runs the full unit test matrix, so you don't need to run the whole suite locally #### UI PRs @@ -71,8 +71,8 @@ make format # Run all linting checks (matches CI exactly) make lint -# Run unit tests to ensure nothing is broken -make test-unit +# Run the tests covering your change (CI runs the full suite) +uv run pytest tests/test_litellm/.py -v # Commit your changes (must follow Conventional Commits — see above) git add . @@ -123,12 +123,13 @@ def test_your_feature(): ### Running Unit Tests -Run all unit tests (uses parallel execution for speed): - +Run the tests covering your change: ```bash -make test-unit +uv run pytest tests/test_litellm/test_your_file.py -v ``` +`tests/test_litellm` holds thousands of tests, so running all of it locally takes a long time. CI runs it as a parallel matrix (`make test-unit-llms`, `make test-unit-proxy-core`, and the other `test-unit-*` targets) on beefier boxes, so if, for whatever reason, you must run the whole suite, it's better to rely on CI to do that. + If you're running broader test suites, proxy tests, or anything that touches PostgreSQL-backed fixtures/plugins, install the full local test environment first: ```bash @@ -137,11 +138,6 @@ make install-test-deps This syncs the locked test environment used across the repo, including `psycopg` v3 plus `psycopg-binary` (used by `pytest-postgresql`), `psycopg2-binary` (used by some proxy E2E tests), and a generated Prisma client for DB-backed proxy tests, so pytest startup matches CI without manual package installs. -Run specific test files: -```bash -uv run pytest tests/test_litellm/test_your_file.py -v -``` - ### Running Linting and Formatting Checks Run all linting checks (matches CI exactly): diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 1ce71c5bd2c..b4c324a2c4c 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5663 }, "reportMissingTypeArgument": { - "limit": 15557 + "limit": 15555 }, "reportMissingTypeStubs": { "limit": 40 @@ -105,13 +105,13 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 39043 + "limit": 39017 }, "reportUnknownParameterType": { - "limit": 19887 + "limit": 19885 }, "reportUnknownVariableType": { - "limit": 30574 + "limit": 30572 }, "reportUnnecessaryCast": { "limit": 117 diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 153fbc0fdc2..252e3675329 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -145,6 +145,11 @@ NUMBER_KEYS: dict[str, JsonSchema] = { "minimum": 1, "description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).", }, + "regional_endpoint_uplift_multiplier": { + "type": "number", + "minimum": 1, + "description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).", + }, } COST_DESCRIPTIONS: dict[str, str] = { diff --git a/helm/litellm-helm/Chart.yaml b/helm/litellm-helm/Chart.yaml index 8ca217825b8..3959d85edf3 100644 --- a/helm/litellm-helm/Chart.yaml +++ b/helm/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.1.1 +version: 1.1.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/litellm-helm/README.md b/helm/litellm-helm/README.md index 4c8712ea7b9..b242373de5d 100644 --- a/helm/litellm-helm/README.md +++ b/helm/litellm-helm/README.md @@ -29,7 +29,7 @@ If `db.useStackgresOperator` is used (not yet implemented): | `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A | | `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | | `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | -| `image.repository` | LiteLLM Proxy image repository | `docker.litellm.ai/berriai/litellm` | +| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` | | `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` | | `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` | | `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` | diff --git a/helm/litellm-helm/tests/deployment_tests.yaml b/helm/litellm-helm/tests/deployment_tests.yaml index f3d62651d8f..b11c445889e 100644 --- a/helm/litellm-helm/tests/deployment_tests.yaml +++ b/helm/litellm-helm/tests/deployment_tests.yaml @@ -15,7 +15,7 @@ tests: pattern: -litellm$ - equal: path: spec.template.spec.containers[0].image - value: ghcr.io/berriai/litellm-database:test + value: ghcr.io/berriai/litellm:test - it: should work with tolerations template: deployment.yaml set: @@ -337,7 +337,7 @@ tests: template: deployment.yaml set: image: - repository: ghcr.io/berriai/litellm-database + repository: ghcr.io/berriai/litellm tag: test extraInitContainers: - name: init-tpl @@ -348,7 +348,7 @@ tests: path: spec.template.spec.initContainers content: name: init-tpl - image: "ghcr.io/berriai/litellm-database:test" + image: "ghcr.io/berriai/litellm:test" command: ["echo", "hello"] - it: should work with extraContainers template: deployment.yaml @@ -366,7 +366,7 @@ tests: template: deployment.yaml set: image: - repository: ghcr.io/berriai/litellm-database + repository: ghcr.io/berriai/litellm tag: test extraContainers: - name: sidecar-tpl @@ -376,12 +376,12 @@ tests: path: spec.template.spec.containers content: name: sidecar-tpl - image: "ghcr.io/berriai/litellm-database:test" + image: "ghcr.io/berriai/litellm:test" - it: should support tpl in podAnnotations template: deployment.yaml set: image: - repository: ghcr.io/berriai/litellm-database + repository: ghcr.io/berriai/litellm tag: test # Mirrors the real-world scenario this feature unblocks: # user disables the built-in ConfigMap (and its built-in checksum/config @@ -398,7 +398,7 @@ tests: value: "test" - equal: path: spec.template.metadata.annotations["example.com/some-key"] - value: "ghcr.io/berriai/litellm-database" + value: "ghcr.io/berriai/litellm" - equal: path: spec.template.metadata.annotations["example.com/literal"] value: "plain-string-value" diff --git a/helm/litellm-helm/tests/migrations-job_tests.yaml b/helm/litellm-helm/tests/migrations-job_tests.yaml index e327a3ec201..1fe545636d4 100644 --- a/helm/litellm-helm/tests/migrations-job_tests.yaml +++ b/helm/litellm-helm/tests/migrations-job_tests.yaml @@ -208,7 +208,7 @@ tests: template: migrations-job.yaml set: image: - repository: ghcr.io/berriai/litellm-database + repository: ghcr.io/berriai/litellm tag: test migrationJob: enabled: true @@ -221,7 +221,7 @@ tests: path: spec.template.spec.initContainers content: name: init-tpl - image: "ghcr.io/berriai/litellm-database:test" + image: "ghcr.io/berriai/litellm:test" command: ["echo", "hello"] - it: should work with extraContainers template: migrations-job.yaml @@ -241,7 +241,7 @@ tests: template: migrations-job.yaml set: image: - repository: ghcr.io/berriai/litellm-database + repository: ghcr.io/berriai/litellm tag: test migrationJob: enabled: true @@ -253,7 +253,7 @@ tests: path: spec.template.spec.containers content: name: sidecar-tpl - image: "ghcr.io/berriai/litellm-database:test" + image: "ghcr.io/berriai/litellm:test" - it: should render the pod-level securityContext from podSecurityContext template: migrations-job.yaml set: diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 628ca038339..4ef8fc97b27 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -6,8 +6,9 @@ replicaCount: 1 # numWorkers: 2 image: - # Use "ghcr.io/berriai/litellm-database" for optimized image with database - repository: ghcr.io/berriai/litellm-database + # Bundles the prisma CLI and engines, which is what lets the migrations job + # and the proxy's own schema check run without network access. + repository: ghcr.io/berriai/litellm pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. # tag: "latest" diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260814000000_add_proxy_worker_heartbeat/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260814000000_add_proxy_worker_heartbeat/migration.sql new file mode 100644 index 00000000000..0a5d9df8aaf --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260814000000_add_proxy_worker_heartbeat/migration.sql @@ -0,0 +1,9 @@ +-- CreateTable +CREATE TABLE "LiteLLM_ProxyWorkerHeartbeat" ( + "worker_id" TEXT NOT NULL, + "hostname" TEXT NOT NULL, + "started_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "last_heartbeat_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_ProxyWorkerHeartbeat_pkey" PRIMARY KEY ("worker_id") +); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql new file mode 100644 index 00000000000..18ef5c40662 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql @@ -0,0 +1,7 @@ +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "group_id" TEXT; + +UPDATE "LiteLLM_ShadowEvalJob" SET "group_id" = "id" WHERE "group_id" IS NULL; + +ALTER TABLE "LiteLLM_ShadowEvalJob" ALTER COLUMN "group_id" SET NOT NULL; + +CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_group_id_idx" ON "LiteLLM_ShadowEvalJob"("group_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql new file mode 100644 index 00000000000..9efa3fdd052 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN "stopped_by" TEXT; + +UPDATE "LiteLLM_ShadowEvalJob" SET stopped_by = 'unknown' +WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc'); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_backfill_spend_log_timestamps/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_backfill_spend_log_timestamps/migration.sql new file mode 100644 index 00000000000..10003afa9db --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_backfill_spend_log_timestamps/migration.sql @@ -0,0 +1,4 @@ +UPDATE "LiteLLM_SpendLogs" +SET "created_at" = "endTime", + "updated_at" = "endTime" +WHERE "created_at" > "endTime" + interval '1 hour'; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 52fb447157b..60058c777ca 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -947,6 +947,17 @@ model LiteLLM_DailyTagSpend { } +// One row per live proxy worker process. Workers upsert their row on a fixed +// heartbeat; counting rows with a recent heartbeat tells how many workers share +// this database, which lets the Admin UI hide its "no Redis" warning for +// deployments that are provably a single worker. +model LiteLLM_ProxyWorkerHeartbeat { + worker_id String @id + hostname String + started_at DateTime @default(now()) + last_heartbeat_at DateTime @default(now()) +} + // Track the status of cron jobs running. Only allow one pod to run the job at a time model LiteLLM_CronJob { cronjob_id String @id @default(cuid()) // Unique ID for the record @@ -1467,28 +1478,38 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } -// Shadow eval: evaluation of an auto-router against a key's live traffic, in either -// direction. forward duplicates the requests the key did not route through the router -// through it, answering whether the key should adopt it; reverse duplicates the requests -// the router did serve against a fixed baseline model, answering whether a key already on -// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge -// compares real vs shadow responses blind. The job row is immutable config plus -// stopped_at; every count, status, and spend figure is derived from the append-only -// attempt rows, so nothing can disagree across pods or stop races. +// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in +// either direction. forward duplicates the requests the keys did not route through the +// router through it, answering whether they should adopt it; reverse duplicates the +// requests the router did serve against a fixed baseline model, answering whether a key +// already on it still benefits. Either way a sampled slice runs in a detached task and an +// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job: +// immutable config plus that key's own turn budget and stop state, so one key exhausting +// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id +// (the id the API reports), written together by one atomic create_many with identical +// config; single-key jobs predating group_id were backfilled group_id = id. "One active +// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE +// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state +// partial indexes; it is what makes a concurrent start on another pod race-safe rather +// than read-then-create. Every count, status, and spend figure is derived from the +// append-only attempt rows, so nothing can disagree across pods or stop races. model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) - api_key_id String // hashed virtual key whose traffic is shadowed + group_id String // legs of one job share this; the API's job id + api_key_id String // hashed virtual key whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // sample budget: judge at most this many turns + max_turns Int // this key's sample budget: judge at most this many turns created_at DateTime @default(now()) created_by String? ends_at DateTime stopped_at DateTime? + stopped_by String? // operator who stopped it early; null when it ended on its own + @@index([group_id]) @@index([api_key_id]) @@index([created_at]) } diff --git a/litellm/__init__.py b/litellm/__init__.py index 1ecb04b6e54..00f67ea0ff5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -221,7 +221,7 @@ overwrite_user_with_key_hash: bool = ( bedrock_request_metadata_fields: Optional[Sequence[str]] = ( None # allow-list of `user_api_key_*` fields (+ `spend_logs_metadata`) sent as Bedrock `requestMetadata` ) -store_audit_logs = False # Enterprise feature, allow users to see audit logs +store_audit_logs: bool | None = None skip_system_message_in_guardrail: bool = False skip_tool_message_in_guardrail: bool = False ### end of callbacks ############# diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index c2cbb9604e5..0cf22d82ca6 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -1,5 +1,5 @@ import json -from collections.abc import Iterable, Iterator +from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass from typing import Any, Final, Literal @@ -87,7 +87,7 @@ async def _handle_completed_batch( return batch_cost, batch_usage, [model_name] return _aggregate_batch_cost_usage_models( - entries=_iter_batch_input_entries(file_content), + entries=_iter_batch_output_entries(file_content), custom_llm_provider=custom_llm_provider, model_name=model_name, model_info=model_info, @@ -111,43 +111,91 @@ def _iter_successful_output_line_stats( model_name: str | None, model_info: ModelInfo | None, ) -> Iterator[_BatchOutputLineStats]: + for entry in entries: + stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info) + if stats is not None: + yield stats + + +def _safe_output_line_stats( + entry: Mapping[str, Any], + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"], + model_name: str | None, + model_info: ModelInfo | None, +) -> _BatchOutputLineStats | None: + """Return the stats for one batch output line, or None for a line that is + unsuccessful or cannot be costed, so a single bad line never aborts the + whole batch's cost accounting.""" + custom_id: Final = entry.get("custom_id") if isinstance(entry, dict) else None + try: + if not _batch_response_was_successful(entry, custom_llm_provider): + return None + return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info) + except Exception as e: # noqa: BLE001 # any single line's costing failure must not abort the whole batch + verbose_logger.warning( + "batch output line could not be costed, so it is billed at $0 and the rest of the batch " + "is still billed. custom_id=%s error=%s", + custom_id, + str(e), + ) + return None + + +def _compute_output_line_stats( + entry: Mapping[str, Any], + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"], + model_name: str | None, + model_info: ModelInfo | None, +) -> _BatchOutputLineStats: + response_body: Final = _get_response_from_batch_job_output_file(entry, custom_llm_provider) + usage: Final = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider) + prompt_details: Final = parse_prompt_tokens_details(usage) + raw_model: Final = response_body.get("model") + response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None + return _BatchOutputLineStats( + cost=_output_line_cost( + response_body=response_body, + usage=usage, + custom_llm_provider=custom_llm_provider, + model_name=model_name, + response_model=response_model, + model_info=model_info, + ), + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + cache_read_tokens=prompt_details["cache_hit_tokens"], + cache_creation_tokens=prompt_details["cache_creation_tokens"], + model=response_model, + ) + + +def _output_line_cost( + response_body: Mapping[str, Any], + usage: Usage, + custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"], + model_name: str | None, + response_model: str | None, + model_info: ModelInfo | None, +) -> float: from litellm.cost_calculator import batch_cost_calculator - for entry in entries: - if not _batch_response_was_successful(entry, custom_llm_provider): - continue - response_body = _get_response_from_batch_job_output_file(entry, custom_llm_provider) - usage = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider) - prompt_details = parse_prompt_tokens_details(usage) - raw_model = response_body.get("model") - response_model = raw_model if isinstance(raw_model, str) and raw_model else None - if model_info is not None or custom_llm_provider in ("anthropic", "bedrock"): - if custom_llm_provider == "bedrock" and model_name: - cost_model = model_name - else: - cost_model = response_model or model_name or "" - prompt_cost, completion_cost = batch_cost_calculator( - usage=usage, - model=cost_model, - custom_llm_provider=custom_llm_provider, - model_info=model_info, - ) - line_cost = prompt_cost + completion_cost - else: - line_cost = litellm.completion_cost( - completion_response=response_body, - custom_llm_provider=custom_llm_provider, - call_type=CallTypes.aretrieve_batch.value, - ) - yield _BatchOutputLineStats( - cost=line_cost, - prompt_tokens=usage.prompt_tokens, - completion_tokens=usage.completion_tokens, - total_tokens=usage.total_tokens, - cache_read_tokens=prompt_details["cache_hit_tokens"], - cache_creation_tokens=prompt_details["cache_creation_tokens"], - model=response_model, + if model_info is None and custom_llm_provider not in ("anthropic", "bedrock"): + return litellm.completion_cost( + completion_response=response_body, + custom_llm_provider=custom_llm_provider, + call_type=CallTypes.aretrieve_batch.value, ) + cost_model: Final = ( + model_name if custom_llm_provider == "bedrock" and model_name else response_model or model_name or "" + ) + prompt_cost, completion_cost = batch_cost_calculator( + usage=usage, + model=cost_model, + custom_llm_provider=custom_llm_provider, + model_info=model_info, + ) + return prompt_cost + completion_cost def _aggregate_batch_cost_usage_models( @@ -338,9 +386,10 @@ def _extract_file_access_credentials(litellm_params: dict | None) -> dict: def _get_file_content_as_dictionary(file_content: bytes) -> list[dict]: """ - Get the file content as a list of dictionaries from JSON Lines format + Get the file content as a list of dictionaries from JSON Lines format, + skipping malformed lines """ - return list(_iter_batch_input_entries(file_content)) + return list(_iter_batch_output_entries(file_content)) def _iter_batch_input_lines(file_content: bytes) -> Iterator[bytes]: @@ -361,15 +410,29 @@ def _iter_batch_input_lines(file_content: bytes) -> Iterator[bytes]: yield line -def _iter_batch_input_entries(file_content: bytes) -> Iterator[dict]: +def _iter_batch_output_entries(file_content: bytes) -> Iterator[dict]: """ - Yield parsed batch input JSONL entries one at a time without materializing the - whole file as a list, so peak memory stays bounded. Raises on a malformed line; - callers that must survive bad rows should iterate ``_iter_batch_input_lines`` - and parse per-row instead. + Yield parsed batch output JSONL entries one at a time without materializing + the whole file as a list, so peak memory stays bounded. A malformed or + non-object line is skipped with a warning so one bad line never aborts the + whole batch's cost accounting. """ for line in _iter_batch_input_lines(file_content): - yield json.loads(line) + entry = _parse_batch_output_line(line) + if entry is not None: + yield entry + + +def _parse_batch_output_line(line: bytes) -> dict | None: + try: + parsed: Final = json.loads(line) + except ValueError as e: + verbose_logger.warning("skipping malformed batch output line: %s", str(e)) + return None + if isinstance(parsed, dict): + return parsed + verbose_logger.warning("skipping non-object batch output line of type %s", type(parsed).__name__) + return None # A batch request's input tokens scale roughly with its serialized size, so this @@ -440,7 +503,9 @@ def _count_prompt_or_input_tokens(model: str, value: Any) -> int: return 0 -def _get_batch_job_usage_from_response_body(response_body: dict, custom_llm_provider: str = "openai") -> Usage: +def _get_batch_job_usage_from_response_body( + response_body: Mapping[str, Any], custom_llm_provider: str = "openai" +) -> Usage: """ Get the tokens of a batch job from the response body """ @@ -472,7 +537,7 @@ def _get_batch_job_usage_from_response_body(response_body: dict, custom_llm_prov return usage -def _get_anthropic_result_from_batch_results_line(batch_results_line: dict) -> dict: +def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> dict: """ Get the ``result`` object from a line of an Anthropic message batch results JSONL file. @@ -482,7 +547,9 @@ def _get_anthropic_result_from_batch_results_line(batch_results_line: dict) -> d return batch_results_line.get("result", None) or {} -def _get_response_from_batch_job_output_file(batch_job_output_file: dict, custom_llm_provider: str = "openai") -> Any: +def _get_response_from_batch_job_output_file( + batch_job_output_file: Mapping[str, Any], custom_llm_provider: str = "openai" +) -> Any: """ Get the response from the batch job output file """ @@ -495,7 +562,9 @@ def _get_response_from_batch_job_output_file(batch_job_output_file: dict, custom return _response_body -def _batch_response_was_successful(batch_job_output_file: dict, custom_llm_provider: str = "openai") -> bool: +def _batch_response_was_successful( + batch_job_output_file: Mapping[str, Any], custom_llm_provider: str = "openai" +) -> bool: """ Check if the batch job response was successful diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 7d7380665d3..8f7cd09d364 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -327,6 +327,8 @@ def cost_per_token( service_tier: str | None = None, # for OpenAI service tier pricing ### DATA RESIDENCY ### data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") + ### VERTEX LOCATION ### + vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") response: Any | None = None, ### REQUEST MODEL ### request_model: str | None = None, # original request model for router detection @@ -587,6 +589,7 @@ def cost_per_token( prompt_characters=prompt_characters, completion_characters=completion_characters, usage=usage_block, + vertex_location=vertex_location, ) elif cost_router == "cost_per_token": return google_cost_per_token( @@ -594,6 +597,7 @@ def cost_per_token( custom_llm_provider=custom_llm_provider, usage=usage_block, service_tier=service_tier, + vertex_location=vertex_location, ) elif custom_llm_provider == "anthropic": return anthropic_cost_per_token(model=model, usage=usage_block, service_tier=service_tier) @@ -1071,6 +1075,7 @@ def _store_cost_breakdown_in_logging_obj( reasoning_cost: float | None = None, service_tier: str | None = None, data_residency: str | None = None, + vertex_location: str | None = None, ) -> None: """ Helper function to store cost breakdown in the logging object. @@ -1090,6 +1095,7 @@ def _store_cost_breakdown_in_logging_obj( margin_total_amount: Total margin added in USD service_tier: Tier the costs above were priced on, already resolved data_residency: Region uplift the costs above were priced on, already resolved + vertex_location: Vertex AI location the costs above were priced on, already resolved """ if litellm_logging_obj is None: return @@ -1113,6 +1119,7 @@ def _store_cost_breakdown_in_logging_obj( reasoning_cost=reasoning_cost, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, ) except Exception as breakdown_error: @@ -1149,6 +1156,8 @@ def completion_cost( service_tier: str | None = None, # for OpenAI service tier pricing ### DATA RESIDENCY ### data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") + ### VERTEX LOCATION ### + vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") ) -> float: """ Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm. @@ -1577,6 +1586,7 @@ def completion_cost( rerank_billed_units=rerank_billed_units, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, response=completion_response, request_model=request_model_for_cost, ) @@ -1664,6 +1674,7 @@ def completion_cost( usage=cost_per_token_usage_object, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, ) _reasoning_cost = _token_type_breakdown.reasoning_cost _cache_read_cost = _token_type_breakdown.cache_read_cost @@ -1686,6 +1697,7 @@ def completion_cost( reasoning_cost=_reasoning_cost, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, ) return _final_cost @@ -1765,6 +1777,8 @@ def response_cost_calculator( service_tier: str | None = None, # for OpenAI service tier pricing ### DATA RESIDENCY ### data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") + ### VERTEX LOCATION ### + vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") ) -> float: """ Returns @@ -1797,6 +1811,7 @@ def response_cost_calculator( litellm_logging_obj=litellm_logging_obj, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, ) return response_cost except Exception as e: diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index f2bd18cd046..78081837ae3 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -42,6 +42,7 @@ from litellm.types.utils import ( # OpenTelemetry imports moved to individual functions to avoid import errors when not installed if TYPE_CHECKING: + from opentelemetry.sdk.resources import Resource as _Resource from opentelemetry.sdk.trace import TracerProvider as _SDKTracerProvider from opentelemetry.sdk.trace.export import SpanExporter as _SpanExporter from opentelemetry.trace import Context as _Context @@ -389,6 +390,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): self._tracer_provider_cache: OrderedDict[str, _CachedTracerProvider] = OrderedDict() self._tracer_provider_cache_lock: Final = threading.Lock() self._max_dynamic_tracer_providers: Final = max(1, max_dynamic_tracer_providers) + self._litellm_resource_memo: _Resource | None = None self._init_tracing(tracer_provider) _debug_otel: Final = str(os.getenv("DEBUG_OTEL", "False")).lower() @@ -414,7 +416,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): self._init_otel_logger_on_litellm_proxy() @staticmethod - def _get_litellm_resource(config: OpenTelemetryConfig): + def _get_litellm_resource(config: OpenTelemetryConfig) -> "_Resource": """Create an OpenTelemetry Resource using config-driven defaults.""" from opentelemetry.sdk.resources import OTELResourceDetector, Resource @@ -429,6 +431,21 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): env_resource: Final = otel_resource_detector.detect() return base_resource.merge(env_resource) + def _litellm_resource(self) -> "_Resource": + """The Resource every provider on this logger is built with, frozen at first use. + + ``Resource.create`` scans every installed distribution's entry points, roughly 3ms and + 200 file opens, and the dynamic providers reach it from the async logging path. Freezing + also keeps them consistent with whatever this logger built at startup. ``cached_property`` + locks class-wide before 3.12, which this file still supports. + """ + memo: Final = self._litellm_resource_memo + if memo is not None: + return memo + built: Final = self._get_litellm_resource(self.config) + self._litellm_resource_memo = built + return built + def _init_otel_logger_on_litellm_proxy(self): """ Initializes OpenTelemetry for litellm proxy server @@ -596,7 +613,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): from opentelemetry.trace import SpanKind def create_tracer_provider(): - provider: Final = TracerProvider(resource=self._get_litellm_resource(self.config)) + provider: Final = TracerProvider(resource=self._litellm_resource()) provider.add_span_processor(self._get_span_processor()) return provider @@ -634,7 +651,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): metric_reader: Final = self._get_metric_reader() return MeterProvider( metric_readers=[metric_reader], - resource=self._get_litellm_resource(self.config), + resource=self._litellm_resource(), ) meter_provider = self._get_or_create_provider( @@ -692,7 +709,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): from opentelemetry.sdk._logs.export import BatchLogRecordProcessor def create_logger_provider(): - provider: Final = OTLoggerProvider(resource=self._get_litellm_resource(self.config)) + provider: Final = OTLoggerProvider(resource=self._litellm_resource()) log_exporter: Final = self._get_log_exporter() provider.add_log_record_processor(BatchLogRecordProcessor(log_exporter)) return provider @@ -1144,9 +1161,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): owns_exporter: Final = _provider_owns_exporter(dynamic_config.exporter) def _build() -> "_SDKTracerProvider": - provider: Final = TracerProvider( - resource=self._get_litellm_resource(self.config), shutdown_on_exit=owns_exporter - ) + provider: Final = TracerProvider(resource=self._litellm_resource(), shutdown_on_exit=owns_exporter) provider.add_span_processor(self._get_span_processor(config_override=dynamic_config)) return provider @@ -1162,9 +1177,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): owns_exporter: Final = _provider_owns_exporter(self.OTEL_EXPORTER) def _build() -> "_SDKTracerProvider": - provider: Final = TracerProvider( - resource=self._get_litellm_resource(self.config), shutdown_on_exit=owns_exporter - ) + provider: Final = TracerProvider(resource=self._litellm_resource(), shutdown_on_exit=owns_exporter) provider.add_span_processor(self._get_span_processor(dynamic_headers=dynamic_headers)) return provider diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 946110abf9e..91a312b4f45 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -13,7 +13,7 @@ import traceback from collections.abc import Callable, Mapping, Sequence from datetime import datetime as dt_object from functools import lru_cache -from types import TracebackType +from types import MappingProxyType, TracebackType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast from httpx import Response @@ -372,6 +372,35 @@ def _published_pricing(deployment_model: str | None) -> ModelInfo | None: return None +def _resolve_vertex_location_for_cost( + custom_llm_provider: str | None, + litellm_params: Mapping[str, object] | None, + optional_params: Mapping[str, object] | None, + model: str, +) -> str | None: + """ + The Vertex AI location a request was served from, resolved the same way + dispatch resolves it, so regional deployments price with the + regional-endpoint uplift. None for non-Vertex providers. + + Chat dispatch reads the location from request kwargs, which reach this + logging object through optional_params: on the proxy the logging object is + created before the router picks a deployment, so the deployment's location + never lands in litellm_params. + """ + if custom_llm_provider is None or not custom_llm_provider.startswith("vertex_ai"): + return None + from litellm.llms.vertex_ai.vertex_llm_base import VertexBase + + empty: Final[Mapping[str, object]] = MappingProxyType({}) + configured_location: Final = ( + VertexBase.explicit_vertex_ai_location(optional_params or empty) + or VertexBase.explicit_vertex_ai_location(litellm_params or empty) + or VertexBase.safe_get_vertex_ai_location(empty) + ) + return VertexBase.get_vertex_region(configured_location, model) + + class Logging(LiteLLMLoggingBaseClass): global \ supabaseClient, \ @@ -1432,6 +1461,7 @@ class Logging(LiteLLMLoggingBaseClass): reasoning_cost: float | None = None, service_tier: str | None = None, data_residency: str | None = None, + vertex_location: str | None = None, ) -> None: """ Helper method to store cost breakdown in the logging object. @@ -1450,6 +1480,7 @@ class Logging(LiteLLMLoggingBaseClass): margin_total_amount: Total margin added in USD service_tier: Tier the costs above were priced on, already resolved data_residency: Region uplift the costs above were priced on, already resolved + vertex_location: Vertex AI location the costs above were priced on, already resolved """ self.cost_breakdown = CostBreakdown( @@ -1459,6 +1490,7 @@ class Logging(LiteLLMLoggingBaseClass): tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, service_tier=service_tier, data_residency=data_residency, + vertex_location=vertex_location, ) if cache_read_cost is not None and cache_read_cost > 0: self.cost_breakdown["cache_read_cost"] = cache_read_cost @@ -1574,6 +1606,12 @@ class Logging(LiteLLMLoggingBaseClass): if hasattr(self, "litellm_params") and self.litellm_params else None ), + "vertex_location": _resolve_vertex_location_for_cost( + custom_llm_provider=self.model_call_details.get("custom_llm_provider", None), + litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None), + optional_params=self.optional_params, + model=litellm_model_name or self.model, + ), } except Exception as e: # error creating kwargs for cost calculation debug_info = StandardLoggingModelCostFailureDebugInformation( diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index f73c4942a1c..0793fe20b21 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -757,6 +757,33 @@ def _get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | return 1.0 +def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float: + """ + Resolve the per-model uplift multiplier for Vertex AI non-global (regional and + multi-region) endpoints. + + Google prices every non-global endpoint at a flat premium over the global + endpoint (e.g. 1.10 = +10%) on all token types for the models that carry + regional pricing. The multiplier is stored on the model entry as + ``regional_endpoint_uplift_multiplier``. + + Returns 1.0 (no uplift) when ``vertex_location`` is ``None`` or ``"global"``, + or when the model has no multiplier configured. + """ + if vertex_location is None or vertex_location.lower() == "global": + return 1.0 + multiplier: Final = model_info.get("regional_endpoint_uplift_multiplier") + if multiplier is None: + return 1.0 + try: + return float(cast(float, multiplier)) + except (TypeError, ValueError): + verbose_logger.exception( + "Invalid regional_endpoint_uplift_multiplier for model; defaulting to 1.0", + ) + return 1.0 + + def get_provider_specific_geo_multiplier(model_info: ModelInfo, usage: Usage) -> float: """ Resolve the provider-specific regional pricing multiplier for the geo the @@ -798,6 +825,7 @@ def generic_cost_per_token( service_tier: str | None = None, data_residency: str | None = None, model_info: ModelInfo | None = None, + vertex_location: str | None = None, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -809,6 +837,9 @@ def generic_cost_per_token( - usage: LiteLLM Usage block, containing anthropic caching information - data_residency: optional OpenAI data-residency region (e.g. "eu", "us"), used to apply the per-model regional-processing uplift multiplier. + - vertex_location: optional Vertex AI location the request was served from + (e.g. "us-east5", "global"), used to apply the per-model + regional-endpoint uplift multiplier when non-global. Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd @@ -968,6 +999,11 @@ def generic_cost_per_token( prompt_cost *= uplift completion_cost *= uplift + vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + if vertex_uplift != 1.0: + prompt_cost *= vertex_uplift + completion_cost *= vertex_uplift + return prompt_cost, completion_cost @@ -988,6 +1024,7 @@ def get_token_type_cost_breakdown( usage: Usage, service_tier: str | None = None, data_residency: str | None = None, + vertex_location: str | None = None, ) -> TokenTypeCostBreakdown: """ Provider-agnostic cost of reasoning and cache tokens, derived from the usage @@ -1069,6 +1106,12 @@ def get_token_type_cost_breakdown( cache_read_cost *= uplift cache_creation_cost *= uplift + vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + if vertex_uplift != 1.0: + reasoning_cost *= vertex_uplift + cache_read_cost *= vertex_uplift + cache_creation_cost *= vertex_uplift + # Mirror the provider-specific geo uplift (e.g. Anthropic us: 1.1) the totals # apply, so cache and reasoning line items stay reconciled with them. geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) diff --git a/litellm/litellm_core_utils/ptu_pricing.py b/litellm/litellm_core_utils/ptu_pricing.py new file mode 100644 index 00000000000..a1f8bb36e27 --- /dev/null +++ b/litellm/litellm_core_utils/ptu_pricing.py @@ -0,0 +1,149 @@ +"""Which deployments accrue PTU flat cost, and what that costs them per token. + +Reserved provisioned throughput is billed by the hour whether or not requests are sent, so +a deployment that accrues flat cost must not also bill per token. The two halves live here +together because they have to agree: a deployment the rollup declines to charge but the +router prices at zero serves its traffic for free. +""" + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from types import MappingProxyType +from typing import Final + +from litellm.secret_managers.main import get_secret_bool +from litellm.types.router import ModelInfo +from litellm.types.utils import CustomPricingLiteLLMParams, MirroredPricingParams + +PTU_COST_ATTRIBUTION_ENV_VAR: Final = "LITELLM_ENABLE_PTU_COST_ATTRIBUTION" + + +def is_ptu_cost_attribution_enabled() -> bool: + """Whether PTU flat-cost attribution is turned on for this process.""" + return get_secret_bool(PTU_COST_ATTRIBUTION_ENV_VAR, False) is True + + +PTU_ZEROED_PRICING_FIELDS: Final = tuple(f for f in MirroredPricingParams.model_fields if f != "tiered_pricing") + ( + "cache_creation_input_token_cost_above_1hr", + "cache_creation_input_token_cost_above_200k_tokens", + "cache_read_input_token_cost_above_200k_tokens", +) +# tiered_pricing is emptied rather than zeroed: its tiers outrank the zeros written beside +# them, so a zero here would leave the cost map's tiers billing the traffic the reserved +# capacity already covers. +PTU_EMPTIED_PRICING_FIELDS: Final = frozenset(("tiered_pricing",)) +# search_context_cost_per_query holds its rates in a table keyed by context size, and an +# absent table means the provider's own default rather than free, so it is zeroed in place +# and written on every PTU deployment rather than only where a table is already stored. +PTU_ZEROED_TABLE_FIELDS: Final = frozenset(("search_context_cost_per_query",)) +SEARCH_CONTEXT_SIZES: Final = ("search_context_size_low", "search_context_size_medium", "search_context_size_high") +# Rate fields only. CustomPricingLiteLLMParams also carries settings that are not charges, +# and zeroing one of those would destroy the deployment's configuration rather than stop a +# charge. +CUSTOM_PRICING_FIELDS: Final = frozenset(f for f in CustomPricingLiteLLMParams.model_fields if "cost" in f) +PTU_ZEROED_PRICING: Final[Mapping[str, float | tuple[()] | Mapping[str, float]]] = MappingProxyType( + { + **dict.fromkeys(PTU_ZEROED_PRICING_FIELDS, 0.0), + **dict.fromkeys(PTU_EMPTIED_PRICING_FIELDS, ()), + **dict.fromkeys(PTU_ZEROED_TABLE_FIELDS, MappingProxyType(dict.fromkeys(SEARCH_CONTEXT_SIZES, 0.0))), + } +) + + +@dataclass(frozen=True, slots=True) +class PTUTerms: + """The reservation a deployment declares, once every field has been validated.""" + + team_id: str + ptu_count: int + cost_per_ptu_per_hour: float + effective_from: datetime + effective_to: datetime | None + + +def _to_utc(parsed: datetime) -> datetime: + """``parsed`` as UTC, reading a naive value as UTC rather than local time.""" + return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed.astimezone(timezone.utc) + + +def _as_utc(value: object) -> datetime | None: + """A model_info datetime as UTC, parsing an ISO string, else None.""" + if isinstance(value, datetime): + return _to_utc(value) + if not isinstance(value, str): + return None + try: + return _to_utc(datetime.fromisoformat(value.replace("Z", "+00:00"))) + except ValueError: + return None + + +def ptu_terms(model_info: Mapping[str, object]) -> PTUTerms | None: + """The reservation this deployment accrues flat cost for, else None. + + A start is required rather than inferred because flat cost accrues from it, and a + present but unparseable bound would read as no bound and widen the window to the whole + day, so either one leaves the deployment unpriced until the config is fixed. + """ + ptu_count: Final = model_info.get("ptu_count") + cost_per_hour: Final = model_info.get("cost_per_ptu_per_hour") + team_id: Final = model_info.get("team_id") + if ptu_count is None or cost_per_hour is None or not team_id: + return None + try: + ptu_count_int: Final = int(ptu_count) + cost_per_hour_float: Final = float(cost_per_hour) + except (TypeError, ValueError, OverflowError): + return None + if not 0 < ptu_count_int <= ModelInfo.MAX_PTU_COUNT: + return None + if not 0 <= cost_per_hour_float <= ModelInfo.MAX_COST_PER_PTU_PER_HOUR: + return None + + raw_from: Final = model_info.get("ptu_effective_from") + raw_to: Final = model_info.get("ptu_effective_to") + effective_from: Final = _as_utc(raw_from) + effective_to: Final = _as_utc(raw_to) + if effective_from is None or (raw_to is not None and effective_to is None): + return None + if effective_to is not None and effective_to <= effective_from: + return None + return PTUTerms( + team_id=str(team_id), + ptu_count=ptu_count_int, + cost_per_ptu_per_hour=cost_per_hour_float, + effective_from=effective_from, + effective_to=effective_to, + ) + + +def zeroed_ptu_pricing( + model_info: Mapping[str, object], declared: Mapping[str, object] +) -> Mapping[str, float | tuple[()] | Mapping[str, float]] | None: + """The pricing a deployment accruing flat cost must carry, else None. + + Both conditions hold or nothing is zeroed. Without the flag no flat cost accrues, so + zeroing would leave the deployment serving for free with nothing charged in its place, + which is what an SDK user who happens to carry ptu_count would otherwise get. The terms + are checked first only because they are a few dict reads, while the flag can resolve + through a configured secret manager, and this runs for every deployment registered. + + Any further rate the deployment itself declares is zeroed alongside the standing set, + since one left standing bills the traffic the reserved capacity already paid for. + """ + if ptu_terms(model_info) is None: + return None + if not is_ptu_cost_attribution_enabled(): + return None + return MappingProxyType( + { + **PTU_ZEROED_PRICING, + **dict.fromkeys( + CUSTOM_PRICING_FIELDS.intersection(declared) + .difference(PTU_ZEROED_TABLE_FIELDS) + .difference(PTU_EMPTIED_PRICING_FIELDS), + 0.0, + ), + } + ) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 67287c903be..ee0518c4aec 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -723,7 +723,7 @@ class ChunkProcessor: for choice in response.choices: if ( hasattr(cast(Choices, choice).message, "reasoning_content") - and cast(Choices, choice).message.reasoning_content is not None + and cast(Choices, choice).message.reasoning_content ): if reasoning_tokens is None: reasoning_tokens = 0 @@ -987,7 +987,12 @@ class ChunkProcessor: returned_usage.completion_tokens_details is not None and returned_usage.completion_tokens_details.reasoning_tokens is None ): - returned_usage.completion_tokens_details.reasoning_tokens = reasoning_tokens + capped_reasoning_tokens: Final = min(max(0, reasoning_tokens), returned_usage.completion_tokens) + returned_usage.completion_tokens_details.reasoning_tokens = capped_reasoning_tokens + if returned_usage.completion_tokens_details.text_tokens is None: + returned_usage.completion_tokens_details.text_tokens = ( + returned_usage.completion_tokens - capped_reasoning_tokens + ) if prompt_tokens_details is not None: returned_usage.prompt_tokens_details = prompt_tokens_details diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 43bcf892865..485091bccd0 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1830,6 +1830,20 @@ class CustomStreamWrapper: return self.chunks.append(model_response.model_copy(update={"choices": []})) + @staticmethod + def _resolve_provider_reported_cost(usage_cost: object) -> float | None: + """ + Providers report usage.cost either as a number or, for Perplexity, as a + breakdown object whose total lives under ``total_cost``. + """ + if isinstance(usage_cost, bool): + return None + if isinstance(usage_cost, (int, float)): + return float(usage_cost) + if isinstance(usage_cost, dict): + return CustomStreamWrapper._resolve_provider_reported_cost(usage_cost.get("total_cost")) + return None + @staticmethod def _propagate_usage_cost_to_hidden_params( response: "ModelResponse", @@ -1840,10 +1854,11 @@ class CustomStreamWrapper: calculator uses it instead of a token-based estimate. """ _usage: Final[Usage | None] = getattr(response, "usage", None) - if _usage is not None and hasattr(_usage, "cost") and _usage.cost is not None: + _cost: Final = CustomStreamWrapper._resolve_provider_reported_cost(getattr(_usage, "cost", None)) + if _cost is not None: if "additional_headers" not in response._hidden_params: response._hidden_params["additional_headers"] = {} - response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(_usage.cost) + response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = _cost def __next__(self) -> "ModelResponseStream": cache_hit = False diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index ef4ad7011c5..414fd23381a 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -5,6 +5,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, NoReturn, cast import httpx +from pydantic import ValidationError import litellm from litellm.constants import ( @@ -39,6 +40,7 @@ from litellm.types.llms.anthropic import ( AnthropicMessagesTool, AnthropicMessagesToolChoice, AnthropicOutputSchema, + AnthropicOutputTokensDetails, AnthropicSystemMessageContent, AnthropicThinkingParam, AnthropicWebSearchTool, @@ -2104,6 +2106,68 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): compaction_blocks, ) + @staticmethod + def _thinking_tokens_from_usage(usage_object: Mapping[str, object]) -> int | None: + details: Final = usage_object.get("output_tokens_details") + if not isinstance(details, Mapping): + return None + try: + return AnthropicOutputTokensDetails.model_validate(details).thinking_tokens + except ValidationError: + return None + + @staticmethod + def _response_has_thinking_block(completion_response: Mapping[str, object] | None) -> bool: + if completion_response is None: + return False + content: Final = completion_response.get("content") + if not isinstance(content, list): + return False + return any( + isinstance(block, Mapping) and block.get("type") in ("thinking", "redacted_thinking") for block in content + ) + + def _build_completion_token_details( + self, + usage_object: Mapping[str, object], + iterations: Sequence[object] | None, + completion_tokens: int, + reasoning_content: str | None, + completion_response: Mapping[str, object] | None, + ) -> CompletionTokensDetailsWrapper: + iteration_thinking_tokens: Final = self._sum_iteration_thinking_tokens(iterations) if iterations else None + reported_thinking_tokens: Final = ( + iteration_thinking_tokens + if iteration_thinking_tokens is not None + else self._thinking_tokens_from_usage(usage_object) + ) + if reported_thinking_tokens is not None: + capped_reported: Final = min(max(0, reported_thinking_tokens), completion_tokens) + return CompletionTokensDetailsWrapper( + reasoning_tokens=capped_reported, + text_tokens=completion_tokens - capped_reported, + ) + if reasoning_content: + estimated: Final = min( + token_counter(text=reasoning_content, count_response_tokens=True), + completion_tokens, + ) + return CompletionTokensDetailsWrapper( + reasoning_tokens=max(0, estimated), + text_tokens=completion_tokens - max(0, estimated), + ) + if self._response_has_thinking_block(completion_response): + return CompletionTokensDetailsWrapper(reasoning_tokens=None, text_tokens=None) + return CompletionTokensDetailsWrapper(reasoning_tokens=0, text_tokens=completion_tokens) + + def _sum_iteration_thinking_tokens(self, iterations: Sequence[object]) -> int | None: + per_iteration: Final = tuple( + self._thinking_tokens_from_usage(iteration) if isinstance(iteration, Mapping) else None + for iteration in iterations + ) + reported: Final = tuple(tokens for tokens in per_iteration if tokens is not None) + return sum(reported) if len(reported) == len(per_iteration) else None + @staticmethod def is_anthropic_usage_object(usage_object: dict) -> bool: """Anthropic reports prompt cache tokens as top-level ``cache_read_input_tokens`` / @@ -2222,14 +2286,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): cache_creation_token_details=cache_creation_token_details, text_tokens=raw_input_tokens, ) - # Always populate completion_token_details, not just when there's reasoning_content - estimated_reasoning_tokens: Final = ( - token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0 - ) - reasoning_tokens: Final = min(estimated_reasoning_tokens, completion_tokens) - completion_token_details: Final = CompletionTokensDetailsWrapper( - reasoning_tokens=max(0, reasoning_tokens), - text_tokens=(completion_tokens - reasoning_tokens if reasoning_tokens > 0 else completion_tokens), + completion_token_details: Final = self._build_completion_token_details( + usage_object=_usage, + iterations=iterations, + completion_tokens=completion_tokens, + reasoning_content=reasoning_content, + completion_response=completion_response, ) total_tokens: Final = prompt_tokens + completion_tokens diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py index f999eae1be6..922769dbbfd 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py @@ -10,6 +10,7 @@ from typing_extensions import TypedDict from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) @@ -134,8 +135,11 @@ class BaseAnthropicMessagesStreamingIterator: if self.completion_start_time is not None: self.litellm_logging_obj.completion_start_time = self.completion_start_time self.litellm_logging_obj.model_call_details["completion_start_time"] = self.completion_start_time - asyncio.create_task( - PassThroughStreamingHandler._route_streaming_logging_to_handler( + # Enqueue on the rooted logging worker rather than asyncio.create_task: + # this also runs during generator teardown after a client disconnect, + # where an unrooted task could be garbage-collected before it bills. + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( + async_coroutine=PassThroughStreamingHandler._route_streaming_logging_to_handler( litellm_logging_obj=self.litellm_logging_obj, passthrough_success_handler_obj=GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ, url_route="/v1/messages", @@ -197,13 +201,21 @@ class BaseAnthropicMessagesStreamingIterator: collected_chunks: Final = [] saw_terminal_event = False - async for chunk in completion_stream: - if self.completion_start_time is None: - self.completion_start_time = datetime.now() - saw_terminal_event = saw_terminal_event or _is_terminal_stream_chunk(chunk) - encoded_chunk = self._convert_chunk_to_sse_format(chunk) - collected_chunks.append(encoded_chunk) - yield encoded_chunk + try: + async for chunk in completion_stream: + if self.completion_start_time is None: + self.completion_start_time = datetime.now() + saw_terminal_event = saw_terminal_event or _is_terminal_stream_chunk(chunk) + encoded_chunk = self._convert_chunk_to_sse_format(chunk) + collected_chunks.append(encoded_chunk) + yield encoded_chunk + except (GeneratorExit, asyncio.CancelledError): + # A client disconnect tears the generator down at the yield, so the + # post-loop logging below never runs and the tokens already streamed + # (and billed by the provider) would never reach spend tracking. See LIT-5839. + if collected_chunks: + await self._handle_streaming_logging(collected_chunks) + raise if not saw_terminal_event: yield _incomplete_stream_error_sse_event() diff --git a/litellm/llms/base_llm/search/transformation.py b/litellm/llms/base_llm/search/transformation.py index dee67e0b100..7668c6132d6 100644 --- a/litellm/llms/base_llm/search/transformation.py +++ b/litellm/llms/base_llm/search/transformation.py @@ -183,6 +183,29 @@ class BaseSearchConfig: """ return headers + def sign_request( + self, + headers: dict[str, str], # mutable-ok: matches the request header dict every other hook on this base takes + optional_params: dict[str, object], # mutable-ok: matches every other hook on this base + request_data: dict[str, object] | list[dict[str, object]], # mutable-ok: transform_search_request's body + api_base: str, + api_key: str | None = None, + ) -> tuple[dict[str, str], bytes | None]: # mutable-ok: the handler passes these headers straight to httpx + """ + OPTIONAL + + Sign the request. Providers like Bedrock AgentCore need to SigV4-sign + the request before sending it to the API. + + For all other providers, this is a no-op and we just return the headers. + + Returns: + Tuple of (headers, signed_json_body). When signed_json_body is not + None, the handler MUST send it verbatim as the request body — + re-serializing the payload would invalidate the signature. + """ + return headers, None + def get_complete_url( self, api_base: str | None, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index cee89f42c2d..4dd3f802638 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1834,6 +1834,7 @@ class AmazonConverseConfig(BaseConfig): self, usage: ConverseTokenUsageBlock, reasoning_content: str | None = None, + thinking_ran: bool = False, ) -> Usage: input_tokens = usage["inputTokens"] output_tokens: Final = usage["outputTokens"] @@ -1854,10 +1855,19 @@ class AmazonConverseConfig(BaseConfig): cache_creation_tokens=cache_creation_input_tokens, text_tokens=raw_input_tokens, ) - reasoning_tokens = token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0 - completion_tokens_details: Final = CompletionTokensDetailsWrapper( - reasoning_tokens=reasoning_tokens, - text_tokens=(output_tokens - reasoning_tokens if reasoning_tokens > 0 else output_tokens), + reasoning_tokens: Final = ( + token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0 + ) + completion_tokens_details: Final = ( + CompletionTokensDetailsWrapper( + reasoning_tokens=reasoning_tokens, + text_tokens=output_tokens - reasoning_tokens, + ) + if reasoning_tokens > 0 + else CompletionTokensDetailsWrapper( + reasoning_tokens=None if thinking_ran else 0, + text_tokens=None if thinking_ran else output_tokens, + ) ) openai_usage: Final = Usage( prompt_tokens=input_tokens, @@ -2254,6 +2264,7 @@ class AmazonConverseConfig(BaseConfig): usage: Final = self.transform_usage( completion_response["usage"], reasoning_content=chat_completion_message.get("reasoning_content"), + thinking_ran=reasoningContentBlocks is not None, ) ## HANDLE TOOL CALLS diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 86f7e9b0d9f..2a125e38a82 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -330,6 +330,7 @@ class AWSEventStreamDecoder: self.response_id: str | None = None self.json_mode = json_mode self._current_tool_name: str | None = None + self._thinking_ran = False def check_empty_tool_call_args(self) -> bool: """ @@ -559,7 +560,12 @@ class AWSEventStreamDecoder: elif "stopReason" in chunk_data: finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop")) elif "usage" in chunk_data: - usage = converse_config.transform_usage(chunk_data.get("usage", {})) + usage = converse_config.transform_usage( + chunk_data.get("usage", {}), + thinking_ran=self._thinking_ran, + ) + if thinking_blocks: + self._thinking_ran = True model_response_provider_specific_fields: Final = {} if "trace" in chunk_data: diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 0161f4fadc9..f74a290d773 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -842,8 +842,15 @@ class AmazonAnthropicClaudeMessagesConfig( patched_stream: Final = self._promote_message_stop_usage(completion_stream) - async for chunk in handler.async_sse_wrapper(patched_stream): - yield chunk + sse_stream: Final = handler.async_sse_wrapper(patched_stream) + try: + async for chunk in sse_stream: + yield chunk + finally: + # Close the inner generator deterministically so a client disconnect + # (GeneratorExit here) reaches async_sse_wrapper's partial-spend logging + # now instead of at garbage collection. See LIT-5839. + await sse_stream.aclose() @staticmethod def _merge_message_start_cache_into_delta_usage( diff --git a/litellm/llms/bedrock/search/__init__.py b/litellm/llms/bedrock/search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bedrock/search/transformation.py b/litellm/llms/bedrock/search/transformation.py new file mode 100644 index 00000000000..920e566c9dd --- /dev/null +++ b/litellm/llms/bedrock/search/transformation.py @@ -0,0 +1,455 @@ +""" +Calls an Amazon Bedrock AgentCore Gateway web-search target (MCP protocol) to search the web. + +Web Search on Amazon Bedrock AgentCore exposes Amazon's managed web index through +an AgentCore Gateway MCP endpoint. + +AWS docs: https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-target-connector-web-search-tool.html + +Authentication (matches the gateway's inbound authorizer type): +- AWS_IAM gateway: the request is SigV4-signed. Credentials come from explicit + params (aws_access_key_id / aws_secret_access_key / aws_session_token / + aws_region_name, also settable in a proxy search_tools entry) or the + standard AWS credential chain (env / profile / IRSA / assumed role) +- CUSTOM_JWT gateway: pass the OAuth2 bearer token (e.g. Cognito + client_credentials) as api_key, or set AGENTCORE_GATEWAY_TOKEN + +Setup: + 1. Create an AgentCore Gateway with a web-search connector target + 2. Set AGENTCORE_GATEWAY_URL (or pass api_base) to the gateway MCP endpoint, e.g. + https://.gateway.bedrock-agentcore..amazonaws.com/mcp + 3. AWS_IAM: ensure the credentials allow bedrock-agentcore:InvokeGateway + CUSTOM_JWT: set AGENTCORE_GATEWAY_TOKEN (or pass api_key) + +Usage: + response = litellm.search( + query="latest AI developments", + search_provider="agentcore", + max_results=5, + aws_access_key_id="...", # optional, omit to use the default chain + aws_secret_access_key="...", + ) +""" + +import json +import re +from collections.abc import Iterator, Mapping, Sequence +from typing import Final + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.secret_managers.main import get_secret_str + +# AgentCore web-search rejects queries longer than 200 characters +AGENTCORE_MAX_QUERY_LENGTH: Final = 200 + +# The provider contract documents a default of 10 results, send it explicitly +# so the gateway can't silently apply a different default. +AGENTCORE_DEFAULT_MAX_RESULTS: Final = 10 + +# Default MCP tool name for a gateway web-search connector target: +# "___". Override with AGENTCORE_SEARCH_TOOL_NAME +# or optional_params["tool_name"] when the target uses a custom name. +AGENTCORE_DEFAULT_TOOL_NAME: Final = "web-search-tool___WebSearch" + +# All web-search connector tools share this suffix; rejecting other names keeps +# a caller-supplied tool_name from invoking unrelated tools on the same gateway +# with the proxy's credentials. +AGENTCORE_TOOL_NAME_SUFFIX: Final = "___WebSearch" + +# MCP revision this provider speaks. Sent on every request because the gateway is +# called statelessly, without an initialize handshake to negotiate a version. +# AgentCore gateways whose protocolConfiguration leaves supportedVersions unset +# accept only 2025-03-26 and reject anything newer with a -32600 error, so that +# is the default; a gateway pinned to another version needs +# AGENTCORE_MCP_PROTOCOL_VERSION set to match. +AGENTCORE_DEFAULT_MCP_PROTOCOL_VERSION: Final = "2025-03-26" + +# Matched against the URL host so a crafted path or query string can't pass for +# a gateway hostname. +_GATEWAY_HOST_PATTERN: Final = re.compile(r"[a-z0-9-]+\.gateway\.bedrock-agentcore\.([a-z0-9-]+)\.amazonaws\.com") + +_SSE_EVENT_SEPARATOR: Final = re.compile(r"\r?\n[ \t]*\r?\n") + +_SSE_LINE_PREFIXES: Final = ("event:", "data:", ":", "id:", "retry:") + + +def _gateway_host_match(api_base: str) -> re.Match[str] | None: + return _GATEWAY_HOST_PATTERN.fullmatch(httpx.URL(api_base).host) + + +_LOOPBACK_HOSTS: Final = frozenset({"localhost", "127.0.0.1", "::1"}) + + +def _credential_safe_transport(api_base: str) -> bool: + url: Final = httpx.URL(api_base) + return url.scheme == "https" or url.host in _LOOPBACK_HOSTS + + +def _string_field(item: Mapping[str, object], *keys: str) -> str | None: + return next( + (value for key in keys if isinstance(value := item.get(key), str) and value), + None, + ) + + +def _to_search_result(item: Mapping[str, object]) -> SearchResult: + return SearchResult( + title=_string_field(item, "title") or "", + url=_string_field(item, "url") or "", + snippet=_string_field(item, "text", "snippet") or "", + date=_string_field(item, "publishedDate", "date"), + last_updated=None, + ) + + +def _result_items(parsed: object) -> tuple[Mapping[str, object], ...]: + items: Final = parsed.get("results", ()) if isinstance(parsed, Mapping) else parsed + if not isinstance(items, Sequence) or isinstance(items, (str, bytes)): + return () + return tuple(item for item in items if isinstance(item, Mapping)) + + +def _parse_result_items(raw_text: object) -> tuple[Mapping[str, object], ...]: + """ + Parse one MCP text block into the search result objects it carries. + + A block holds either a JSON list of results or a {"results": [...]} object; + anything unparseable is skipped rather than failing the whole response. + """ + if not isinstance(raw_text, str): + return () + try: + parsed: Final = json.loads(raw_text) + except json.JSONDecodeError: + return () + return _result_items(parsed) + + +def _iter_sse_events(text: str) -> Iterator[Mapping[str, object]]: + """ + Yield the JSON payload of each SSE event in a Streamable HTTP MCP response. + + Per the SSE spec an event's data is the concatenation of all its ``data:`` + lines (joined with newlines), and a stream may carry several events, e.g. + progress notifications before the JSON-RPC response. + """ + for chunk in _SSE_EVENT_SEPARATOR.split(text): + payload = "\n".join(line[len("data:") :].lstrip() for line in chunk.splitlines() if line.startswith("data:")) + if not payload: + continue + try: + parsed = json.loads(payload) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + yield parsed + + +class AgentCoreSearchConfig(BaseSearchConfig, BaseAWSLLM): + def __init__(self) -> None: + BaseSearchConfig.__init__(self) + BaseAWSLLM.__init__(self) + + @staticmethod + def ui_friendly_name() -> str: + return "Web Search on Amazon Bedrock" + + def validate_environment( + self, + headers: dict, # mutable-ok: BaseSearchConfig hands providers the mutable request header dict + api_key: str | None = None, + api_base: str | None = None, + **kwargs: object, # kwargs-ok: BaseSearchConfig.validate_environment forwards provider-specific extras + ) -> dict: # mutable-ok: the handler passes these headers straight to httpx, which wants a dict + """ + Set MCP transport headers. Per the MCP Streamable HTTP transport spec, + the client MUST accept both application/json and text/event-stream, and + declare its protocol revision with MCP-Protocol-Version. + + Authentication itself happens in sign_request(): bearer token for + CUSTOM_JWT gateways, AWS SigV4 for AWS_IAM gateways. + """ + return { # mutable-ok: httpx request headers are a dict + **headers, + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + "MCP-Protocol-Version": get_secret_str("AGENTCORE_MCP_PROTOCOL_VERSION") + or AGENTCORE_DEFAULT_MCP_PROTOCOL_VERSION, + } + + def get_complete_url( + self, + api_base: str | None, + optional_params: dict, # mutable-ok: BaseSearchConfig passes optional params as a dict + data: dict | list[dict] | None = None, # mutable-ok: BaseSearchConfig request bodies are JSON dicts + **kwargs: object, # kwargs-ok: BaseSearchConfig.get_complete_url forwards provider-specific extras + ) -> str: + gateway_url: Final = api_base or get_secret_str("AGENTCORE_GATEWAY_URL") + if not gateway_url: + raise ValueError( + "AGENTCORE_GATEWAY_URL is not set. Set it to your AgentCore Gateway MCP " + "endpoint (https://.gateway.bedrock-agentcore." + ".amazonaws.com/mcp) or pass api_base." + ) + return gateway_url + + def transform_search_request( + self, + query: str | list[str], # mutable-ok: BaseSearchConfig accepts a list of queries + optional_params: dict, # mutable-ok: BaseSearchConfig passes optional params as a dict + **kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_request forwards provider-specific extras + ) -> dict: # mutable-ok: the JSON-RPC body is serialized as a JSON object + """ + Transform Search request to an MCP tools/call request. + + Args: + query: Search query (string or list of strings). AgentCore only + supports single string queries; lists are joined with spaces. + optional_params: Optional parameters for the request + - max_results: Maximum number of results (1-25), default 10 + - tool_name: Override the MCP tool name of the gateway target + + Returns: + Dict with the JSON-RPC 2.0 request body + """ + joined_query: Final = " ".join(query) if isinstance(query, list) else query + tool_name: Final = ( + optional_params.get("tool_name") + or get_secret_str("AGENTCORE_SEARCH_TOOL_NAME") + or AGENTCORE_DEFAULT_TOOL_NAME + ) + if not tool_name.endswith(AGENTCORE_TOOL_NAME_SUFFIX): + raise ValueError( + f"Invalid AgentCore search tool_name '{tool_name}': must end with " + f"'{AGENTCORE_TOOL_NAME_SUFFIX}' (a web-search connector tool). " + "Other gateway tools cannot be invoked through this provider." + ) + + return { # mutable-ok: JSON-RPC request bodies are JSON objects + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { # mutable-ok: JSON-RPC request bodies are JSON objects + "name": tool_name, + "arguments": { # mutable-ok: JSON-RPC request bodies are JSON objects + "query": joined_query[:AGENTCORE_MAX_QUERY_LENGTH], + "maxResults": optional_params.get("max_results", AGENTCORE_DEFAULT_MAX_RESULTS), + }, + }, + } + + def sign_request( + self, + headers: dict[str, str], # mutable-ok: BaseSearchConfig hands providers the mutable request header dict + optional_params: dict[str, object], # mutable-ok: BaseSearchConfig passes optional params as a dict + request_data: dict[str, object] | list[dict[str, object]], # mutable-ok: request bodies are JSON dicts + api_base: str, + api_key: str | None = None, + ) -> tuple[dict[str, str], bytes | None]: # mutable-ok: BaseSearchConfig.sign_request returns httpx headers + """ + Authenticate the MCP request. + + CUSTOM_JWT gateways: attach the caller's OAuth2 bearer token (api_key + or AGENTCORE_GATEWAY_TOKEN), no AWS credentials involved. + + AWS_IAM gateways: SigV4-sign with the bedrock-agentcore service name. + """ + if not isinstance(request_data, dict): + raise TypeError("AgentCore search expects a single dict request body") + + if not _credential_safe_transport(api_base): + raise ValueError( + f"Refusing to send AgentCore credentials over plaintext HTTP to '{api_base}': a bearer " + "token or SigV4 signature would be readable in transit. Use an https gateway URL " + "(plain http is allowed only for localhost)." + ) + + # Server-managed credentials only go to a trusted host, otherwise an + # authenticated caller could point api_base at their own server (e.g. via + # /search_tools/test_connection) and collect AGENTCORE_GATEWAY_TOKEN or a + # SigV4 signature with the proxy's credential scope and session token. + gateway_host_match: Final = _gateway_host_match(api_base) + bearer_token: Final = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("AGENTCORE_GATEWAY_TOKEN",), + base_env_var="AGENTCORE_GATEWAY_URL", + default_api_base=api_base if gateway_host_match else None, + ) + if bearer_token: + bearer_headers: Final = { # mutable-ok: httpx request headers are a dict + **headers, + "Authorization": f"Bearer {bearer_token}", + } + return bearer_headers, json.dumps(request_data).encode() + + if gateway_host_match is None and not self._is_configured_gateway(api_base): + raise ValueError( + f"Refusing to send SigV4-signed AgentCore requests to '{api_base}': it is neither an " + "AgentCore gateway hostname nor the host in AGENTCORE_GATEWAY_URL. Set " + "AGENTCORE_GATEWAY_URL to authorize a custom gateway hostname." + ) + + signing_params: Final = ( + optional_params + if optional_params.get("aws_region_name") is not None + else { # mutable-ok: BaseAWSLLM._sign_request takes optional params as a dict + **optional_params, + "aws_region_name": self._signing_region(api_base), + } + ) + + # api_key="" (not None, but falsy) disables BaseAWSLLM's fallback to the + # AWS_BEARER_TOKEN_BEDROCK env var: that token is a Bedrock Runtime + # credential and must not be sent to an AgentCore gateway. + return self._sign_request( + service_name="bedrock-agentcore", + headers=headers, + optional_params=signing_params, + request_data=request_data, + api_base=api_base, + api_key="", + ) + + @staticmethod + def _is_configured_gateway(api_base: str) -> bool: + configured: Final = get_secret_str("AGENTCORE_GATEWAY_URL") + if not configured: + return False + return httpx.URL(configured).host == httpx.URL(api_base).host + + @staticmethod + def _signing_region(api_base: str) -> str: + """ + Resolve the SigV4 signing region, which must match the gateway's region. + + Standard gateway hostnames carry it, so callers don't have to set + aws_region_name to a region different from their default. For custom or + private hostnames, defer to the AWS configuration chain (env vars and + the shared config / profile region), and error out when that yields + nothing rather than silently signing for a guessed region the gateway + would reject with a confusing auth error. + """ + match: Final = _gateway_host_match(api_base) + if match: + return match.group(1) + + # boto3's session resolution covers env vars AND the AWS shared config + # (profile region), unlike BaseAWSLLM's helper, which silently defaults + # to us-west-2 when nothing is configured. + import boto3 + + configured_region: Final = boto3.Session().region_name + if configured_region: + return configured_region + raise ValueError( + f"Cannot derive the SigV4 signing region from api_base '{api_base}' " + "or the AWS configuration chain. Set aws_region_name (or AWS_DEFAULT_REGION / " + "a profile region) to the gateway's region when using a custom hostname." + ) + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_response forwards provider-specific extras + ) -> SearchResponse: + """ + Transform an MCP tools/call response to LiteLLM unified SearchResponse. + + The gateway returns JSON-RPC (as plain JSON or a single-message SSE + stream) whose result.content[] text blocks contain a JSON list of + {title, url, date/publishedDate, text} entries. Web-search connector + 1.1.0 and later repeat that list in result.structuredContent, which is + the only machine-readable copy when the text block holds prose instead. + """ + response_json: Final = self._parse_mcp_body(raw_response) + + error: Final = response_json.get("error") + if error is not None: + raise BedrockError( + status_code=raw_response.status_code if raw_response.status_code >= 400 else 502, + message=f"AgentCore gateway MCP error: {error}", + ) + + # A failed tools/call is reported in-band, as HTTP 200 with result.isError + # and the failure text where the results would be. + result: Final = response_json.get("result") + if isinstance(result, dict) and result.get("isError"): + raise BedrockError( + status_code=raw_response.status_code if raw_response.status_code >= 400 else 502, + message=f"AgentCore web search tool error: {self._tool_error_message(response_json)}", + ) + + text_items: Final = tuple( + item for block in self._text_blocks(response_json) for item in _parse_result_items(block.get("text")) + ) + structured: Final = result.get("structuredContent") if isinstance(result, Mapping) else None + items: Final = text_items or _result_items(structured) + + results: Final = [_to_search_result(item) for item in items] # mutable-ok: pydantic list field + + return SearchResponse(results=results, object="search") + + def _tool_error_message(self, response_json: Mapping[str, object]) -> str: + texts: Final = tuple( + text for block in self._text_blocks(response_json) if isinstance(text := block.get("text"), str) + ) + return " ".join(texts) if texts else json.dumps(response_json.get("result"))[:500] + + @staticmethod + def _text_blocks(response_json: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + result: Final = response_json.get("result") + content: Final = result.get("content") if isinstance(result, dict) else None + if not isinstance(content, Sequence) or isinstance(content, (str, bytes)): + return () + return tuple(block for block in content if isinstance(block, dict) and block.get("type") == "text") + + @staticmethod + def _parse_mcp_body(raw_response: httpx.Response) -> Mapping[str, object]: + """ + Parse a JSON or SSE-framed (Streamable HTTP transport) MCP response. + + Return the event whose payload carries the JSON-RPC response, i.e. one + containing ``result`` or ``error``, falling back to the last event when + the stream carries only notifications. + """ + text: Final = raw_response.text + if not text.lstrip().startswith(_SSE_LINE_PREFIXES): + return raw_response.json() + + events: Final = tuple(_iter_sse_events(text)) + response_event: Final = next( + (event for event in events if "result" in event or "error" in event), + None, + ) + if response_event is not None: + return response_event + if events: + return events[-1] + raise BedrockError( + status_code=502, + message=f"AgentCore gateway returned SSE without a JSON data frame: {text[:200]}", + ) + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict, # mutable-ok: BaseSearchConfig.get_error_class takes the response headers as a dict + ) -> Exception: + return BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 3ee803646a9..cc522aed1ee 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5,7 +5,7 @@ import ssl from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from contextlib import asynccontextmanager from functools import lru_cache -from types import ModuleType +from types import MappingProxyType, ModuleType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, TypeVar, Union, cast, get_type_hints from urllib.parse import parse_qs, urlencode, urlparse, urlunparse @@ -1788,6 +1788,14 @@ class BaseLLMHTTPHandler: api_key=api_key, ) + signed_headers, signed_json_body = provider_config.sign_request( + headers=headers, + optional_params=optional_params, + request_data=data, + api_base=complete_url, + api_key=api_key, + ) + ## LOGGING logging_obj.pre_call( input=query if isinstance(query, str) else str(query), @@ -1811,14 +1819,15 @@ class BaseLLMHTTPHandler: # Note: timeout is set on the client itself, not per-request for GET response = client.get( url=complete_url, - headers=headers, + headers=signed_headers, ) else: - # Make POST request with JSON data + # A signed body must be sent verbatim, re-serializing it would break the signature response = client.post( url=complete_url, - headers=headers, - json=data, + headers=signed_headers, + data=signed_json_body, + json=data if signed_json_body is None else None, timeout=timeout, ) except Exception as e: @@ -1872,6 +1881,14 @@ class BaseLLMHTTPHandler: api_key=api_key, ) + signed_headers, signed_json_body = provider_config.sign_request( + headers=headers, + optional_params=optional_params, + request_data=data, + api_base=complete_url, + api_key=api_key, + ) + ## LOGGING logging_obj.pre_call( input=query if isinstance(query, str) else str(query), @@ -1900,14 +1917,15 @@ class BaseLLMHTTPHandler: # Note: timeout is set on the client itself, not per-request for GET response = await async_httpx_client.get( url=complete_url, - headers=headers, + headers=signed_headers, ) else: - # Make async POST request with JSON data + # A signed body must be sent verbatim, re-serializing it would break the signature response = await async_httpx_client.post( url=complete_url, - headers=headers, - json=data, + headers=signed_headers, + data=signed_json_body, + json=data if signed_json_body is None else None, timeout=timeout, ) except Exception as e: @@ -2069,6 +2087,14 @@ class BaseLLMHTTPHandler: if anthropic_messages_provider_config.should_filter_anthropic_beta_headers(): headers = update_headers_with_filtered_beta(headers=headers, provider=custom_llm_provider) + from litellm.llms.vertex_ai.vertex_llm_base import VertexBase + + explicit_vertex_location: Final = VertexBase.explicit_vertex_ai_location(MappingProxyType(dict(litellm_params))) + vertex_location_params: Final = ( + MappingProxyType({"vertex_location": explicit_vertex_location}) + if explicit_vertex_location + else MappingProxyType({}) + ) logging_obj.update_from_kwargs( kwargs=kwargs, model=model, @@ -2077,6 +2103,7 @@ class BaseLLMHTTPHandler: "preset_cache_key": None, "stream_response": {}, "model_info": kwargs.get("model_info"), + **vertex_location_params, **anthropic_messages_optional_request_params, }, custom_llm_provider=custom_llm_provider, diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index 86a5bb207ec..23cb1e5b580 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -7,6 +7,7 @@ from litellm import verbose_logger from litellm.litellm_core_utils.llm_cost_calc.utils import ( _is_above_128k, generic_cost_per_token, + get_vertex_regional_endpoint_uplift, ) from litellm.types.utils import ModelInfo, Usage @@ -63,6 +64,7 @@ def cost_per_character( usage: Usage, prompt_characters: float | None = None, completion_characters: float | None = None, + vertex_location: str | None = None, ) -> tuple[float, float]: """ Calculates the cost per character for a given VertexAI model, input messages, and response object. @@ -72,6 +74,8 @@ def cost_per_character( - custom_llm_provider: str, "vertex_ai-*" - prompt_characters: float, the number of input characters - completion_characters: float, the number of output characters + - vertex_location: the Vertex AI location serving the request; non-global + locations apply the model's regional-endpoint uplift multiplier Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd @@ -79,8 +83,6 @@ def cost_per_character( Raises: Exception if model requires >128k pricing, but model cost not mapped """ - model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) - ## GET MODEL INFO model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) @@ -162,7 +164,8 @@ def cost_per_character( usage=usage, ) - return prompt_cost, completion_cost + vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + return prompt_cost * vertex_uplift, completion_cost * vertex_uplift def _handle_128k_pricing( @@ -196,6 +199,7 @@ def cost_per_token( custom_llm_provider: str, usage: Usage, service_tier: str | None = None, + vertex_location: str | None = None, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -207,6 +211,8 @@ def cost_per_token( - completion_tokens: float, the number of output tokens - service_tier: optional tier derived from Gemini trafficType ("priority" for ON_DEMAND_PRIORITY, "flex" for FLEX/batch). + - vertex_location: the Vertex AI location serving the request; non-global + locations apply the model's regional-endpoint uplift multiplier Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd @@ -222,14 +228,17 @@ def cost_per_token( input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens") output_cost_per_token_above_128k_tokens: Final = model_info.get("output_cost_per_token_above_128k_tokens") if input_cost_per_token_above_128k_tokens is not None or output_cost_per_token_above_128k_tokens is not None: - return _handle_128k_pricing( + prompt_cost_128k, completion_cost_128k = _handle_128k_pricing( model_info=model_info, usage=usage, ) + vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + return prompt_cost_128k * vertex_uplift, completion_cost_128k * vertex_uplift return generic_cost_per_token( model=model, custom_llm_provider=custom_llm_provider, usage=usage, service_tier=service_tier, + vertex_location=vertex_location, ) diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 445e34966a9..75098515deb 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -8,6 +8,7 @@ import asyncio import json import os import threading +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, Literal from urllib.parse import urlparse @@ -68,7 +69,8 @@ class VertexBase: # re-acquire it without deadlocking the current thread. self._sync_refresh_lock = threading.RLock() - def get_vertex_region(self, vertex_region: str | None, model: str) -> str: + @staticmethod + def get_vertex_region(vertex_region: str | None, model: str) -> str: import litellm # Try to get supported_regions directly from model_cost @@ -1191,7 +1193,18 @@ class VertexBase: ) @staticmethod - def safe_get_vertex_ai_location(litellm_params: dict) -> str | None: + def explicit_vertex_ai_location(params: Mapping[str, object]) -> str | None: + """ + The location explicitly configured in the given params, without any + module-level or environment fallback. None when not configured. + """ + for configured in (params.get("vertex_location"), params.get("vertex_ai_location")): + if isinstance(configured, str) and configured: + return configured + return None + + @staticmethod + def safe_get_vertex_ai_location(litellm_params: Mapping[str, object]) -> str | None: """ Safely get Vertex AI location without mutating the litellm_params dict. @@ -1205,8 +1218,7 @@ class VertexBase: Vertex AI location/region or None """ return ( - litellm_params.get("vertex_location") - or litellm_params.get("vertex_ai_location") + VertexBase.explicit_vertex_ai_location(litellm_params) or litellm.vertex_location or get_secret_str("VERTEXAI_LOCATION") or get_secret_str("VERTEX_LOCATION") diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 07f9027313b..d0eca17272d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -54,6 +54,7 @@ "output_cost_per_image": 0.04 }, "1024-x-1024/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 1.9e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -67,6 +68,7 @@ "output_cost_per_image": 0.08 }, "256-x-256/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 2.4414e-07, "litellm_provider": "openai", "mode": "image_generation", @@ -80,6 +82,7 @@ "output_cost_per_image": 0.018 }, "512-x-512/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.86e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -2887,6 +2890,7 @@ "supports_function_calling": true }, "azure_ai/claude-haiku-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -2908,6 +2912,7 @@ "supports_vision": true }, "azure_ai/claude-opus-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -2930,6 +2935,7 @@ "supports_output_config": true }, "azure_ai/claude-opus-4-6": { + "deprecation_date": "2027-02-02", "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -2959,6 +2965,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-7": { + "deprecation_date": "2027-04-06", "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -3083,6 +3090,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-1": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -3104,6 +3112,7 @@ "supports_vision": true }, "azure_ai/claude-sonnet-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -3156,6 +3165,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-sonnet-4-6": { + "deprecation_date": "2027-02-10", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -3226,6 +3236,7 @@ "supports_tool_choice": true }, "azure_ai/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -3318,6 +3329,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -3364,6 +3376,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-2026-03-05": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -3410,6 +3423,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-pro": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -3455,6 +3469,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-pro-2026-03-05": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -3500,6 +3515,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-mini": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -3540,6 +3556,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-mini-2026-03-17": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -3580,6 +3597,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-nano": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -3620,6 +3638,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-nano-2026-03-17": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -3849,6 +3868,7 @@ "supports_vision": true }, "azure/eu/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -3918,6 +3938,7 @@ "supports_none_reasoning_effort": true }, "azure/eu/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -3948,6 +3969,7 @@ "supports_vision": true }, "azure/eu/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -4107,6 +4129,7 @@ "supports_vision": true }, "azure/global-standard/gpt-4o-mini": { + "deprecation_date": "2027-04-14", "input_cost_per_token": 1.5e-07, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -4155,6 +4178,7 @@ "supports_vision": true }, "azure/global/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -4224,6 +4248,7 @@ "supports_none_reasoning_effort": true }, "azure/global/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -4254,6 +4279,7 @@ "supports_vision": true }, "azure/global/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -4492,6 +4518,7 @@ "supports_vision": true }, "azure/gpt-4.1": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -4559,6 +4586,7 @@ "supports_web_search": false }, "azure/gpt-4.1-mini": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, @@ -4626,6 +4654,7 @@ "supports_web_search": false }, "azure/gpt-4.1-nano": { + "deprecation_date": "2026-10-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -4902,6 +4931,7 @@ "supports_vision": false }, "azure/gpt-4o-mini": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.65e-07, "litellm_provider": "azure", @@ -5344,6 +5374,7 @@ "supports_vision": true }, "azure/gpt-5": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5507,6 +5538,7 @@ "supports_vision": true }, "azure/gpt-5-mini": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -5572,6 +5604,7 @@ "supports_vision": true }, "azure/gpt-5-nano": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5e-09, "input_cost_per_token": 5e-08, "litellm_provider": "azure", @@ -5667,6 +5700,7 @@ "supports_vision": true }, "azure/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5736,6 +5770,7 @@ "supports_none_reasoning_effort": true }, "azure/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5797,6 +5832,7 @@ "supports_vision": true }, "azure/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -5827,6 +5863,7 @@ "supports_vision": true }, "azure/gpt-5.2": { + "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.75e-07, "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", @@ -6136,6 +6173,7 @@ "supports_web_search": true }, "azure/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -6180,6 +6218,7 @@ "supports_minimal_reasoning_effort": true }, "azure/us/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.8e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, @@ -6218,6 +6257,7 @@ "supports_minimal_reasoning_effort": true }, "azure/eu/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.8e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, @@ -6379,6 +6419,7 @@ "supports_minimal_reasoning_effort": true }, "azure/gpt-5.4-pro": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -7045,6 +7086,7 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -7095,6 +7137,7 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -7142,6 +7185,7 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -7408,6 +7452,7 @@ "supports_web_search": true }, "azure/gpt-5.4-mini": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", @@ -7489,6 +7534,7 @@ "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -7601,6 +7647,7 @@ "output_cost_per_token": 0.0 }, "azure/high/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7610,6 +7657,7 @@ ] }, "azure/high/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7619,6 +7667,7 @@ ] }, "azure/high/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7628,6 +7677,7 @@ ] }, "azure/low/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7637,6 +7687,7 @@ ] }, "azure/low/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7646,6 +7697,7 @@ ] }, "azure/low/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7655,6 +7707,7 @@ ] }, "azure/medium/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7664,6 +7717,7 @@ ] }, "azure/medium/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7673,6 +7727,7 @@ ] }, "azure/medium/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7695,6 +7750,7 @@ ] }, "azure/gpt-image-1.5": { + "deprecation_date": "2027-06-16", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -7720,6 +7776,7 @@ ] }, "azure/gpt-image-2": { + "deprecation_date": "2027-10-21", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -7751,6 +7808,7 @@ "supports_pdf_input": true }, "azure/low/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7760,6 +7818,7 @@ ] }, "azure/low/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7769,6 +7828,7 @@ ] }, "azure/low/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0345052083e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7778,6 +7838,7 @@ ] }, "azure/medium/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7787,6 +7848,7 @@ ] }, "azure/medium/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7796,6 +7858,7 @@ ] }, "azure/medium/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 7.9752604167e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7805,6 +7868,7 @@ ] }, "azure/high/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7814,6 +7878,7 @@ ] }, "azure/high/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7823,6 +7888,7 @@ ] }, "azure/high/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.1575520833e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7850,6 +7916,7 @@ "supports_function_calling": true }, "azure/o1": { + "deprecation_date": "2026-10-21", "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "azure", @@ -7944,6 +8011,7 @@ "supports_vision": false }, "azure/o3": { + "deprecation_date": "2026-10-21", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "litellm_provider": "azure", @@ -8041,6 +8109,7 @@ "supports_web_search": true }, "azure/o3-mini": { + "deprecation_date": "2026-10-01", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure", @@ -8071,6 +8140,7 @@ "supports_vision": false }, "azure/o3-pro": { + "deprecation_date": "2026-12-17", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "azure", @@ -8132,6 +8202,7 @@ "supports_vision": true }, "azure/o4-mini": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure", @@ -8580,6 +8651,7 @@ "supports_vision": true }, "azure/us/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -8649,6 +8721,7 @@ "supports_none_reasoning_effort": true }, "azure/us/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -8679,6 +8752,7 @@ "supports_vision": true }, "azure/us/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -8876,6 +8950,7 @@ ] }, "azure_ai/FW-DeepSeek-V3.2": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.1e-07, "input_cost_per_token": 6.2e-07, "litellm_provider": "azure_ai", @@ -8906,6 +8981,7 @@ "supports_tool_choice": true }, "azure_ai/FW-GLM-5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 2.2e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure_ai", @@ -8921,6 +8997,7 @@ "supports_tool_choice": true }, "azure_ai/FW-GLM-5.1": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 2.86e-07, "input_cost_per_token": 1.54e-06, "litellm_provider": "azure_ai", @@ -8987,6 +9064,7 @@ "supports_tool_choice": true }, "azure_ai/FW-Kimi-K2.5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 6.6e-07, "litellm_provider": "azure_ai", @@ -9079,6 +9157,7 @@ "supports_vision": true }, "azure_ai/FW-MiniMax-M2.5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.3e-08, "input_cost_per_token": 3.3e-07, "litellm_provider": "azure_ai", @@ -9164,6 +9243,7 @@ ] }, "azure_ai/MAI-Image-2e": { + "deprecation_date": "2026-08-15", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", "mode": "image_generation", @@ -9175,6 +9255,7 @@ ] }, "azure_ai/Llama-3.2-11B-Vision-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 3.7e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9188,6 +9269,7 @@ "supports_vision": true }, "azure_ai/Llama-3.2-90B-Vision-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 2.04e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9249,6 +9331,7 @@ "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-405B-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 5.33e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9271,6 +9354,7 @@ "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-8B-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 3e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9452,6 +9536,7 @@ "supports_reasoning": true }, "azure_ai/mistral-document-ai-2505": { + "deprecation_date": "2026-07-20", "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.003, "mode": "ocr", @@ -9529,6 +9614,7 @@ "output_cost_per_token": 0.0 }, "azure_ai/cohere-rerank-v3.5": { + "deprecation_date": "2026-05-14", "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, "litellm_provider": "azure_ai", @@ -9591,6 +9677,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-r1": { + "deprecation_date": "2026-08-13", "input_cost_per_token": 1.35e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9614,6 +9701,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v3-0324": { + "deprecation_date": "2026-07-13", "input_cost_per_token": 1.14e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9626,6 +9714,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v3.1": { + "deprecation_date": "2026-07-13", "input_cost_per_token": 1.23e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9639,6 +9728,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v4-pro": { + "deprecation_date": "2028-02-20", "input_cost_per_token": 1.74e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -9652,6 +9742,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v4-flash": { + "deprecation_date": "2028-02-20", "input_cost_per_token": 1.9e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -9683,6 +9774,7 @@ "supports_embedding_image_input": true }, "azure_ai/global/grok-3": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9697,6 +9789,7 @@ "supports_web_search": true }, "azure_ai/global/grok-3-mini": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9712,6 +9805,7 @@ "supports_web_search": true }, "azure_ai/grok-3": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9726,6 +9820,7 @@ "supports_web_search": true }, "azure_ai/grok-3-mini": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9773,6 +9868,7 @@ "supports_web_search": true }, "azure_ai/grok-4-fast-non-reasoning": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -9786,6 +9882,7 @@ "supports_web_search": true }, "azure_ai/grok-4-fast-reasoning": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -9863,6 +9960,7 @@ "supports_tool_choice": true }, "azure_ai/kimi-k2.5": { + "deprecation_date": "2027-01-26", "input_cost_per_token": 6e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -9877,6 +9975,7 @@ "supports_vision": true }, "azure_ai/kimi-k2.6": { + "deprecation_date": "2027-04-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -10004,6 +10103,7 @@ "supports_vision": true }, "babbage-002": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 4e-07, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, @@ -12014,6 +12114,7 @@ ] }, "claude-haiku-4-5-20251001": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -12037,6 +12138,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-haiku-4-5": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -12185,6 +12287,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -12218,6 +12321,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -12252,6 +12356,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-5": { + "deprecation_date": "2027-06-30", "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -12288,6 +12393,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-6": { + "deprecation_date": "2027-02-17", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -12434,6 +12540,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-opus-4-5-20251101": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12463,6 +12570,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-5": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12492,6 +12600,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6": { + "deprecation_date": "2027-02-05", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12528,6 +12637,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6-20260205": { + "deprecation_date": "2027-02-05", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12564,6 +12674,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "deprecation_date": "2027-04-16", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12602,6 +12713,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-opus-4-7-20260416": { + "deprecation_date": "2027-04-16", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12640,6 +12752,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "deprecation_date": "2027-06-09", "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -12675,6 +12788,7 @@ "prompt_cache_min_tokens": 512 }, "claude-opus-5": { + "deprecation_date": "2027-07-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12713,6 +12827,7 @@ "prompt_cache_min_tokens": 512 }, "claude-opus-4-8": { + "deprecation_date": "2027-05-28", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15042,6 +15157,7 @@ "mode": "search" }, "davinci-002": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 2e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, @@ -16676,6 +16792,14 @@ "notes": "APISerpent deep search (/api/search), multi-engine (Google, Bing, Yahoo, DuckDuckGo). Pricing: $0.60/1k searches." } }, + "agentcore/search": { + "input_cost_per_query": 0.0, + "litellm_provider": "agentcore", + "mode": "search", + "metadata": { + "notes": "Web Search on Amazon Bedrock AgentCore, billed by AWS on the gateway" + } + }, "tinyfish/search": { "input_cost_per_query": 0.0, "litellm_provider": "tinyfish", @@ -18594,6 +18718,7 @@ } }, "gemini-2.5-flash": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -18639,6 +18764,7 @@ "supports_image_size": false }, "gemini-2.5-flash-image": { + "deprecation_date": "2026-10-02", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -18683,6 +18809,7 @@ "supports_image_size": false }, "gemini-3-pro-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -18763,6 +18890,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -18887,6 +19015,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-lite": { + "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -18943,6 +19072,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.5-flash-lite": { + "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, @@ -19032,6 +19162,7 @@ "supports_web_search": true }, "gemini-2.5-flash-lite": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1e-08, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, @@ -19303,6 +19434,7 @@ "supports_image_size": false }, "gemini-2.5-pro": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -19403,6 +19535,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-pro-preview": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -19460,6 +19593,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-pro-preview-customtools": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -19614,6 +19748,8 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash": { + "prompt_cache_min_tokens": 4096, + "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1e-06, @@ -19624,6 +19760,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -19665,6 +19802,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.6-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -19679,6 +19817,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -19719,6 +19858,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.7-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -19733,6 +19873,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -19773,6 +19914,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-pro-preview": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -19830,6 +19972,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-pro-preview-customtools": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -20050,6 +20193,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-robotics-er-1.6-preview": { + "deprecation_date": "2026-08-31", "input_cost_per_audio_token": 2e-06, "input_cost_per_token": 1e-06, "litellm_provider": "gemini", @@ -20120,6 +20264,7 @@ "supports_vision": true }, "gemini-embedding-001": { + "deprecation_date": "2028-05-20", "input_cost_per_token": 1.5e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, @@ -20562,8 +20707,8 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-3.1-flash-image": { - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 32768, @@ -20571,8 +20716,8 @@ "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, - "output_cost_per_token": 1.5e-06, - "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "rpm": 1000, "tpm": 4000000, "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image", @@ -20605,8 +20750,8 @@ }, "gemini/gemini-3.1-flash-image-preview": { "deprecation_date": "2026-06-25", - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 32768, @@ -20614,8 +20759,8 @@ "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, - "output_cost_per_token": 1.5e-06, - "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "rpm": 1000, "tpm": 4000000, "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image-preview", @@ -21337,6 +21482,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-3.5-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 1.5e-07, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1.5e-06, @@ -21391,6 +21537,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-3.6-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -21448,6 +21595,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-3.7-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -21538,6 +21686,7 @@ "tpm": 800000 }, "gemini/gemini-3.1-pro-preview": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 2e-06, @@ -21595,6 +21744,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-3.1-pro-preview-customtools": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 2e-06, @@ -21733,6 +21883,8 @@ "supports_vision": true }, "gemini-3.5-flash": { + "prompt_cache_min_tokens": 4096, + "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1.5e-06, @@ -21785,6 +21937,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.6-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -21840,6 +21993,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.7-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -23245,6 +23399,7 @@ "supports_tool_choice": true }, "gpt-3.5-turbo-instruct": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 1.5e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, @@ -24376,6 +24531,7 @@ "supports_pdf_input": true }, "low/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24387,6 +24543,7 @@ "supports_pdf_input": true }, "low/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24398,6 +24555,7 @@ "supports_pdf_input": true }, "low/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24409,6 +24567,7 @@ "supports_pdf_input": true }, "medium/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.034, "litellm_provider": "openai", "mode": "image_generation", @@ -24420,6 +24579,7 @@ "supports_pdf_input": true }, "medium/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -24431,6 +24591,7 @@ "supports_pdf_input": true }, "medium/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -24442,6 +24603,7 @@ "supports_pdf_input": true }, "high/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.133, "litellm_provider": "openai", "mode": "image_generation", @@ -24453,6 +24615,7 @@ "supports_pdf_input": true }, "high/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -24464,6 +24627,7 @@ "supports_pdf_input": true }, "high/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -24475,6 +24639,7 @@ "supports_pdf_input": true }, "standard/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24486,6 +24651,7 @@ "supports_pdf_input": true }, "standard/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24497,6 +24663,7 @@ "supports_pdf_input": true }, "standard/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24508,6 +24675,7 @@ "supports_pdf_input": true }, "1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24519,6 +24687,7 @@ "supports_pdf_input": true }, "1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24530,6 +24699,7 @@ "supports_pdf_input": true }, "1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -27443,18 +27613,21 @@ "output_cost_per_second": 0.0 }, "hd/1024-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 7.629e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "hd/1024-x-1792/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.539e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "hd/1792-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.539e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -27501,6 +27674,7 @@ "max_output_tokens": 8192 }, "high/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.167, "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "openai", @@ -27511,6 +27685,7 @@ ] }, "high/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -27521,6 +27696,7 @@ ] }, "high/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -28308,6 +28484,7 @@ "supports_tool_choice": true }, "low/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.011, "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "openai", @@ -28318,6 +28495,7 @@ ] }, "low/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -28328,6 +28506,7 @@ ] }, "low/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -28352,6 +28531,7 @@ "output_cost_per_image": 0.072 }, "medium/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.042, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28362,6 +28542,7 @@ ] }, "medium/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28372,6 +28553,7 @@ ] }, "medium/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28382,6 +28564,7 @@ ] }, "low/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.005, "litellm_provider": "openai", "mode": "image_generation", @@ -28390,6 +28573,7 @@ ] }, "low/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -28398,6 +28582,7 @@ ] }, "low/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -28406,6 +28591,7 @@ ] }, "medium/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.011, "litellm_provider": "openai", "mode": "image_generation", @@ -28414,6 +28600,7 @@ ] }, "medium/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -28422,6 +28609,7 @@ ] }, "medium/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -30315,6 +30503,7 @@ ] }, "multimodalembedding@001": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2e-07, "input_cost_per_image": 0.0001, "input_cost_per_token": 8e-07, @@ -36013,18 +36202,21 @@ "output_cost_per_image": 0.14 }, "standard/1024-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 3.81469e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "standard/1024-x-1792/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 4.359e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "standard/1792-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 4.359e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -36088,6 +36280,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" }, "text-embedding-005": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -36161,6 +36354,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "text-moderation-007": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36170,6 +36364,7 @@ "output_cost_per_token": 0.0 }, "text-moderation-latest": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36179,6 +36374,7 @@ "output_cost_per_token": 0.0 }, "text-moderation-stable": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36188,6 +36384,7 @@ "output_cost_per_token": 0.0 }, "text-multilingual-embedding-002": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -38675,6 +38872,7 @@ "supports_tool_choice": true }, "vertex_ai/claude-haiku-4-5": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -38685,6 +38883,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", "supports_assistant_prefill": true, "supports_function_calling": true, @@ -38698,6 +38897,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-haiku-4-5@20251001": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -38708,6 +38908,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", "supports_assistant_prefill": true, "supports_function_calling": true, @@ -38850,6 +39051,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38877,6 +39079,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-1": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38895,6 +39098,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4-1@20250805": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38913,6 +39117,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4-5": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -38923,6 +39128,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38941,6 +39147,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-5@20251101": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -38951,6 +39158,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38970,6 +39178,8 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6": { + "deprecation_date": "2027-02-05", + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39000,6 +39210,8 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6@default": { + "deprecation_date": "2027-02-05", + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39030,6 +39242,8 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-7": { + "deprecation_date": "2027-04-16", + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39061,6 +39275,8 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-opus-4-7@default": { + "deprecation_date": "2027-04-16", + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39092,6 +39308,8 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { + "deprecation_date": "2027-06-08", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39123,6 +39341,8 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-fable-5@default": { + "deprecation_date": "2027-06-08", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39154,6 +39374,8 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-5": { + "deprecation_date": "2027-01-24", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39186,6 +39408,8 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-5@default": { + "deprecation_date": "2027-01-24", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39218,6 +39442,8 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-4-8": { + "deprecation_date": "2027-05-28", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39250,6 +39476,8 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { + "deprecation_date": "2027-05-28", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39282,6 +39510,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39298,6 +39527,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -39310,6 +39540,8 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { + "deprecation_date": "2026-12-24", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -39342,6 +39574,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -39372,6 +39605,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5@20250929": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39388,6 +39622,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -39401,6 +39636,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4@20250514": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -39428,6 +39664,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39459,6 +39696,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4@20250514": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39623,6 +39861,7 @@ "supports_tool_choice": true }, "vertex_ai/gemini-2.5-flash-image": { + "deprecation_date": "2026-10-02", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -39668,6 +39907,7 @@ "supports_image_size": false }, "vertex_ai/gemini-3-pro-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -39700,6 +39940,7 @@ "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, "vertex_ai/gemini-3.1-flash-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -39776,6 +40017,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-flash-lite": { + "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -39794,6 +40036,7 @@ "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, "output_cost_per_token_priority": 2.7e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", "supported_endpoints": [ "/v1/chat/completions", @@ -39832,6 +40075,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash-lite": { + "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, @@ -39849,6 +40093,7 @@ "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, "output_cost_per_token_priority": 4.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", "supported_endpoints": [ "/v1/chat/completions", @@ -40549,6 +40794,7 @@ "supports_tool_choice": true }, "vertex_ai/veo-2.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40563,6 +40809,7 @@ ] }, "vertex_ai/veo-3.0-fast-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40577,6 +40824,7 @@ ] }, "vertex_ai/veo-3.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40619,6 +40867,7 @@ ] }, "vertex_ai/veo-3.1-generate-001": { + "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40633,6 +40882,7 @@ ] }, "vertex_ai/veo-3.1-fast-generate-001": { + "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -47014,6 +47264,8 @@ } }, "vertex_ai/claude-sonnet-5@default": { + "deprecation_date": "2026-12-24", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -47046,6 +47298,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -47975,15 +48228,15 @@ }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 2.8e-09, - "input_cost_per_token": 1.4e-07, - "input_cost_per_token_cache_hit": 2.8e-09, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 2.8e-07, + "output_cost_per_token": 1.32e-06, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -48001,15 +48254,15 @@ }, "deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 3.625e-09, - "input_cost_per_token": 4.35e-07, - "input_cost_per_token_cache_hit": 3.625e-09, + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 8.7e-07, + "output_cost_per_token": 3.96e-06, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -48027,15 +48280,15 @@ }, "deepseek/deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 2.8e-09, - "input_cost_per_token": 1.4e-07, - "input_cost_per_token_cache_hit": 2.8e-09, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 2.8e-07, + "output_cost_per_token": 1.32e-06, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -48053,15 +48306,15 @@ }, "deepseek/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 3.625e-09, - "input_cost_per_token": 4.35e-07, - "input_cost_per_token_cache_hit": 3.625e-09, + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 8.7e-07, + "output_cost_per_token": 3.96e-06, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" diff --git a/litellm/proxy/_experimental/mcp_server/exceptions.py b/litellm/proxy/_experimental/mcp_server/exceptions.py index 8c704c0fe93..a1b3b167a4a 100644 --- a/litellm/proxy/_experimental/mcp_server/exceptions.py +++ b/litellm/proxy/_experimental/mcp_server/exceptions.py @@ -75,6 +75,24 @@ class MCPUpstreamAuthError(Exception): ) +class MCPOpenApiUpstreamError(Exception): + """An OpenAPI-backed MCP tool's upstream answered with a non-2xx that is not a 401. + + Carries the status only. The upstream's response body is deliberately dropped rather than served + as tool content: it crosses a trust boundary and may hold prose, urls, or an error document that + reads as data, which is how these failures came to be reported as successful tool output. This + matches ``outcome_wire_value``'s contract for listing faults, category and status and nothing + else. A 401 is raised as ``MCPUpstreamAuthError`` instead, so the caller learns to + re-authenticate; every other status stays here, mirroring the regular MCP path where a 403 + deliberately does not produce a challenge. + """ + + def __init__(self, status_code: int, server_name: str) -> None: + self.status_code = status_code + self.server_name = server_name + super().__init__(f"upstream returned HTTP {status_code}") + + class MCPToolResultError(Exception): """An MCP tool call completed with ``isError=True`` in its result. diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 65855df89f6..26a6f8d1251 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -2330,7 +2330,15 @@ class MCPServerManager: input_schema = build_input_schema(resolved_operation) # Create tool function with headers using imported function - tool_func = create_tool_function(path, method, resolved_operation, base_url, headers=headers) + tool_func = create_tool_function( + path, + method, + resolved_operation, + base_url, + headers=headers, + server_label=server.name or server.server_name or server.alias or server.server_id, + relays_upstream_auth=server.is_client_forwarded_token, + ) tool_func.__name__ = prefixed_tool_name tool_func.__doc__ = description @@ -4979,6 +4987,12 @@ class MCPServerManager: return result + except MCPUpstreamAuthError: + # The caller must re-authenticate upstream, so this keeps its type all the way to the + # renderers: the streamable path turns it into an isError result naming the status, and + # the REST path relays a real 401 with the upstream's WWW-Authenticate. Flattening it + # into the generic message below would lose both. + raise except Exception as e: error_msg = f"Error calling OpenAPI tool {tool_name}: {e}" verbose_logger.error(error_msg) diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index eb78aaeca0b..083a98cdd36 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -15,6 +15,12 @@ from urllib.parse import quote import httpx from typing_extensions import ReadOnly, Required +from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError +from litellm.proxy._experimental.mcp_server.exceptions import ( + MCPOpenApiUpstreamError, + MCPUpstreamAuthError, +) + # Tool names emitted from OpenAPI specs must work across all major LLM providers. # OpenAI/Anthropic/Bedrock all enforce a character class roughly equivalent to # ^[a-zA-Z0-9_-]+$ on tool names. Many specs (notably GitHub's REST API) use @@ -392,12 +398,40 @@ def _merge_openapi_tool_request_headers( return effective_headers +def _raise_for_upstream_failure( + response: httpx.Response, + upstream: str, + relays_upstream_auth: bool, +) -> None: + """Turn a non-2xx upstream response into the right typed failure, or return for a 2xx. + + Both call sites feed this: ``get`` hands back the response for a 4xx, while post/put/patch/delete + raise ``MaskedHTTPStatusError`` from inside the HTTP handler, so without one classifier the + non-GET tools would keep serving an error body as tool output. + + Only the client-forwarded modes carry the caller's own upstream token, so only they can act on a + 401 by re-authenticating; ``_call_regular_mcp_tool`` gates its re-auth signal the same way. Every + other status carries the code alone, never the upstream's body, which crosses a trust boundary. + """ + if response.status_code < 400: + return + if response.status_code == 401 and relays_upstream_auth: + raise MCPUpstreamAuthError( + status_code=response.status_code, + www_authenticate=response.headers.get("www-authenticate"), + server_name=upstream, + ) + raise MCPOpenApiUpstreamError(response.status_code, upstream) + + def create_tool_function( path: str, method: str, operation: _OpenAPIOperation, base_url: str, headers: dict[str, str] | None = None, + server_label: str | None = None, + relays_upstream_auth: bool = False, ): """Create a tool function for an OpenAPI operation. @@ -477,20 +511,26 @@ def create_tool_function( json_body = {"data": body_value} client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) + upstream: Final = server_label or f"{original_method.upper()} {path}" - if original_method == "get": - response = await client.get(url, params=params, headers=effective_headers) - elif original_method == "post": - response = await client.post(url, params=params, json=json_body, headers=effective_headers) - elif original_method == "put": - response = await client.put(url, params=params, json=json_body, headers=effective_headers) - elif original_method == "delete": - response = await client.delete(url, params=params, headers=effective_headers) - elif original_method == "patch": - response = await client.patch(url, params=params, json=json_body, headers=effective_headers) - else: - return f"Unsupported HTTP method: {original_method}" + try: + if original_method == "get": + response = await client.get(url, params=params, headers=effective_headers) + elif original_method == "post": + response = await client.post(url, params=params, json=json_body, headers=effective_headers) + elif original_method == "put": + response = await client.put(url, params=params, json=json_body, headers=effective_headers) + elif original_method == "delete": + response = await client.delete(url, params=params, headers=effective_headers) + elif original_method == "patch": + response = await client.patch(url, params=params, json=json_body, headers=effective_headers) + else: + return f"Unsupported HTTP method: {original_method}" + except MaskedHTTPStatusError as e: + _raise_for_upstream_failure(e.response, upstream, relays_upstream_auth) + raise + _raise_for_upstream_failure(response, upstream, relays_upstream_auth) return response.text return tool_function diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 8d69d84e492..0dc85c0318c 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -407,8 +407,6 @@ if MCP_AVAILABLE: StreamableHTTPSessionManager = None from mcp.types import ( CallToolResult, - EmbeddedResource, - ImageContent, ListToolsResult, Prompt, TextContent, @@ -2861,12 +2859,11 @@ if MCP_AVAILABLE: _extra_token: Final = _request_extra_headers.set(forwarded_headers) _resolved_token: Final = _request_resolved_auth_headers.set(resolved_auth_headers) try: - local_content = await _handle_local_mcp_tool(name, arguments) + response = await _handle_local_mcp_tool(name, arguments) finally: _request_auth_header.reset(_auth_token) _request_extra_headers.reset(_extra_token) _request_resolved_auth_headers.reset(_resolved_token) - response = CallToolResult(content=local_content, isError=False) # Try managed MCP server tool (the name is bare; the prefix boundary was # already resolved above against this server's registered prefixes) @@ -2940,8 +2937,7 @@ if MCP_AVAILABLE: if "arguments" in hook_result: arguments = hook_result["arguments"] # pyright: ignore[reportAny] # hook returns untyped args - local_content = await _handle_local_mcp_tool(original_tool_name, arguments) - response = CallToolResult(content=local_content, isError=False) + response = await _handle_local_mcp_tool(original_tool_name, arguments) return await _run_post_mcp_call_guardrails( result=response, @@ -3319,11 +3315,18 @@ if MCP_AVAILABLE: verbose_logger.debug("CALL TOOL RESULT: %s", call_tool_result) return call_tool_result - async def _handle_local_mcp_tool( - name: str, arguments: dict[str, object] - ) -> list[TextContent | ImageContent | EmbeddedResource]: - """ - Handle tool execution for local registry tools + async def _handle_local_mcp_tool(name: str, arguments: dict[str, object]) -> CallToolResult: + """Execute a local-registry tool and report whether it succeeded. + + Returns the result rather than bare content because the verdict is part of it: the content + alone cannot say whether the handler failed, so callers used to stamp isError=False on every + outcome and an upstream rejection was served as tool output. + + A failure is reported as ``isError=True`` here rather than raised, because the REST surface + turns an unrecognized exception into a 500 and an upstream 403 or 429 is not a gateway crash. + ``MCPUpstreamAuthError`` is the exception: it propagates so the caller is told to + re-authenticate, which both renderers already know how to say. + Note: Local tools don't use prefixes, so we use the original name """ import inspect @@ -3333,15 +3336,16 @@ if MCP_AVAILABLE: raise HTTPException(status_code=404, detail=f"Tool '{name}' not found") try: - # Check if handler is async or sync if inspect.iscoroutinefunction(tool.handler): result = await tool.handler(**arguments) else: result = tool.handler(**arguments) - return [TextContent(text=str(result), type="text")] + except MCPUpstreamAuthError: + raise except Exception as e: verbose_logger.exception("Error executing local tool %s: %s", name, e) - return [TextContent(text=f"Error: {e}", type="text")] + return CallToolResult(content=[TextContent(text=f"Error: {e}", type="text")], isError=True) + return CallToolResult(content=[TextContent(text=str(result), type="text")], isError=False) def _get_mcp_servers_in_path(path: str) -> list[str] | None: """ diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8e57327b31b..6f352b73290 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2439,6 +2439,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="max request size in MB, if a request is larger than this size it will be rejected", ) + max_batch_file_size_mb: int | None = Field( + None, + description="max batch input file size in MB for /v1/files uploads with purpose=batch, if a file is larger than this size it will be rejected before being forwarded to the provider", + ) max_response_size_mb: int | None = Field( None, description="max response size in MB, if a response is larger than this size it will be rejected", diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index 72ed67728d8..9fe52b7a27d 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -521,6 +521,20 @@ This is a one-time file patch and restore, not a live traffic interceptor. A Cla Cursor is not supported: it has no equivalent file-based config to hot-patch this way, since its model routing lives in its own app storage and is configured through its GUI. +#### Making It Permanent at Login + +`lite up` holds the patch only for as long as it runs. To wire Claude Code up once and leave it that way, pass `--config-claude` to `lite login`: + +```bash +lite --base-url https://your-proxy.example.com login --config-claude +``` + +It writes the same two settings `lite up` does, `env.ANTHROPIC_BASE_URL` and `apiKeyHelper`, but persistently: there is no backup, nothing to restore, and no foreground process to keep alive. Every other key in `~/.claude/settings.json` is preserved, the file is created if it does not exist, and it is written atomically with owner-only permissions. Plain `lite login` is unchanged; nothing happens to your Claude Code config unless you pass the flag. + +Because the credential is reached through `apiKeyHelper` rather than copied into the file, a later `lite login` refreshes it with no further action: Claude Code re-runs the helper on every request and picks up whatever token the most recent login stored. Nothing secret is written to `settings.json`. + +Run it again to point Claude Code at a different proxy; the base URL and the helper are both rewritten. `lite up` and `--config-claude` manage the same file, so the flag refuses to run while a `lite up` session holds a backup, and tells you to run `lite down` first, rather than writing settings that `lite up` would silently revert when it stops. + ### QA Complexity-Based Auto-Routing Against Your Real Proxy `lite autoroute` lets you try LiteLLM's complexity-based auto-routing -- picking a cheaper or more expensive model depending on how complex a prompt looks -- against models your key already has access to on your real, running proxy, without editing that proxy's `config.yaml` and without any real request ever bypassing it. It builds a second, throwaway proxy locally that forwards every request back to your real proxy, and points Claude Code at that local proxy for the duration of the session. diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py index 1cac515f9f2..0a0bcf80ee5 100644 --- a/litellm/proxy/client/cli/commands/auth.py +++ b/litellm/proxy/client/cli/commands/auth.py @@ -16,6 +16,12 @@ from typing_extensions import NotRequired, TypedDict from litellm.constants import CLI_JWT_EXPIRATION_HOURS from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh +from .claude_settings import ( + CLAUDE_SETTINGS_PATH, + SETTINGS_FILE_OWNERS, + ClaudeSettingsError, + write_claude_settings, +) from .private_json import write_private_json @@ -629,9 +635,28 @@ def _render_and_prompt_for_team_selection(teams: list[CliTeam]) -> str | None: return None +def _configure_claude_code(base_url: str) -> None: + """Point Claude Code at base_url by patching ~/.claude/settings.json.""" + try: + write_claude_settings(base_url, CLAUDE_SETTINGS_PATH, SETTINGS_FILE_OWNERS) + except ClaudeSettingsError as e: + raise click.ClickException(f"Logged in, but could not configure Claude Code: {e}") + click.echo(f"\nConfigured Claude Code: {CLAUDE_SETTINGS_PATH} now routes through {base_url.rstrip('/')}.") + click.echo("Your other Claude Code settings were left untouched. Restart Claude Code to pick this up.") + + @click.command(name="login") +@click.option( + "--config-claude", + is_flag=True, + default=False, + help=( + "After logging in, update ~/.claude/settings.json so Claude Code routes through this proxy. " + "Unrelated settings are preserved." + ), +) @click.pass_context -def login(ctx: click.Context): +def login(ctx: click.Context, config_claude: bool): """Login to LiteLLM proxy using SSO authentication""" from litellm.constants import LITELLM_CLI_SOURCE_IDENTIFIER from litellm.proxy.client.cli.interface import show_commands @@ -683,6 +708,9 @@ def login(ctx: click.Context): click.echo(f"JWT Token: {api_key[:20]}...") click.echo("You can now use the CLI without specifying --api-key") + if config_claude: + _configure_claude_code(base_url) + # Show available commands after successful login click.echo("\n" + "=" * 60) show_commands() @@ -698,6 +726,10 @@ def login(ctx: click.Context): except KeyboardInterrupt: click.echo("\nAuthentication cancelled by user.") return + except click.ClickException: + # Login itself already succeeded; only the post-login step failed, so this + # must not be relabelled as an authentication failure by the handler below. + raise except Exception as e: click.echo(f"Authentication failed: {e}") return diff --git a/litellm/proxy/client/cli/commands/autoroute/commands.py b/litellm/proxy/client/cli/commands/autoroute/commands.py index 09e5a53b92f..26d45138a27 100644 --- a/litellm/proxy/client/cli/commands/autoroute/commands.py +++ b/litellm/proxy/client/cli/commands/autoroute/commands.py @@ -10,11 +10,16 @@ import click import yaml from pydantic import JsonValue, TypeAdapter, ValidationError -from ..up import CLAUDE_SETTINGS_PATH, UpError, load_json_or_empty, restore_claude_settings, write_backup +from ..claude_settings import ( + AUTOROUTE_BACKUP_PATH, + CLAUDE_SETTINGS_PATH, + ClaudeSettingsError, + load_json_or_empty, +) from ..up import BackupRecord as ClaudeBackupRecord +from ..up import restore_claude_settings, write_backup from .config import master_key_from_config from .process import ( - AUTOROUTE_DIR, CONFIG_PATH, DEFAULT_AUTOROUTE_PORT, LOG_PATH, @@ -35,8 +40,6 @@ from .process import ( from .settings import merge_claude_settings_static_token from .wizard import run_configure_wizard -AUTOROUTE_BACKUP_PATH: Final = AUTOROUTE_DIR / "claude_settings_backup.json" - _GENERATED_CONFIG_ADAPTER: Final = TypeAdapter(dict[str, JsonValue]) @@ -108,7 +111,7 @@ def up(port: int) -> None: try: existing_pid: Final = read_pid_record() - except UpError as e: + except ClaudeSettingsError as e: raise click.ClickException(str(e)) if existing_pid is not None and is_running(existing_pid.pid): raise click.ClickException( @@ -157,7 +160,7 @@ def up(port: int) -> None: CLAUDE_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True) with secure_create(CLAUDE_SETTINGS_PATH) as f: json.dump(merged, f, indent=2) - except UpError as e: + except ClaudeSettingsError as e: terminate(process.pid) clear_pid_record() raise click.ClickException(str(e)) @@ -175,7 +178,7 @@ def up(port: int) -> None: clear_pid_record() try: restore_claude_settings(CLAUDE_SETTINGS_PATH, AUTOROUTE_BACKUP_PATH) - except UpError as e: + except ClaudeSettingsError as e: # Runs from atexit/a signal handler too, outside Click's own exception # handling -- raising here would only produce an unhandled-exception # warning on stderr, not a clean message. @@ -207,7 +210,7 @@ def down() -> None: """Restore Claude Code settings and stop a leftover ephemeral proxy, if any""" try: record: PidRecord | None = read_pid_record() - except UpError as e: + except ClaudeSettingsError as e: # down is the crash-recovery path -- a corrupt pid record must not block it; clear the # unusable record and keep going rather than leaving the user with no way to clean up. click.echo(f"{e} Clearing it and continuing cleanup.", err=True) @@ -219,7 +222,7 @@ def down() -> None: try: restored: Final = restore_claude_settings(CLAUDE_SETTINGS_PATH, AUTOROUTE_BACKUP_PATH) - except UpError as e: + except ClaudeSettingsError as e: raise click.ClickException(str(e)) if restored is None: click.echo("Nothing to restore.") diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py new file mode 100644 index 00000000000..e9a6a25a064 --- /dev/null +++ b/litellm/proxy/client/cli/commands/claude_settings.py @@ -0,0 +1,155 @@ +"""Shared handling of Claude Code's ~/.claude/settings.json. + +`lite up` patches this file temporarily and restores it on exit; `lite login +--config-claude` patches it persistently. Both need the same merge and the same +apiKeyHelper command, and `up` already imports from `auth`, so the shared parts +live here rather than in either command module. +""" + +import shlex +import shutil +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +from pydantic import JsonValue, TypeAdapter, ValidationError + +from .private_json import write_private_json + +ENV_KEY: Final = "env" +API_KEY_HELPER_KEY: Final = "apiKeyHelper" +ANTHROPIC_BASE_URL_KEY: Final = "ANTHROPIC_BASE_URL" +ANTHROPIC_API_KEY_KEY: Final = "ANTHROPIC_API_KEY" + +CLAUDE_SETTINGS_PATH: Final = Path.home() / ".claude" / "settings.json" +BACKUP_PATH: Final = Path.home() / ".litellm" / "claude_settings_backup.json" +AUTOROUTE_BACKUP_PATH: Final = Path.home() / ".litellm" / "autorouter" / "claude_settings_backup.json" + + +@dataclass(frozen=True, slots=True) +class SettingsFileOwner: + """A command that takes temporary ownership of CLAUDE_SETTINGS_PATH and restores it later.""" + + backup_path: Path + start_command: str + stop_command: str + + +SETTINGS_FILE_OWNERS: Final = ( + SettingsFileOwner(BACKUP_PATH, "lite up", "lite down"), + SettingsFileOwner(AUTOROUTE_BACKUP_PATH, "lite autoroute up", "lite autoroute down"), +) + +_SETTINGS_ADAPTER: Final = TypeAdapter(dict[str, JsonValue]) + + +class ClaudeSettingsError(Exception): + """Raised for any user-actionable failure while reading or writing Claude Code settings.""" + + +def load_json_or_empty(path: Path) -> dict[str, JsonValue]: + try: + content: Final = path.read_bytes() if path.exists() else b"" + except OSError as e: + raise ClaudeSettingsError(f"Could not read {path}: {e}") from e + if not content.strip(): + return {} + try: + return _SETTINGS_ADAPTER.validate_json(content) + except ValidationError: + raise ClaudeSettingsError( + f"{path} contains invalid JSON (or its root is not an object); cannot proceed safely." + ) + + +def merge_claude_settings( + settings: Mapping[str, JsonValue], base_url: str, api_key_helper: str +) -> dict[str, JsonValue]: + """Return a new settings dict wired to route Claude Code through the proxy. + + Only env.ANTHROPIC_BASE_URL and the top-level apiKeyHelper are overridden; a + stray env.ANTHROPIC_API_KEY is dropped so it cannot outrank the helper-issued + token (same reasoning as build_agent_env in agents.py). Every other key is + preserved untouched. + """ + raw_env: Final = settings.get(ENV_KEY, {}) + base_env: Final = raw_env if isinstance(raw_env, dict) else {} + env: Final = { + **{key: value for key, value in base_env.items() if key != ANTHROPIC_API_KEY_KEY}, + ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"), + } + return {**settings, ENV_KEY: env, API_KEY_HELPER_KEY: api_key_helper} + + +def resolve_api_key_helper(base_url: str) -> str: + """Build the shell command Claude Code should run for its apiKeyHelper. + + Resolves `lite` to an absolute path so the helper works regardless of the + PATH visible to whatever subprocess Claude Code spawns it from. Passing + --base-url explicitly (rather than relying on the bare invocation Claude + Code would otherwise use) makes `print-token` enforce that the cached + token was actually issued for this proxy -- without it, a token minted + for a different, previously-logged-into proxy would be handed to + whichever server the settings currently point at. + + --base-url belongs to the top-level `lite` group, so it has to precede the + subcommand; click rejects it outright after `print-token`. + """ + lite_path: Final = shutil.which("lite") + if lite_path is None: + raise ClaudeSettingsError( + "Could not find `lite` on your PATH. Claude Code's apiKeyHelper needs an absolute path to it." + ) + return f"{shlex.quote(lite_path)} --base-url {shlex.quote(base_url)} auth print-token" + + +def write_claude_settings(base_url: str, settings_path: Path, owners: Sequence[SettingsFileOwner]) -> None: + """Persistently point Claude Code at base_url, preserving every unrelated setting. + + Refuses while any owner holds a backup: each restores its backup when it + stops, which would silently undo this write. + """ + for owner in owners: + if owner.backup_path.exists(): + raise ClaudeSettingsError( + f"`{owner.start_command}` is currently managing {settings_path} (backup at " + f"{owner.backup_path}) and will restore it when it stops. " + f"Run `{owner.stop_command}` first, then retry." + ) + normalized_base_url: Final = base_url.rstrip("/") + api_key_helper: Final = resolve_api_key_helper(normalized_base_url) + existing: Final = load_json_or_empty(settings_path) + raw_env: Final = existing.get(ENV_KEY) + if raw_env is not None and not isinstance(raw_env, dict): + raise ClaudeSettingsError( + f'{settings_path} has a non-object "{ENV_KEY}" value, which this would discard. ' + "Fix or remove it, then retry." + ) + merged: Final = merge_claude_settings(existing, normalized_base_url, api_key_helper) + # os.replace() swaps the symlink itself for a regular file, silently detaching a + # settings.json that is symlinked into a dotfiles repo. There is no backup to undo + # that here, unlike `lite up`, so write through to the link's target instead. + target: Final = settings_path.resolve() if settings_path.is_symlink() else settings_path + try: + write_private_json(str(target), merged) + except OSError as e: + raise ClaudeSettingsError(f"Could not write {target}: {e}") from e + + +__all__ = ( + "ANTHROPIC_API_KEY_KEY", + "ANTHROPIC_BASE_URL_KEY", + "API_KEY_HELPER_KEY", + "AUTOROUTE_BACKUP_PATH", + "BACKUP_PATH", + "CLAUDE_SETTINGS_PATH", + "ENV_KEY", + "SETTINGS_FILE_OWNERS", + "ClaudeSettingsError", + "SettingsFileOwner", + "load_json_or_empty", + "merge_claude_settings", + "resolve_api_key_helper", + "write_claude_settings", +) diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index 7023241cf06..dd266b4afa1 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -2,12 +2,10 @@ import atexit import contextlib import json import os -import shlex -import shutil import signal import sys import threading -from collections.abc import Iterator, Mapping +from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path from types import FrameType @@ -20,17 +18,17 @@ from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh from .agents import AgentRunError, resolve_api_key, verify_proxy_key from .auth import load_token, login - -ENV_KEY: Final = "env" -API_KEY_HELPER_KEY: Final = "apiKeyHelper" -ANTHROPIC_BASE_URL_KEY: Final = "ANTHROPIC_BASE_URL" -ANTHROPIC_API_KEY_KEY: Final = "ANTHROPIC_API_KEY" - -CLAUDE_SETTINGS_PATH: Final = Path.home() / ".claude" / "settings.json" -BACKUP_PATH: Final = Path.home() / ".litellm" / "claude_settings_backup.json" +from .claude_settings import ( + BACKUP_PATH, + CLAUDE_SETTINGS_PATH, + ClaudeSettingsError, + load_json_or_empty, + merge_claude_settings, + resolve_api_key_helper, +) -class UpError(Exception): +class UpError(ClaudeSettingsError): """Raised for any user-actionable failure while starting/stopping interception.""" @@ -42,40 +40,9 @@ class BackupRecord: content: dict[str, JsonValue] | None -_SETTINGS_ADAPTER: Final = TypeAdapter(dict[str, JsonValue]) _BACKUP_RECORD_ADAPTER: Final = TypeAdapter(BackupRecord) -def load_json_or_empty(path: Path) -> dict[str, JsonValue]: - if not path.exists(): - return {} - with open(path, "r") as f: - content: Final = f.read() - if not content.strip(): - return {} - try: - return _SETTINGS_ADAPTER.validate_json(content) - except ValidationError: - raise UpError(f"{path} contains invalid JSON (or its root is not an object); cannot proceed safely.") - - -def merge_claude_settings( - settings: Mapping[str, JsonValue], base_url: str, api_key_helper: str -) -> dict[str, JsonValue]: - """Return a new settings dict wired to route Claude Code through the proxy. - - Only env.ANTHROPIC_BASE_URL and the top-level apiKeyHelper are overridden; a - stray env.ANTHROPIC_API_KEY is dropped so it cannot outrank the helper-issued - token (same reasoning as build_agent_env in agents.py). Every other key is - preserved untouched. - """ - raw_env: Final = settings.get(ENV_KEY, {}) - base_env: Final = raw_env if isinstance(raw_env, dict) else {} - env: Final = {**base_env, ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/")} - env.pop(ANTHROPIC_API_KEY_KEY, None) - return {**settings, ENV_KEY: env, API_KEY_HELPER_KEY: api_key_helper} - - @contextlib.contextmanager def secure_create(path: Path) -> Iterator[IO[str]]: """Open path for writing with mode 0600 fixed up before any content is written. @@ -136,26 +103,6 @@ def restore_claude_settings(settings_path: Path | None = None, backup_path: Path return record -def resolve_api_key_helper(base_url: str) -> str: - """Build the shell command Claude Code should run for its apiKeyHelper. - - Resolves `lite` to an absolute path so the helper works regardless of the - PATH visible to whatever subprocess Claude Code spawns it from. Passing - --base-url explicitly (rather than relying on the bare invocation Claude - Code would otherwise use) makes `print-token` enforce that the cached - token was actually issued for this proxy -- without it, a token minted - for a different, previously-logged-into proxy would be handed to - whichever server `up` currently points at. - """ - lite_path: Final = shutil.which("lite") - if lite_path is None: - raise UpError( - "Could not find `lite` on your PATH. Claude Code's apiKeyHelper needs " - "an absolute path to it, so `lite up` cannot continue." - ) - return f"{shlex.quote(lite_path)} auth print-token --base-url {shlex.quote(base_url)}" - - def _ensure_fresh_login(ctx: click.Context) -> None: base_url: Final = ctx.obj["base_url"].rstrip("/") token_data = load_token() @@ -224,7 +171,7 @@ def up(ctx: click.Context) -> None: merged: Final = merge_claude_settings(original_settings, base_url, api_key_helper) with open(CLAUDE_SETTINGS_PATH, "w") as f: json.dump(merged, f, indent=2) - except (AgentRunError, UpError) as e: + except (AgentRunError, ClaudeSettingsError) as e: raise click.ClickException(str(e)) click.echo(f"litellm: routing Claude Code through proxy at {base_url.rstrip('/')}") @@ -241,7 +188,7 @@ def up(ctx: click.Context) -> None: return try: _restore_and_report() - except UpError as e: + except ClaudeSettingsError as e: # Runs from atexit/a signal handler, outside Click's own exception # handling -- raising here would only produce an unhandled-exception # warning on stderr, not a clean message. @@ -264,7 +211,7 @@ def down() -> None: """ try: _restore_and_report() - except UpError as e: + except ClaudeSettingsError as e: raise click.ClickException(str(e)) @@ -272,6 +219,7 @@ __all__ = [ "BACKUP_PATH", "CLAUDE_SETTINGS_PATH", "BackupRecord", + "ClaudeSettingsError", "UpError", "down", "load_json_or_empty", diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index b6dddbb029d..65a271d4029 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1204,6 +1204,29 @@ class DBSpendUpdateWriter: except Exception as e: verbose_proxy_logger.debug("_flush_tool_discovery_queue error (non-blocking): %s", e) + @staticmethod + async def _handle_spend_update_failure( + e: Exception, + attempt: int, + n_retry_times: int, + start_time: float, + proxy_logging_obj: ProxyLogging, + ) -> None: + """Retry a failed spend-update transaction on connection errors or deadlocks, else re-raise.""" + from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler + from litellm.proxy.utils import _raise_failed_update_spend_exception + + is_retryable = isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) or PrismaDBExceptionHandler.is_deadlock_error(e) + if not is_retryable or attempt >= n_retry_times: + _raise_failed_update_spend_exception(e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj) + verbose_proxy_logger.warning( + "Retrying spend update after retryable DB error (attempt %s/%s): %s", + attempt + 1, + n_retry_times, + e, + ) + await asyncio.sleep(random.uniform(2**attempt, 2 ** (attempt + 1))) + async def _commit_spend_updates_to_db( self, prisma_client: PrismaClient, @@ -1215,10 +1238,7 @@ class DBSpendUpdateWriter: Commits all the spend `UPDATE` transactions to the Database """ - from litellm.proxy.utils import ( - ProxyUpdateSpend, - _raise_failed_update_spend_exception, - ) + from litellm.proxy.utils import ProxyUpdateSpend ### UPDATE USER TABLE ### user_list_transactions: Final = db_spend_update_transactions["user_list_transactions"] @@ -1238,18 +1258,13 @@ class DBSpendUpdateWriter: data={"spend": {"increment": response_cost}}, ) break - except DB_RETRY_SAFE_ERROR_TYPES as e: - if i >= n_retry_times: # If we've reached the maximum number of retries - _raise_failed_update_spend_exception( - e=e, - start_time=start_time, - proxy_logging_obj=proxy_logging_obj, - ) - # Optionally, sleep for a bit before retrying - await asyncio.sleep(2**i) # Exponential backoff except Exception as e: - _raise_failed_update_spend_exception( - e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj + await self._handle_spend_update_failure( + e=e, + attempt=i, + n_retry_times=n_retry_times, + start_time=start_time, + proxy_logging_obj=proxy_logging_obj, ) ### UPDATE END-USER TABLE ### @@ -1281,18 +1296,13 @@ class DBSpendUpdateWriter: }, ) break - except DB_RETRY_SAFE_ERROR_TYPES as e: - if i >= n_retry_times: # If we've reached the maximum number of retries - _raise_failed_update_spend_exception( - e=e, - start_time=start_time, - proxy_logging_obj=proxy_logging_obj, - ) - # Optionally, sleep for a bit before retrying - await asyncio.sleep(2**i) # Exponential backoff except Exception as e: - _raise_failed_update_spend_exception( - e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj + await self._handle_spend_update_failure( + e=e, + attempt=i, + n_retry_times=n_retry_times, + start_time=start_time, + proxy_logging_obj=proxy_logging_obj, ) ### UPDATE TEAM TABLE ### @@ -1314,18 +1324,13 @@ class DBSpendUpdateWriter: data={"spend": {"increment": response_cost}}, ) break - except DB_RETRY_SAFE_ERROR_TYPES as e: - if i >= n_retry_times: # If we've reached the maximum number of retries - _raise_failed_update_spend_exception( - e=e, - start_time=start_time, - proxy_logging_obj=proxy_logging_obj, - ) - # Optionally, sleep for a bit before retrying - await asyncio.sleep(2**i) # Exponential backoff except Exception as e: - _raise_failed_update_spend_exception( - e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj + await self._handle_spend_update_failure( + e=e, + attempt=i, + n_retry_times=n_retry_times, + start_time=start_time, + proxy_logging_obj=proxy_logging_obj, ) ### UPDATE TEAM Membership TABLE with spend ### @@ -1361,18 +1366,13 @@ class DBSpendUpdateWriter: ) # Transaction succeeded, break out of retry loop break - except DB_RETRY_SAFE_ERROR_TYPES as e: - if i >= n_retry_times: # If we've reached the maximum number of retries - _raise_failed_update_spend_exception( - e=e, - start_time=start_time, - proxy_logging_obj=proxy_logging_obj, - ) - # Optionally, sleep for a bit before retrying - await asyncio.sleep(2**i) # Exponential backoff except Exception as e: - _raise_failed_update_spend_exception( - e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj + await self._handle_spend_update_failure( + e=e, + attempt=i, + n_retry_times=n_retry_times, + start_time=start_time, + proxy_logging_obj=proxy_logging_obj, ) # Invalidate cache for updated team memberships @@ -1403,25 +1403,13 @@ class DBSpendUpdateWriter: data={"spend": {"increment": response_cost}}, ) break - except DB_RETRY_SAFE_ERROR_TYPES as e: - if i >= n_retry_times: # If we've reached the maximum number of retries - _raise_failed_update_spend_exception( - e=e, - start_time=start_time, - proxy_logging_obj=proxy_logging_obj, - ) - # Optionally, sleep for a bit before retrying - await asyncio.sleep( - # Sleep a random amount to avoid retrying and deadlocking again: when two transactions deadlock they are - # cancelled basically at the same time, so if they wait the same time they will also retry at the same time - # and thus they are more likely to deadlock again. - # Instead, we sleep a random amount so that they retry at slightly different times, lowering the chance of - # repeated deadlocks, and therefore of exceeding the retry limit. - random.uniform(2**i, 2 ** (i + 1)) - ) except Exception as e: - _raise_failed_update_spend_exception( - e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj + await self._handle_spend_update_failure( + e=e, + attempt=i, + n_retry_times=n_retry_times, + start_time=start_time, + proxy_logging_obj=proxy_logging_obj, ) ### UPDATE TAG TABLE ### @@ -1470,8 +1458,6 @@ class DBSpendUpdateWriter: prisma_client: Prisma client instance proxy_logging_obj: Proxy logging object """ - from litellm.proxy.utils import _raise_failed_update_spend_exception - verbose_proxy_logger.debug("%s Spend transactions: %s", entity_name, transactions) if transactions is not None and len(transactions.keys()) > 0: for i in range(n_retry_times + 1): @@ -1493,17 +1479,13 @@ class DBSpendUpdateWriter: data={"spend": {"increment": response_cost}}, ) break - except DB_RETRY_SAFE_ERROR_TYPES as e: - if i >= n_retry_times: - _raise_failed_update_spend_exception( - e=e, - start_time=start_time, - proxy_logging_obj=proxy_logging_obj, - ) - await asyncio.sleep(2**i) # Exponential backoff except Exception as e: - _raise_failed_update_spend_exception( - e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj + await DBSpendUpdateWriter._handle_spend_update_failure( + e=e, + attempt=i, + n_retry_times=n_retry_times, + start_time=start_time, + proxy_logging_obj=proxy_logging_obj, ) # fmt: off @@ -1672,7 +1654,16 @@ class DBSpendUpdateWriter: break - except DB_RETRY_SAFE_ERROR_TYPES as e: + except Exception as e: + from litellm.proxy.db.exception_handler import ( + PrismaDBExceptionHandler, + ) + + is_retryable = isinstance( + e, DB_RETRY_SAFE_ERROR_TYPES + ) or PrismaDBExceptionHandler.is_deadlock_error(e) + if not is_retryable: + raise if i >= n_retry_times: _raise_failed_update_spend_exception( e=e, diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index e0a21ceed26..f7a39aaa50f 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -166,6 +166,22 @@ class PrismaDBExceptionHandler: return True return False + @staticmethod + def is_deadlock_error(e: Exception) -> bool: + """True iff ``e`` is a Postgres deadlock (P2034 / 40P01) surfaced through prisma.""" + import prisma + + if not isinstance(e, prisma.errors.PrismaError): + return False + if getattr(e, "code", None) == "P2034": + return True + error_message = str(e).lower() + return ( + "deadlock detected" in error_message + or "40p01" in error_message + or "write conflict or a deadlock" in error_message + ) + @staticmethod def is_prisma_engine_internal_error(e: Exception) -> bool: """True iff ``e`` is a non-``PrismaError`` exception raised from inside diff --git a/litellm/proxy/db/proxy_worker_heartbeat.py b/litellm/proxy/db/proxy_worker_heartbeat.py new file mode 100644 index 00000000000..990ff48eb18 --- /dev/null +++ b/litellm/proxy/db/proxy_worker_heartbeat.py @@ -0,0 +1,93 @@ +""" +Live proxy worker census, one row per worker process. + +Every uvicorn worker upserts its own row on a fixed heartbeat, so counting +rows with a recent heartbeat answers "how many workers share this database?" +without any coordination. The Admin UI's "no Redis" banner uses that count to +hide itself for deployments that are provably a single worker, where per-worker +rate limits, budgets, and router state are already global. All timestamps are +written and compared with the database's own clock, so pods with skewed clocks +still agree. +""" + +from __future__ import annotations + +import socket +from typing import TYPE_CHECKING, Final + +from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict + +from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid +from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS: Final = 60 +PROXY_WORKER_LIVENESS_WINDOW_SECONDS: Final = 3 * PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS +STALE_ROW_RETENTION_SECONDS: Final = 3600 + +BEAT_SQL: Final = """ +INSERT INTO "LiteLLM_ProxyWorkerHeartbeat" (worker_id, hostname, last_heartbeat_at) +VALUES ($1, $2, NOW()) +ON CONFLICT (worker_id) DO UPDATE SET last_heartbeat_at = NOW() +""" + +PRUNE_SQL: Final = """ +DELETE FROM "LiteLLM_ProxyWorkerHeartbeat" +WHERE last_heartbeat_at < NOW() - make_interval(secs => $1) +""" + +COUNT_SQL: Final = """ +SELECT COUNT(*)::int AS live_workers FROM "LiteLLM_ProxyWorkerHeartbeat" +WHERE last_heartbeat_at > NOW() - make_interval(secs => $1) +""" + +DEREGISTER_SQL: Final = """ +DELETE FROM "LiteLLM_ProxyWorkerHeartbeat" WHERE worker_id = $1 +""" + + +class _LiveWorkerCountRow(TypedDict): + live_workers: ReadOnly[int] + + +_COUNT_ROWS_ADAPTER: Final = TypeAdapter(tuple[_LiveWorkerCountRow, ...]) + + +class ProxyWorkerHeartbeat: + def __init__(self, prisma_client: PrismaClient, worker_id: str | None = None) -> None: + self.prisma_client: Final = prisma_client + self.worker_id: Final[str] = worker_id or str(uuid.uuid4()) + self.hostname: Final = socket.gethostname() + + async def beat(self) -> None: + try: + await self.prisma_client.db.execute_raw(BEAT_SQL, self.worker_id, self.hostname) + await self.prisma_client.db.execute_raw(PRUNE_SQL, STALE_ROW_RETENTION_SECONDS) + except Exception as beat_err: # noqa: BLE001 # a missed heartbeat must never take down the worker + verbose_proxy_logger.debug("Proxy worker heartbeat write failed: %s", beat_err) + + async def deregister(self) -> None: + try: + await self.prisma_client.db.execute_raw(DEREGISTER_SQL, self.worker_id) + except Exception as deregister_err: # noqa: BLE001 # best-effort cleanup; the liveness window ages the row out anyway + verbose_proxy_logger.debug("Proxy worker heartbeat deregister failed: %s", deregister_err) + + +async def count_live_proxy_workers(prisma_client: PrismaClient) -> int | None: + """ + The number of workers with a recent heartbeat, or None when the database + cannot answer. Callers must treat None as "unknown", not as zero. Always + counts on the primary: a lagging read replica must never undercount. + """ + try: + db: Final = prisma_client.db + primary_db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) else db + rows: Final = await primary_db.query_raw(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS) + return _COUNT_ROWS_ADAPTER.validate_python(rows)[0]["live_workers"] + except Exception as count_err: # noqa: BLE001 # an unknown count must degrade to "warn", never to a 503 + verbose_proxy_logger.debug("Live proxy worker count unavailable: %s", count_err) + return None diff --git a/litellm/proxy/example_config_yaml/agentcore_websearch_config.yaml b/litellm/proxy/example_config_yaml/agentcore_websearch_config.yaml new file mode 100644 index 00000000000..12402095c4d --- /dev/null +++ b/litellm/proxy/example_config_yaml/agentcore_websearch_config.yaml @@ -0,0 +1,40 @@ +# Claude Code / Anthropic-native web search on Bedrock, backed by +# Amazon Bedrock AgentCore Web Search (AWS-managed web index, no third-party +# search API). See litellm/llms/bedrock/search/transformation.py for details. + +model_list: + - model_name: claude-sonnet + litellm_params: + model: bedrock/us.anthropic.claude-sonnet-5 + aws_region_name: us-east-1 + +search_tools: + - search_tool_name: agentcore-search + litellm_params: + search_provider: agentcore + # Your AgentCore Gateway MCP endpoint (gateway must have a `web-search` + # connector target). Alternatively set the AGENTCORE_GATEWAY_URL env var. + api_base: https://.gateway.bedrock-agentcore.us-east-1.amazonaws.com/mcp + + # The gateway exposes the connector as "___WebSearch". + # Default is "web-search-tool___WebSearch", matching the target name used + # in the AWS docs' boto3/CLI setup examples. If your target was created + # with a different name (misconfiguration surfaces as an MCP "tool not + # found" error), set the AGENTCORE_SEARCH_TOOL_NAME env var or pass + # tool_name in the request body. The search router forwards only + # search_provider / api_key / api_base from this litellm_params block, + # so a tool_name set here would be silently ignored. + + # AWS_IAM gateway (default): SigV4-signed using the standard AWS + # credential chain (env / profile / IRSA / instance role). Explicit + # aws_access_key_id / aws_secret_access_key set here would be silently + # ignored for the same reason; pass them per request instead. + + # CUSTOM_JWT gateway alternative — OAuth2 bearer token instead of SigV4: + # api_key: os.environ/AGENTCORE_GATEWAY_TOKEN + +litellm_settings: + callbacks: ["websearch_interception"] + websearch_interception_params: + enabled_providers: ["bedrock"] + search_tool_name: agentcore-search diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index e814ec42d26..33894777bc3 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -34,6 +34,7 @@ from litellm.proxy.auth.auth_utils import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.db.proxy_worker_heartbeat import count_live_proxy_workers from litellm.proxy.health_check import ( ADMIN_ONLY_HEALTH_DISPLAY_PARAMS, _clean_endpoint_data, @@ -1451,7 +1452,7 @@ def callback_name(callback): DISABLE_NO_REDIS_WARNING_ENV_VAR: Final = "LITELLM_DISABLE_NO_REDIS_WARNING" -def _show_no_redis_warning() -> bool: +async def _show_no_redis_warning() -> bool: """ Whether the UI should warn that no Redis is configured. @@ -1461,16 +1462,22 @@ def _show_no_redis_warning() -> bool: coordination cache (from a Redis response cache, general_settings. coordination_redis, or the REDIS_* env fallback) and the router's own Redis (router_settings.redis_host), which backs cooldowns and usage-based - routing on its own. Operators who know they run one worker can silence the - warning with LITELLM_DISABLE_NO_REDIS_WARNING=true. + routing on its own. A deployment whose worker-heartbeat census proves it + is exactly one worker needs no cross-worker coordination, so it never + warns; when the census is unavailable or shows more than one worker, the + warning stands unless LITELLM_DISABLE_NO_REDIS_WARNING=true silences it. """ - from litellm.proxy.proxy_server import llm_router, redis_usage_cache + from litellm.proxy.proxy_server import llm_router, prisma_client, redis_usage_cache if redis_usage_cache is not None: return False if llm_router is not None and llm_router.cache.redis_cache is not None: return False - return get_secret_bool(DISABLE_NO_REDIS_WARNING_ENV_VAR, False) is not True + if get_secret_bool(DISABLE_NO_REDIS_WARNING_ENV_VAR, False) is True: + return False + if prisma_client is None: + return True + return await count_live_proxy_workers(prisma_client) != 1 async def _get_health_readiness_details( @@ -1513,7 +1520,7 @@ async def _get_health_readiness_details( # check log level log_level_name: Final = logging.getLevelName(verbose_logger.getEffectiveLevel()) is_detailed_debug: Final = verbose_logger.isEnabledFor(logging.DEBUG) - show_no_redis_warning: Final = _show_no_redis_warning() + show_no_redis_warning: Final = await _show_no_redis_warning() # check DB if prisma_client is not None: # if db passed in, check if it's connected diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index 6231563450b..88803d6442d 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -43,6 +43,7 @@ class KeyManagementEventHooks: from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, get_audit_log_changed_by, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name @@ -53,8 +54,7 @@ class KeyManagementEventHooks: except Exception as e: verbose_proxy_logger.warning("Failed to send key created email: %s", e) - # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True - if litellm.store_audit_logs is True: + if is_audit_logging_enabled(): _updated_values: Final = response.model_dump_json(exclude_none=True) asyncio.create_task( create_audit_log_for_update( @@ -103,11 +103,11 @@ class KeyManagementEventHooks: from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, get_audit_log_changed_by, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name - # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True - if litellm.store_audit_logs is True: + if is_audit_logging_enabled(): _updated_values: Final = json.dumps(data.json(exclude_none=True), default=str) _before_value = existing_key_row.json(exclude_none=True) @@ -144,6 +144,7 @@ class KeyManagementEventHooks: from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, get_audit_log_changed_by, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name @@ -180,7 +181,7 @@ class KeyManagementEventHooks: verbose_proxy_logger.warning("Failed to send key rotated email: %s", e) # store the audit log - if litellm.store_audit_logs is True and existing_key_row.token is not None: + if is_audit_logging_enabled() and existing_key_row.token is not None: asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( @@ -218,12 +219,12 @@ class KeyManagementEventHooks: from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, get_audit_log_changed_by, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name - # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True # we do this after the first for loop, since first for loop is for validation. we only want this inserted after validation passes - if litellm.store_audit_logs is True and data.keys is not None: + if is_audit_logging_enabled() and data.keys is not None: # make an audit log for each key deleted for key in keys_being_deleted: if key.token is None: diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index 929df2a778c..6d978929c05 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -20,7 +20,10 @@ from litellm.proxy._types import ( UserAPIKeyAuth, WebhookEvent, ) -from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update +from litellm.proxy.management_helpers.audit_logs import ( + create_audit_log_for_update, + is_audit_logging_enabled, +) from litellm.repositories.user_repository import UserRepository @@ -203,7 +206,7 @@ class UserManagementEventHooks: - user_api_key_dict: UserAPIKeyAuth - The user api key dictionary. - litellm_proxy_admin_name: Optional[str] - The name of the proxy admin. """ - if not litellm.store_audit_logs: + if not is_audit_logging_enabled(): return from litellm.proxy.management_helpers.audit_logs import ( diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index d8b7414f32c..0112ad1f6ed 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -6,10 +6,13 @@ POST /auto_router/test_routing - Route one prompt through an unsaved complexity- from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone +from itertools import groupby +from operator import attrgetter from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Protocol +from uuid import uuid4 -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator from litellm._logging import verbose_proxy_logger from litellm.exceptions import BudgetExceededError @@ -41,6 +44,8 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( AutoRouterRoutingTestRequest, AutoRouterRoutingTestResponse, RequestComplexityRouterConfig, + ShadowEvalDirection, + ShadowEvalJobKeyResponse, ShadowEvalJobResponse, ShadowEvalResult, ShadowEvalSlice, @@ -89,17 +94,9 @@ class _ShadowEvalJobRow(Protocol): class _ShadowEvalJobTable(Protocol): - async def find_unique(self, *, where: Mapping[str, object]) -> _ShadowEvalJobRow | None: ... + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_ShadowEvalJobRow]: ... - async def find_first(self, *, where: Mapping[str, object]) -> _ShadowEvalJobRow | None: ... - - async def find_many( - self, *, where: Mapping[str, object], order: Mapping[str, str], take: int - ) -> Sequence[_ShadowEvalJobRow]: ... - - async def create(self, data: Mapping[str, object]) -> _ShadowEvalJobRow: ... - - async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> _ShadowEvalJobRow | None: ... + async def create_many(self, data: Sequence[Mapping[str, object]]) -> int: ... class _ShadowEvalAttemptRow(Protocol): @@ -606,18 +603,19 @@ _ATTEMPT_AGG_SELECT: Final = """ COUNT(*) FILTER (WHERE outcome = 'tie')::int AS ties, AVG(confidence)::float AS avg_confidence FROM "LiteLLM_ShadowEvalAttempt" -WHERE job_id = $1 AND outcome != 'error' +WHERE job_id = ANY($1::text[]) AND outcome != 'error' GROUP BY 1 """ _ATTEMPT_AGG_BY_TIER_SQL: Final = "SELECT COALESCE(tier, 'UNCLASSIFIED') AS grp," + _ATTEMPT_AGG_SELECT _ATTEMPT_AGG_BY_MODEL_SQL: Final = "SELECT COALESCE(real_model, 'unknown') AS grp," + _ATTEMPT_AGG_SELECT +_ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT _SWEEP_FINISHED_JOBS_SQL: Final = """ -UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = NOW() -WHERE j.api_key_id = $1 AND j.stopped_at IS NULL +UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = (NOW() AT TIME ZONE 'utc') +WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL AND ( - j.ends_at <= NOW() + j.ends_at <= (NOW() AT TIME ZONE 'utc') OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns ) """ @@ -628,7 +626,52 @@ SELECT COUNT(*) FILTER (WHERE outcome = 'error')::int AS error_count, COALESCE(SUM(judge_cost), 0)::float AS judge_spend FROM "LiteLLM_ShadowEvalAttempt" -WHERE job_id = $1 +WHERE job_id = ANY($1::text[]) +""" + +_ATTEMPT_COUNTS_SQL: Final = """ +SELECT a.job_id, COUNT(*)::int AS attempt_count +FROM "LiteLLM_ShadowEvalAttempt" a +JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id +WHERE a.job_id = ANY($1::text[]) AND (j.stopped_at IS NULL OR a.created_at <= j.stopped_at) +GROUP BY a.job_id +""" + +_STOP_JOB_SQL: Final = """ +UPDATE "LiteLLM_ShadowEvalJob" +SET stopped_by = $2, stopped_at = COALESCE(stopped_at, $3::timestamp) +WHERE group_id = $1 AND stopped_by IS NULL + AND ends_at > (NOW() AT TIME ZONE 'utc') + AND EXISTS ( + SELECT 1 FROM "LiteLLM_ShadowEvalJob" k + WHERE k.group_id = $1 AND k.stopped_at IS NULL + AND (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_turns + ) +""" + + +class _AttemptCountRow(BaseModel): + job_id: str + attempt_count: int + + +_ATTEMPT_COUNT_ROWS: Final = TypeAdapter(list[_AttemptCountRow]) + + +_LIST_LEGS_SQL: Final = """ +SELECT * FROM "LiteLLM_ShadowEvalJob" +WHERE group_id IN ( + SELECT group_id FROM "LiteLLM_ShadowEvalJob" + GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int +) +""" + +_LIST_LEGS_BY_KEY_SQL: Final = """ +SELECT * FROM "LiteLLM_ShadowEvalJob" +WHERE group_id IN ( + SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE api_key_id = $2 + GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int +) """ @@ -659,18 +702,98 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]: ) +class _LegRow(BaseModel): + """One LiteLLM_ShadowEvalJob row, validated off the untyped prisma record. A row is + one key's leg of a job; the legs of a job share group_id and identical config, written + together by one create_many. The API's job id is the group id, so leg ids never leave + the server (attempts reference them internally).""" + + model_config = ConfigDict(from_attributes=True) + + id: str + group_id: str + api_key_id: str + router_name: str + direction: ShadowEvalDirection + baseline_model: str | None = None + judge_model: str + shadow_percentage: float + max_turns: int + created_at: datetime + ends_at: datetime + stopped_at: datetime | None = None + stopped_by: str | None = None + + @field_validator("created_at", "ends_at", "stopped_at") + @classmethod + def _as_aware_utc(cls, value: datetime | None) -> datetime | None: + """The columns store naive UTC wall time (prisma's convention); prisma reads hand + back aware datetimes while raw SQL reads hand back naive ones, so this boundary + makes every read aware UTC before anything compares or serializes them.""" + if value is None or value.tzinfo is not None: + return value + return value.replace(tzinfo=timezone.utc) + + +_LEG_ROWS: Final = TypeAdapter(list[_LegRow]) + + +async def _leg_attempt_counts(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> Mapping[str, int]: + """Each leg's attempt count by leg id, judged and errored alike, in one grouped read. + It is the same count the sampler budgets against max_turns, so the derived status + flips to completed exactly when sampling actually ends. A stamped leg's count freezes + at its stopped_at: in-flight attempts that land after the stamp are excluded, so they + can never reclassify a leg that was stopped under budget as budget-spent.""" + if not legs: + return MappingProxyType({}) + rows: Final = _ATTEMPT_COUNT_ROWS.validate_python( + await _query_raw(prisma_client, _ATTEMPT_COUNTS_SQL, [leg.id for leg in legs]) # mutable-ok: query param + or () + ) + return MappingProxyType({row.job_id: row.attempt_count for row in rows}) + + +def _group_response(group_id: str, legs: Sequence[_LegRow], attempt_counts: Mapping[str, int]) -> ShadowEvalJobResponse: + """The one constructor of a job response: the caller names the group and passes that + group's legs. Config is read off the first leg because every leg carries the same copy, + written by one create_many. No caller may serialize a raw row (that would leak a leg id + as the job id).""" + first: Final = legs[0] + return ShadowEvalJobResponse( + job_id=group_id, + keys=tuple( + ShadowEvalJobKeyResponse( + api_key_id=leg.api_key_id, + max_turns=leg.max_turns, + stopped_at=leg.stopped_at, + attempt_count=attempt_counts.get(leg.id, 0), + ) + for leg in sorted(legs, key=lambda leg: leg.api_key_id) + ), + router_name=first.router_name, + direction=first.direction, + baseline_model=first.baseline_model, + judge_model=first.judge_model, + shadow_percentage=first.shadow_percentage, + created_at=first.created_at, + ends_at=first.ends_at, + stopped_by=next((leg.stopped_by for leg in legs if leg.stopped_by is not None), None), + ) + + _NO_KEY_LABELS: Final[tuple[str | None, str | None]] = (None, None) async def _with_key_labels( prisma_client: "PrismaClient", responses: Sequence[ShadowEvalJobResponse] ) -> tuple[ShadowEvalJobResponse, ...]: - """Resolve each job's key hash to the key's alias and masked name in one batched read, + """Resolve every scoped key's hash to its alias and masked name in one batched read, so the UI can say whose traffic a job shadows. Deleted keys resolve to None.""" if not responses: return () + tokens: Final = sorted(frozenset(key.api_key_id for response in responses for key in response.keys)) key_rows: Final = await _verification_tokens(prisma_client).find_many( - where={"token": {"in": sorted({response.api_key_id for response in responses})}} # mutable-ok: Prisma filter + where={"token": {"in": tokens}} # mutable-ok: Prisma filter ) labels: Final[Mapping[str, tuple[str | None, str | None]]] = { row.token: (row.key_alias, row.key_name) for row in key_rows or () @@ -678,32 +801,50 @@ async def _with_key_labels( return tuple( response.model_copy( update={ # mutable-ok: pydantic update payload - "key_alias": labels.get(response.api_key_id, _NO_KEY_LABELS)[0], - "key_name": labels.get(response.api_key_id, _NO_KEY_LABELS)[1], + "keys": tuple( + key.model_copy( + update={ # mutable-ok: pydantic update payload + "key_alias": labels.get(key.api_key_id, _NO_KEY_LABELS)[0], + "key_name": labels.get(key.api_key_id, _NO_KEY_LABELS)[1], + } + ) + for key in response.keys + ) } ) for response in responses ) -async def _shadow_eval_results(prisma_client: "PrismaClient", job_id: str) -> ShadowEvalResult | None: - """Both stratifications of one job's verdicts. Tier answers "where does the router do - well"; the model stratification groups by whichever model served the real arm, so it - answers "which of the models this key uses today would the router beat" forward, and - "for the turns the router sent to X, did X beat the baseline" in reverse. Reads are - bounded by the job's own attempts (<= max_turns) via the job_id index.""" +async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> ShadowEvalResult | None: + """All three stratifications of one job's verdicts. Tier answers "where does the router + do well"; the model stratification groups by whichever model served the real arm, so it + answers "which of the models these keys use today would the router beat" forward, and + "for the turns the router sent to X, did X beat the baseline" in reverse; key answers + "which key's traffic does the router suit". Reads are bounded by the job's own attempts + (<= the sum of its keys' max_turns) via the job_id index.""" + leg_ids: Final = [leg.id for leg in legs] # mutable-ok: query param by_tier: Final = _ATTEMPT_AGG_ROWS.validate_python( - await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, job_id) or () + await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, leg_ids) or () ) if not by_tier: return None by_model: Final = _ATTEMPT_AGG_ROWS.validate_python( - await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, job_id) or () + await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, leg_ids) or () + ) + key_by_leg: Final = MappingProxyType({leg.id: leg.api_key_id for leg in legs}) + by_leg: Final = _ATTEMPT_AGG_ROWS.validate_python( + await _query_raw(prisma_client, _ATTEMPT_AGG_BY_LEG_SQL, leg_ids) or () + ) + by_key: Final = tuple( + row.model_copy(update={"grp": key_by_leg[row.grp]}) # mutable-ok: pydantic update payload + for row in by_leg ) total_turns: Final = sum(r.turn_count for r in by_tier) return ShadowEvalResult( by_tier=_slices(by_tier), by_current_model=_slices(by_model), + by_key=_slices(by_key), overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns), overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns), ) @@ -721,20 +862,21 @@ async def start_shadow_eval( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> ShadowEvalJobResponse: """ - Start a shadow eval: duplicate a sampled slice of a key's live traffic against a second - arm, judge the two responses blind, and stratify win rates by tier and by the model that - served the real arm. + Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against + a second arm, judge the two responses blind, and stratify win rates by tier, by the model + that served the real arm, and by key. - A forward job answers whether the key should adopt router_name: it samples the requests + A forward job answers whether the keys should adopt router_name: it samples the requests the router did not serve and duplicates them through it. A reverse job answers whether a key already on the router still gains from it: it samples the requests the router did serve and duplicates them against baseline_model. A key can hold one active job per direction, so both questions can run at once. - Shadow responses are never served to users. The job samples until it has judged - max_turns turns, reaches the end of its window, or is stopped; sampling changes - propagate to pods within about 10 seconds. Shadow and judge calls bill to the - shadowed key but are excluded from request counts and auto-router adoption metrics. + Shadow responses are never served to users. Each key samples until it has judged + max_turns turns of its own traffic, the job's window ends, or the job is stopped, so one + key running out of budget does not end sampling for the others; sampling changes + propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed + key but are excluded from request counts and auto-router adoption metrics. """ from litellm.proxy.proxy_server import llm_router, prisma_client @@ -746,48 +888,58 @@ async def start_shadow_eval( _validate_plain_model(llm_router, data.judge_model, "judge_model") if data.baseline_model is not None: _validate_plain_model(llm_router, data.baseline_model, "baseline_model") - key_row: Final = await _verification_tokens(prisma_client).find_unique( - where={"token": data.api_key_id} # mutable-ok: Prisma filter + token_rows: Final = await _verification_tokens(prisma_client).find_many( + where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter ) - if key_row is None: + unknown: Final = tuple(sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ()))) + if unknown: raise HTTPException( status_code=400, detail=( - f"api_key_id '{data.api_key_id}' is not a key on this proxy; pass the key's token hash, " + f"api_key_ids not on this proxy: {', '.join(unknown)}; pass each key's token hash, " "the value the key list and key info endpoints report" ), ) - # A job that expired or exhausted its turn budget stopped sampling on its own, but - # still holds its slot in the per-key, per-direction partial unique index until - # stamped; free it so a new eval can start. Sweeping both directions is deliberate. - await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, data.api_key_id) - active: Final = await _shadow_eval_jobs(prisma_client).find_first( + # A job whose window passed or whose turn budget ran out stopped sampling on its own, + # but its legs still hold their slots in the per-key, per-direction partial unique index + # until stamped; free them so a new eval can start. Sweeping both directions is deliberate. + requested: Final = list(data.api_key_ids) # mutable-ok: query param + await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, requested) + claimed: Final = await _shadow_eval_jobs(prisma_client).find_many( where={ # mutable-ok: Prisma filter - "api_key_id": data.api_key_id, + "api_key_id": {"in": requested}, # mutable-ok: Prisma filter "direction": data.direction, "stopped_at": None, }, ) - if active is not None: + if claimed: raise HTTPException( status_code=409, - detail=f"Key already has an active {data.direction} shadow eval job ({active.id}). Stop it first.", + detail=( + f"Already in an active {data.direction} shadow eval job: " + + ", ".join(sorted(f"{row.api_key_id} (job {row.group_id})" for row in claimed)) + + ". Stop it first." + ), ) now: Final = datetime.now(timezone.utc) + group_id: Final = str(uuid4()) + ends_at: Final = now + timedelta(days=data.duration_days) + shared_config: Final = { # mutable-ok: Prisma payload + "group_id": group_id, + "router_name": data.router_name, + "direction": data.direction, + "baseline_model": data.baseline_model, + "judge_model": data.judge_model, + "shadow_percentage": data.shadow_percentage, + "max_turns": data.max_turns, + "created_by": user_api_key_dict.user_id, + "created_at": now, + "ends_at": ends_at, + } try: - job: Final = await _shadow_eval_jobs(prisma_client).create( - data={ # mutable-ok: Prisma payload - "api_key_id": data.api_key_id, - "router_name": data.router_name, - "direction": data.direction, - "baseline_model": data.baseline_model, - "judge_model": data.judge_model, - "shadow_percentage": data.shadow_percentage, - "max_turns": data.max_turns, - "created_by": user_api_key_dict.user_id, - "ends_at": now + timedelta(days=data.duration_days), - } + await _shadow_eval_jobs(prisma_client).create_many( + data=[{**shared_config, "api_key_id": key} for key in data.api_key_ids] # mutable-ok: Prisma payload ) except Exception as e: if not _is_unique_violation(e): @@ -795,11 +947,28 @@ async def start_shadow_eval( raise HTTPException( status_code=409, detail=( - f"Key already has an active {data.direction} shadow eval job (started concurrently). Stop it first." + f"A requested key was claimed by another {data.direction} shadow eval job concurrently. Stop it first." ), ) from e - return ShadowEvalJobResponse.model_validate(job, from_attributes=True).model_copy( - update={"key_alias": key_row.key_alias, "key_name": key_row.key_name} # mutable-ok: pydantic update payload + labels: Final = MappingProxyType({row.token: row for row in token_rows}) + return ShadowEvalJobResponse( + job_id=group_id, + keys=tuple( + ShadowEvalJobKeyResponse( + api_key_id=api_key_id, + max_turns=data.max_turns, + key_alias=labels[api_key_id].key_alias, + key_name=labels[api_key_id].key_name, + ) + for api_key_id in sorted(data.api_key_ids) + ), + router_name=data.router_name, + direction=data.direction, + baseline_model=data.baseline_model, + judge_model=data.judge_model, + shadow_percentage=data.shadow_percentage, + created_at=now, + ends_at=ends_at, ) @@ -811,23 +980,38 @@ async def start_shadow_eval( ) async def list_shadow_eval_jobs( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], - api_key_id: Annotated[str | None, Query(description="Filter to jobs shadowing this key")] = None, + api_key_id: Annotated[ + str | None, Query(description="Filter to jobs that shadow this key, alone or alongside others") + ] = None, limit: Annotated[int, Query(ge=1, le=200, description="Newest jobs to return")] = 50, ) -> tuple[ShadowEvalJobResponse, ...]: - """List shadow eval jobs, newest first. Counts and results ride the detail endpoint only.""" + """List shadow eval jobs, newest first, each key with its attempt count so status is + accurate. Judged counts, spend, and results ride the detail endpoint only.""" from litellm.proxy.proxy_server import prisma_client _require_admin_viewer(user_api_key_dict, "view shadow evals") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - records: Final = await _shadow_eval_jobs(prisma_client).find_many( - where={"api_key_id": api_key_id} if api_key_id else {}, # mutable-ok: Prisma filter - order={"created_at": "desc"}, # mutable-ok: Prisma order - take=limit, + legs: Final = _LEG_ROWS.validate_python( + ( + await _query_raw(prisma_client, _LIST_LEGS_BY_KEY_SQL, limit, api_key_id) + if api_key_id + else await _query_raw(prisma_client, _LIST_LEGS_SQL, limit) + ) + or () ) + by_group: Final[Mapping[str, tuple[_LegRow, ...]]] = MappingProxyType( + { + group_id: tuple(group) + for group_id, group in groupby(sorted(legs, key=attrgetter("group_id")), key=attrgetter("group_id")) + } + ) + newest_first: Final = sorted( + by_group, key=lambda group_id: max(leg.created_at for leg in by_group[group_id]), reverse=True + ) + counts: Final = await _leg_attempt_counts(prisma_client, legs) return await _with_key_labels( - prisma_client, - tuple(ShadowEvalJobResponse.model_validate(record, from_attributes=True) for record in records or ()), + prisma_client, tuple(_group_response(group_id, by_group[group_id], counts) for group_id in newest_first) ) @@ -847,20 +1031,24 @@ async def get_shadow_eval_job( _require_admin_viewer(user_api_key_dict, "view shadow evals") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - record: Final = await _shadow_eval_jobs(prisma_client).find_unique( - where={"id": job_id} # mutable-ok: Prisma filter + legs: Final = _LEG_ROWS.validate_python( + await _shadow_eval_jobs(prisma_client).find_many( + where={"group_id": job_id} # mutable-ok: Prisma filter + ) + or () ) - if record is None: + if not legs: raise HTTPException(status_code=404, detail=f"No shadow eval job {job_id}") + leg_ids: Final = [leg.id for leg in legs] # mutable-ok: query param totals: Final = _ATTEMPT_TOTALS_ROWS.validate_python( - await _query_raw(prisma_client, _ATTEMPT_TOTALS_SQL, job_id) or () + await _query_raw(prisma_client, _ATTEMPT_TOTALS_SQL, leg_ids) or () ) latest_error: Final = await _shadow_eval_attempts(prisma_client).find_first( - where={"job_id": job_id, "outcome": "error"}, # mutable-ok: Prisma filter + where={"job_id": {"in": leg_ids}, "outcome": "error"}, # mutable-ok: Prisma filter order={"created_at": "desc"}, # mutable-ok: Prisma order ) labeled: Final = await _with_key_labels( - prisma_client, (ShadowEvalJobResponse.model_validate(record, from_attributes=True),) + prisma_client, (_group_response(job_id, legs, await _leg_attempt_counts(prisma_client, legs)),) ) return labeled[0].model_copy( update={ # mutable-ok: pydantic update payload @@ -868,7 +1056,7 @@ async def get_shadow_eval_job( "error_count": totals[0].error_count if totals else 0, "judge_spend": round(totals[0].judge_spend, 6) if totals else 0.0, "last_error": latest_error.error if latest_error else None, - "results": await _shadow_eval_results(prisma_client, job_id), + "results": await _shadow_eval_results(prisma_client, legs), } ) @@ -883,25 +1071,33 @@ async def stop_shadow_eval_job( job_id: str, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> ShadowEvalJobResponse: - """Stop an active shadow eval job. Attempts are kept; sampling halts within ~10s.""" + """Stop an active shadow eval job, every key it scopes at once. Attempts are kept; + sampling halts within ~10s. Keys that already stopped on their own budget keep the + stopped_at they earned. The statement is the whole state machine: it claims the job + only while a leg still samples inside the window with no stop recorded, so a racing + operator, a same-instant budget spend, and a repeat stop all read the same 400 with + the status the job actually holds.""" from litellm.proxy.proxy_server import prisma_client _require_admin_writer(user_api_key_dict, "stop a shadow eval") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - record: Final = await _shadow_eval_jobs(prisma_client).find_unique( - where={"id": job_id} # mutable-ok: Prisma filter + stamp: Final = datetime.now(timezone.utc) + operator: Final = user_api_key_dict.user_id or "operator" + claimed: Final = await prisma_client.db.execute_raw( + _STOP_JOB_SQL, job_id, operator, stamp.replace(tzinfo=None).isoformat() ) - if record is None: + legs: Final = _LEG_ROWS.validate_python( + await _shadow_eval_jobs(prisma_client).find_many( + where={"group_id": job_id} # mutable-ok: Prisma filter + ) + or () + ) + if not legs: raise HTTPException(status_code=404, detail=f"No shadow eval job {job_id}") - current: Final = ShadowEvalJobResponse.model_validate(record, from_attributes=True) - if current.status != "running": + counts: Final = await _leg_attempt_counts(prisma_client, legs) + current: Final = _group_response(job_id, legs, counts) + if claimed == 0: raise HTTPException(status_code=400, detail=f"Job {job_id} is already {current.status}") - updated: Final = await _shadow_eval_jobs(prisma_client).update( - where={"id": job_id}, # mutable-ok: Prisma filter - data={"stopped_at": datetime.now(timezone.utc)}, # mutable-ok: Prisma payload - ) - labeled: Final = await _with_key_labels( - prisma_client, (ShadowEvalJobResponse.model_validate(updated, from_attributes=True),) - ) + labeled: Final = await _with_key_labels(prisma_client, (current,)) return labeled[0] diff --git a/litellm/proxy/management_endpoints/cache_settings_endpoints.py b/litellm/proxy/management_endpoints/cache_settings_endpoints.py index 53d03bc7ba6..385073edc90 100644 --- a/litellm/proxy/management_endpoints/cache_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/cache_settings_endpoints.py @@ -17,7 +17,6 @@ from typing import TYPE_CHECKING, Any, Final, Protocol from fastapi import APIRouter, Depends, Header, HTTPException from pydantic import BaseModel, Field -import litellm from litellm._logging import verbose_proxy_logger from litellm._redis import _redis_kwargs_from_environment from litellm._uuid import uuid @@ -299,14 +298,15 @@ async def _emit_cache_settings_audit_log( exception. Captured under ``LiteLLM_CacheConfig`` so the row co-locates with the table it mutates. """ - if litellm.store_audit_logs is not True: - return - from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name + if not is_audit_logging_enabled(): + return + task: Final = asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( diff --git a/litellm/proxy/management_endpoints/config_override_endpoints.py b/litellm/proxy/management_endpoints/config_override_endpoints.py index 12c99477d3c..dde0751d98d 100644 --- a/litellm/proxy/management_endpoints/config_override_endpoints.py +++ b/litellm/proxy/management_endpoints/config_override_endpoints.py @@ -100,16 +100,15 @@ async def _emit_hashicorp_vault_audit_log( ``LiteLLM_ConfigOverrides`` so the row co-locates with the table it mutates. """ - import litellm - - if litellm.store_audit_logs is not True: - return - from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name + if not is_audit_logging_enabled(): + return + task: Final = asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( diff --git a/litellm/proxy/management_endpoints/coordination_redis_endpoints.py b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py index fe9a613656d..86ce336c7a3 100644 --- a/litellm/proxy/management_endpoints/coordination_redis_endpoints.py +++ b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py @@ -243,12 +243,15 @@ async def _emit_coordination_redis_audit_log( litellm_changed_by: str | None, ) -> None: """Emit an audit-log row for a /coordination_redis/settings mutation.""" - if litellm.store_audit_logs is not True: - return - - from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update + from litellm.proxy.management_helpers.audit_logs import ( + create_audit_log_for_update, + is_audit_logging_enabled, + ) from litellm.proxy.proxy_server import litellm_proxy_admin_name + if not is_audit_logging_enabled(): + return + task: Final = asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 2b88658e1b4..99a85e02b52 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -2220,6 +2220,7 @@ async def delete_user( ) from litellm.proxy.management_helpers.audit_logs import ( get_audit_log_changed_by, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import ( create_audit_log_for_update, @@ -2298,9 +2299,8 @@ async def delete_user( }, ) - # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True # we do this after the first for loop, since first for loop is for validation. we only want this inserted after validation passes - if litellm.store_audit_logs is True: + if is_audit_logging_enabled(): # make an audit log for each team deleted _user_row = user_row.json(exclude_none=True) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index ade568ad07f..71218d6114b 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1423,6 +1423,11 @@ async def _check_team_key_limits( ) +_INHERITED_MODEL_SENTINELS: Final = frozenset( + {SpecialModelNames.all_team_models.value, SpecialModelNames.all_proxy_models.value} +) + + async def _check_project_key_limits( project_id: str, data: GenerateKeyRequest | UpdateKeyRequest, @@ -1432,7 +1437,8 @@ async def _check_project_key_limits( """ Validate that key's models and budget respect its project's limits. - - Key models must be a subset of project models + - Key models must be a subset of project models, except the all-team-models / all-proxy-models + sentinels, which inherit a parent scope and are narrowed by the project at request time - Key max_budget must be <= project max_budget """ project_obj: Final = await get_project_object( @@ -1450,7 +1456,7 @@ async def _check_project_key_limits( # Validate key models are a subset of project models if data.models and len(project_obj.models) > 0: for m in data.models: - if m not in project_obj.models: + if m not in project_obj.models and m not in _INHERITED_MODEL_SENTINELS: raise HTTPException( status_code=400, detail={ @@ -6266,6 +6272,7 @@ async def block_key( """ from litellm.proxy.management_helpers.audit_logs import ( get_audit_log_changed_by, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import ( create_audit_log_for_update, @@ -6312,7 +6319,7 @@ async def block_key( code=status.HTTP_404_NOT_FOUND, ) - if litellm.store_audit_logs is True: + if is_audit_logging_enabled(): asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( @@ -6379,6 +6386,7 @@ async def unblock_key( """ from litellm.proxy.management_helpers.audit_logs import ( get_audit_log_changed_by, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import ( create_audit_log_for_update, @@ -6425,7 +6433,7 @@ async def unblock_key( code=status.HTTP_404_NOT_FOUND, ) - if litellm.store_audit_logs is True: + if is_audit_logging_enabled(): asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 06c32af2dc2..54a591a5e1a 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -64,7 +64,10 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) -from litellm.proxy.management_helpers.audit_logs import get_audit_log_changed_by +from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + is_audit_logging_enabled, +) from litellm.repositories.table_repositories import ( MCPServerRepository, MCPUserCredentialsRepository, @@ -2018,7 +2021,7 @@ if MCP_AVAILABLE: await global_mcp_server_manager.reload_servers_from_database() # TODO: Enterprise: Finish audit log trail - if litellm.store_audit_logs: + if is_audit_logging_enabled(): pass # TODO: Delete from virtual keys @@ -2613,7 +2616,7 @@ if MCP_AVAILABLE: ) # TODO: Enterprise: Finish audit log trail - if litellm.store_audit_logs: + if is_audit_logging_enabled(): pass return _redact_mcp_credentials(mcp_server_record_updated) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index ade24d194d2..1b49e2455e4 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -24,6 +24,13 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import LITELLM_PROXY_ADMIN_NAME +from litellm.litellm_core_utils.ptu_pricing import ( + CUSTOM_PRICING_FIELDS, + PTU_EMPTIED_PRICING_FIELDS, + PTU_ZEROED_PRICING_FIELDS, + PTU_ZEROED_TABLE_FIELDS, + SEARCH_CONTEXT_SIZES, +) from litellm.proxy._types import ( BlockModelRequest, CommonProxyErrors, @@ -89,7 +96,6 @@ from litellm.types.router import ( ModelInfo, updateDeployment, ) -from litellm.types.utils import CustomPricingLiteLLMParams from litellm.utils import get_utc_datetime router: Final = APIRouter() @@ -346,12 +352,8 @@ def _validate_ptu_model_info(model_info: Mapping[str, object]) -> None: # tiered_pricing is the one mirrored field that is a table of ranges, not a rate, so it is stored # empty (see _PTU_EMPTIED_PRICING_FIELDS): its tiers outrank the zeros written beside them, so # dropping it would leave the cost map's tiers billing the traffic the reserved capacity covers. -_PTU_ZEROED_PRICING_FIELDS: Final = tuple(f for f in SPECIAL_MODEL_INFO_PARAMS if f != "tiered_pricing") + ( - "cache_creation_input_token_cost_above_1hr", - "cache_creation_input_token_cost_above_200k_tokens", - "cache_read_input_token_cost_above_200k_tokens", -) -_PTU_EMPTIED_PRICING_FIELDS: Final = frozenset({"tiered_pricing"}) +_PTU_ZEROED_PRICING_FIELDS: Final = PTU_ZEROED_PRICING_FIELDS +_PTU_EMPTIED_PRICING_FIELDS: Final = PTU_EMPTIED_PRICING_FIELDS _PTU_ZEROED_PRICING: Final[Mapping[str, float | tuple[()]]] = MappingProxyType( { **dict.fromkeys(_PTU_ZEROED_PRICING_FIELDS, 0.0), @@ -363,13 +365,13 @@ _EMPTY_MODEL_INFO: Final[Mapping[str, object]] = _NO_PRICING_OVERRIDE # Rate fields only. CustomPricingLiteLLMParams also carries settings that are not charges # (an embedding's output_vector_size, the regional uplift multipliers), and zeroing one of # those would destroy the deployment's configuration rather than stop a charge. -_CUSTOM_PRICING_FIELDS: Final = frozenset(f for f in CustomPricingLiteLLMParams.model_fields if "cost" in f) +_CUSTOM_PRICING_FIELDS: Final = CUSTOM_PRICING_FIELDS # search_context_cost_per_query holds its rates in a table keyed by context size, and an absent # table means the provider's own default rate rather than free (litellm/llms/gemini/cost_calculator # falls back to $0.035), so it is zeroed in place rather than emptied like tiered_pricing, and # written on every PTU deployment rather than only where a table is already stored. -_PTU_ZEROED_TABLE_FIELDS: Final = frozenset({"search_context_cost_per_query"}) -_SEARCH_CONTEXT_SIZES: Final = ("search_context_size_low", "search_context_size_medium", "search_context_size_high") +_PTU_ZEROED_TABLE_FIELDS: Final = PTU_ZEROED_TABLE_FIELDS +_SEARCH_CONTEXT_SIZES: Final = SEARCH_CONTEXT_SIZES def _is_nonzero_rate(value: object) -> bool: diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 472e25bbc28..0e8c4e1825d 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -13,7 +13,6 @@ from typing import Annotated, Any, Final from fastapi import APIRouter, Depends, Header, HTTPException, Request, status -import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.proxy._types import ( @@ -182,14 +181,15 @@ async def _emit_team_callback_audit_log( Callback secrets are redacted before serialization so the audit table cannot itself become a credential-harvest sink. """ - if litellm.store_audit_logs is not True: - return - from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name + if not is_audit_logging_enabled(): + return + redacted_before: Final = _redact_callback_secrets(before_metadata) redacted_after: Final = _redact_callback_secrets(after_metadata) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 31006958ba1..82e22bb5bbf 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -1252,6 +1252,7 @@ async def new_team( try: from litellm.proxy.management_helpers.audit_logs import ( get_audit_log_changed_by, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import ( _license_check, @@ -1560,8 +1561,7 @@ async def new_team( litellm_proxy_admin_name=litellm_proxy_admin_name, ) - # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True - if litellm.store_audit_logs is True: + if is_audit_logging_enabled(): _updated_values = complete_team_data.json(exclude_none=True) _updated_values = json.dumps(_updated_values, default=str) @@ -1953,6 +1953,7 @@ async def update_team( ``` """ try: + from litellm.proxy.management_helpers.audit_logs import is_audit_logging_enabled from litellm.proxy.proxy_server import ( litellm_proxy_admin_name, llm_router, @@ -2261,8 +2262,7 @@ async def update_team( proxy_logging_obj=proxy_logging_obj, ) - # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True - if litellm.store_audit_logs is True: + if is_audit_logging_enabled(): await _create_team_update_audit_log( existing_team_row=existing_team_row, updated_kv=updated_kv, @@ -3727,6 +3727,7 @@ async def delete_team( """ from litellm.proxy.management_helpers.audit_logs import ( get_audit_log_changed_by, + is_audit_logging_enabled, ) from litellm.proxy.proxy_server import ( create_audit_log_for_update, @@ -3771,9 +3772,8 @@ async def delete_team( litellm_changed_by=litellm_changed_by, ) - # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True # we do this after the first for loop, since first for loop is for validation. we only want this inserted after validation passes - if litellm.store_audit_logs is True: + if is_audit_logging_enabled(): # make an audit log for each team deleted for team_id in data.team_ids: team_row: LiteLLM_TeamTable | None = await prisma_client.get_data( diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index 2b714f06413..ecd6abea3c3 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -24,6 +24,22 @@ _audit_log_callback_cache: Final[dict[str, CustomLogger]] = {} ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY: Final = "allow_litellm_changed_by_header" +def is_audit_logging_enabled(store_audit_logs: bool | None = None) -> bool: + from litellm.secret_managers.main import get_secret_bool + + configured_value: Final[bool | None] = litellm.store_audit_logs if store_audit_logs is None else store_audit_logs + if configured_value is not None: + return configured_value + + environment_value: Final[bool | None] = get_secret_bool("LITELLM_STORE_AUDIT_LOGS") + if environment_value is not None: + return environment_value + + from litellm.proxy.proxy_server import premium_user + + return premium_user is True + + def _allows_litellm_changed_by_header(user_api_key_dict: UserAPIKeyAuth) -> bool: for admin_metadata in (user_api_key_dict.metadata, user_api_key_dict.team_metadata): if ( @@ -164,11 +180,7 @@ async def create_object_audit_log( - user_api_key_dict: UserAPIKeyAuth - The user api key dictionary. - litellm_proxy_admin_name: Optional[str] - The name of the proxy admin. """ - from litellm.secret_managers.main import get_secret_bool - - _store_audit_logs: Final[bool | None] = litellm.store_audit_logs or get_secret_bool("LITELLM_STORE_AUDIT_LOGS") - - if _store_audit_logs is not True: + if not is_audit_logging_enabled(): return _changed_by: Final = get_audit_log_changed_by( @@ -196,10 +208,7 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): """ Create an audit log for an object. """ - from litellm.secret_managers.main import get_secret_bool - - _store_audit_logs: Final[bool | None] = litellm.store_audit_logs or get_secret_bool("LITELLM_STORE_AUDIT_LOGS") - if _store_audit_logs is not True: + if not is_audit_logging_enabled(): return from litellm.proxy.proxy_server import premium_user, prisma_client diff --git a/litellm/proxy/openai_files_endpoints/batch_file_validation.py b/litellm/proxy/openai_files_endpoints/batch_file_validation.py new file mode 100644 index 00000000000..0aee5e8cc54 --- /dev/null +++ b/litellm/proxy/openai_files_endpoints/batch_file_validation.py @@ -0,0 +1,182 @@ +import json +from collections.abc import Iterator +from dataclasses import dataclass +from itertools import chain +from typing import BinaryIO, Final, NoReturn, assert_never + +from litellm.proxy._types import ProxyException + +BATCH_LINE_REQUIRED_KEYS: Final = ("custom_id", "method", "url", "body") +_MB: Final = 1024 * 1024 + + +@dataclass(frozen=True, slots=True) +class BatchFileTooLarge: + size_bytes: int + limit_mb: int + + +@dataclass(frozen=True, slots=True) +class BatchFileWrongExtension: + filename: str + + +@dataclass(frozen=True, slots=True) +class BatchFileEmpty: + pass + + +@dataclass(frozen=True, slots=True) +class BatchFileInvalidJsonLine: + line_number: int + + +@dataclass(frozen=True, slots=True) +class BatchFileLineNotObject: + line_number: int + + +@dataclass(frozen=True, slots=True) +class BatchFileMissingLineKey: + line_number: int + key: str + + +BatchFileValidationFailure = ( + BatchFileTooLarge + | BatchFileWrongExtension + | BatchFileEmpty + | BatchFileInvalidJsonLine + | BatchFileLineNotObject + | BatchFileMissingLineKey +) + + +def _file_size_bytes(file_source: bytes | BinaryIO) -> int: + if isinstance(file_source, bytes): + return len(file_source) + file_source.seek(0, 2) + size: Final = file_source.tell() + file_source.seek(0) + return size + + +def _iter_lines(file_source: bytes | BinaryIO) -> Iterator[bytes]: + if isinstance(file_source, bytes): + return iter(file_source.splitlines()) + file_source.seek(0) + return iter(file_source) + + +def _check_line(line_number: int, raw_line: bytes) -> BatchFileValidationFailure | None: + try: + parsed: Final = json.loads(raw_line) + except (json.JSONDecodeError, UnicodeDecodeError): + return BatchFileInvalidJsonLine(line_number=line_number) + if not isinstance(parsed, dict): + return BatchFileLineNotObject(line_number=line_number) + missing: Final = next((key for key in BATCH_LINE_REQUIRED_KEYS if key not in parsed), None) + if missing is None: + return None + return BatchFileMissingLineKey(line_number=line_number, key=missing) + + +def _scan_lines(file_source: bytes | BinaryIO) -> BatchFileValidationFailure | None: + content_lines: Final = ( + (line_number, raw_line) + for line_number, raw_line in enumerate(_iter_lines(file_source), start=1) + if raw_line.strip() + ) + first_line: Final = next(content_lines, None) + if first_line is None: + return BatchFileEmpty() + return next( + ( + failure + for line_number, raw_line in chain((first_line,), content_lines) + for failure in (_check_line(line_number, raw_line),) + if failure is not None + ), + None, + ) + + +def check_batch_file_upload( + filename: str | None, + file_source: bytes | BinaryIO, + max_batch_file_size_mb: int | None, +) -> BatchFileValidationFailure | None: + if filename is None or not filename.lower().endswith(".jsonl"): + return BatchFileWrongExtension(filename=filename or "") + if max_batch_file_size_mb is not None and max_batch_file_size_mb > 0: + size_bytes: Final = _file_size_bytes(file_source) + if size_bytes > max_batch_file_size_mb * _MB: + return BatchFileTooLarge(size_bytes=size_bytes, limit_mb=max_batch_file_size_mb) + scan_failure: Final = _scan_lines(file_source) + if not isinstance(file_source, bytes): + file_source.seek(0) + return scan_failure + + +def raise_batch_file_validation_failure(failure: BatchFileValidationFailure) -> NoReturn: + match failure: + case BatchFileTooLarge(size_bytes=size_bytes, limit_mb=limit_mb): + raise ProxyException( + message=( + f"Batch input file is {size_bytes / _MB:.1f} MB, which exceeds the configured " + f"max_batch_file_size_mb of {limit_mb} MB. The file was not forwarded to the provider." + ), + type="invalid_request_error", + param="file", + code=413, + ) + case BatchFileWrongExtension(filename=filename): + raise ProxyException( + message=( + f"Invalid file format for Batch API: '{filename}'. " + "Batch input files must be .jsonl files. The file was not forwarded to the provider." + ), + type="invalid_request_error", + param="file", + code=400, + ) + case BatchFileEmpty(): + raise ProxyException( + message="Batch input file has no request lines. The file was not forwarded to the provider.", + type="invalid_request_error", + param="file", + code=400, + ) + case BatchFileInvalidJsonLine(line_number=line_number): + raise ProxyException( + message=( + f"Batch input file line {line_number} is not valid JSON. " + "The file was not forwarded to the provider." + ), + type="invalid_request_error", + param="file", + code=400, + ) + case BatchFileLineNotObject(line_number=line_number): + raise ProxyException( + message=( + f"Batch input file line {line_number} must be a JSON object. " + "The file was not forwarded to the provider." + ), + type="invalid_request_error", + param="file", + code=400, + ) + case BatchFileMissingLineKey(line_number=line_number, key=key): + raise ProxyException( + message=( + f"Missing required parameter: '{key}' (batch input file line {line_number}). " + f"Each line must be a JSON object with keys {', '.join(BATCH_LINE_REQUIRED_KEYS)}. " + "The file was not forwarded to the provider." + ), + type="invalid_request_error", + param=key, + code=400, + ) + case _: + assert_never(failure) diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 361b5b920e2..b7200de8fb6 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -21,6 +21,7 @@ from fastapi import ( UploadFile, status, ) +from pydantic import TypeAdapter import litellm from litellm import CreateFileRequest, get_secret_str @@ -41,6 +42,10 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( get_custom_llm_provider_from_request_headers, get_custom_llm_provider_from_request_query, ) +from litellm.proxy.openai_files_endpoints.batch_file_validation import ( + check_batch_file_upload, + raise_batch_file_validation_failure, +) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, add_internal_model_credentials, @@ -65,6 +70,8 @@ from litellm.types.llms.openai import ( router: Final = APIRouter() +_MAX_BATCH_FILE_SIZE_MB_ADAPTER: Final = TypeAdapter(int | None) + files_config = None @@ -361,18 +368,27 @@ async def create_file( # Prepare the data for forwarding - # Replace with: valid_purposes: Final = get_args(OpenAIFilesPurpose) if purpose not in valid_purposes: - raise HTTPException( - status_code=400, - detail={ - "error": f"Invalid purpose: {purpose}. Must be one of: {valid_purposes}", - }, + raise ProxyException( + message=f"Invalid purpose: {purpose}. Must be one of: {valid_purposes}", + type="invalid_request_error", + param="purpose", + code=400, ) # Cast purpose to OpenAIFilesPurpose type purpose = cast(OpenAIFilesPurpose, purpose) + if purpose == "batch": + batch_file_failure: Final = await asyncio.to_thread( + check_batch_file_upload, + file.filename, + file_source, + _MAX_BATCH_FILE_SIZE_MB_ADAPTER.validate_python(general_settings.get("max_batch_file_size_mb")), + ) + if batch_file_failure is not None: + raise_batch_file_validation_failure(batch_file_failure) + data = {} # Parse expires_after if provided @@ -552,6 +568,8 @@ async def create_file( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) verbose_proxy_logger.exception("litellm.proxy.proxy_server.create_file(): Exception occured - %s", e) + if isinstance(e, ProxyException): + raise e if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "message", str(e.detail)), diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 621b3ff9c83..ddcca1d372b 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -10,6 +10,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import VERTEX_BATCH_PREDICTION_JOBS_ROUTE from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.vertex_ai.common_utils import get_vertex_location_from_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( ModelResponseIterator as VertexModelResponseIterator, ) @@ -60,6 +61,9 @@ class VertexPassthroughLoggingHandler: request_body: dict | None = None, **kwargs, ) -> PassThroughEndpointLoggingTypedDict: + vertex_location: Final = get_vertex_location_from_url(url_route) + if vertex_location is not None: + logging_obj.optional_params["vertex_location"] = vertex_location if "predictLongRunning" in url_route: model = VertexPassthroughLoggingHandler.extract_model_from_url(url_route) @@ -82,6 +86,7 @@ class VertexPassthroughLoggingHandler: model=model, custom_llm_provider="vertex_ai", call_type="create_video", + vertex_location=vertex_location, ) # Set response_cost in _hidden_params to prevent recalculation @@ -123,6 +128,7 @@ class VertexPassthroughLoggingHandler: end_time=end_time, logging_obj=logging_obj, custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route), + vertex_location=vertex_location, ) return { @@ -190,6 +196,7 @@ class VertexPassthroughLoggingHandler: end_time=end_time, logging_obj=logging_obj, custom_llm_provider="vertex_ai", + vertex_location=vertex_location, ) return { @@ -206,6 +213,7 @@ class VertexPassthroughLoggingHandler: model="vertex_ai/search_api", custom_llm_provider="vertex_ai", call_type="vector_store_search", + vertex_location=vertex_location, ) standard_pass_through_response_object: Final[StandardPassThroughResponseObject] = { @@ -302,6 +310,7 @@ class VertexPassthroughLoggingHandler: completion_response=litellm_prediction_response, model=model, custom_llm_provider="vertex_ai", + vertex_location=get_vertex_location_from_url(url_route), ) kwargs["response_cost"] = response_cost @@ -381,6 +390,7 @@ class VertexPassthroughLoggingHandler: completion_response=litellm_embedding_response, model=model, custom_llm_provider=custom_llm_provider, + vertex_location=get_vertex_location_from_url(url_route), ) kwargs["response_cost"] = response_cost @@ -413,6 +423,9 @@ class VertexPassthroughLoggingHandler: - Logs in litellm callbacks """ kwargs: dict[str, Any] = {} + vertex_location: Final = get_vertex_location_from_url(url_route) + if vertex_location is not None: + litellm_logging_obj.optional_params["vertex_location"] = vertex_location model = model or VertexPassthroughLoggingHandler.extract_model_from_url(url_route) complete_streaming_response: Final = VertexPassthroughLoggingHandler._build_complete_streaming_response( all_chunks=all_chunks, @@ -438,6 +451,7 @@ class VertexPassthroughLoggingHandler: end_time=end_time, logging_obj=litellm_logging_obj, custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(url_route), + vertex_location=vertex_location, ) return { @@ -591,6 +605,7 @@ class VertexPassthroughLoggingHandler: end_time: datetime, logging_obj: LiteLLMLoggingObj, custom_llm_provider: str, + vertex_location: str | None, ) -> dict: """ Create the standard logging object for Vertex passthrough generateContent (streaming and non-streaming) @@ -601,6 +616,7 @@ class VertexPassthroughLoggingHandler: completion_response=litellm_model_response, model=model, custom_llm_provider="vertex_ai", + vertex_location=vertex_location, ) kwargs["response_cost"] = response_cost diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index cae735be988..af082f04706 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -384,6 +384,10 @@ from litellm.proxy.db.gateway_request_tracking import ( GatewayRequestAccumulator, flush_gateway_requests, ) +from litellm.proxy.db.proxy_worker_heartbeat import ( + PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS, + ProxyWorkerHeartbeat, +) from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router @@ -635,6 +639,7 @@ from litellm.secret_managers.main import ( get_secret_bool, get_secret_str, normalize_nonempty_secret_str, + secret_manager_would_be_consulted, str_to_bool, ) from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs @@ -874,9 +879,11 @@ async def _flush_spend_logs_queue_on_shutdown() -> None: verbose_proxy_logger.exception("Error flushing spend logs queue on shutdown: %s", e) -async def proxy_shutdown_event() -> None: +async def proxy_shutdown_event(worker_heartbeat: ProxyWorkerHeartbeat | None = None) -> None: global prisma_client, master_key, user_custom_auth, user_custom_key_generate, user_custom_key_update verbose_proxy_logger.info("Shutting down LiteLLM Proxy Server") + if worker_heartbeat is not None and prisma_client: + await worker_heartbeat.deregister() if prisma_client: # Drain the SGR fold first: it lives in memory, so an un-drained interval # is lost, and a write attempted after disconnect raises @@ -1210,7 +1217,7 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: ) ### START BATCH WRITING DB + CHECKING NEW MODELS### - if prisma_client is not None: + worker_heartbeat: Final = ( await ProxyStartupEvent.initialize_scheduled_background_jobs( general_settings=general_settings, prisma_client=prisma_client, @@ -1219,7 +1226,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: proxy_batch_write_at=proxy_batch_write_at, proxy_logging_obj=proxy_logging_obj, ) - + if prisma_client is not None + else None + ) + if prisma_client is not None: await ProxyStartupEvent._update_default_team_member_budget() ## SYNC UI SETTINGS ## @@ -1290,7 +1300,7 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: await proxy_config.stop_auth_cache_invalidation_subscriber() - await proxy_shutdown_event() + await proxy_shutdown_event(worker_heartbeat=worker_heartbeat) def _generate_stable_operation_id(route: "APIRoute") -> str: @@ -4371,9 +4381,55 @@ class ProxyConfig: item = self._check_for_os_environ_vars(config=item, depth=depth + 1, max_depth=max_depth) # if the value is a string and starts with "os.environ/" - then it's an environment variable elif isinstance(value, str) and value.startswith("os.environ/"): - config[key] = get_secret(value) + resolved = get_secret(value) + if resolved is None and secret_manager_would_be_consulted(value): + verbose_proxy_logger.warning("%s is absent from the configured secret manager", value) + config[key] = resolved return config + def _initialize_secret_manager_from_raw_config( + self, config: Mapping[str, object], config_file_path: str | None + ) -> None: + """ + Bring the secret manager up before `os.environ/` references are resolved. + + `_check_for_os_environ_vars` writes whatever it resolves back into the config, so a key + held only by the secret manager would otherwise become a permanent `None` that the later + fallbacks in `load_config` can no longer recover from. + + `get_config` also runs on management-endpoint request paths, so this returns early once a + manager exists rather than rebuilding the client on every request. + + The manager's own settings can only come from real environment variables, so they are + resolved against a throwaway copy and the config is left untouched for the main pass. + """ + if litellm.secret_manager_client is not None: + return + + general_settings: Final = config.get("general_settings") + if not isinstance(general_settings, dict): + return + + raw_system: Final = general_settings.get("key_management_system") + key_management_system: Final = ( + get_secret(raw_system) + if isinstance(raw_system, str) and raw_system.startswith("os.environ/") + else raw_system + ) + if not isinstance(key_management_system, str): + return + + raw_settings: Final = general_settings.get("key_management_settings") + if isinstance(raw_settings, dict): + litellm._key_management_settings = KeyManagementSettings( + **self._check_for_os_environ_vars(config=copy.deepcopy(raw_settings)) + ) + + self.initialize_secret_manager( + key_management_system=key_management_system, + config_file_path=config_file_path, + ) + def _get_team_config(self, team_id: str, all_teams_config: list[dict]) -> dict: team_config: dict = {} for team in all_teams_config: @@ -4544,6 +4600,8 @@ class ProxyConfig: printed_yaml: Final = copy.deepcopy(config) printed_yaml.pop("environment_variables", None) + self._initialize_secret_manager_from_raw_config(config=config, config_file_path=config_file_path) + config = self._check_for_os_environ_vars(config=config) self.update_config_state(config=config) @@ -4977,6 +5035,7 @@ class ProxyConfig: ) elif key == "audit_log_callbacks": from litellm.proxy.management_helpers.audit_logs import ( + is_audit_logging_enabled, reset_audit_log_callback_cache, ) @@ -4995,14 +5054,14 @@ class ProxyConfig: litellm.audit_log_callbacks.append(callback) _store_audit_logs = litellm_settings.get("store_audit_logs", litellm.store_audit_logs) - if _store_audit_logs: + if is_audit_logging_enabled(store_audit_logs=_store_audit_logs): print( # noqa: T201 f"{blue_color_code} Initialized Audit Log Callbacks - {litellm.audit_log_callbacks} {reset_color_code}" ) else: verbose_proxy_logger.warning( - "'audit_log_callbacks' is configured but 'store_audit_logs' is not enabled. " - "Audit log callbacks will not fire until 'store_audit_logs: true' is added to litellm_settings." + "'audit_log_callbacks' is configured but audit logging is not enabled. " + "Audit log callbacks will not fire." ) elif key == "cache_params": # this is set in the cache branch @@ -5114,17 +5173,14 @@ class ProxyConfig: key: general_settings[key] for key in SPEND_LOG_CLEANUP_BOUND_SETTINGS if key in general_settings } - ### LOAD KEY MANAGEMENT SETTINGS FIRST (needed for custom secret manager) ### + ### LOAD KEY MANAGEMENT SETTINGS ### + # The secret manager itself is brought up by get_config(), which runs before the + # `os.environ/` references in this config were resolved. Re-reading the settings here + # picks up any of them that were themselves secret-manager backed. key_management_settings: Final = general_settings.get("key_management_settings", None) if key_management_settings is not None: litellm._key_management_settings = KeyManagementSettings(**key_management_settings) - ### LOAD SECRET MANAGER ### - key_management_system: Final = general_settings.get("key_management_system", None) - self.initialize_secret_manager( - key_management_system=key_management_system, - config_file_path=config_file_path, - ) ### [DEPRECATED] LOAD FROM GOOGLE KMS ### old way of loading from google kms use_google_kms: Final = general_settings.get("use_google_kms", False) load_google_kms(use_google_kms=use_google_kms) @@ -6316,6 +6372,9 @@ class ProxyConfig: if "global_max_parallel_requests" in _general_settings: general_settings["global_max_parallel_requests"] = _general_settings["global_max_parallel_requests"] + if "max_batch_file_size_mb" not in self._yaml_general_settings_keys: + general_settings["max_batch_file_size_mb"] = _general_settings.get("max_batch_file_size_mb") + ## ALERTING ARGS ## if "alerting_args" in _general_settings: general_settings["alerting_args"] = _general_settings["alerting_args"] @@ -8733,7 +8792,7 @@ class ProxyStartupEvent: proxy_budget_rescheduler_max_time: int, proxy_batch_write_at: int, proxy_logging_obj: ProxyLogging, - ): + ) -> ProxyWorkerHeartbeat: """Initializes scheduled background jobs""" global store_model_in_db, scheduler @@ -8778,6 +8837,18 @@ class ProxyStartupEvent: # Ensure minimum interval of 30 seconds for batch writing to prevent memory issues batch_writing_interval: Final = proxy_batch_write_at + random.randint(0, 5) + ### PROXY WORKER HEARTBEAT ### + worker_heartbeat: Final = ProxyWorkerHeartbeat(prisma_client=prisma_client) + await worker_heartbeat.beat() + scheduler.add_job( + worker_heartbeat.beat, + "interval", + seconds=PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS, + id="proxy_worker_heartbeat_job", + replace_existing=True, + misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, + ) + ### RESET BUDGET ### if general_settings.get("disable_reset_budget", False) is False: budget_reset_job: Final = ResetBudgetJob( @@ -9117,6 +9188,7 @@ class ProxyStartupEvent: "APScheduler started with memory leak prevention settings: removed jitter, increased intervals, misfire_grace_time=%s", APSCHEDULER_MISFIRE_GRACE_TIME, ) + return worker_heartbeat @classmethod async def _initialize_spend_tracking_background_jobs(cls, scheduler: AsyncIOScheduler): @@ -15689,6 +15761,7 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro "max_parallel_requests": "Integer", "global_max_parallel_requests": "Integer", "max_request_size_mb": "Integer", + "max_batch_file_size_mb": "Integer", "max_response_size_mb": "Integer", "proxy_config_reload_interval_seconds": "Integer", "pass_through_endpoints": "PydanticModel", diff --git a/litellm/proxy/read_model_list.py b/litellm/proxy/read_model_list.py index cdd6680aa40..a1830e7f2bc 100644 --- a/litellm/proxy/read_model_list.py +++ b/litellm/proxy/read_model_list.py @@ -9,7 +9,8 @@ effects. Instead we reuse ``ProxyConfig.get_config`` — the actual config reader — so the gateway inherits the same heavy lifting the proxy does: ``include:`` merging, ``os.environ/`` + secret-manager resolution, and DB-stored models (when a DB is -configured). It has no proxy-setup side effects. Returns the resolved +configured). Its only proxy-setup side effect is bringing up the configured +secret manager, which is what makes that resolution work. Returns the resolved ``model_list``; the Rust side deserializes each entry into its ``Deployment``. """ diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 52fb447157b..60058c777ca 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -947,6 +947,17 @@ model LiteLLM_DailyTagSpend { } +// One row per live proxy worker process. Workers upsert their row on a fixed +// heartbeat; counting rows with a recent heartbeat tells how many workers share +// this database, which lets the Admin UI hide its "no Redis" warning for +// deployments that are provably a single worker. +model LiteLLM_ProxyWorkerHeartbeat { + worker_id String @id + hostname String + started_at DateTime @default(now()) + last_heartbeat_at DateTime @default(now()) +} + // Track the status of cron jobs running. Only allow one pod to run the job at a time model LiteLLM_CronJob { cronjob_id String @id @default(cuid()) // Unique ID for the record @@ -1467,28 +1478,38 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } -// Shadow eval: evaluation of an auto-router against a key's live traffic, in either -// direction. forward duplicates the requests the key did not route through the router -// through it, answering whether the key should adopt it; reverse duplicates the requests -// the router did serve against a fixed baseline model, answering whether a key already on -// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge -// compares real vs shadow responses blind. The job row is immutable config plus -// stopped_at; every count, status, and spend figure is derived from the append-only -// attempt rows, so nothing can disagree across pods or stop races. +// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in +// either direction. forward duplicates the requests the keys did not route through the +// router through it, answering whether they should adopt it; reverse duplicates the +// requests the router did serve against a fixed baseline model, answering whether a key +// already on it still benefits. Either way a sampled slice runs in a detached task and an +// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job: +// immutable config plus that key's own turn budget and stop state, so one key exhausting +// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id +// (the id the API reports), written together by one atomic create_many with identical +// config; single-key jobs predating group_id were backfilled group_id = id. "One active +// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE +// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state +// partial indexes; it is what makes a concurrent start on another pod race-safe rather +// than read-then-create. Every count, status, and spend figure is derived from the +// append-only attempt rows, so nothing can disagree across pods or stop races. model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) - api_key_id String // hashed virtual key whose traffic is shadowed + group_id String // legs of one job share this; the API's job id + api_key_id String // hashed virtual key whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // sample budget: judge at most this many turns + max_turns Int // this key's sample budget: judge at most this many turns created_at DateTime @default(now()) created_by String? ends_at DateTime stopped_at DateTime? + stopped_by String? // operator who stopped it early; null when it ended on its own + @@index([group_id]) @@index([api_key_id]) @@index([created_at]) } diff --git a/litellm/proxy/spend_tracking/ptu_feature_flag.py b/litellm/proxy/spend_tracking/ptu_feature_flag.py index 9078079b676..7f52dfa155d 100644 --- a/litellm/proxy/spend_tracking/ptu_feature_flag.py +++ b/litellm/proxy/spend_tracking/ptu_feature_flag.py @@ -1,18 +1,12 @@ -"""Opt-in flag for PTU (provisioned throughput unit) flat-cost attribution. +"""Re-exported from ``litellm.litellm_core_utils.ptu_pricing``. -The whole feature is inert unless an operator sets -``LITELLM_ENABLE_PTU_COST_ATTRIBUTION``: the daily rollup is not scheduled, the -model endpoints reject PTU config, the daily activity read path reports zero flat -cost, and the model form hides the PTU inputs. +The flag lives in core because the router reads it while registering a deployment, and +router code cannot import from the proxy. """ -from typing import Final +from litellm.litellm_core_utils.ptu_pricing import ( + PTU_COST_ATTRIBUTION_ENV_VAR, + is_ptu_cost_attribution_enabled, +) -from litellm.secret_managers.main import get_secret_bool - -PTU_COST_ATTRIBUTION_ENV_VAR: Final = "LITELLM_ENABLE_PTU_COST_ATTRIBUTION" - - -def is_ptu_cost_attribution_enabled() -> bool: - """Report whether this deployment opted into PTU flat-cost attribution.""" - return get_secret_bool(PTU_COST_ATTRIBUTION_ENV_VAR, False) is True +__all__ = ("PTU_COST_ATTRIBUTION_ENV_VAR", "is_ptu_cost_attribution_enabled") diff --git a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py index 25de9f6d065..381641be96d 100644 --- a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py +++ b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py @@ -14,6 +14,7 @@ and share the existing unique constraint. import asyncio import json +import sys from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import date, datetime, time, timedelta, timezone @@ -29,14 +30,15 @@ from litellm.constants import ( PTU_ROLLUP_MAX_BACKFILL_DAYS, PTU_SENTINEL_API_KEY, ) +from litellm.litellm_core_utils.ptu_pricing import ptu_terms from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled -from litellm.types.router import ModelInfo if TYPE_CHECKING: from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.utils import PrismaClient _HOURS_PER_DAY: Final = 24 +_PRUNE_ID_CHUNK_SIZE: Final = 5_000 _UPSERT_ATTEMPTS: Final = 3 _UPSERT_RETRY_BACKOFF_SECONDS: Final = 0.5 @@ -72,28 +74,6 @@ class PTUModel: effective_to: datetime | None = None -def _parse_utc_datetime(value: object) -> datetime | None: - """Parse a model_info datetime (ISO string or datetime) into a UTC-aware datetime, else None.""" - parsed: Final = _coerce_datetime(value) - if parsed is None: - return None - if parsed.tzinfo is None: - return parsed.replace(tzinfo=timezone.utc) - return parsed.astimezone(timezone.utc) - - -def _coerce_datetime(value: object) -> datetime | None: - """``value`` as a datetime, parsing an ISO string, else None.""" - if isinstance(value, datetime): - return value - if not isinstance(value, str): - return None - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: - return None - - def _public_model_name(row: object, model_info: Mapping[str, object]) -> str: """The name an operator recognises for this deployment. @@ -167,46 +147,20 @@ def _parse_ptu_model(row: object) -> PTUModel | None: Valid means model_info has a positive ptu_count, a non-negative cost_per_ptu_per_hour, and a team_id (1 model -> 1 team). """ - raw_model_info: Final = getattr(row, "model_info", None) - model_info: Final = _decode_model_info(raw_model_info) + model_info: Final = _decode_model_info(getattr(row, "model_info", None)) if model_info is None: return None - ptu_count: Final = model_info.get("ptu_count") - cost_per_hour: Final = model_info.get("cost_per_ptu_per_hour") - team_id: Final = model_info.get("team_id") - if ptu_count is None or cost_per_hour is None or not team_id: - return None - try: - ptu_count_int: Final = int(ptu_count) - cost_per_hour_float: Final = float(cost_per_hour) - except (TypeError, ValueError, OverflowError): - return None - if not 0 < ptu_count_int <= ModelInfo.MAX_PTU_COUNT: - return None - if not 0 <= cost_per_hour_float <= ModelInfo.MAX_COST_PER_PTU_PER_HOUR: - return None - if model_info.get("ptu_effective_from") is None: - # The endpoints require a start; a row without one predates that rule or was - # written around them, and inferring one would bill days the deployment did not exist - return None - raw_from: Final = model_info.get("ptu_effective_from") - raw_to: Final = model_info.get("ptu_effective_to") - effective_from: Final = _parse_utc_datetime(raw_from) - effective_to: Final = _parse_utc_datetime(raw_to) - # A present-but-unparseable bound would read as "no bound" and silently widen the - # window to the whole day, so the deployment is skipped until the config is fixed - if (raw_from is not None and effective_from is None) or (raw_to is not None and effective_to is None): - return None - if effective_from is not None and effective_to is not None and effective_to <= effective_from: + terms: Final = ptu_terms(model_info) + if terms is None: return None return PTUModel( model_id=str(getattr(row, "model_id", "") or ""), model_name=_public_model_name(row, model_info), - team_id=str(team_id), - ptu_count=ptu_count_int, - cost_per_ptu_per_hour=cost_per_hour_float, - effective_from=effective_from, - effective_to=effective_to, + team_id=terms.team_id, + ptu_count=terms.ptu_count, + cost_per_ptu_per_hour=terms.cost_per_ptu_per_hour, + effective_from=terms.effective_from, + effective_to=terms.effective_to, ) @@ -358,10 +312,70 @@ async def _upsert_charge_with_retry( return False -async def _load_ptu_models(prisma_client: "PrismaClient") -> tuple[PTUModel, ...]: - """Every model deployment currently carrying valid manual PTU config.""" +@dataclass(frozen=True, slots=True) +class _LoadedDeployments: + """The deployments a run will price, and every deployment id it looked at. + + The id set is deliberately wider than the priced set. A deployment whose PTU config + was removed produces no charge and still has to be prunable, so bounding the prune on + what priced would strand its old rows forever. It is also a guaranteed superset of the + priced set, or a run could write a charge that falls outside its own delete filter. + """ + + models: tuple[PTUModel, ...] + scanned_ids: frozenset[str] + config_sourced: bool + + +def _running_router() -> object | None: + """The proxy's router, or None outside a running proxy. + + Read out of ``sys.modules`` rather than imported, so a rollup driven from a test or a + script does not pull the whole proxy server in behind it. + """ + proxy_server: Final = sys.modules.get("litellm.proxy.proxy_server") + return getattr(proxy_server, "llm_router", None) if proxy_server is not None else None + + +def _config_deployments(router: object | None, *, owned_by_db: frozenset[str]) -> tuple[_PTUDeployment, ...]: + """Deployments the router holds that no ``LiteLLM_ProxyModelTable`` row owns. + + ``db_model`` is forced True on every deployment loaded from that table and defaults to + False on ModelInfo, so the complement is what config.yaml declared. A per-request + credential clone carries ``original_model_id`` and reuses its source's PTU config under + a fresh id, so pricing it would bill one reservation once per distinct client key. + """ + entries: Final = tuple(getattr(router, "model_list", None) or ()) + records: Final = tuple(_router_deployment(entry) for entry in entries) + return tuple( + record + for record in records + if record is not None + and record.model_info.get("db_model") is not True + and record.model_info.get("original_model_id") is None + and record.model_id not in owned_by_db + ) + + +async def _load_ptu_models(prisma_client: "PrismaClient") -> _LoadedDeployments: + """Every deployment carrying valid manual PTU config, and every id the scan saw. + + Reserved capacity is billed by the provider whichever file declared it, so a + deployment the proxy only knows from config.yaml accrues alongside the stored ones. + """ rows: Final = await prisma_client.db.litellm_proxymodeltable.find_many() - return tuple(parsed for parsed in (_parse_ptu_model(row) for row in rows) if parsed is not None) + db_ids: Final = frozenset(model_id for row in rows if (model_id := str(getattr(row, "model_id", "") or ""))) + config_records: Final = _config_deployments(_running_router(), owned_by_db=db_ids) + models: Final = tuple( + parsed for parsed in (_parse_ptu_model(row) for row in (*rows, *config_records)) if parsed is not None + ) + return _LoadedDeployments( + models=models, + config_sourced=bool(config_records), + scanned_ids=db_ids + | frozenset(record.model_id for record in config_records) + | frozenset(model.model_id for model in models), + ) async def run_ptu_flat_cost_rollup( @@ -378,8 +392,10 @@ async def run_ptu_flat_cost_rollup( The prune predicate is ``updated_at < run_started`` rather than "not in the charge set I computed", which matters under concurrency: whether a row is garbage becomes a property of the row instead of one run's in-memory config snapshot, so a run can - never delete a row a concurrent run just wrote. It is still skipped when any charge - failed to write, since a row whose replacement never landed would look unrefreshed. + never delete a row a concurrent run just wrote. It is bounded to the deployments this + run looked at, so a row it cannot account for is out of reach either way. It is still + skipped when any charge failed to write, since a row whose replacement never landed + would look unrefreshed. """ day: Final = target_date or (datetime.now(timezone.utc).date() - timedelta(days=1)) @@ -390,7 +406,8 @@ async def run_ptu_flat_cost_rollup( date_str: Final = day.isoformat() run_started: Final = datetime.now(timezone.utc) - ptu_models: Final = await _load_ptu_models(prisma_client) + loaded: Final = await _load_ptu_models(prisma_client) + ptu_models: Final = loaded.models charges: Final = _aggregate_charges(ptu_models, day) landed: Final = tuple( @@ -415,7 +432,12 @@ async def run_ptu_flat_cost_rollup( date_str, ) else: - await _prune_unrefreshed_sentinel_rows(prisma_client, date_str=date_str, run_started=run_started) + await _prune_unrefreshed_sentinel_rows( + prisma_client, + date_str=date_str, + run_started=run_started, + scanned_ids=loaded.scanned_ids if loaded.config_sourced else None, + ) verbose_proxy_logger.info( "PTU rollup for %s: %d PTU models processed, %d rows written, %d rows failed", @@ -524,7 +546,7 @@ async def run_ptu_flat_cost_backfill( verbose_proxy_logger.warning("PTU backfill: prisma_client is None, skipping") return BackfillResult(start=end, end=end, days_scanned=0, rows_written=0) - ptu_models: Final = await _load_ptu_models(prisma_client) + ptu_models: Final = (await _load_ptu_models(prisma_client)).models days: Final = _backfill_window(ptu_models, end) if not days: @@ -707,26 +729,61 @@ async def _prune_unrefreshed_sentinel_rows( *, date_str: str, run_started: datetime, + scanned_ids: frozenset[str] | None, ) -> None: - """Delete the day's PTU sentinel rows this run did not refresh. + """Delete the day's PTU sentinel rows this run looked at and did not refresh. - Every charge the run wrote bumps ``updated_at`` past ``run_started``, so anything - left below that mark is a (team, model) the current config no longer prices. The mark - is pulled back by ``PTU_PRUNE_SKEW_GRACE_SECONDS`` because the two timestamps come - from different hosts: a stale row is hours old, a concurrently written one is seconds - old, and the grace separates them without waiting on clocks agreeing. The - predicate reads only the row, never the caller's config snapshot, which is what - makes it safe to run twice, out of order, or beside another pod: a row written - after this run began is out of reach of its delete. Mirrors the retention predicate - ``SpendLogCleanup`` deletes by.""" + Two conditions, and a row survives unless it meets both. It must be stale: every + charge the run wrote bumps ``updated_at`` past ``run_started``, so anything left below + that mark is a (team, model) the current config no longer prices. The mark is pulled + back by ``PTU_PRUNE_SKEW_GRACE_SECONDS`` because the two timestamps come from + different hosts, and the grace separates a row that is hours old from one written + seconds ago without waiting on clocks agreeing. + + A run that priced a deployment only its own host declares must also name the + deployments it scanned. Staleness alone is sufficient while every run derives its + charges from the same table, because then any two runs compute the same set, so a + database-only run still sweeps by timestamp exactly as it always has. Once one host's + charges come from a file the others cannot read, a row it never considered is not + evidence of anything, and deleting it drops a charge that host is responsible for. + + Where the bound applies the ids go out in chunks, because each is one bind variable and + the server rejects a statement carrying more than 32767 of them, which a proxy holding + that many deployments would otherwise hit every night with no handler above here. + """ cutoff: Final = run_started - timedelta(seconds=PTU_PRUNE_SKEW_GRACE_SECONDS) - await prisma_client.db.litellm_dailyteamspend.delete_many( - where={ # mutable-ok: prisma delete filter - "date": date_str, - "api_key": PTU_SENTINEL_API_KEY, - "updated_at": {"lt": cutoff}, # mutable-ok: prisma comparison filter - } + unbounded: Final = { # mutable-ok: prisma delete filter + "date": date_str, + "api_key": PTU_SENTINEL_API_KEY, + "updated_at": {"lt": cutoff}, # mutable-ok: prisma comparison filter + } + ordered: Final = () if scanned_ids is None else tuple(sorted(scanned_ids)) + filters: Final = ( + (unbounded,) + if scanned_ids is None + else tuple( + MappingProxyType( + { + **unbounded, + "model": { # mutable-ok: prisma membership filter + "in": ordered[start : start + _PRUNE_ID_CHUNK_SIZE] + }, + } + ) + for start in range(0, len(ordered), _PRUNE_ID_CHUNK_SIZE) + ) ) + deletions: Final = tuple( + [await prisma_client.db.litellm_dailyteamspend.delete_many(where=where) for where in filters] + ) + deleted: Final = sum(deletions) + if deleted: + verbose_proxy_logger.info( + "PTU rollup for %s: pruned %s stale sentinel row(s) across %s deployment(s)", + date_str, + deleted, + "every" if scanned_ids is None else len(scanned_ids), + ) __all__ = ( diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 448723ab3bc..997180efdde 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -130,6 +130,7 @@ class PricingBasis(NamedTuple): service_tier: str | None = None data_residency: str | None = None + vertex_location: str | None = None _STANDARD_RATES: Final = PricingBasis() @@ -141,8 +142,8 @@ def _pricing_basis(cost_breakdown: Mapping[str, object] | None) -> PricingBasis: Rows written before this field shipped carry neither key, and there is no backfill: they price at standard rates, which is what they already did. - Both values survive a JSON round trip on the way here, so neither is guaranteed to be - a string. `generic_cost_per_token` calls `.lower()` on both without a type check, and + These values survive a JSON round trip on the way here, so none is guaranteed to be + a string. `generic_cost_per_token` calls `.lower()` on them without a type check, and the resulting `AttributeError` would be swallowed into a silent zero by the caller's `except`, so anything that is not a string is dropped here instead. """ @@ -150,9 +151,11 @@ def _pricing_basis(cost_breakdown: Mapping[str, object] | None) -> PricingBasis: return _STANDARD_RATES service_tier: Final = cost_breakdown.get("service_tier") data_residency: Final = cost_breakdown.get("data_residency") + vertex_location: Final = cost_breakdown.get("vertex_location") return PricingBasis( service_tier=service_tier if isinstance(service_tier, str) else None, data_residency=data_residency if isinstance(data_residency, str) else None, + vertex_location=vertex_location if isinstance(vertex_location, str) else None, ) @@ -193,6 +196,7 @@ def _cost_of_usage( service_tier=basis.service_tier, data_residency=basis.data_residency, model_info=model_info, + vertex_location=basis.vertex_location, ) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings verbose_proxy_logger.debug( diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 3146d8bccfb..0b56f0d8246 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -216,6 +216,15 @@ def _extract_usage_for_ocr_call(response_obj: Any, response_obj_dict: dict) -> d return {} +def _sl_attribution_fallback( + standard_logging_payload: StandardLoggingPayload | None, + field: Literal["model_id", "model_group", "api_base", "custom_llm_provider"], +) -> str: + if standard_logging_payload is None: + return "" + return standard_logging_payload.get(field) or "" + + def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogsPayload: if kwargs is None: kwargs = {} @@ -288,8 +297,15 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs ): # use 'tags' from standard logging payload instead request_tags = safe_dumps(standard_logging_payload["request_tags"]) - _model_id: Final = metadata.get("model_info", {}).get("id", "") - _model_group: Final = metadata.get("model_group", "") + _model_id: Final = metadata.get("model_info", {}).get("id", "") or _sl_attribution_fallback( + standard_logging_payload, "model_id" + ) + _model_group: Final = metadata.get("model_group", "") or _sl_attribution_fallback( + standard_logging_payload, "model_group" + ) + _api_base: Final = litellm_params.get("api_base", "") or _sl_attribution_fallback( + standard_logging_payload, "api_base" + ) # Extract overhead from hidden_params if available litellm_overhead_time_ms = None @@ -389,7 +405,11 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs # Extract agent_id for A2A requests (set directly on model_call_details) agent_id: Final[str | None] = kwargs.get("agent_id") or metadata.get("agent_id") - custom_llm_provider: Final = kwargs.get("custom_llm_provider") + custom_llm_provider: Final = ( + kwargs.get("custom_llm_provider") + or _sl_attribution_fallback(standard_logging_payload, "custom_llm_provider") + or None + ) raw_model: Final = cast(str, kwargs.get("model") or "") model_name: Final = reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) @@ -414,13 +434,13 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs completion_tokens=usage.get("completion_tokens", standard_logging_completion_tokens), request_tags=request_tags, end_user=end_user_id or "", - api_base=litellm_params.get("api_base", ""), + api_base=_api_base, model_group=_model_group, model_id=_model_id, mcp_namespaced_tool_name=mcp_namespaced_tool_name, agent_id=agent_id, requester_ip_address=clean_metadata.get("requester_ip_address", None), - custom_llm_provider=kwargs.get("custom_llm_provider", ""), + custom_llm_provider=custom_llm_provider or "", messages=_get_messages_for_spend_logs_payload( standard_logging_payload=standard_logging_payload, metadata=metadata ), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a743526e975..1d042e2521b 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -30,7 +30,6 @@ from litellm.constants import ( SPEND_LOG_WRITE_BATCH_MAX_BYTES, ) from litellm.proxy._types import ( - DB_RETRY_SAFE_ERROR_TYPES, CommonProxyErrors, ProxyErrorTypes, ProxyException, @@ -517,6 +516,34 @@ def _failure_usage_to_lift( return estimated_usage, 0.0 +_EMPTY_LIFT: Final = MappingProxyType({}) + + +def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, object]: + """Failure-path callbacks run after ``litellm_logging_obj`` is popped from + request_data (it is not serialisable), so the caller merges these fields + onto request_data first: the first-handoff instant for preprocessing + latency, recovered or estimated usage for token counts, and the standard + logging object for deployment attribution on failed-request spend logs.""" + _logging_obj: Final = request_data.get("litellm_logging_obj") + if _logging_obj is None: + return _EMPTY_LIFT + _model_call_details: Final = getattr(_logging_obj, "model_call_details", {}) + _first_handoff: Final = _model_call_details.get("first_api_call_start_time") + _usage_to_lift: Final = _failure_usage_to_lift( + model_call_details=_model_call_details, + request_body=request_data, + dispatched=_first_handoff is not None, + ) + _entries: Final = ( + ("first_api_call_start_time", _first_handoff), + ("combined_usage_object", None if _usage_to_lift is None else _usage_to_lift[0]), + ("response_cost", None if _usage_to_lift is None else (_usage_to_lift[1] or 0.0)), + ("standard_logging_object", _model_call_details.get("standard_logging_object")), + ) + return MappingProxyType({key: value for key, value in _entries if value is not None}) + + @dataclass(frozen=True) class _CallbackCapabilities: """Cached per-hook capability flags derived from ``litellm.callbacks``. @@ -2281,6 +2308,11 @@ class ProxyLogging: ) ) + # Auth and pass-through failure bodies are unstripped client input, and + # the logging handler below flattens body keys into model_call_details, + # so drop the key before it can masquerade as the built payload. + request_data.pop("standard_logging_object", None) + ### LOGGING ### if self._is_proxy_only_llm_api_error( original_exception=original_exception, @@ -2294,29 +2326,7 @@ class ProxyLogging: original_exception=original_exception, ) - # Lift the first-handoff instant onto request_data (top-level - # internal key, not metadata) so failure-path callbacks can still - # compute preprocessing latency after the logging object is popped. - _logging_obj: Final = request_data.get("litellm_logging_obj") - if _logging_obj is not None: - _model_call_details: Final = getattr(_logging_obj, "model_call_details", {}) - _first_handoff: Final = _model_call_details.get("first_api_call_start_time") - if _first_handoff is not None: - request_data["first_api_call_start_time"] = _first_handoff - - # Lift recovered partial-stream usage, or an estimated input-side - # usage for a dispatched failure, onto request_data so the - # failure-path spend callbacks (which run after the logging object - # is popped) record real token counts instead of zero. - _usage_to_lift: Final = _failure_usage_to_lift( - model_call_details=_model_call_details, - request_body=request_data, - dispatched=_first_handoff is not None, - ) - if _usage_to_lift is not None: - _lifted_usage, _lifted_cost = _usage_to_lift - request_data["combined_usage_object"] = _lifted_usage - request_data["response_cost"] = _lifted_cost + request_data.update(_failure_fields_to_lift(request_data)) # Remove before callbacks iterate — not serialisable request_data.pop("litellm_logging_obj", None) @@ -5960,15 +5970,14 @@ class ProxyUpdateSpend: ) break - except DB_RETRY_SAFE_ERROR_TYPES as e: - if i >= n_retry_times: # If we've reached the maximum number of retries - _raise_failed_update_spend_exception( - e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj - ) - # Optionally, sleep for a bit before retrying - await asyncio.sleep(2**i) # Exponential backoff except Exception as e: - _raise_failed_update_spend_exception(e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj) + await DBSpendUpdateWriter._handle_spend_update_failure( + e=e, + attempt=i, + n_retry_times=n_retry_times, + start_time=start_time, + proxy_logging_obj=proxy_logging_obj, + ) @staticmethod async def update_spend_logs( diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 4892e3b348c..64084bfb063 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1996,6 +1996,12 @@ class LiteLLMCompletionResponsesConfig: output_items.append(item) return output_items + @staticmethod + def _encode_thinking_blocks(message: Message) -> str | None: + thinking_blocks: Final[Sequence[Mapping[str, object]]] = getattr(message, "thinking_blocks", None) or () + preserved: Final = tuple(block for block in thinking_blocks if block.get("signature") or block.get("data")) + return json.dumps(preserved, separators=(",", ":")) if preserved else None + @staticmethod def _extract_reasoning_output_items( chat_completion_response: ModelResponse, @@ -2004,12 +2010,14 @@ class LiteLLMCompletionResponsesConfig: for choice in choices: if hasattr(choice, "message") and choice.message: message = choice.message - if hasattr(message, "reasoning_content") and message.reasoning_content: + reasoning_content = getattr(message, "reasoning_content", None) or "" + encrypted_content = LiteLLMCompletionResponsesConfig._encode_thinking_blocks(message) + if reasoning_content or encrypted_content: # Only check the first choice for reasoning content return [ GenericResponseOutputItem( type="reasoning", - id=f"rs_{hash(str(message.reasoning_content))}", + id=f"rs_{hash(reasoning_content or encrypted_content)}", status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status( choice.finish_reason ), @@ -2017,10 +2025,13 @@ class LiteLLMCompletionResponsesConfig: content=[ OutputText( type="output_text", - text=message.reasoning_content, + text=text, annotations=[], ) + for text in (reasoning_content,) + if text ], + encrypted_content=encrypted_content, ) ] return [] @@ -2292,18 +2303,19 @@ class LiteLLMCompletionResponsesConfig: # Translate completion_tokens_details to output_tokens_details if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details is not None: completion_details: Final = usage.completion_tokens_details - output_details_dict: Final[dict[str, int]] = {} - if hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None: - output_details_dict["reasoning_tokens"] = completion_details.reasoning_tokens - - if hasattr(completion_details, "text_tokens") and completion_details.text_tokens is not None: - output_details_dict["text_tokens"] = completion_details.text_tokens - - if hasattr(completion_details, "image_tokens") and completion_details.image_tokens is not None: - output_details_dict["image_tokens"] = completion_details.image_tokens - - if output_details_dict: - response_usage.output_tokens_details = OutputTokensDetails(**output_details_dict) + reasoning_token_count: Final = getattr(completion_details, "reasoning_tokens", None) + optional_output_details: Final[dict[str, int]] = { + field: value + for field, value in ( + ("text_tokens", getattr(completion_details, "text_tokens", None)), + ("image_tokens", getattr(completion_details, "image_tokens", None)), + ) + if value is not None + } + response_usage.output_tokens_details = OutputTokensDetails( + reasoning_tokens=reasoning_token_count if reasoning_token_count is not None else 0, + **optional_output_details, + ) return response_usage diff --git a/litellm/router.py b/litellm/router.py index 00c6c4c8b6f..26158ae0a56 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -64,6 +64,7 @@ from litellm.litellm_core_utils.coroutine_checker import coroutine_checker from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.ptu_pricing import zeroed_ptu_pricing from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) @@ -7695,7 +7696,16 @@ class Router: - None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params) """ try: - litellm_params: Final[LiteLLM_Params] = LiteLLM_Params(**_litellm_params) + zeroed_pricing: Final = ( + zeroed_ptu_pricing(_model_info, _litellm_params) if _model_info.get("db_model") is not True else None + ) + litellm_params: Final[LiteLLM_Params] = LiteLLM_Params( + **( + _litellm_params + if zeroed_pricing is None + else MappingProxyType({**_litellm_params, **zeroed_pricing}) + ) + ) warn_on_provider_credential_mismatch(model_name=_model_name, litellm_params=_litellm_params) deployment = Deployment( **deployment_info, @@ -10253,11 +10263,13 @@ class Router: returned_models.extend(self.get_model_list_from_routing_groups(model_name=model_name)) if len(returned_models) == 0: # check if wildcard route - potential_wildcard_models: Final = self.pattern_router.route(model_name) or [] + potential_wildcard_models: Final = self.pattern_router.get_deployments_by_pattern(model=model_name or "") ## check for team-specific wildcard models if team_id is not None and team_id in self.team_pattern_routers: - potential_team_only_wildcard_models: Final = self.team_pattern_routers[team_id].route(model_name) or [] + potential_team_only_wildcard_models: Final = self.team_pattern_routers[ + team_id + ].get_deployments_by_pattern(model=model_name or "") potential_wildcard_models.extend(potential_team_only_wildcard_models) if model_name is not None and potential_wildcard_models is not None: @@ -11189,6 +11201,8 @@ class Router: if pre_routing_hook_response is not None: model = pre_routing_hook_response.model messages = pre_routing_hook_response.messages + if pre_routing_hook_response.litellm_params: + request_kwargs.update(pre_routing_hook_response.litellm_params) ######################################################### # Resolve the strategy and logger AFTER the pre-routing hook, since @@ -11298,6 +11312,8 @@ class Router: if pre_routing_hook_response is not None: model = pre_routing_hook_response.model messages = pre_routing_hook_response.messages + if pre_routing_hook_response.litellm_params: + request_kwargs.update(pre_routing_hook_response.litellm_params) # 2. Get healthy deployments healthy_deployments: Final = await self.async_get_healthy_deployments( diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 259933dbb9e..cf7bde93360 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -53,6 +53,21 @@ model_list: REASONING: o1-preview ``` +Each tier can also use a model entry with request parameter overrides. A tier value may be +a model string, a single object, or a list mixing strings and objects. Object entries must +contain a model name and may contain any LiteLLM request parameters. The model name must +still resolve to a deployment in `model_list`; this configuration does not create one + +```yaml + tiers: + COMPLEX: opus + REASONING: + - model_name: opus + litellm_params: + reasoning_effort: xhigh + - abc +``` + ### Renaming the tiers `tier_labels` puts your own vocabulary on the four tiers: @@ -165,7 +180,7 @@ response = litellm.completion( ### Reasoning Override -If 2+ reasoning markers are detected in the user message, the request is automatically routed to the REASONING tier regardless of the weighted score. This ensures complex reasoning tasks get the appropriate model. +If 2+ reasoning markers are detected in the user message, the request is promoted to the REASONING tier even when the weighted score maps lower, so complex reasoning tasks get the appropriate model. The promotion requires the score to reach `reasoning_override_min_score`, which tracks `tier_boundaries.simple_medium` unless set, so stock phrases on an otherwise trivial prompt cannot buy the top tier. Set it to `0` to promote on the markers alone. ### System Prompt Handling diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index d16063b9bd4..0cb50cf3a3d 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -30,6 +30,7 @@ from litellm.constants import EMPTY_MAPPING, RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata +from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, @@ -663,6 +664,35 @@ class ClassificationOutcome(NamedTuple): classifier_cost: float | None = None +class _SessionAffinityPin(NamedTuple): + model: str + tier: ComplexityTier | None + + +def _parse_session_affinity_pin(value: object) -> _SessionAffinityPin | None: + if isinstance(value, str): + return _SessionAffinityPin(model=value, tier=None) + parts: Final[tuple[object, object] | None] = ( + (value.get("model"), value.get("tier")) + if isinstance(value, Mapping) + else (value[0], value[1]) + if isinstance(value, (list, tuple)) and len(value) == 2 + else None + ) + if parts is None: + return None + model, tier_value = parts + if not isinstance(model, str): + return None + tier: Final = ComplexityTier(tier_value) if isinstance(tier_value, str) else None + return _SessionAffinityPin(model=model, tier=tier) + + +def _session_affinity_cache_value(model: str, tier: ComplexityTier | str | None) -> Mapping[str, str | None]: + tier_value: Final = _tier_name(tier) if tier is not None else None + return {"model": model, "tier": tier_value} # mutable-ok: cache requires JSON mapping + + class ComplexityRouter(CustomLogger): """ Complexity router that classifies requests and routes to appropriate models. @@ -1020,13 +1050,14 @@ class ComplexityRouter(CustomLogger): weights: Final = self.config.dimension_weights weighted_score: Final = sum(d.score * weights.get(d.name, 0) for d in dimensions) - # Check for reasoning override (2+ reasoning markers) + boundaries: Final = self._effective_tier_boundaries() + clears_override_floor: Final = weighted_score >= self._effective_reasoning_override_min_score() + # Reuse match count from _score_keyword_match to avoid scanning twice - if reasoning_match_count >= 2: + if reasoning_match_count >= 2 and clears_override_floor: return ComplexityTier.REASONING, weighted_score, tuple(signals), "reasoning_override" # Map score to tier - boundaries: Final = self._effective_tier_boundaries() if weighted_score < boundaries["simple_medium"]: tier = ComplexityTier.SIMPLE elif weighted_score < boundaries["medium_complex"]: @@ -1038,6 +1069,18 @@ class ComplexityRouter(CustomLogger): return tier, weighted_score, tuple(signals), "heuristic_scorer" + def _effective_reasoning_override_min_score(self) -> float: + """The score a request must reach before the reasoning-marker override may promote it. + + Unset tracks the SIMPLE/MEDIUM boundary, so moving that boundary moves this floor with it + and the override still cannot rescue a request the mapping would call SIMPLE. An explicit + 0 is a real floor, not an absent one, so the comparison is against None. + """ + configured: Final = self.config.reasoning_override_min_score + if configured is None: + return self._effective_tier_boundaries()["simple_medium"] + return configured + def _effective_tier_boundaries(self) -> StandardLoggingRoutingDecisionTierBoundaries: """The tier boundaries in effect, with the documented defaults filled in. @@ -1065,6 +1108,7 @@ class ComplexityRouter(CustomLogger): classifier_model: str | None = None, classifier_cost: float | None = None, conversation_continuing: bool = True, + tier_litellm_params: Mapping[str, object] | None = None, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -1094,6 +1138,7 @@ class ComplexityRouter(CustomLogger): if score is not None: decision["score"] = score decision["tier_boundaries"] = self._effective_tier_boundaries() + decision["reasoning_override_min_score"] = self._effective_reasoning_override_min_score() if signals: # Stored as a list because this record is serialized to JSON for the spend # log and read back as an array by the dashboard; a sequence type that only @@ -1113,6 +1158,10 @@ class ComplexityRouter(CustomLogger): decision["classifier_model"] = classifier_model if classifier_cost is not None: decision["classifier_cost"] = classifier_cost + if tier_litellm_params: + masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params) + if isinstance(masked_tier_litellm_params, Mapping): + decision["tier_litellm_params"] = masked_tier_litellm_params return decision async def aclassify( @@ -1443,6 +1492,13 @@ class ComplexityRouter(CustomLogger): raise ValueError(f"No model configured for tier {tier_key} and no default_model set") + def _litellm_params_for_model(self, tier: ComplexityTier | str | None, model: str) -> Mapping[str, object]: + if tier is None: + return MappingProxyType({}) + entries: Final = self.config.tier_model_configs.get(_tier_name(tier), ()) + entry: Final = next((candidate for candidate in entries if candidate.model_name == model), None) + return entry.litellm_params if entry is not None else MappingProxyType({}) + @staticmethod def _pick_from_tier_value(model: str | list[str], tier_key: str) -> str: if isinstance(model, str): @@ -2054,9 +2110,10 @@ class ComplexityRouter(CustomLogger): cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None if cache_key is not None: - pinned_model: Final = await self.litellm_router_instance.cache.async_get_cache(key=cache_key) - if isinstance(pinned_model, str): - routed_model: str | None = pinned_model + pinned_value: Final = await self.litellm_router_instance.cache.async_get_cache(key=cache_key) + pinned_pin: Final = _parse_session_affinity_pin(pinned_value) + if pinned_pin is not None: + routed_model: str | None = pinned_pin.model pin_escalation_keyword: str | None = None if self.escalation_keywords: user_message: Final = ( @@ -2065,16 +2122,21 @@ class ComplexityRouter(CustomLogger): if user_message is not None: pin_escalation_keyword = self._matched_escalation_keyword(user_message) if pin_escalation_keyword is not None: - routed_model = self._escalated_pin(pinned_model) + routed_model = self._escalated_pin(pinned_pin.model) if routed_model is not None: - escalated: Final = routed_model != pinned_model + escalated: Final = routed_model != pinned_pin.model + resolved_pin_tier: Final = ( + pinned_pin.tier + if not escalated and pinned_pin.tier is not None + else self._tier_for_model(routed_model) + ) # The floor outranks the pin because plan mode is a transient state of the # session, not a request to move it: the turns carrying the sentinel route at # the floor, and the stored pin deliberately keeps the session's own model so # the first turn after plan mode exits auto-routes exactly as it would have. # Escalation is the opposite on purpose -- an explicit ask to re-pin higher. pin_plan_sentinel: Final = self._matched_plan_mode_signal(request_kwargs, resolved_messages) - pinned_tier: Final = self._tier_for_model(routed_model) if pin_plan_sentinel is not None else None + pinned_tier: Final = resolved_pin_tier if pin_plan_sentinel is not None else None plan_floored: Final = ( pinned_tier is not None and self._apply_plan_mode_floor(pinned_tier) != pinned_tier ) @@ -2085,7 +2147,7 @@ class ComplexityRouter(CustomLogger): # pin mid-conversation just because it outlives the original write. await self.litellm_router_instance.cache.async_set_cache( key=cache_key, - value=session_model, + value=_session_affinity_cache_value(session_model, resolved_pin_tier), ttl=self.config.session_affinity_ttl_seconds, ) if self.config.adaptive: @@ -2104,19 +2166,23 @@ class ComplexityRouter(CustomLogger): verbose_router_logger.info( "ComplexityRouter: routing decision cause=%s, routed_model=%s", cause, routed_model ) + routed_pin_tier: Final = self._tier_for_model(routed_model) if plan_floored else resolved_pin_tier + session_tier_litellm_params: Final = self._litellm_params_for_model(routed_pin_tier, routed_model) has_original_messages: Final = messages is not None and len(messages) > 0 return self._with_session_deployment_affinity( PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, + litellm_params=session_tier_litellm_params, routing_decision=self._build_routing_decision( routed_model=routed_model, cause=cause, - tier=self._tier_for_model(routed_model), + tier=routed_pin_tier, matched_keyword=pin_plan_sentinel if plan_floored else None, escalation_keyword=pin_escalation_keyword, escalated=escalated, conversation_continuing=conversation_continuing, + tier_litellm_params=session_tier_litellm_params, ), ) ) @@ -2143,7 +2209,10 @@ class ComplexityRouter(CustomLogger): if pinnable and cache_key is not None and response is not None: await self.litellm_router_instance.cache.async_set_cache( key=cache_key, - value=response.model, + value=_session_affinity_cache_value( + response.model, + response.routing_decision.get("tier") if response.routing_decision is not None else None, + ), ttl=self.config.session_affinity_ttl_seconds, ) return self._with_session_deployment_affinity(response) @@ -2257,6 +2326,7 @@ class ComplexityRouter(CustomLogger): ) keyword_plan_floored: Final = routed_tier != escalated_tier routed_model = await self._pick_model_for_tier(routed_tier, messages, resolved_messages, request_kwargs) + keyword_tier_litellm_params: Final = self._litellm_params_for_model(routed_tier, routed_model) keyword_cause: Final[RoutingDecisionCause] = ( "plan_mode" if keyword_plan_floored @@ -2272,6 +2342,7 @@ class ComplexityRouter(CustomLogger): return PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, + litellm_params=keyword_tier_litellm_params, routing_decision=self._build_routing_decision( routed_model=routed_model, conversation_continuing=conversation_continuing, @@ -2280,6 +2351,7 @@ class ComplexityRouter(CustomLogger): matched_keyword=plan_mode_sentinel if keyword_plan_floored else override.matched_keyword, escalation_keyword=escalation_keyword, escalated=keyword_escalated, + tier_litellm_params=keyword_tier_litellm_params, ), ) @@ -2366,6 +2438,7 @@ class ComplexityRouter(CustomLogger): routed_model, ) + tier_litellm_params: Final = self._litellm_params_for_model(tier, routed_model) classifier_model: Final = ( self.config.classifier_llm_config.model if outcome.cause == "llm_classifier" and self.config.classifier_llm_config is not None @@ -2391,6 +2464,7 @@ class ComplexityRouter(CustomLogger): return PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, + litellm_params=tier_litellm_params, routing_decision=self._build_routing_decision( routed_model=routed_model, conversation_continuing=conversation_continuing, @@ -2403,5 +2477,6 @@ class ComplexityRouter(CustomLogger): escalated=escalated, classifier_model=classifier_model, classifier_cost=outcome.classifier_cost, + tier_litellm_params=tier_litellm_params, ), ) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 6d43199c948..73f1378e5f7 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -5,10 +5,12 @@ Contains default keyword lists, weights, tier boundaries, and configuration clas All values are configurable via proxy config.yaml. """ +from collections.abc import Mapping from enum import Enum -from typing import Final, Literal +from types import MappingProxyType +from typing import Annotated, Final, Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, SkipValidation, field_serializer, field_validator, model_validator from litellm.types.router import AdaptiveRouterWeights, ClassifierPlugin, RoutingPlugin @@ -159,6 +161,44 @@ class ReminderMarkerPair(BaseModel): return self +class ComplexityTierModel(BaseModel): + model_config = ConfigDict(frozen=True) + + model_name: str + litellm_params: Annotated[Mapping[str, object], SkipValidation()] = Field( + default_factory=lambda: MappingProxyType({}) + ) + + @field_validator("litellm_params", mode="before") + @classmethod + def _freeze_litellm_params(cls, value: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType(dict(value)) + + @field_serializer("litellm_params") + def _serialize_litellm_params(self, value: Mapping[str, object]) -> Mapping[str, object]: + return dict(value) # mutable-ok: Pydantic JSON serialization requires a concrete mapping + + +def _normalize_tier_entries( + raw_value: object, + tier: str, +) -> tuple[str | list[str], tuple[ComplexityTierModel, ...]]: + raw_entries: Final = raw_value if isinstance(raw_value, (list, tuple)) else (raw_value,) + entries: Final = tuple( + ComplexityTierModel(model_name=entry) if isinstance(entry, str) else ComplexityTierModel.model_validate(entry) + for entry in raw_entries + ) + model_names: Final = tuple(entry.model_name for entry in entries) + if len(model_names) != len(frozenset(model_names)): + raise ValueError(f"tier {tier} contains duplicate model_name values; each pool entry needs distinct parameters") + normalized: Final = ( + entries[0].model_name + if not isinstance(raw_value, (list, tuple)) + else list(model_names) # mutable-ok: config.tiers must preserve its existing list contract + ) + return normalized, entries + + # ─── Default Keyword Lists ─── # Note: Keywords should be full words/phrases to avoid substring false positives. # The matching logic uses word boundary detection for single-word keywords. @@ -425,6 +465,9 @@ class ComplexityRouterConfig(BaseModel): "A list is randomly picked from when adaptive=False, and used as a soft-floor home pool when adaptive=True" ), ) + tier_model_configs: Mapping[str, tuple[ComplexityTierModel, ...]] = Field( + default_factory=dict, + ) tier_definitions: tuple[TierDefinition, ...] | None = Field( default=None, @@ -481,6 +524,15 @@ class ComplexityRouterConfig(BaseModel): ), ) + reasoning_override_min_score: float | None = Field( + default=None, + description=( + "Minimum weighted score a request must reach before 2+ reasoning markers may promote it to the " + "reasoning tier. Unset tracks tier_boundaries.simple_medium, so the override never rescues a " + "request the scorer placed in the cheapest tier; 0 restores the unconditional override" + ), + ) + # Token count thresholds token_thresholds: dict[str, int] = Field( default_factory=lambda: DEFAULT_TOKEN_THRESHOLDS.copy(), @@ -768,6 +820,55 @@ class ComplexityRouterConfig(BaseModel): coerced[key] = item return coerced + @model_validator(mode="before") + @classmethod + def _normalize_tier_model_configs(cls, value: object) -> object: + if not isinstance(value, dict): + return value + raw_tiers: Final = value.get("tiers") + if not isinstance(raw_tiers, dict): + return value + existing_configs: Final = value.get("tier_model_configs") + normalized_entries: Final = MappingProxyType( + {tier: _normalize_tier_entries(raw_value, tier) for tier, raw_value in raw_tiers.items()} + ) + normalized_tiers: Final = MappingProxyType( + {tier: normalized for tier, (normalized, _) in normalized_entries.items()} + ) + incoming_params: Final = ( + MappingProxyType( + { + (tier, entry.model_name): entry.litellm_params + for tier, entries in existing_configs.items() + for entry in (ComplexityTierModel.model_validate(item) for item in entries) + } + ) + if isinstance(existing_configs, dict) + else MappingProxyType({}) + ) + tier_model_configs: Final = MappingProxyType( + { + tier: tuple( + entry.model_copy( + update=MappingProxyType( + { + "litellm_params": incoming_params.get((tier, entry.model_name), entry.litellm_params), + } + ) + ) + for entry in entries + ) + for tier, (_, entries) in normalized_entries.items() + if any(entry.litellm_params for entry in entries) + or (isinstance(existing_configs, dict) and tier in existing_configs) + } + ) + return { # mutable-ok: Pydantic before-validator requires a concrete mapping + **value, + "tiers": normalized_tiers, + "tier_model_configs": tier_model_configs, + } + @field_validator("escalation_keywords") @classmethod def _normalize_escalation_keywords(cls, value: list[str] | None) -> list[str] | None: diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index d1e4b3bb2ce..e89fbbdab65 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -365,6 +365,22 @@ def get_secret( raise e +def secret_manager_would_be_consulted(secret_name: str) -> bool: + """ + Returns True if a `get_secret` read for `secret_name` would actually reach the hosted manager. + + Mirrors the gating `get_secret` applies below: the manager has to be up and readable, and + `hosted_keys`, when set, is an allowlist of the names it is consulted for. Callers use this to + tell "the manager does not have this key" apart from "the manager was never asked". + """ + if not _should_read_secret_from_secret_manager(): + return False + key_management_settings: Final = litellm._key_management_settings + if key_management_settings is None or key_management_settings.hosted_keys is None: + return True + return secret_name.removeprefix("os.environ/") in key_management_settings.hosted_keys + + def _should_read_secret_from_secret_manager() -> bool: """ Returns True if the secret manager should be used to read the secret, False otherwise @@ -373,11 +389,7 @@ def _should_read_secret_from_secret_manager() -> bool: - If the `_key_management_settings` access mode is "read_only" or "read_and_write", return True - Otherwise, return False """ - if litellm.secret_manager_client is not None: - if litellm._key_management_settings is not None: - if ( - litellm._key_management_settings.access_mode == "read_only" - or litellm._key_management_settings.access_mode == "read_and_write" - ): - return True - return False + key_management_settings: Final = litellm._key_management_settings + if litellm.secret_manager_client is None or key_management_settings is None: + return False + return key_management_settings.access_mode in ("read_only", "read_and_write") diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 17ba78b0190..5179210f942 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -620,6 +620,12 @@ class AnthropicResponseUsageBlock(BaseModel): output_tokens: int +class AnthropicOutputTokensDetails(BaseModel): + model_config = ConfigDict(extra="allow") + + thinking_tokens: int | None = None + + AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 9461297feca..63c93e0f268 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -6,7 +6,7 @@ from collections.abc import Mapping from datetime import datetime, timezone from typing import Final, Literal, TypeAlias -from pydantic import AliasChoices, BaseModel, ConfigDict, Field, computed_field, field_validator, model_validator +from pydantic import BaseModel, Field, computed_field, field_validator, model_validator from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig from litellm.types.utils import StandardLoggingRoutingDecision @@ -155,13 +155,17 @@ DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5" class StartShadowEvalRequest(BaseModel): - """Start duplicating a key's traffic for blind comparison against an auto-router.""" + """Start duplicating one or more keys' traffic for blind comparison against an auto-router.""" - api_key_id: str = Field( + api_key_ids: tuple[str, ...] = Field( + min_length=1, + max_length=100, description=( - "The hashed virtual key whose traffic will be shadowed. Shadow evaluation runs ONLY on this " - "key's traffic; requests made with any other key are not sampled." - ) + "The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these " + "keys' traffic; requests made with any other key are not sampled. Each key carries its own " + "max_turns budget, so one key exhausting its budget leaves the others sampling. At most 100 " + "keys per job, which also bounds every read the job's endpoints make." + ), ) router_name: str = Field(description="The auto-router under evaluation, in either direction") direction: ShadowEvalDirection = Field( @@ -204,8 +208,9 @@ class StartShadowEvalRequest(BaseModel): ge=1, le=2000, description=( - "Sample budget: the job judges at most this many turns, then completes. This is also the spend " - "bound; expected judge cost is roughly max_turns times one judge call" + "Per-key sample budget: the job judges at most this many turns of EACH scoped key's traffic, " + "so a job over N keys judges at most N times max_turns turns. This is also the spend bound; " + "expected judge cost is roughly that turn ceiling times one judge call" ), ) @@ -214,6 +219,12 @@ class StartShadowEvalRequest(BaseModel): def _round_percentage(cls, value: float) -> float: return round(value, 2) + @field_validator("api_key_ids") + @classmethod + def _dedupe_keys(cls, value: tuple[str, ...]) -> tuple[str, ...]: + """A key named twice would collide with itself on the one-active-per-(key, direction) index.""" + return tuple(dict.fromkeys(value)) + @model_validator(mode="after") def _baseline_model_matches_direction(self) -> "StartShadowEvalRequest": if self.direction == "reverse" and self.baseline_model is None: @@ -251,24 +262,46 @@ class ShadowEvalResult(BaseModel): by_tier: tuple[ShadowEvalSlice, ...] by_current_model: tuple[ShadowEvalSlice, ...] = Field( description=( - "Sliced by the model that served the real arm: the key's incumbent models in forward mode, " + "Sliced by the model that served the real arm: the keys' incumbent models in forward mode, " "and in reverse the models the router itself picked" ) ) + by_key: tuple[ShadowEvalSlice, ...] = Field( + description=( + "One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job " + "scopes but has not judged a turn for yet are absent rather than reported as zero" + ), + ) overall_shadow_win_rate_pct: float overall_tie_rate_pct: float -class ShadowEvalJobResponse(BaseModel): - """A shadow-eval job. Validates directly from the prisma record (job_id reads the - row's id); status is derived from stopped_at and ends_at, never stored, so no writer - anywhere can produce an inconsistent one. Aggregate fields are populated by the - detail endpoint only and stay None on list responses.""" +class ShadowEvalJobKeyResponse(BaseModel): + """One key a job shadows, with its own budget and stop state.""" - model_config = ConfigDict(from_attributes=True, populate_by_name=True) + api_key_id: str = Field(description="The hashed virtual key whose traffic this entry scopes") + max_turns: int = Field(description="This key's own sample budget, independent of its siblings'") + stopped_at: datetime | None = Field( + default=None, + description=( + "When this key's slot was stamped free, whether its own budget ran out, the window closed, " + "or an operator stopped the job; status is derived, so a spent budget reads completed even " + "while this is still unset" + ), + ) + attempt_count: int | None = Field( + default=None, + description=( + "This key's sampled attempts so far, judged and errored alike, the same count the sampler " + "budgets against max_turns; populated on list and detail responses. Frozen at stopped_at " + "once the key is stamped, so in-flight attempts landing after a stop never reclassify it" + ), + ) + + @property + def budget_spent(self) -> bool: + return self.attempt_count is not None and self.attempt_count >= self.max_turns - job_id: str = Field(validation_alias=AliasChoices("id", "job_id")) - api_key_id: str = Field(description="The hashed virtual key whose traffic this job evaluates, and only that key's") key_alias: str | None = Field( default=None, description="Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted", @@ -277,15 +310,34 @@ class ShadowEvalJobResponse(BaseModel): default=None, description="Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias", ) + + +class ShadowEvalJobResponse(BaseModel): + """A shadow-eval job over one or more keys, each with its own budget and stop state; + status is derived from stopped_by, the keys' stop and budget state, and ends_at, + never stored, so no writer anywhere can produce an inconsistent one. Aggregate + fields are populated by the detail endpoint only and stay None on list responses.""" + + job_id: str + keys: tuple[ShadowEvalJobKeyResponse, ...] = Field( + min_length=1, + description="The keys whose traffic this job evaluates, and only those keys', each with its own budget", + ) router_name: str direction: ShadowEvalDirection = "forward" baseline_model: str | None = None judge_model: str shadow_percentage: float - max_turns: int created_at: datetime ends_at: datetime - stopped_at: datetime | None = None + stopped_by: str | None = Field( + default=None, + description=( + "The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled " + "by migration for jobs that displayed stopped when the column arrived; None when the job " + "ended on its own. Its presence is what makes a job read stopped rather than completed" + ), + ) judged_count: int | None = Field(default=None, description="Verdicts recorded; detail endpoint only") error_count: int | None = Field(default=None, description="Sampled attempts that errored; detail endpoint only") @@ -296,12 +348,19 @@ class ShadowEvalJobResponse(BaseModel): @computed_field @property def status(self) -> ShadowEvalStatus: - """A job whose window has passed reads completed even if a later sweep stamped - stopped_at; stopped means sampling ended before the window did.""" + """Three recorded facts, no history-guessing: a stop is stopped_by (the migration + backfills it for every job that displayed stopped when the column arrived, so the + pre-column population is closed), completion is the window passing or every key + spending its budget, and anything else is running. The all-keys-stamped fallback + covers only stops written by pre-column pods during a rolling deploy.""" + if self.stopped_by is not None: + return "stopped" if datetime.now(timezone.utc) >= ( self.ends_at if self.ends_at.tzinfo else self.ends_at.replace(tzinfo=timezone.utc) ): return "completed" - if self.stopped_at is not None: + if all(key.budget_spent for key in self.keys): + return "completed" + if all(key.stopped_at is not None for key in self.keys): return "stopped" return "running" diff --git a/litellm/types/router.py b/litellm/types/router.py index 7d1dd1358d5..99a4603ae49 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,6 +4,7 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum +from collections.abc import Mapping from dataclasses import dataclass from typing import Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints @@ -897,6 +898,7 @@ class PreRoutingHookResponse(BaseModel): messages: list[dict[str, Any]] | None routing_decision: StandardLoggingRoutingDecision | None = None session_affinity_ttl_seconds: int | None = None + litellm_params: Mapping[str, object] | None = None _PreRoutingStrategyT_co = TypeVar("_PreRoutingStrategyT_co", covariant=True) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 07005d7f9ad..96b9343353d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -248,6 +248,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): regional_processing_uplift_multiplier_us: ( float | None ) # OpenAI US data-residency uplift multiplier applied to all token costs (e.g. 1.10 = +10%) + regional_endpoint_uplift_multiplier: ReadOnly[ + float | None + ] # Vertex AI non-global (regional) endpoint uplift multiplier applied to all token costs (e.g. 1.10 = +10%) output_cost_per_character: float | None # only for vertex ai models output_cost_per_audio_token: float | None output_cost_per_token_above_128k_tokens: float | None # only for vertex ai models @@ -2836,9 +2839,11 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): classifier_cost: float escalated: bool tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries + reasoning_override_min_score: ReadOnly[float] conversation_continuing: bool savings_baseline_model: str savings_baseline_deployment_id: str + tier_litellm_params: Mapping[str, object] # writable-ok: Pydantic warns on ReadOnly TypedDict fields # Fields whose values quote the caller's prompt. Dropped when an operator turns message @@ -2860,9 +2865,11 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset( "classifier_cost", "escalated", "tier_boundaries", + "reasoning_override_min_score", "conversation_continuing", "savings_baseline_model", "savings_baseline_deployment_id", + "tier_litellm_params", } ) @@ -3113,16 +3120,17 @@ class CostBreakdown(TypedDict, total=False): """ Detailed cost breakdown for a request. - ``service_tier`` and ``data_residency`` record the pricing basis the cost was - computed on, not what the caller asked for. A consumer that has to price a - counterfactual against this request (what another model would have charged for - it) needs the same basis to compare like with like, and re-deriving it from the - request is not possible after the fact: the tier the biller used comes from - ``optional_params``, which no log record carries. + ``service_tier``, ``data_residency``, and ``vertex_location`` record the pricing + basis the cost was computed on, not what the caller asked for. A consumer that has + to price a counterfactual against this request (what another model would have + charged for it) needs the same basis to compare like with like, and re-deriving it + from the request is not possible after the fact: the tier the biller used comes + from ``optional_params``, which no log record carries. """ service_tier: str | None data_residency: str | None + vertex_location: ReadOnly[str | None] input_cost: float # Cost of raw (non-cached) input tokens only cache_read_cost: float # Cost of cache-read tokens (discounted rate) cache_creation_cost: float # Cost of cache-write tokens (premium rate) @@ -3388,6 +3396,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): annotation_cost_per_page: float | None = None regional_processing_uplift_multiplier_eu: float | None = None regional_processing_uplift_multiplier_us: float | None = None + regional_endpoint_uplift_multiplier: float | None = None @classmethod def strip_custom_pricing_fields(cls, model_info: dict[str, Any]) -> dict[str, Any]: @@ -3818,6 +3827,7 @@ class SearchProviders(str, Enum): YOU_COM = "you_com" APISERPENT = "apiserpent" TINYFISH = "tinyfish" + AGENTCORE = "agentcore" NIMBLE = "nimble" diff --git a/litellm/utils.py b/litellm/utils.py index a7b70c4129a..d1b0cb882ac 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5662,6 +5662,7 @@ def _get_model_info_helper( regional_processing_uplift_multiplier_us=_model_info.get( "regional_processing_uplift_multiplier_us", None ), + regional_endpoint_uplift_multiplier=_model_info.get("regional_endpoint_uplift_multiplier", None), output_cost_per_audio_token=_model_info.get("output_cost_per_audio_token", None), output_cost_per_character=_model_info.get("output_cost_per_character", None), output_cost_per_reasoning_token=_model_info.get("output_cost_per_reasoning_token", None), @@ -9077,6 +9078,7 @@ class ProviderConfigManager: from litellm.llms.apiserpent.search.transformation import ( APISerpentSearchConfig, ) + from litellm.llms.bedrock.search.transformation import AgentCoreSearchConfig from litellm.llms.brave.search.transformation import BraveSearchConfig from litellm.llms.dataforseo.search.transformation import DataForSEOSearchConfig from litellm.llms.duckduckgo.search.transformation import DuckDuckGoSearchConfig @@ -9115,6 +9117,7 @@ class ProviderConfigManager: SearchProviders.YOU_COM: YouComSearchConfig, SearchProviders.APISERPENT: APISerpentSearchConfig, SearchProviders.TINYFISH: TinyfishSearchConfig, + SearchProviders.AGENTCORE: AgentCoreSearchConfig, SearchProviders.NIMBLE: NimbleSearchConfig, } config_class: Final = PROVIDER_TO_CONFIG_MAP.get(provider, None) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 07f9027313b..d0eca17272d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -54,6 +54,7 @@ "output_cost_per_image": 0.04 }, "1024-x-1024/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 1.9e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -67,6 +68,7 @@ "output_cost_per_image": 0.08 }, "256-x-256/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 2.4414e-07, "litellm_provider": "openai", "mode": "image_generation", @@ -80,6 +82,7 @@ "output_cost_per_image": 0.018 }, "512-x-512/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.86e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -2887,6 +2890,7 @@ "supports_function_calling": true }, "azure_ai/claude-haiku-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -2908,6 +2912,7 @@ "supports_vision": true }, "azure_ai/claude-opus-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -2930,6 +2935,7 @@ "supports_output_config": true }, "azure_ai/claude-opus-4-6": { + "deprecation_date": "2027-02-02", "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -2959,6 +2965,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-7": { + "deprecation_date": "2027-04-06", "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -3083,6 +3090,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-1": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -3104,6 +3112,7 @@ "supports_vision": true }, "azure_ai/claude-sonnet-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -3156,6 +3165,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-sonnet-4-6": { + "deprecation_date": "2027-02-10", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -3226,6 +3236,7 @@ "supports_tool_choice": true }, "azure_ai/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -3318,6 +3329,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -3364,6 +3376,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-2026-03-05": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -3410,6 +3423,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-pro": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -3455,6 +3469,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-pro-2026-03-05": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -3500,6 +3515,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-mini": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -3540,6 +3556,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-mini-2026-03-17": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -3580,6 +3597,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-nano": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -3620,6 +3638,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-nano-2026-03-17": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -3849,6 +3868,7 @@ "supports_vision": true }, "azure/eu/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -3918,6 +3938,7 @@ "supports_none_reasoning_effort": true }, "azure/eu/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -3948,6 +3969,7 @@ "supports_vision": true }, "azure/eu/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -4107,6 +4129,7 @@ "supports_vision": true }, "azure/global-standard/gpt-4o-mini": { + "deprecation_date": "2027-04-14", "input_cost_per_token": 1.5e-07, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -4155,6 +4178,7 @@ "supports_vision": true }, "azure/global/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -4224,6 +4248,7 @@ "supports_none_reasoning_effort": true }, "azure/global/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -4254,6 +4279,7 @@ "supports_vision": true }, "azure/global/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -4492,6 +4518,7 @@ "supports_vision": true }, "azure/gpt-4.1": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -4559,6 +4586,7 @@ "supports_web_search": false }, "azure/gpt-4.1-mini": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, @@ -4626,6 +4654,7 @@ "supports_web_search": false }, "azure/gpt-4.1-nano": { + "deprecation_date": "2026-10-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -4902,6 +4931,7 @@ "supports_vision": false }, "azure/gpt-4o-mini": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.65e-07, "litellm_provider": "azure", @@ -5344,6 +5374,7 @@ "supports_vision": true }, "azure/gpt-5": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5507,6 +5538,7 @@ "supports_vision": true }, "azure/gpt-5-mini": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -5572,6 +5604,7 @@ "supports_vision": true }, "azure/gpt-5-nano": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5e-09, "input_cost_per_token": 5e-08, "litellm_provider": "azure", @@ -5667,6 +5700,7 @@ "supports_vision": true }, "azure/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5736,6 +5770,7 @@ "supports_none_reasoning_effort": true }, "azure/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5797,6 +5832,7 @@ "supports_vision": true }, "azure/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -5827,6 +5863,7 @@ "supports_vision": true }, "azure/gpt-5.2": { + "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.75e-07, "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", @@ -6136,6 +6173,7 @@ "supports_web_search": true }, "azure/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -6180,6 +6218,7 @@ "supports_minimal_reasoning_effort": true }, "azure/us/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.8e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, @@ -6218,6 +6257,7 @@ "supports_minimal_reasoning_effort": true }, "azure/eu/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.8e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, @@ -6379,6 +6419,7 @@ "supports_minimal_reasoning_effort": true }, "azure/gpt-5.4-pro": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -7045,6 +7086,7 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -7095,6 +7137,7 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -7142,6 +7185,7 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -7408,6 +7452,7 @@ "supports_web_search": true }, "azure/gpt-5.4-mini": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", @@ -7489,6 +7534,7 @@ "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -7601,6 +7647,7 @@ "output_cost_per_token": 0.0 }, "azure/high/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7610,6 +7657,7 @@ ] }, "azure/high/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7619,6 +7667,7 @@ ] }, "azure/high/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7628,6 +7677,7 @@ ] }, "azure/low/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7637,6 +7687,7 @@ ] }, "azure/low/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7646,6 +7697,7 @@ ] }, "azure/low/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7655,6 +7707,7 @@ ] }, "azure/medium/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7664,6 +7717,7 @@ ] }, "azure/medium/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7673,6 +7727,7 @@ ] }, "azure/medium/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7695,6 +7750,7 @@ ] }, "azure/gpt-image-1.5": { + "deprecation_date": "2027-06-16", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -7720,6 +7776,7 @@ ] }, "azure/gpt-image-2": { + "deprecation_date": "2027-10-21", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -7751,6 +7808,7 @@ "supports_pdf_input": true }, "azure/low/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7760,6 +7818,7 @@ ] }, "azure/low/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7769,6 +7828,7 @@ ] }, "azure/low/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0345052083e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7778,6 +7838,7 @@ ] }, "azure/medium/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7787,6 +7848,7 @@ ] }, "azure/medium/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7796,6 +7858,7 @@ ] }, "azure/medium/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 7.9752604167e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7805,6 +7868,7 @@ ] }, "azure/high/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7814,6 +7878,7 @@ ] }, "azure/high/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7823,6 +7888,7 @@ ] }, "azure/high/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.1575520833e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7850,6 +7916,7 @@ "supports_function_calling": true }, "azure/o1": { + "deprecation_date": "2026-10-21", "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "azure", @@ -7944,6 +8011,7 @@ "supports_vision": false }, "azure/o3": { + "deprecation_date": "2026-10-21", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "litellm_provider": "azure", @@ -8041,6 +8109,7 @@ "supports_web_search": true }, "azure/o3-mini": { + "deprecation_date": "2026-10-01", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure", @@ -8071,6 +8140,7 @@ "supports_vision": false }, "azure/o3-pro": { + "deprecation_date": "2026-12-17", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "azure", @@ -8132,6 +8202,7 @@ "supports_vision": true }, "azure/o4-mini": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure", @@ -8580,6 +8651,7 @@ "supports_vision": true }, "azure/us/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -8649,6 +8721,7 @@ "supports_none_reasoning_effort": true }, "azure/us/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -8679,6 +8752,7 @@ "supports_vision": true }, "azure/us/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -8876,6 +8950,7 @@ ] }, "azure_ai/FW-DeepSeek-V3.2": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.1e-07, "input_cost_per_token": 6.2e-07, "litellm_provider": "azure_ai", @@ -8906,6 +8981,7 @@ "supports_tool_choice": true }, "azure_ai/FW-GLM-5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 2.2e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure_ai", @@ -8921,6 +8997,7 @@ "supports_tool_choice": true }, "azure_ai/FW-GLM-5.1": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 2.86e-07, "input_cost_per_token": 1.54e-06, "litellm_provider": "azure_ai", @@ -8987,6 +9064,7 @@ "supports_tool_choice": true }, "azure_ai/FW-Kimi-K2.5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 6.6e-07, "litellm_provider": "azure_ai", @@ -9079,6 +9157,7 @@ "supports_vision": true }, "azure_ai/FW-MiniMax-M2.5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.3e-08, "input_cost_per_token": 3.3e-07, "litellm_provider": "azure_ai", @@ -9164,6 +9243,7 @@ ] }, "azure_ai/MAI-Image-2e": { + "deprecation_date": "2026-08-15", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", "mode": "image_generation", @@ -9175,6 +9255,7 @@ ] }, "azure_ai/Llama-3.2-11B-Vision-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 3.7e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9188,6 +9269,7 @@ "supports_vision": true }, "azure_ai/Llama-3.2-90B-Vision-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 2.04e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9249,6 +9331,7 @@ "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-405B-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 5.33e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9271,6 +9354,7 @@ "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-8B-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 3e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9452,6 +9536,7 @@ "supports_reasoning": true }, "azure_ai/mistral-document-ai-2505": { + "deprecation_date": "2026-07-20", "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.003, "mode": "ocr", @@ -9529,6 +9614,7 @@ "output_cost_per_token": 0.0 }, "azure_ai/cohere-rerank-v3.5": { + "deprecation_date": "2026-05-14", "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, "litellm_provider": "azure_ai", @@ -9591,6 +9677,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-r1": { + "deprecation_date": "2026-08-13", "input_cost_per_token": 1.35e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9614,6 +9701,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v3-0324": { + "deprecation_date": "2026-07-13", "input_cost_per_token": 1.14e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9626,6 +9714,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v3.1": { + "deprecation_date": "2026-07-13", "input_cost_per_token": 1.23e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9639,6 +9728,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v4-pro": { + "deprecation_date": "2028-02-20", "input_cost_per_token": 1.74e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -9652,6 +9742,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v4-flash": { + "deprecation_date": "2028-02-20", "input_cost_per_token": 1.9e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -9683,6 +9774,7 @@ "supports_embedding_image_input": true }, "azure_ai/global/grok-3": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9697,6 +9789,7 @@ "supports_web_search": true }, "azure_ai/global/grok-3-mini": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9712,6 +9805,7 @@ "supports_web_search": true }, "azure_ai/grok-3": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9726,6 +9820,7 @@ "supports_web_search": true }, "azure_ai/grok-3-mini": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9773,6 +9868,7 @@ "supports_web_search": true }, "azure_ai/grok-4-fast-non-reasoning": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -9786,6 +9882,7 @@ "supports_web_search": true }, "azure_ai/grok-4-fast-reasoning": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -9863,6 +9960,7 @@ "supports_tool_choice": true }, "azure_ai/kimi-k2.5": { + "deprecation_date": "2027-01-26", "input_cost_per_token": 6e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -9877,6 +9975,7 @@ "supports_vision": true }, "azure_ai/kimi-k2.6": { + "deprecation_date": "2027-04-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -10004,6 +10103,7 @@ "supports_vision": true }, "babbage-002": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 4e-07, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, @@ -12014,6 +12114,7 @@ ] }, "claude-haiku-4-5-20251001": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -12037,6 +12138,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-haiku-4-5": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -12185,6 +12287,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -12218,6 +12321,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -12252,6 +12356,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-5": { + "deprecation_date": "2027-06-30", "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -12288,6 +12393,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-6": { + "deprecation_date": "2027-02-17", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -12434,6 +12540,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-opus-4-5-20251101": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12463,6 +12570,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-5": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12492,6 +12600,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6": { + "deprecation_date": "2027-02-05", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12528,6 +12637,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6-20260205": { + "deprecation_date": "2027-02-05", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12564,6 +12674,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "deprecation_date": "2027-04-16", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12602,6 +12713,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-opus-4-7-20260416": { + "deprecation_date": "2027-04-16", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12640,6 +12752,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "deprecation_date": "2027-06-09", "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -12675,6 +12788,7 @@ "prompt_cache_min_tokens": 512 }, "claude-opus-5": { + "deprecation_date": "2027-07-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12713,6 +12827,7 @@ "prompt_cache_min_tokens": 512 }, "claude-opus-4-8": { + "deprecation_date": "2027-05-28", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15042,6 +15157,7 @@ "mode": "search" }, "davinci-002": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 2e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, @@ -16676,6 +16792,14 @@ "notes": "APISerpent deep search (/api/search), multi-engine (Google, Bing, Yahoo, DuckDuckGo). Pricing: $0.60/1k searches." } }, + "agentcore/search": { + "input_cost_per_query": 0.0, + "litellm_provider": "agentcore", + "mode": "search", + "metadata": { + "notes": "Web Search on Amazon Bedrock AgentCore, billed by AWS on the gateway" + } + }, "tinyfish/search": { "input_cost_per_query": 0.0, "litellm_provider": "tinyfish", @@ -18594,6 +18718,7 @@ } }, "gemini-2.5-flash": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -18639,6 +18764,7 @@ "supports_image_size": false }, "gemini-2.5-flash-image": { + "deprecation_date": "2026-10-02", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -18683,6 +18809,7 @@ "supports_image_size": false }, "gemini-3-pro-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -18763,6 +18890,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -18887,6 +19015,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-lite": { + "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -18943,6 +19072,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.5-flash-lite": { + "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, @@ -19032,6 +19162,7 @@ "supports_web_search": true }, "gemini-2.5-flash-lite": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1e-08, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, @@ -19303,6 +19434,7 @@ "supports_image_size": false }, "gemini-2.5-pro": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -19403,6 +19535,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-pro-preview": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -19460,6 +19593,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-pro-preview-customtools": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -19614,6 +19748,8 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash": { + "prompt_cache_min_tokens": 4096, + "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1e-06, @@ -19624,6 +19760,7 @@ "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -19665,6 +19802,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.6-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -19679,6 +19817,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -19719,6 +19858,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.7-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -19733,6 +19873,7 @@ "output_cost_per_token": 3.75e-06, "output_cost_per_token_batches": 1.875e-06, "output_cost_per_token_flex": 1.875e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "supported_endpoints": [ "/v1/chat/completions", @@ -19773,6 +19914,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-pro-preview": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -19830,6 +19972,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-pro-preview-customtools": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -20050,6 +20193,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-robotics-er-1.6-preview": { + "deprecation_date": "2026-08-31", "input_cost_per_audio_token": 2e-06, "input_cost_per_token": 1e-06, "litellm_provider": "gemini", @@ -20120,6 +20264,7 @@ "supports_vision": true }, "gemini-embedding-001": { + "deprecation_date": "2028-05-20", "input_cost_per_token": 1.5e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, @@ -20562,8 +20707,8 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-3.1-flash-image": { - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 32768, @@ -20571,8 +20716,8 @@ "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, - "output_cost_per_token": 1.5e-06, - "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "rpm": 1000, "tpm": 4000000, "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image", @@ -20605,8 +20750,8 @@ }, "gemini/gemini-3.1-flash-image-preview": { "deprecation_date": "2026-06-25", - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 32768, @@ -20614,8 +20759,8 @@ "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, - "output_cost_per_token": 1.5e-06, - "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "rpm": 1000, "tpm": 4000000, "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image-preview", @@ -21337,6 +21482,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-3.5-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 1.5e-07, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1.5e-06, @@ -21391,6 +21537,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-3.6-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -21448,6 +21595,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-3.7-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -21538,6 +21686,7 @@ "tpm": 800000 }, "gemini/gemini-3.1-pro-preview": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 2e-06, @@ -21595,6 +21744,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-3.1-pro-preview-customtools": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 2e-06, @@ -21733,6 +21883,8 @@ "supports_vision": true }, "gemini-3.5-flash": { + "prompt_cache_min_tokens": 4096, + "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1.5e-06, @@ -21785,6 +21937,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.6-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -21840,6 +21993,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.7-flash": { + "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -23245,6 +23399,7 @@ "supports_tool_choice": true }, "gpt-3.5-turbo-instruct": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 1.5e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, @@ -24376,6 +24531,7 @@ "supports_pdf_input": true }, "low/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24387,6 +24543,7 @@ "supports_pdf_input": true }, "low/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24398,6 +24555,7 @@ "supports_pdf_input": true }, "low/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24409,6 +24567,7 @@ "supports_pdf_input": true }, "medium/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.034, "litellm_provider": "openai", "mode": "image_generation", @@ -24420,6 +24579,7 @@ "supports_pdf_input": true }, "medium/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -24431,6 +24591,7 @@ "supports_pdf_input": true }, "medium/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -24442,6 +24603,7 @@ "supports_pdf_input": true }, "high/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.133, "litellm_provider": "openai", "mode": "image_generation", @@ -24453,6 +24615,7 @@ "supports_pdf_input": true }, "high/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -24464,6 +24627,7 @@ "supports_pdf_input": true }, "high/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -24475,6 +24639,7 @@ "supports_pdf_input": true }, "standard/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24486,6 +24651,7 @@ "supports_pdf_input": true }, "standard/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24497,6 +24663,7 @@ "supports_pdf_input": true }, "standard/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24508,6 +24675,7 @@ "supports_pdf_input": true }, "1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24519,6 +24687,7 @@ "supports_pdf_input": true }, "1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24530,6 +24699,7 @@ "supports_pdf_input": true }, "1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -27443,18 +27613,21 @@ "output_cost_per_second": 0.0 }, "hd/1024-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 7.629e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "hd/1024-x-1792/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.539e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "hd/1792-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.539e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -27501,6 +27674,7 @@ "max_output_tokens": 8192 }, "high/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.167, "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "openai", @@ -27511,6 +27685,7 @@ ] }, "high/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -27521,6 +27696,7 @@ ] }, "high/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -28308,6 +28484,7 @@ "supports_tool_choice": true }, "low/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.011, "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "openai", @@ -28318,6 +28495,7 @@ ] }, "low/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -28328,6 +28506,7 @@ ] }, "low/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -28352,6 +28531,7 @@ "output_cost_per_image": 0.072 }, "medium/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.042, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28362,6 +28542,7 @@ ] }, "medium/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28372,6 +28553,7 @@ ] }, "medium/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28382,6 +28564,7 @@ ] }, "low/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.005, "litellm_provider": "openai", "mode": "image_generation", @@ -28390,6 +28573,7 @@ ] }, "low/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -28398,6 +28582,7 @@ ] }, "low/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -28406,6 +28591,7 @@ ] }, "medium/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.011, "litellm_provider": "openai", "mode": "image_generation", @@ -28414,6 +28600,7 @@ ] }, "medium/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -28422,6 +28609,7 @@ ] }, "medium/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -30315,6 +30503,7 @@ ] }, "multimodalembedding@001": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2e-07, "input_cost_per_image": 0.0001, "input_cost_per_token": 8e-07, @@ -36013,18 +36202,21 @@ "output_cost_per_image": 0.14 }, "standard/1024-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 3.81469e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "standard/1024-x-1792/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 4.359e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "standard/1792-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 4.359e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -36088,6 +36280,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" }, "text-embedding-005": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -36161,6 +36354,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "text-moderation-007": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36170,6 +36364,7 @@ "output_cost_per_token": 0.0 }, "text-moderation-latest": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36179,6 +36374,7 @@ "output_cost_per_token": 0.0 }, "text-moderation-stable": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36188,6 +36384,7 @@ "output_cost_per_token": 0.0 }, "text-multilingual-embedding-002": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -38675,6 +38872,7 @@ "supports_tool_choice": true }, "vertex_ai/claude-haiku-4-5": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -38685,6 +38883,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", "supports_assistant_prefill": true, "supports_function_calling": true, @@ -38698,6 +38897,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-haiku-4-5@20251001": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -38708,6 +38908,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5", "supports_assistant_prefill": true, "supports_function_calling": true, @@ -38850,6 +39051,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38877,6 +39079,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-1": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38895,6 +39098,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4-1@20250805": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38913,6 +39117,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4-5": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -38923,6 +39128,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38941,6 +39147,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-5@20251101": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -38951,6 +39158,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, + "regional_endpoint_uplift_multiplier": 1.1, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, @@ -38970,6 +39178,8 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6": { + "deprecation_date": "2027-02-05", + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39000,6 +39210,8 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6@default": { + "deprecation_date": "2027-02-05", + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39030,6 +39242,8 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-7": { + "deprecation_date": "2027-04-16", + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39061,6 +39275,8 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-opus-4-7@default": { + "deprecation_date": "2027-04-16", + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39092,6 +39308,8 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { + "deprecation_date": "2027-06-08", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39123,6 +39341,8 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-fable-5@default": { + "deprecation_date": "2027-06-08", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39154,6 +39374,8 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-5": { + "deprecation_date": "2027-01-24", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39186,6 +39408,8 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-5@default": { + "deprecation_date": "2027-01-24", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39218,6 +39442,8 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-4-8": { + "deprecation_date": "2027-05-28", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39250,6 +39476,8 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { + "deprecation_date": "2027-05-28", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39282,6 +39510,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39298,6 +39527,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -39310,6 +39540,8 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { + "deprecation_date": "2026-12-24", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -39342,6 +39574,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -39372,6 +39605,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5@20250929": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39388,6 +39622,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -39401,6 +39636,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4@20250514": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -39428,6 +39664,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39459,6 +39696,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4@20250514": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39623,6 +39861,7 @@ "supports_tool_choice": true }, "vertex_ai/gemini-2.5-flash-image": { + "deprecation_date": "2026-10-02", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -39668,6 +39907,7 @@ "supports_image_size": false }, "vertex_ai/gemini-3-pro-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -39700,6 +39940,7 @@ "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, "vertex_ai/gemini-3.1-flash-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -39776,6 +40017,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-flash-lite": { + "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -39794,6 +40036,7 @@ "output_cost_per_token_batches": 7.5e-07, "output_cost_per_token_flex": 7.5e-07, "output_cost_per_token_priority": 2.7e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", "supported_endpoints": [ "/v1/chat/completions", @@ -39832,6 +40075,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash-lite": { + "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, @@ -39849,6 +40093,7 @@ "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, "output_cost_per_token_priority": 4.5e-06, + "regional_endpoint_uplift_multiplier": 1.1, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", "supported_endpoints": [ "/v1/chat/completions", @@ -40549,6 +40794,7 @@ "supports_tool_choice": true }, "vertex_ai/veo-2.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40563,6 +40809,7 @@ ] }, "vertex_ai/veo-3.0-fast-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40577,6 +40824,7 @@ ] }, "vertex_ai/veo-3.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40619,6 +40867,7 @@ ] }, "vertex_ai/veo-3.1-generate-001": { + "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40633,6 +40882,7 @@ ] }, "vertex_ai/veo-3.1-fast-generate-001": { + "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -47014,6 +47264,8 @@ } }, "vertex_ai/claude-sonnet-5@default": { + "deprecation_date": "2026-12-24", + "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -47046,6 +47298,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6@default": { + "regional_endpoint_uplift_multiplier": 1.1, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -47975,15 +48228,15 @@ }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 2.8e-09, - "input_cost_per_token": 1.4e-07, - "input_cost_per_token_cache_hit": 2.8e-09, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 2.8e-07, + "output_cost_per_token": 1.32e-06, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -48001,15 +48254,15 @@ }, "deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 3.625e-09, - "input_cost_per_token": 4.35e-07, - "input_cost_per_token_cache_hit": 3.625e-09, + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 8.7e-07, + "output_cost_per_token": 3.96e-06, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -48027,15 +48280,15 @@ }, "deepseek/deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 2.8e-09, - "input_cost_per_token": 1.4e-07, - "input_cost_per_token_cache_hit": 2.8e-09, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 2.8e-07, + "output_cost_per_token": 1.32e-06, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -48053,15 +48306,15 @@ }, "deepseek/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 3.625e-09, - "input_cost_per_token": 4.35e-07, - "input_cost_per_token_cache_hit": 3.625e-09, + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 8.7e-07, + "output_cost_per_token": 3.96e-06, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index cd02fde595f..82854a3b717 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -514,6 +514,11 @@ "type": "object", "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema)." }, + "regional_endpoint_uplift_multiplier": { + "type": "number", + "minimum": 1, + "description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%)." + }, "regional_processing_uplift_multiplier_eu": { "type": "number", "minimum": 1, diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index dadbf33cbfe..5c312dcf1c8 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -18,7 +18,7 @@ "limit": 711 }, "ANN205": { - "limit": 113 + "limit": 112 }, "ANN206": { "limit": 133 diff --git a/schema.prisma b/schema.prisma index 52fb447157b..60058c777ca 100644 --- a/schema.prisma +++ b/schema.prisma @@ -947,6 +947,17 @@ model LiteLLM_DailyTagSpend { } +// One row per live proxy worker process. Workers upsert their row on a fixed +// heartbeat; counting rows with a recent heartbeat tells how many workers share +// this database, which lets the Admin UI hide its "no Redis" warning for +// deployments that are provably a single worker. +model LiteLLM_ProxyWorkerHeartbeat { + worker_id String @id + hostname String + started_at DateTime @default(now()) + last_heartbeat_at DateTime @default(now()) +} + // Track the status of cron jobs running. Only allow one pod to run the job at a time model LiteLLM_CronJob { cronjob_id String @id @default(cuid()) // Unique ID for the record @@ -1467,28 +1478,38 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } -// Shadow eval: evaluation of an auto-router against a key's live traffic, in either -// direction. forward duplicates the requests the key did not route through the router -// through it, answering whether the key should adopt it; reverse duplicates the requests -// the router did serve against a fixed baseline model, answering whether a key already on -// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge -// compares real vs shadow responses blind. The job row is immutable config plus -// stopped_at; every count, status, and spend figure is derived from the append-only -// attempt rows, so nothing can disagree across pods or stop races. +// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in +// either direction. forward duplicates the requests the keys did not route through the +// router through it, answering whether they should adopt it; reverse duplicates the +// requests the router did serve against a fixed baseline model, answering whether a key +// already on it still benefits. Either way a sampled slice runs in a detached task and an +// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job: +// immutable config plus that key's own turn budget and stop state, so one key exhausting +// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id +// (the id the API reports), written together by one atomic create_many with identical +// config; single-key jobs predating group_id were backfilled group_id = id. "One active +// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE +// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state +// partial indexes; it is what makes a concurrent start on another pod race-safe rather +// than read-then-create. Every count, status, and spend figure is derived from the +// append-only attempt rows, so nothing can disagree across pods or stop races. model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) - api_key_id String // hashed virtual key whose traffic is shadowed + group_id String // legs of one job share this; the API's job id + api_key_id String // hashed virtual key whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // sample budget: judge at most this many turns + max_turns Int // this key's sample budget: judge at most this many turns created_at DateTime @default(now()) created_by String? ends_at DateTime stopped_at DateTime? + stopped_by String? // operator who stopped it early; null when it ended on its own + @@index([group_id]) @@index([api_key_id]) @@index([created_at]) } diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 680e0dff67b..d7334552d0c 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -71,6 +71,16 @@ Request and response bodies are typed pydantic models in `models.py`; only the f Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure coverage of the harness itself carries no marker and runs regardless. Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache +## Record and replay fixtures + +`E2E_FIXTURE_MODE` selects the transport every client is built on: `live` (the default, and what an unset variable means: nothing changes), `record` (run against the live proxy and write every interaction to a fixture bundle), or `replay` (serve every interaction back from the bundle with no HTTP at all, so a replay run needs no proxy and cannot bill a provider). The seam is `select_transport` in `fixture_transport.py`, applied inside `build_proxy_client`; both transports fulfil the same `Transport` protocol, so no test or client changes shape in any mode + +A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per transport call in call order (`0000-post-chat-completions.json`). Auth header values and credential request fields (`api_key`, `*_secret_key`, `static_headers`, and the like; the list is `fixture_canonical.py`'s) are redacted on write, and file uploads store a sha256 digest instead of the bytes; response bodies are stored verbatim (a /key/generate response keeps the ephemeral virtual key it minted), which is part of why bundles are gitignored. `fixture_bundle.py` owns the format + +Replay matches calls per test by canonical key: `fixture_canonical.py` canonicalizes the recorded request (volatile headers and credential fields out, unique markers, generated ids, uuids, and timestamps replaced with fixed placeholders, object keys sorted) and the key is the method, path, and a content hash, so identity survives re-records and machine changes while any real content drift is a `ReplayMiss` that names the computed key, the closest recorded key with its file, and a content diff, and never falls through to a live call. Matching is order-independent across distinct keys (concurrent calls may interleave) and FIFO within one key (a poll loop replays its responses in recorded order); a passed test must also consume its whole recording, or teardown fails it naming a leftover key. Either way the fix is always to re-record with `E2E_FIXTURE_MODE=record`. Every rewrite rule lives in `fixture_canonical.py`, so a new volatile header, credential field name, or generated-id shape is one edit there. Record starts fresh every time: it wipes the previous bundle (refusing to wipe a directory that is not a bundle) and never reads it. A replay bundle whose manifest is older than seven days hard-fails at collection time naming the bundle's age, so replay can never certify against fixtures that have drifted more than a week from the live proxy + +Deliberately not here yet: streaming chunk fidelity (LIT-5742) and scoping record/replay to provider-bound traffic (LIT-5745) + ## Typing The harness is fully typed with no error budget: `make lint-e2e-basedpyright` must report zero basedpyright errors, and CI enforces that on any PR touching `tests/e2e/**/*.py`. When a response field is untyped, model it in `models.py` (just the fields you read) and let pydantic validate it, rather than threading a `dict` or `Any` through the test diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index dc69bd42171..67da1be9562 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -52,6 +52,17 @@ The suites run against a live proxy, so bring one up first by running the litell Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy +### Record and replay + +`E2E_FIXTURE_MODE=record` runs a suite against the live proxy as usual while writing every request/response pair to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`); `E2E_FIXTURE_MODE=replay` then runs the same suite entirely from that bundle, with no proxy traffic and no provider spend; the proxy liveness gate is skipped, so replay runs with no proxy up at all. Unset (or `live`) behaves exactly as before the knob existed + +```bash +E2E_FIXTURE_MODE=record uv run pytest tests/e2e/llm_translation/ -v +E2E_FIXTURE_MODE=replay uv run pytest tests/e2e/llm_translation/ -v +``` + +Replay fails hard (`ReplayMiss`) when the tests drift from the recording, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. See `CLAUDE.md` in this directory for the bundle format and the transport seam + Tests marked `@pytest.mark.e2e` hard-fail when no proxy answers `/health/liveliness`, so a run that goes red with `No live proxy` at setup means the proxy isn't up; they never skip for a missing proxy, so an absent proxy can't be mistaken for a pass ## What a complete test looks like diff --git a/tests/e2e/access_control/access_control_client.py b/tests/e2e/access_control/access_control_client.py index 7ace036f433..634a96bb0bd 100644 --- a/tests/e2e/access_control/access_control_client.py +++ b/tests/e2e/access_control/access_control_client.py @@ -2,12 +2,13 @@ from __future__ import annotations +import time from dataclasses import dataclass from pydantic import BaseModel, ValidationError from proxy_client import ProxyClient -from e2e_http import StreamingResponse +from e2e_http import NoBody, StreamingResponse, is_ok, unwrap from models import ( ChatBody, ChatMessage, @@ -15,9 +16,16 @@ from models import ( LiteLLMParamsBody, ModelInfoBody, ModelNewBody, + TeamDeleteBody, + TeamInfoParams, + TeamInfoResponse, + TeamNewBody, + TeamNewResponse, + TeamUpdateBody, ) MODEL_ACCESS_DENIED_MARKER = "key_model_access_denied" +TEAM_MODEL_ACCESS_DENIED_MARKER = "team_model_access_denied" ROUTE_NOT_ALLOWED_MARKER = "not allowed to call this route" @@ -31,6 +39,14 @@ class ApiErrorEnvelope(BaseModel): error: ApiErrorDetail +class AccessGroupInfoResponse(BaseModel): + """GET /access_group/{name}/info: the deployments a model access group grants.""" + + access_group: str + model_names: list[str] + deployment_count: int + + def error_envelope(body: str) -> ApiErrorEnvelope | None: """The OpenAI-shaped `{"error": {...}}` a client parses, or None if absent.""" try: @@ -51,15 +67,75 @@ class AccessControlClient: def delete_key(self, key: str) -> None: self.proxy.delete_key(key) - def chat_status(self, key: str, model: str, content: str) -> StreamingResponse: + def chat_status( + self, key: str, model: str, content: str, max_completion_tokens: int | None = None + ) -> StreamingResponse: return self.proxy.transport.send( "/chat/completions", headers=self.proxy.transport.bearer(key), json=ChatBody( - model=model, messages=[ChatMessage(role="user", content=content)] + model=model, + messages=[ChatMessage(role="user", content=content)], + max_completion_tokens=max_completion_tokens, ), ) + def create_team(self, team_alias: str, models: list[str]) -> str: + team_id = unwrap( + self.proxy.transport.post( + "/team/new", + headers=self.proxy.transport.master, + json=TeamNewBody(team_alias=team_alias, models=models), + response_type=TeamNewResponse, + ) + ).team_id + self._await_team(team_id) + return team_id + + def set_team_models(self, team_id: str, team_alias: str, models: list[str]) -> None: + """Replace the team's allow-list. /model/new appends a team-scoped deployment's + public name to it, so a test that means to grant only an access group has to + put the allow-list back afterwards.""" + _ = unwrap( + self.proxy.transport.post( + "/team/update", + headers=self.proxy.transport.master, + json=TeamUpdateBody(team_id=team_id, team_alias=team_alias, models=models), + response_type=NoBody, + ) + ) + + def delete_team(self, team_id: str) -> None: + _ = self.proxy.transport.post( + "/team/delete", + headers=self.proxy.transport.master, + json=TeamDeleteBody(team_ids=[team_id]), + response_type=NoBody, + ) + + def access_group_info(self, access_group: str) -> AccessGroupInfoResponse | None: + result = self.proxy.transport.get( + f"/access_group/{access_group}/info", + headers=self.proxy.transport.master, + params=NoBody(), + response_type=AccessGroupInfoResponse, + ) + return unwrap(result) if is_ok(result) else None + + def _await_team(self, team_id: str) -> None: + deadline = time.monotonic() + self.proxy.poll_timeout + while time.monotonic() < deadline: + result = self.proxy.transport.get( + "/team/info", + headers=self.proxy.transport.master, + params=TeamInfoParams(team_id=team_id), + response_type=TeamInfoResponse, + ) + if is_ok(result): + return + time.sleep(self.proxy.poll_interval) + raise AssertionError(f"/team/info never resolved team {team_id!r} created by /team/new") + def create_model_status(self, key: str, model_name: str) -> StreamingResponse: return self.proxy.transport.send( "/model/new", diff --git a/tests/e2e/access_control/test_model_access_group_e2e.py b/tests/e2e/access_control/test_model_access_group_e2e.py new file mode 100644 index 00000000000..5cc062ea096 --- /dev/null +++ b/tests/e2e/access_control/test_model_access_group_e2e.py @@ -0,0 +1,277 @@ +"""Live e2e: a model access group as the grant on a key and on a team. + +Whoever holds the group can call every deployment in it and nothing else, whether +the request names a deployment exactly, names a model that a wildcard deployment +in the group covers, or spells that model with its provider prefix. The bare-name +spelling is the LIT-5813 regression: the group-membership lookup skipped the +provider-prefix retry every other model-resolution path performs, so a group +holding `openai/gpt-5.4*` denied `gpt-5.4-nano` while allowing `openai/gpt-5.4-nano`. +""" + +from __future__ import annotations + +import os +import time +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from typing import Final + +import pytest + +from access_control_client import ( + AccessControlClient, + MODEL_ACCESS_DENIED_MARKER, + TEAM_MODEL_ACCESS_DENIED_MARKER, +) +from e2e_config import unique_marker +from lifecycle import ResourceManager +from models import ( + ChatResponse, + KeyGenerateBody, + LiteLLMParamsBody, + ModelInfoBody, + ModelNewBody, +) + +pytestmark = pytest.mark.e2e + +WILDCARD_PATTERN: Final = "openai/gpt-5.4*" +WILDCARD_BARE_MODEL: Final = "gpt-5.4-nano" +WILDCARD_PREFIXED_MODEL: Final = "openai/gpt-5.4-nano" +GROUP_BACKEND: Final = "openai/gpt-5.4-nano" +UNCOVERED_OPENAI_MODEL: Final = "gpt-5.2" + +TEAM_WILDCARD_PATTERN: Final = "openai/gpt-5.6*" +TEAM_WILDCARD_BARE_MODEL: Final = "gpt-5.6-luna" + +MAX_COMPLETION_TOKENS: Final = 16 +PROMPT: Final = "Reply with exactly: OK" + + +@dataclass(frozen=True, slots=True) +class GroupedDeployments: + """A wildcard deployment and an exactly-named one inside `access_group`, plus a + deployment left out of it.""" + + access_group: str + member_model: str + outsider_model: str + + +@dataclass(frozen=True, slots=True) +class TeamGrant: + """A team whose whole allow-list is `access_group`, holding one team-scoped + wildcard deployment, and a key that belongs to it.""" + + access_group: str + team_id: str + key: str + + +ModelSelector = Callable[[GroupedDeployments], str] + +ALLOWED: Final[tuple[tuple[str, ModelSelector], ...]] = ( + ("bare name the group's wildcard covers", lambda grouped: WILDCARD_BARE_MODEL), + ("provider-prefixed name the group's wildcard covers", lambda grouped: WILDCARD_PREFIXED_MODEL), + ("exactly-named deployment in the group", lambda grouped: grouped.member_model), +) + +DENIED: Final[tuple[tuple[str, ModelSelector], ...]] = ( + ("deployment outside the group", lambda grouped: grouped.outsider_model), + ("provider model outside the group's wildcard", lambda grouped: UNCOVERED_OPENAI_MODEL), + ("name no provider claims", lambda grouped: f"e2e-ag-unknown-{unique_marker()}"), +) + + +def _provider_key(env_var: str) -> str: + return os.environ.get(env_var) or f"os.environ/{env_var}" + + +def _grouped_model(model_name: str, backend: str, access_groups: list[str] | None) -> ModelNewBody: + return ModelNewBody( + model_name=model_name, + litellm_params=LiteLLMParamsBody(model=backend, api_key=_provider_key("OPENAI_API_KEY")), + model_info=ModelInfoBody(access_groups=access_groups), + ) + + +def _await_group_members(client: AccessControlClient, access_group: str, expected: frozenset[str]) -> None: + """The grant under test is the group's membership, so prove the proxy recorded it + before asserting on what the group lets through.""" + deadline = time.monotonic() + client.proxy.poll_timeout + listed: list[str] = [] + while time.monotonic() < deadline: + info = client.access_group_info(access_group) + listed = info.model_names if info is not None else [] + if expected.issubset(listed): + return + time.sleep(client.proxy.poll_interval) + pytest.fail( + f"/access_group/{access_group}/info never listed {sorted(expected)} as members; last read {listed}" + ) + + +def _await_team_allowlist(client: AccessControlClient, grant_key: str, access_group: str) -> None: + """Registering a team-scoped deployment appends its public name to the team's + allow-list, and a wildcard sitting there directly would grant the model under test + on its own. Poll a denial until the message enumerates the allow-list the test + means to exercise: the group, and nothing else.""" + allowlist: Final = f"models=['{access_group}']" + deadline = time.monotonic() + client.proxy.poll_timeout + body = "" + while time.monotonic() < deadline: + body = client.chat_status( + grant_key, UNCOVERED_OPENAI_MODEL, f"{PROMPT} {unique_marker()}", MAX_COMPLETION_TOKENS + ).body + if allowlist in body: + return + time.sleep(client.proxy.poll_interval) + pytest.fail(f"the team's allow-list never settled to {allowlist}; last denial read {body[:300]}") + + +@pytest.fixture(scope="module") +def grouped(client: AccessControlClient) -> Iterator[GroupedDeployments]: + marker: Final = unique_marker() + deployments: Final = GroupedDeployments( + access_group=f"e2e-ag-{marker}", + member_model=f"e2e-ag-member-{marker}", + outsider_model=f"e2e-ag-outsider-{marker}", + ) + registrations: Final = ( + _grouped_model(WILDCARD_PATTERN, WILDCARD_PATTERN, [deployments.access_group]), + _grouped_model(deployments.member_model, GROUP_BACKEND, [deployments.access_group]), + _grouped_model(deployments.outsider_model, GROUP_BACKEND, None), + ) + created: Final = tuple(client.proxy.register_model(body) for body in registrations) + try: + _await_group_members( + client, + deployments.access_group, + frozenset({WILDCARD_PATTERN, deployments.member_model}), + ) + yield deployments + finally: + for model_id in created: + client.proxy.delete_model(model_id) + + +@pytest.fixture(scope="module") +def team_grant(client: AccessControlClient) -> Iterator[TeamGrant]: + marker: Final = unique_marker() + access_group: Final = f"e2e-agt-{marker}" + team_alias: Final = f"e2e-ag-team-{marker}" + team_id: Final = client.create_team(team_alias, [access_group]) + key: Final = client.proxy.generate_key(KeyGenerateBody(models=[], team_id=team_id)) + model_id: Final = client.proxy.register_model( + ModelNewBody( + model_name=TEAM_WILDCARD_PATTERN, + litellm_params=LiteLLMParamsBody( + model=TEAM_WILDCARD_PATTERN, api_key=_provider_key("OPENAI_API_KEY") + ), + model_info=ModelInfoBody(team_id=team_id, access_groups=[access_group]), + ), + listed_for=key, + ) + client.set_team_models(team_id, team_alias, [access_group]) + try: + _await_team_allowlist(client, key, access_group) + yield TeamGrant(access_group=access_group, team_id=team_id, key=key) + finally: + client.proxy.delete_model(model_id) + client.proxy.delete_key(key) + client.delete_team(team_id) + + +class TestKeyScopedToAccessGroup: + @pytest.mark.covers( + "other.auth.model_access_group.wildcard_bare_name_allowed", + "other.auth.model_access_group.member_allowed", + ) + @pytest.mark.parametrize(("case", "select_model"), ALLOWED) + def test_group_grants_every_deployment_in_it( + self, + case: str, + select_model: ModelSelector, + client: AccessControlClient, + resources: ResourceManager, + grouped: GroupedDeployments, + ) -> None: + key = resources.key(models=[grouped.access_group]) + model = select_model(grouped) + + result = client.chat_status( + key, model, f"{PROMPT} {unique_marker()}", MAX_COMPLETION_TOKENS + ) + + assert result.status_code == 200, ( + f"a key holding access group {grouped.access_group!r} must be able to call " + f"{model!r} ({case}), got {result.status_code}: {result.body[:300]}" + ) + assert ChatResponse.model_validate_json(result.body).choices, ( + f"200 must carry a real completion, not an error envelope: {result.body[:300]}" + ) + + @pytest.mark.covers("other.auth.model_access_group.non_member_denied") + @pytest.mark.parametrize(("case", "select_model"), DENIED) + def test_group_grants_nothing_outside_it( + self, + case: str, + select_model: ModelSelector, + client: AccessControlClient, + resources: ResourceManager, + grouped: GroupedDeployments, + ) -> None: + key = resources.key(models=[grouped.access_group]) + model = select_model(grouped) + + result = client.chat_status( + key, model, f"{PROMPT} {unique_marker()}", MAX_COMPLETION_TOKENS + ) + + assert result.status_code == 403, ( + f"a key holding only access group {grouped.access_group!r} must be denied 403 on " + f"{model!r} ({case}), got {result.status_code}: {result.body[:300]}" + ) + assert MODEL_ACCESS_DENIED_MARKER in result.body, ( + f"403 body must be a key model-access denial, got: {result.body[:300]}" + ) + + +class TestTeamScopedToAccessGroup: + @pytest.mark.covers("other.auth.model_access_group.team_wildcard_bare_name_allowed") + def test_group_grants_the_teams_own_wildcard( + self, client: AccessControlClient, team_grant: TeamGrant + ) -> None: + result = client.chat_status( + team_grant.key, + TEAM_WILDCARD_BARE_MODEL, + f"{PROMPT} {unique_marker()}", + MAX_COMPLETION_TOKENS, + ) + + assert result.status_code == 200, ( + f"a team whose allow-list is access group {team_grant.access_group!r} must be able to " + f"call {TEAM_WILDCARD_BARE_MODEL!r} through its team-scoped {TEAM_WILDCARD_PATTERN!r} " + f"deployment, got {result.status_code}: {result.body[:300]}" + ) + assert ChatResponse.model_validate_json(result.body).choices, ( + f"200 must carry a real completion, not an error envelope: {result.body[:300]}" + ) + + @pytest.mark.covers("other.auth.model_access_group.team_non_member_denied") + def test_group_grants_the_team_nothing_outside_it( + self, client: AccessControlClient, team_grant: TeamGrant + ) -> None: + model = f"e2e-ag-unknown-{unique_marker()}" + + result = client.chat_status( + team_grant.key, model, f"{PROMPT} {unique_marker()}", MAX_COMPLETION_TOKENS + ) + + assert result.status_code == 403, ( + f"a team holding only access group {team_grant.access_group!r} must be denied 403 on " + f"{model!r}, got {result.status_code}: {result.body[:300]}" + ) + assert TEAM_MODEL_ACCESS_DENIED_MARKER in result.body, ( + f"403 body must be a team model-access denial, got: {result.body[:300]}" + ) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index eff3b4ddf58..da2a7da0bfa 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -15,19 +15,27 @@ shared fixtures build on it. import functools import os -from collections.abc import Iterator +from collections.abc import Generator, Iterator +from datetime import datetime, timezone import pytest import requests -from e2e_config import CONTROL_PLANE_BASE_URL, PROXY_BASE_URL +from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup +from fixture_transport import ( + fixture_mode_collection_error, + fixture_report_lines, + parse_fixture_mode, + replay_leftover_error, +) from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager from proxy_client import ProxyClient, build_proxy_client _E2E_TEST_RAN = pytest.StashKey[bool]() +_CALL_PASSED = pytest.StashKey[bool]() def pytest_configure(config: pytest.Config) -> None: @@ -49,6 +57,21 @@ def pytest_configure(config: pytest.Config) -> None: ) +def pytest_sessionstart(session: pytest.Session) -> None: + """Abort before collection when E2E_FIXTURE_MODE can never work: an unknown + mode value, or replay against a missing, unreadable, or stale bundle (the + stale message names the bundle's age). Live and record modes pass through.""" + reason = fixture_mode_collection_error( + FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc) + ) + if reason is not None: + raise pytest.UsageError(reason) + + +def pytest_report_header(config: pytest.Config) -> list[str]: + return fixture_report_lines(FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc)) + + def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: """Attach the two custom signals (suite package and covered cell ids) to every test's user_properties so the standard JUnit report (`--junitxml`) records them @@ -91,9 +114,12 @@ def _proxy_fail_reason() -> str | None: def pytest_runtest_setup(item: pytest.Item) -> None: """Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe. Unmarked tests (unit coverage of the harness) don't touch the proxy, so they - run even when none is up. Never skip for a missing proxy.""" + run even when none is up. Never skip for a missing proxy. Replay mode serves + every call from the fixture bundle, so it needs no live proxy either.""" if item.get_closest_marker("e2e") is None: return + if parse_fixture_mode(FIXTURE_MODE_RAW) == "replay": + return reason = _proxy_fail_reason() if reason is not None: pytest.fail(reason) @@ -110,6 +136,36 @@ def pytest_runtest_call(item: pytest.Item) -> None: item.session.stash[_E2E_TEST_RAN] = True +@pytest.hookimpl(wrapper=True) +def pytest_runtest_makereport( + item: pytest.Item, call: pytest.CallInfo[None] +) -> Generator[None, pytest.TestReport, pytest.TestReport]: + """Stash the call-phase outcome so teardown can tell a passed test from a + failed one without re-deriving it.""" + report = yield + if report.when == "call": + item.stash[_CALL_PASSED] = report.passed + return report + + +@pytest.hookimpl(wrapper=True) +def pytest_runtest_teardown(item: pytest.Item) -> Generator[None, None, None]: + """In replay mode a passing test must consume its whole recording: leftover + interactions mean the test now makes fewer calls than it did at record time, + so the replay proved less than the bundle claims. The check runs after the + yield so fixture finalizers replay their recorded calls first. Failed tests + are left alone - their own failure already explains any unconsumed tail.""" + result = yield + if not item.stash.get(_CALL_PASSED, False): + return result + reason = replay_leftover_error( + mode_raw=FIXTURE_MODE_RAW, bundle_dir=FIXTURE_DIR, test_key=item.nodeid + ) + if reason is not None: + pytest.fail(reason) + return result + + def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: """Once the whole e2e session is done (all suites), optionally truncate the spend logs so the DB doesn't accumulate test rows. The truncate is destructive diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index c7140a4503b..814ebae2e0b 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -12,6 +12,11 @@ - {id: other.auth.jwt.valid_token_allows, module: other, tier: P0, area: auth, assertions: [valid_token_allows], source: "handle_jwt.py:77-150", rationale: "Valid JWT with correct issuer + claims grants access"} - {id: other.auth.jwt.expired_denied, module: other, tier: P0, area: auth, assertions: [expired_denied], source: "handle_jwt.py:125-135", rationale: "Expired JWT rejected even with valid signature"} - {id: other.auth.jwt.invalid_signature_denied, module: other, tier: P0, area: auth, assertions: [invalid_signature_denied], source: "handle_jwt.py:145-150", rationale: "Bad/missing signature fails verification"} +- {id: other.auth.model_access_group.wildcard_bare_name_allowed, module: other, tier: P0, area: auth, assertions: [wildcard_bare_name_allowed], source: "auth_checks.py:3232 / LIT-5813", fail_before_fix: proven, rationale: "A grant of a group holding a wildcard deployment covers the bare model names callers actually send, not only the provider-prefixed spelling"} +- {id: other.auth.model_access_group.member_allowed, module: other, tier: P0, area: auth, assertions: [member_allowed], source: "auth_checks.py:3232", rationale: "A key whose allow-list is a model access group can call the deployments in that group"} +- {id: other.auth.model_access_group.non_member_denied, module: other, tier: P0, area: auth, assertions: [non_member_denied], source: "auth_checks.py:3232", rationale: "That same grant reaches nothing outside the group, including provider models the group's wildcard does not cover"} +- {id: other.auth.model_access_group.team_wildcard_bare_name_allowed, module: other, tier: P1, area: auth, assertions: [team_wildcard_bare_name_allowed], source: "auth_checks.py:3232 / LIT-5813", fail_before_fix: proven, rationale: "The same bare-name grant holds when the wildcard deployment is team-scoped and the team's allow-list is the group"} +- {id: other.auth.model_access_group.team_non_member_denied, module: other, tier: P1, area: auth, assertions: [team_non_member_denied], source: "auth_checks.py:3232", rationale: "A team-level group grant reaches nothing outside the group"} - {id: other.auth.virtual_key.route_permission_enforced, module: other, tier: P0, area: auth, assertions: [route_permission_enforced], source: "route_checks.py:89-151", rationale: "allowed_routes whitelist denies disallowed routes"} - {id: other.auth.virtual_key.route_group_allowed, module: other, tier: P1, area: auth, assertions: [route_group_allowed], source: "route_checks.py:106-128", rationale: "allowed_routes=[llm_api_routes] grants all LLM endpoints"} - {id: other.auth.passthrough.model_allowlist_enforced, module: other, tier: P1, area: auth, assertions: [model_allowlist_enforced], source: "route_checks.py:135-151", rationale: "Passthrough enforces per-key model allow-lists"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 277478eebaf..a5c3729f4be 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -13,6 +13,8 @@ from pathlib import Path from dotenv import load_dotenv +from fixture_transport import deterministic_marker, parse_fixture_mode + # Local runs keep provider / DataDog keys in tests/e2e/.env (see CONTRIBUTING.md). # Compose injects them into the proxy container, but pytest on the host does not # inherit that file unless we load it. override=False so a real shell export wins. @@ -90,6 +92,15 @@ PROPAGATION_TIMEOUT = float(os.environ.get("E2E_PROPAGATION_TIMEOUT", "15")) EXPECT_RUST = os.environ.get("E2E_EXPECT_RUST", "").strip().lower() in ("1", "true", "yes") +# Record/replay fixture selection (see fixture_transport.py). The raw mode value +# is parsed and validated there; "live" (the default, also for empty values) +# means the harness behaves exactly as before this knob existed. +FIXTURE_MODE_RAW = os.environ.get("E2E_FIXTURE_MODE", "live") +FIXTURE_DIR = Path( + os.environ.get("E2E_FIXTURE_DIR", "").strip() + or str(Path(__file__).resolve().parent / ".fixtures") +) + # Deliberately modest concurrency. The suite shares its proxy with every other # suite in the run, and 750 users at spawn rate 50 saturated the request path hard # enough to distort latency-sensitive neighbours (and to spend real provider money @@ -148,7 +159,11 @@ def datadog_mcp_url(*, toolsets: str = "core") -> str: def unique_marker() -> str: """A short unique token per call/run, so concurrent runs and the shared - response cache never collide on prompts, tags, or customer ids.""" + response cache never collide on prompts, tags, or customer ids. In record + and replay modes the token is deterministic per test instead, so a replay + run regenerates the exact requests the record run sent.""" + if parse_fixture_mode(FIXTURE_MODE_RAW) in ("record", "replay"): + return deterministic_marker() return uuid.uuid4().hex[:12] diff --git a/tests/e2e/fixture_bundle.py b/tests/e2e/fixture_bundle.py new file mode 100644 index 00000000000..615ae8df1a4 --- /dev/null +++ b/tests/e2e/fixture_bundle.py @@ -0,0 +1,315 @@ +"""On-disk fixture bundle format for record/replay e2e runs (LIT-5729). + +A bundle is a directory: one ``manifest.json`` (record timestamp + harness +version + format version) plus one subdirectory per test, holding one JSON file +per transport interaction in call order. Bundles older than +``MAX_BUNDLE_AGE`` hard-fail replay at collection time (see conftest), so a +green replay run can never certify against fixtures that have drifted more than +a week from the live proxy. + +This module owns the format only. The transports that produce and consume it +live in fixture_transport.py and the canonical match keys they compute live in +fixture_canonical.py (LIT-5741); streaming chunk fidelity and provider-scoping +are follow-ups (LIT-5742/5745). Every interaction file stores the full redacted +request because replay matches on its canonicalized content. +""" + +from __future__ import annotations + +import hashlib +import re +import shutil +import subprocess +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Annotated, Final, Literal + +from pydantic import BaseModel, Field, JsonValue, TypeAdapter + +from e2e_http import ( + BinaryStream, + NetworkError, + ProbeResult, + RateLimitedError, + Result, + StreamingResponse, + Success, + UnauthorizedError, + UnknownApiError, + ValidationError, +) + +BUNDLE_FORMAT_VERSION: Final = 1 +MAX_BUNDLE_AGE: Final = timedelta(days=7) +MANIFEST_FILENAME: Final = "manifest.json" + +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +class Manifest(BaseModel): + format_version: int + recorded_at: datetime + harness_version: str + + +class RecordedRequest(BaseModel): + """The request as the transport saw it, auth header values and credential + body/form fields redacted. + + Replay matches on the canonical content key fixture_canonical.py computes + over ``method`` (the transport verb, not the HTTP verb), ``path``, and the + canonicalized headers, params, body, form, and file identity. File uploads + store a content digest instead of the bytes.""" + + method: str + path: str + headers: dict[str, str] + params: dict[str, str] = {} + body: JsonValue | None = None + form: dict[str, str] | None = None + file_name: str | None = None + file_sha256: str | None = None + file_bytes: int | None = None + + +class RecordedResult(BaseModel): + """A ``Result[R]`` flattened for disk. ``data`` holds the success payload as + raw JSON; replay re-validates it against the ``response_type`` the caller + passes, exactly like a live response body.""" + + shape: Literal["result"] = "result" + kind: Literal["success", "network", "unauthorized", "rate_limited", "validation", "unknown"] + status_code: int | None = None + data: JsonValue | None = None + message: str | None = None + body: str | None = None + retry_after_seconds: int | None = None + + +class RecordedStreaming(BaseModel): + shape: Literal["streaming"] = "streaming" + payload: StreamingResponse + + +class RecordedBinary(BaseModel): + shape: Literal["binary"] = "binary" + payload: BinaryStream + + +class RecordedProbe(BaseModel): + shape: Literal["probe"] = "probe" + payload: ProbeResult + + +type RecordedResponse = RecordedResult | RecordedStreaming | RecordedBinary | RecordedProbe + + +class Interaction(BaseModel): + request: RecordedRequest + response: Annotated[ + RecordedResult | RecordedStreaming | RecordedBinary | RecordedProbe, + Field(discriminator="shape"), + ] + + +def to_json_value(model: BaseModel) -> JsonValue: + return _JSON.validate_json(model.model_dump_json(by_alias=True)) + + +def from_result[R: BaseModel](result: Result[R]) -> RecordedResult: + match result: + case Success(status_code=status_code, data=data): + return RecordedResult(kind="success", status_code=status_code, data=to_json_value(data)) + case NetworkError(message=message): + return RecordedResult(kind="network", message=message) + case UnauthorizedError(): + return RecordedResult(kind="unauthorized") + case RateLimitedError(retry_after_seconds=retry_after_seconds, body=body): + return RecordedResult(kind="rate_limited", retry_after_seconds=retry_after_seconds, body=body) + case ValidationError(message=message): + return RecordedResult(kind="validation", message=message) + case UnknownApiError(status_code=status_code, body=body): + return RecordedResult(kind="unknown", status_code=status_code, body=body) + + +def to_result[R: BaseModel](recorded: RecordedResult, response_type: type[R]) -> Result[R]: + match recorded.kind: + case "success": + return Success( + status_code=recorded.status_code or 200, + data=response_type.model_validate(recorded.data), + ) + case "network": + return NetworkError(message=recorded.message or "") + case "unauthorized": + return UnauthorizedError() + case "rate_limited": + return RateLimitedError( + retry_after_seconds=recorded.retry_after_seconds, body=recorded.body or "" + ) + case "validation": + return ValidationError(message=recorded.message or "") + case "unknown": + return UnknownApiError(status_code=recorded.status_code or 0, body=recorded.body or "") + + +def slugify(raw: str, *, limit: int = 60) -> str: + clean = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw).strip("-") + return clean[:limit].rstrip("-") + + +def slug_for_test(test_key: str) -> str: + """Directory name for one test's interactions: a readable tail plus a short + digest of the full node id, so same-named methods in different classes or + files never collide.""" + digest = hashlib.sha1(test_key.encode()).hexdigest()[:8] + tail = slugify(test_key.rsplit("::", 1)[-1]) + return f"{tail}-{digest}" if tail else digest + + +def interaction_filename(ordinal: int, request: RecordedRequest) -> str: + path_part = slugify(request.path, limit=40) or "root" + return f"{ordinal:04d}-{request.method}-{path_part}.json" + + +def harness_version() -> str: + try: + proc = subprocess.run( + ("git", "rev-parse", "--short", "HEAD"), + cwd=Path(__file__).resolve().parent, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return "unknown" + return proc.stdout.strip() or "unknown" + + +@dataclass(slots=True) +class BundleRecorder: + """Appends interaction files under ``root``, one subdirectory per test, with + a per-test ordinal that fixes replay order. ``prepare_bundle`` is the only + constructor: it guarantees the directory started empty with a fresh + manifest, so record mode never reads (or merges into) an existing bundle.""" + + root: Path + _ordinals: dict[str, int] = field(default_factory=dict) + + def record(self, *, test_key: str, request: RecordedRequest, response: RecordedResponse) -> None: + slug = slug_for_test(test_key) + ordinal = self._ordinals.get(slug, 0) + self._ordinals[slug] = ordinal + 1 + directory = self.root / slug + directory.mkdir(parents=True, exist_ok=True) + interaction = Interaction(request=request, response=response) + target = directory / interaction_filename(ordinal, request) + target.write_text(interaction.model_dump_json(indent=2), encoding="utf-8") + + +@dataclass(frozen=True, slots=True) +class UnsafeBundleDir: + path: Path + reason: str + + +def prepare_bundle(root: Path) -> BundleRecorder | UnsafeBundleDir: + """Start a fresh bundle at ``root`` for record mode: wipe whatever bundle is + there and write a new manifest. Refuses to wipe a directory that is neither + empty nor a bundle (no manifest.json), so a mistyped E2E_FIXTURE_DIR can + never delete unrelated files.""" + if root.exists(): + if not root.is_dir(): + return UnsafeBundleDir(path=root, reason="exists and is not a directory") + entries = tuple(root.iterdir()) + if entries and not (root / MANIFEST_FILENAME).is_file(): + return UnsafeBundleDir( + path=root, + reason=f"is not empty and has no {MANIFEST_FILENAME}; refusing to wipe a non-bundle directory", + ) + shutil.rmtree(root) + root.mkdir(parents=True) + manifest = Manifest( + format_version=BUNDLE_FORMAT_VERSION, + recorded_at=datetime.now(timezone.utc), + harness_version=harness_version(), + ) + (root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(indent=2), encoding="utf-8") + return BundleRecorder(root=root) + + +@dataclass(frozen=True, slots=True) +class FreshBundle: + manifest: Manifest + + +@dataclass(frozen=True, slots=True) +class StaleBundle: + recorded_at: datetime + age: timedelta + limit: timedelta + + +@dataclass(frozen=True, slots=True) +class UnreadableBundle: + reason: str + + +type BundleFreshness = FreshBundle | StaleBundle | UnreadableBundle + + +def _read_manifest(root: Path) -> Manifest | UnreadableBundle: + manifest_path = root / MANIFEST_FILENAME + if not manifest_path.is_file(): + return UnreadableBundle(reason=f"no {MANIFEST_FILENAME} found (record one with E2E_FIXTURE_MODE=record)") + try: + return Manifest.model_validate_json(manifest_path.read_text(encoding="utf-8")) + except ValueError as exc: + return UnreadableBundle(reason=f"{MANIFEST_FILENAME} is invalid: {exc}") + + +def check_freshness(root: Path, *, now: datetime) -> BundleFreshness: + manifest = _read_manifest(root) + if isinstance(manifest, UnreadableBundle): + return manifest + if manifest.format_version != BUNDLE_FORMAT_VERSION: + return UnreadableBundle( + reason=f"format_version {manifest.format_version} != supported {BUNDLE_FORMAT_VERSION}" + ) + recorded_at = ( + manifest.recorded_at + if manifest.recorded_at.tzinfo is not None + else manifest.recorded_at.replace(tzinfo=timezone.utc) + ) + age = now - recorded_at + if age > MAX_BUNDLE_AGE: + return StaleBundle(recorded_at=recorded_at, age=age, limit=MAX_BUNDLE_AGE) + return FreshBundle(manifest=manifest) + + +def format_age(age: timedelta) -> str: + total_hours = int(age.total_seconds()) // 3600 + return f"{total_hours // 24}d{total_hours % 24}h" + + +@dataclass(frozen=True, slots=True) +class LoadedBundle: + manifest: Manifest + interactions: dict[str, tuple[Interaction, ...]] + + +def load_bundle(root: Path) -> LoadedBundle | UnreadableBundle: + manifest = _read_manifest(root) + if isinstance(manifest, UnreadableBundle): + return manifest + interactions = { + directory.name: tuple( + Interaction.model_validate_json(file.read_text(encoding="utf-8")) + for file in sorted(directory.glob("*.json")) + ) + for directory in sorted(root.iterdir()) + if directory.is_dir() + } + return LoadedBundle(manifest=manifest, interactions=interactions) diff --git a/tests/e2e/fixture_canonical.py b/tests/e2e/fixture_canonical.py new file mode 100644 index 00000000000..427f06bf8fb --- /dev/null +++ b/tests/e2e/fixture_canonical.py @@ -0,0 +1,150 @@ +"""Canonical request identity for replay matching (LIT-5741). + +Matching a replayed call against the raw recorded request never hits: unique +markers salt prompts, model names, and tags; every run mints fresh virtual +keys; request ids and timestamps differ on every call. Matching on transport +verb + path alone collides: two different requests to the same route silently +swap responses, which passes when it should miss. The canonicalizer strips +exactly the volatile material (volatile headers, credential fields, markers, +generated ids, timestamps) and hashes what remains with sorted object keys, so +identity is content-based and stable across runs and machines. + +Every rewrite rule lives in this module, next to the transports that apply it: +a new volatile header, credential field name, or generated-id shape is one +edit here, never a per-suite change. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from functools import reduce +from typing import Final + +from pydantic import JsonValue + +from fixture_bundle import RecordedRequest + +VOLATILE_HEADER_NAMES: Final[frozenset[str]] = frozenset( + { + "authorization", + "x-litellm-api-key", + "x-api-key", + "x-goog-api-key", + "x-request-id", + "traceparent", + "tracestate", + } +) + +SECRET_FIELD_NAMES: Final[frozenset[str]] = frozenset( + {"api_key", "aws_access_key_id", "static_headers", "vertex_credentials"} +) +SECRET_FIELD_SUFFIXES: Final[tuple[str, ...]] = ( + "_api_key", + "_secret_key", + "_secret_access_key", + "_session_token", + "_credentials", + "_password", +) +SECRET_PLACEHOLDER: Final = "" + +PLACEHOLDER_RULES: Final[tuple[tuple[re.Pattern[str], str], ...]] = ( + (re.compile(r"(?"), + ( + re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"), + "", + ), + (re.compile(r"sk-[A-Za-z0-9_-]{16,}"), ""), + ( + re.compile(r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?"), + "", + ), + (re.compile(r"(?"), + ( + re.compile(r"\b(?:chatcmpl|msgbatch|msg|resp|batch|call|req|ftjob|gen|file)[-_][A-Za-z0-9]{8,}\b"), + "", + ), + (re.compile(r"(?"), +) + + +def is_secret_field(name: str) -> bool: + lowered: Final = name.lower() + return lowered in SECRET_FIELD_NAMES or lowered.endswith(SECRET_FIELD_SUFFIXES) + + +def canonical_string(value: str) -> str: + return reduce(lambda acc, rule: rule[0].sub(rule[1], acc), PLACEHOLDER_RULES, value) + + +def _canonical_flat(fields: dict[str, str]) -> dict[str, JsonValue]: + return { + key: SECRET_PLACEHOLDER if is_secret_field(key) else canonical_string(value) + for key, value in fields.items() + } + + +def _canonical_value(value: JsonValue) -> JsonValue: + match value: + case str(): + return canonical_string(value) + case dict(): + return { + key: SECRET_PLACEHOLDER + if is_secret_field(key) and item is not None + else _canonical_value(item) + for key, item in value.items() + } + case list(): + return [_canonical_value(item) for item in value] + case _: + return value + + +@dataclass(frozen=True, slots=True) +class CanonicalRequest: + method: str + path: str + content: str + + @property + def key(self) -> str: + digest: Final = hashlib.sha256( + f"{self.method} {self.path}\n{self.content}".encode() + ).hexdigest()[:16] + return f"{self.method} {self.path} #{digest}" + + def pretty_content(self) -> str: + return json.dumps(json.loads(self.content), indent=2, sort_keys=True) + + +def canonicalize(request: RecordedRequest) -> CanonicalRequest: + file_identity: Final[JsonValue | None] = ( + None + if request.file_name is None and request.file_sha256 is None + else { + "name": None if request.file_name is None else canonical_string(request.file_name), + "sha256": request.file_sha256, + "bytes": request.file_bytes, + } + ) + content: Final[dict[str, JsonValue]] = { + "headers": { + name.lower(): canonical_string(value) + for name, value in request.headers.items() + if name.lower() not in VOLATILE_HEADER_NAMES + }, + "params": _canonical_flat(request.params), + "body": _canonical_value(request.body), + "form": None if request.form is None else _canonical_flat(request.form), + "file": file_identity, + } + return CanonicalRequest( + method=request.method, + path=canonical_string(request.path), + content=json.dumps(content, sort_keys=True, separators=(",", ":")), + ) diff --git a/tests/e2e/fixture_transport.py b/tests/e2e/fixture_transport.py new file mode 100644 index 00000000000..ce4eec701ca --- /dev/null +++ b/tests/e2e/fixture_transport.py @@ -0,0 +1,724 @@ +"""Record/replay transports behind the same ``Transport`` protocol (LIT-5729). + +``RecordingTransport`` decorates the live transport: every call passes through +unchanged and its request/response pair is appended to the fixture bundle. +``ReplayTransport`` implements the protocol from a recorded bundle alone: no +HTTP, no proxy, no provider spend. Because both fulfil ``Transport``, no test +or client changes shape; ``build_proxy_client`` picks the transport from +``E2E_FIXTURE_MODE`` (live | record | replay, default live). + +Replay matches each call by test node id and canonical content key +(fixture_canonical.py, LIT-5741): volatile headers, credential fields, unique +markers, generated ids, and timestamps are canonicalized out before hashing, so +matching is order-independent across distinct keys, FIFO within a key, and a +miss fails hard (``ReplayMiss``) printing the computed key and the closest +recorded key without ever falling through to a live call. Streaming chunk +fidelity is LIT-5742; scoping record/replay to provider-bound traffic is +LIT-5745. +""" + +from __future__ import annotations + +import difflib +import functools +import hashlib +import os +from collections import deque +from dataclasses import dataclass, field +from datetime import datetime +from itertools import islice +from pathlib import Path +from typing import Final, Literal, assert_never + +from pydantic import BaseModel, JsonValue + +from e2e_http import AuthHeaders, BinaryStream, ProbeResult, Result, StreamingResponse +from fixture_bundle import ( + BundleRecorder, + FreshBundle, + Interaction, + LoadedBundle, + RecordedBinary, + RecordedProbe, + RecordedRequest, + RecordedResponse, + RecordedResult, + RecordedStreaming, + StaleBundle, + UnreadableBundle, + UnsafeBundleDir, + check_freshness, + format_age, + from_result, + interaction_filename, + load_bundle, + prepare_bundle, + slug_for_test, + to_json_value, + to_result, +) +from fixture_canonical import CanonicalRequest, canonicalize, is_secret_field +from transport import Transport + +type FixtureMode = Literal["live", "record", "replay"] + +FIXTURE_MODES: Final[tuple[FixtureMode, ...]] = ("live", "record", "replay") + +SESSION_TEST_KEY: Final = "session" + +REDACTED_HEADER_NAMES: Final[frozenset[str]] = frozenset({"authorization", "x-litellm-api-key"}) +REDACTED_VALUE: Final = "" + + +@dataclass(frozen=True, slots=True) +class InvalidFixtureMode: + value: str + + +def parse_fixture_mode(raw: str) -> FixtureMode | InvalidFixtureMode: + normalized = raw.strip().lower() or "live" + match normalized: + case "live" | "record" | "replay": + return normalized + case _: + return InvalidFixtureMode(value=raw) + + +def current_test_key() -> str: + """The pytest node id of the running test, from the PYTEST_CURRENT_TEST env + var pytest maintains (`` (setup|call|teardown)``); ``session`` for + calls outside any test (e.g. session-finish cleanup).""" + raw = os.environ.get("PYTEST_CURRENT_TEST", "") + if not raw: + return SESSION_TEST_KEY + return raw.rsplit(" (", 1)[0] + + +class ReplayMiss(AssertionError): + """Replay had no recorded interaction for a call the suite made. The test + drifted from the bundle (or the bundle from the suite): re-record.""" + + +_marker_ordinals: Final[dict[str, int]] = {} + + +def deterministic_marker() -> str: + """Stable stand-in for uuid-based unique markers in record and replay modes: + the Nth marker of a test is a pure function of the test's node id and N, so a + replay run regenerates exactly the model names, prompts, and tags the record + run sent and every recorded poll response still satisfies its predicate.""" + test_key = current_test_key() + ordinal = _marker_ordinals.get(test_key, 0) + _marker_ordinals[test_key] = ordinal + 1 + return hashlib.sha1(f"{test_key}#{ordinal}".encode()).hexdigest()[:12] + + +def _dump_flat(model: BaseModel | None) -> dict[str, str]: + if model is None: + return {} + dumped: dict[str, object] = model.model_dump(by_alias=True, exclude_none=True) + return {key: str(value) for key, value in dumped.items()} + + +def _redact(headers: dict[str, str]) -> dict[str, str]: + return { + name: REDACTED_VALUE if name.lower() in REDACTED_HEADER_NAMES else value + for name, value in headers.items() + } + + +def _redact_secret_fields(value: JsonValue) -> JsonValue: + match value: + case dict(): + return { + key: REDACTED_VALUE + if is_secret_field(key) and item is not None + else _redact_secret_fields(item) + for key, item in value.items() + } + case list(): + return [_redact_secret_fields(item) for item in value] + case _: + return value + + +def _redact_flat(fields: dict[str, str]) -> dict[str, str]: + return { + key: REDACTED_VALUE if is_secret_field(key) else value for key, value in fields.items() + } + + +def recorded_request( + method: str, + path: str, + *, + headers: BaseModel, + body: BaseModel | None = None, + params: BaseModel | None = None, + form: BaseModel | None = None, + file_name: str | None = None, + file_content: bytes | None = None, +) -> RecordedRequest: + return RecordedRequest( + method=method, + path=path, + headers=_redact(_dump_flat(headers)), + params=_redact_flat(_dump_flat(params)), + body=None if body is None else _redact_secret_fields(to_json_value(body)), + form=None if form is None else _redact_flat(_dump_flat(form)), + file_name=file_name, + file_sha256=None if file_content is None else hashlib.sha256(file_content).hexdigest(), + file_bytes=None if file_content is None else len(file_content), + ) + + +@dataclass(frozen=True, slots=True) +class RecordingTransport: + """Decorator over the live transport: forwards every call and appends the + interaction to the bundle, so a green live run leaves behind exactly the + traffic replay needs.""" + + inner: Transport + recorder: BundleRecorder + + def _record(self, request: RecordedRequest, response: RecordedResponse) -> None: + self.recorder.record(test_key=current_test_key(), request=request, response=response) + + def bearer(self, key: str) -> AuthHeaders: + return self.inner.bearer(key) + + @property + def master(self) -> AuthHeaders: + return self.inner.master + + def post[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + result = self.inner.post(path, headers=headers, json=json, response_type=response_type) + self._record(recorded_request("post", path, headers=headers, body=json), from_result(result)) + return result + + def get[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + params: BaseModel, + response_type: type[R], + timeout: float | None = None, + ) -> Result[R]: + result = self.inner.get( + path, headers=headers, params=params, response_type=response_type, timeout=timeout + ) + self._record(recorded_request("get", path, headers=headers, params=params), from_result(result)) + return result + + def delete[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + response_type: type[R], + params: BaseModel | None = None, + ) -> Result[R]: + result = self.inner.delete( + path, headers=headers, json=json, response_type=response_type, params=params + ) + self._record( + recorded_request("delete", path, headers=headers, body=json, params=params), + from_result(result), + ) + return result + + def patch[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + result = self.inner.patch(path, headers=headers, json=json, response_type=response_type) + self._record(recorded_request("patch", path, headers=headers, body=json), from_result(result)) + return result + + def put[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + result = self.inner.put(path, headers=headers, json=json, response_type=response_type) + self._record(recorded_request("put", path, headers=headers, body=json), from_result(result)) + return result + + def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: + response = self.inner.stream(path, headers=headers, json=json) + self._record( + recorded_request("stream", path, headers=headers, body=json), + RecordedStreaming(payload=response), + ) + return response + + def stream_binary( + self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 + ) -> BinaryStream: + response = self.inner.stream_binary(path, headers=headers, json=json, chunk_size=chunk_size) + self._record( + recorded_request("stream_binary", path, headers=headers, body=json), + RecordedBinary(payload=response), + ) + return response + + def send( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + params: BaseModel | None = None, + stream: bool = False, + ) -> StreamingResponse: + response = self.inner.send(path, headers=headers, json=json, params=params, stream=stream) + self._record( + recorded_request("send", path, headers=headers, body=json, params=params), + RecordedStreaming(payload=response), + ) + return response + + def probe(self, path: str, *, params: BaseModel) -> ProbeResult: + response = self.inner.probe(path, params=params) + self._record( + recorded_request("probe", path, headers=self.master, params=params), + RecordedProbe(payload=response), + ) + return response + + def upload[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + form: BaseModel, + filename: str, + content: bytes, + file_content_type: str = "application/jsonl", + file_field: str = "file", + params: BaseModel | None = None, + response_type: type[R], + ) -> Result[R]: + result = self.inner.upload( + path, + headers=headers, + form=form, + filename=filename, + content=content, + file_content_type=file_content_type, + file_field=file_field, + params=params, + response_type=response_type, + ) + self._record( + recorded_request( + "upload", + path, + headers=headers, + params=params, + form=form, + file_name=filename, + file_content=content, + ), + from_result(result), + ) + return result + + def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: + response = self.inner.download(path, headers=headers) + self._record( + recorded_request("download", path, headers=headers), + RecordedStreaming(payload=response), + ) + return response + + +def _build_pool(recorded: tuple[Interaction, ...]) -> dict[str, deque[Interaction]]: + keys: Final = tuple(canonicalize(interaction.request).key for interaction in recorded) + return { + key: deque( + interaction + for candidate_key, interaction in zip(keys, recorded, strict=True) + if candidate_key == key + ) + for key in dict.fromkeys(keys) + } + + +def _closest_recorded( + canonical: CanonicalRequest, recorded: tuple[Interaction, ...] +) -> tuple[CanonicalRequest, str]: + candidates: Final = tuple(canonicalize(interaction.request) for interaction in recorded) + ratios: Final = tuple( + difflib.SequenceMatcher( + None, f"{canonical.method} {canonical.path}\n{canonical.content}", + f"{candidate.method} {candidate.path}\n{candidate.content}", + ).ratio() + for candidate in candidates + ) + best: Final = max(range(len(candidates)), key=lambda index: ratios[index]) + return candidates[best], interaction_filename(best, recorded[best].request) + + +def _miss_message(test_key: str, slug: str, canonical: CanonicalRequest, bundle: LoadedBundle) -> str: + recorded: Final = bundle.interactions.get(slug, ()) + if not recorded: + return ( + f"replay miss for {test_key}: computed key {canonical.key} but nothing is recorded " + f"under {slug}; re-record with E2E_FIXTURE_MODE=record" + ) + closest, closest_file = _closest_recorded(canonical, recorded) + diff: Final = "\n".join( + islice( + difflib.unified_diff( + closest.pretty_content().splitlines(), + canonical.pretty_content().splitlines(), + fromfile=f"closest recorded ({closest_file})", + tofile="test made", + lineterm="", + ), + 60, + ) + ) + return ( + f"replay miss for {test_key}: no recorded interaction matches key {canonical.key}; " + f"closest recorded key is {closest.key} ({closest_file})\n{diff}\n" + "re-record with E2E_FIXTURE_MODE=record" + ) + + +@dataclass(slots=True) +class ReplaySource: + """One shared pool per test over a loaded bundle, so every client built in + the session consumes the same recorded interactions. Every pool is built + once at construction and per-key consumption is a single atomic deque pop, + so concurrent replay calls never race. Calls match by canonical content + key: order-independent across distinct keys (concurrent tests interleave + calls nondeterministically), FIFO within one key (a poll loop replays its + recorded responses in recorded order).""" + + bundle: LoadedBundle + _pools: dict[str, dict[str, deque[Interaction]]] = field(init=False) + + def __post_init__(self) -> None: + self._pools = { + slug: _build_pool(recorded) for slug, recorded in self.bundle.interactions.items() + } + + def _pool(self, slug: str) -> dict[str, deque[Interaction]]: + return self._pools.get(slug, {}) + + def next_interaction(self, request: RecordedRequest) -> Interaction: + test_key: Final = current_test_key() + slug: Final = slug_for_test(test_key) + pool: Final = self._pool(slug) + canonical: Final = canonicalize(request) + queue: Final = pool.get(canonical.key) + if queue is None: + raise ReplayMiss(_miss_message(test_key, slug, canonical, self.bundle)) + try: + return queue.popleft() + except IndexError: + raise ReplayMiss( + f"replay exhausted for {test_key}: every recorded interaction for key " + f"{canonical.key} is already consumed; re-record with E2E_FIXTURE_MODE=record" + ) from None + + def leftover_error(self, test_key: str) -> str | None: + """Non-None when the test consumed fewer interactions than were recorded, + meaning a passing replay proved less than the bundle claims.""" + slug: Final = slug_for_test(test_key) + recorded: Final = self.bundle.interactions.get(slug, ()) + if not recorded: + return None + leftover: Final = tuple( + interaction for queue in self._pool(slug).values() for interaction in queue + ) + if not leftover: + return None + return ( + f"replay incomplete for {test_key}: {len(leftover)} of {len(recorded)} recorded " + f"interactions never consumed, e.g. {canonicalize(leftover[0].request).key}; " + "re-record with E2E_FIXTURE_MODE=record" + ) + + +def _expect_result(interaction: Interaction) -> RecordedResult: + match interaction.response: + case RecordedResult() as recorded: + return recorded + case RecordedStreaming() | RecordedBinary() | RecordedProbe(): + raise ReplayMiss( + f"recorded {interaction.request.method} {interaction.request.path} is not a typed result" + ) + + +def _expect_streaming(interaction: Interaction) -> StreamingResponse: + match interaction.response: + case RecordedStreaming(payload=payload): + return payload + case RecordedResult() | RecordedBinary() | RecordedProbe(): + raise ReplayMiss( + f"recorded {interaction.request.method} {interaction.request.path} is not a streaming response" + ) + + +@dataclass(frozen=True, slots=True) +class ReplayTransport: + """A ``Transport`` served entirely from a recorded bundle: never opens a + connection, so a replay run cannot bill a provider.""" + + source: ReplaySource + master_key: str + + def bearer(self, key: str) -> AuthHeaders: + return AuthHeaders(authorization=f"Bearer {key}") + + @property + def master(self) -> AuthHeaders: + return self.bearer(self.master_key) + + def post[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + return to_result( + _expect_result( + self.source.next_interaction(recorded_request("post", path, headers=headers, body=json)) + ), + response_type, + ) + + def get[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + params: BaseModel, + response_type: type[R], + timeout: float | None = None, + ) -> Result[R]: + return to_result( + _expect_result( + self.source.next_interaction(recorded_request("get", path, headers=headers, params=params)) + ), + response_type, + ) + + def delete[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + response_type: type[R], + params: BaseModel | None = None, + ) -> Result[R]: + return to_result( + _expect_result( + self.source.next_interaction( + recorded_request("delete", path, headers=headers, body=json, params=params) + ) + ), + response_type, + ) + + def patch[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + return to_result( + _expect_result( + self.source.next_interaction(recorded_request("patch", path, headers=headers, body=json)) + ), + response_type, + ) + + def put[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + return to_result( + _expect_result( + self.source.next_interaction(recorded_request("put", path, headers=headers, body=json)) + ), + response_type, + ) + + def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: + return _expect_streaming( + self.source.next_interaction(recorded_request("stream", path, headers=headers, body=json)) + ) + + def stream_binary( + self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 + ) -> BinaryStream: + interaction = self.source.next_interaction( + recorded_request("stream_binary", path, headers=headers, body=json) + ) + match interaction.response: + case RecordedBinary(payload=payload): + return payload + case RecordedResult() | RecordedStreaming() | RecordedProbe(): + raise ReplayMiss( + f"recorded stream_binary {interaction.request.path} is not a binary stream" + ) + + def send( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + params: BaseModel | None = None, + stream: bool = False, + ) -> StreamingResponse: + return _expect_streaming( + self.source.next_interaction( + recorded_request("send", path, headers=headers, body=json, params=params) + ) + ) + + def probe(self, path: str, *, params: BaseModel) -> ProbeResult: + interaction = self.source.next_interaction( + recorded_request("probe", path, headers=self.master, params=params) + ) + match interaction.response: + case RecordedProbe(payload=payload): + return payload + case RecordedResult() | RecordedStreaming() | RecordedBinary(): + raise ReplayMiss(f"recorded probe {interaction.request.path} is not a probe result") + + def upload[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + form: BaseModel, + filename: str, + content: bytes, + file_content_type: str = "application/jsonl", + file_field: str = "file", + params: BaseModel | None = None, + response_type: type[R], + ) -> Result[R]: + return to_result( + _expect_result( + self.source.next_interaction( + recorded_request( + "upload", + path, + headers=headers, + params=params, + form=form, + file_name=filename, + file_content=content, + ) + ) + ), + response_type, + ) + + def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: + return _expect_streaming( + self.source.next_interaction(recorded_request("download", path, headers=headers)) + ) + + +@functools.lru_cache(maxsize=8) +def _shared_recorder(root: Path) -> BundleRecorder: + prepared = prepare_bundle(root) + if isinstance(prepared, UnsafeBundleDir): + raise ValueError(f"E2E_FIXTURE_DIR {prepared.path} {prepared.reason}") + return prepared + + +@functools.lru_cache(maxsize=8) +def _shared_replay_source(root: Path) -> ReplaySource: + loaded = load_bundle(root) + if isinstance(loaded, UnreadableBundle): + raise ValueError(f"cannot replay from {root}: {loaded.reason}") + return ReplaySource(bundle=loaded) + + +def replay_leftover_error(*, mode_raw: str, bundle_dir: Path, test_key: str) -> str | None: + """Teardown-time completeness check: in replay mode a passed test with + unconsumed recorded interactions must fail instead of passing against a + recording it no longer matches. Inert in every other mode.""" + if parse_fixture_mode(mode_raw) != "replay": + return None + return _shared_replay_source(bundle_dir).leftover_error(test_key) + + +def select_transport( + live: Transport, *, mode_raw: str, bundle_dir: Path, master_key: str +) -> Transport: + """The one seam every client build goes through: wraps (record), replaces + (replay), or passes through (live) the transport per E2E_FIXTURE_MODE. The + recorder and replay cursors are process-wide singletons per bundle dir, so + every client in a session shares one bundle and one recorded sequence.""" + mode = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode(value=value): + raise ValueError(f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}") + case "live": + return live + case "record": + return RecordingTransport(inner=live, recorder=_shared_recorder(bundle_dir)) + case "replay": + return ReplayTransport(source=_shared_replay_source(bundle_dir), master_key=master_key) + case _: + assert_never(mode) + + +def fixture_mode_collection_error(mode_raw: str, bundle_dir: Path, *, now: datetime) -> str | None: + """Session-abort reason for a fixture-mode setup that can never work, or None. + Called at collection time (conftest pytest_sessionstart) so a stale or missing + bundle fails the whole run up front, naming the bundle age, instead of failing + every test individually.""" + mode = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode(value=value): + return f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}" + case "live" | "record": + return None + case "replay": + freshness = check_freshness(bundle_dir, now=now) + match freshness: + case FreshBundle(): + return None + case StaleBundle(recorded_at=recorded_at, age=age, limit=limit): + return ( + f"fixture bundle at {bundle_dir} is stale: recorded {recorded_at.isoformat()}, " + f"age {format_age(age)} exceeds the {limit.days}-day limit; " + "re-record with E2E_FIXTURE_MODE=record" + ) + case UnreadableBundle(reason=reason): + return f"E2E_FIXTURE_MODE=replay cannot use bundle at {bundle_dir}: {reason}" + case _: + assert_never(freshness) + case _: + assert_never(mode) + + +def fixture_report_lines(mode_raw: str, bundle_dir: Path, *, now: datetime) -> list[str]: + """pytest report-header lines; empty in live mode so an unset + E2E_FIXTURE_MODE keeps today's output byte-identical.""" + mode = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode() | "live": + return [] + case "record": + return [f"e2e fixture mode: record -> {bundle_dir}"] + case "replay": + freshness = check_freshness(bundle_dir, now=now) + match freshness: + case FreshBundle(manifest=manifest): + return [ + f"e2e fixture mode: replay <- {bundle_dir} " + f"(recorded {manifest.recorded_at.isoformat()}, harness {manifest.harness_version})" + ] + case StaleBundle() | UnreadableBundle(): + return [f"e2e fixture mode: replay <- {bundle_dir}"] + case _: + assert_never(freshness) + case _: + assert_never(mode) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 150be8966ee..ac41971a2c8 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -767,6 +767,8 @@ class ModelInfoBody(BaseModel): # constraint when a prior run's teardown had not removed the row. id: str | None = None mode: ModelMode | None = None + access_groups: list[str] | None = None + team_id: str | None = None class ModelNewBody(BaseModel): @@ -862,6 +864,7 @@ class TeamNewResponse(BaseModel): class TeamUpdateBody(BaseModel): team_id: str team_alias: str + models: list[str] | None = None class TeamInfoParams(BaseModel): diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 5050b6fce68..3cae337a5ff 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -65,6 +65,8 @@ from models import ( ) from e2e_config import ( CONTROL_PLANE_BASE_URL, + FIXTURE_DIR, + FIXTURE_MODE_RAW, MASTER_KEY, POLL_INTERVAL, POLL_TIMEOUT, @@ -72,6 +74,7 @@ from e2e_config import ( REQUEST_TIMEOUT, settle_propagation, ) +from fixture_transport import select_transport from transport import HttpTransport, SplitTransport, Transport RowsPredicate = Callable[[list[SpendLogRow]], bool] @@ -275,7 +278,21 @@ class ProxyClient: mode: ModelMode | None = None, ) -> str: """Register a deployment under `model_name` and return its proxy-assigned - model_id, once the model is actually servable on the data plane. + model_id, once the model is actually servable on the data plane.""" + return self.register_model( + ModelNewBody( + model_name=model_name, + litellm_params=litellm_params, + model_info=ModelInfoBody(mode=mode), + ) + ) + + def register_model(self, body: ModelNewBody, listed_for: str | None = None) -> str: + """`create_model` for deployments that carry more than a mode: access groups, + team scoping, a pinned id. `listed_for` is the virtual key whose /v1/models + view must list the deployment before it counts as servable, because a + team-scoped deployment is listed to its own team and to nobody else, master + key included; leave it unset for a proxy-wide model. /model/new is a control-plane route; the data plane (which serves /chat, /ocr, ...) only picks the new model up on its next DB reload, so a call @@ -293,25 +310,22 @@ class ProxyClient: self.transport.post( "/model/new", headers=self.transport.master, - json=ModelNewBody( - model_name=model_name, - litellm_params=litellm_params, - model_info=ModelInfoBody(mode=mode), - ), + json=body, response_type=ModelNewResponse, ) ).model_id written_at = time.monotonic() - self._await_model_servable(model_name) + self._await_model_servable(body.model_name, listed_for) settle_propagation(written_at) return model_id - def _await_model_servable(self, model_name: str) -> None: + def _await_model_servable(self, model_name: str, listed_for: str | None = None) -> None: """Block until the data plane lists `model_name`, or fail at model_servable_timeout.""" + headers = self.transport.master if listed_for is None else self.transport.bearer(listed_for) outcome = await_servable( lambda poll_timeout: self.transport.get( "/v1/models", - headers=self.transport.master, + headers=headers, params=NoBody(), response_type=ModelsListResponse, timeout=poll_timeout, @@ -531,19 +545,29 @@ def build_proxy_client( The endpoints are injectable for callers that resolve the proxy some other way than ``e2e_config``'s env names (see ``claude_code/_env.py``); they must pass all three together, since a caller that overrides only the data plane - would leave management calls pointed at the env default.""" + would leave management calls pointed at the env default. + + E2E_FIXTURE_MODE wraps (record) or replaces (replay) the transport here, so + every client built from this seam records or replays without changing shape; + unset it stays the plain SplitTransport (see fixture_transport.py).""" + split = SplitTransport( + data=HttpTransport( + base_url=base_url, + master_key=master_key, + request_timeout=REQUEST_TIMEOUT, + ), + control=HttpTransport( + base_url=control_plane_base_url, + master_key=master_key, + request_timeout=REQUEST_TIMEOUT, + ), + ) return ProxyClient( - transport=SplitTransport( - data=HttpTransport( - base_url=base_url, - master_key=master_key, - request_timeout=REQUEST_TIMEOUT, - ), - control=HttpTransport( - base_url=control_plane_base_url, - master_key=master_key, - request_timeout=REQUEST_TIMEOUT, - ), + transport=select_transport( + split, + mode_raw=FIXTURE_MODE_RAW, + bundle_dir=FIXTURE_DIR, + master_key=master_key, ), poll_timeout=POLL_TIMEOUT, poll_interval=POLL_INTERVAL, diff --git a/tests/e2e/test_fixture_bundle.py b/tests/e2e/test_fixture_bundle.py new file mode 100644 index 00000000000..fd4cca6451f --- /dev/null +++ b/tests/e2e/test_fixture_bundle.py @@ -0,0 +1,218 @@ +"""Harness coverage for the on-disk fixture bundle format (LIT-5729). + +No proxy and no ``e2e`` marker: these pin the bundle CONTRACT - the seven-day +freshness gate that names the bundle's age, record mode's wipe safety (never +delete a directory that is not a bundle), collision-free per-test slugs, and +lossless Result round-trips - so replay can never silently drift from what +record wrote. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +from pydantic import BaseModel + +from e2e_http import ( + NetworkError, + RateLimitedError, + Result, + Success, + UnauthorizedError, + UnknownApiError, + ValidationError, +) +from fixture_bundle import ( + BUNDLE_FORMAT_VERSION, + MANIFEST_FILENAME, + MAX_BUNDLE_AGE, + BundleRecorder, + FreshBundle, + LoadedBundle, + Manifest, + RecordedRequest, + RecordedResult, + StaleBundle, + UnreadableBundle, + UnsafeBundleDir, + check_freshness, + format_age, + from_result, + interaction_filename, + load_bundle, + prepare_bundle, + slug_for_test, + to_result, +) + +NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc) + + +class Payload(BaseModel): + value: str + + +def write_manifest( + root: Path, recorded_at: datetime, *, format_version: int = BUNDLE_FORMAT_VERSION +) -> None: + root.mkdir(parents=True, exist_ok=True) + manifest = Manifest( + format_version=format_version, recorded_at=recorded_at, harness_version="abc1234" + ) + (root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(), encoding="utf-8") + + +def prepared(root: Path) -> BundleRecorder: + recorder = prepare_bundle(root) + assert isinstance(recorder, BundleRecorder) + return recorder + + +def plain_request(path: str) -> RecordedRequest: + return RecordedRequest(method="post", path=path, headers={}) + + +class TestResultRoundTrip: + @pytest.mark.parametrize( + "result", + [ + Success(status_code=201, data=Payload(value="ok")), + NetworkError(message="connection refused"), + UnauthorizedError(), + RateLimitedError(retry_after_seconds=7, body="slow down"), + ValidationError(message="bad shape"), + UnknownApiError(status_code=502, body="upstream exploded"), + ], + ) + def test_every_result_kind_survives_disk_and_back(self, result: Result[Payload]) -> None: + assert to_result(from_result(result), Payload) == result + + +class TestFreshness: + def test_bundle_at_the_limit_is_still_fresh(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - MAX_BUNDLE_AGE) + assert isinstance(check_freshness(root, now=NOW), FreshBundle) + + def test_stale_bundle_reports_age_and_limit(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=8, hours=3)) + freshness = check_freshness(root, now=NOW) + assert isinstance(freshness, StaleBundle) + assert freshness.age == timedelta(days=8, hours=3) + assert format_age(freshness.age) == "8d3h" + assert freshness.limit == MAX_BUNDLE_AGE + + def test_naive_recorded_at_is_read_as_utc(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, (NOW - timedelta(days=1)).replace(tzinfo=None)) + assert isinstance(check_freshness(root, now=NOW), FreshBundle) + + def test_missing_manifest_is_unreadable_with_recording_hint(self, tmp_path: Path) -> None: + freshness = check_freshness(tmp_path / "absent", now=NOW) + assert isinstance(freshness, UnreadableBundle) + assert MANIFEST_FILENAME in freshness.reason + assert "E2E_FIXTURE_MODE=record" in freshness.reason + + def test_corrupt_manifest_is_unreadable(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + root.mkdir() + (root / MANIFEST_FILENAME).write_text("{not json", encoding="utf-8") + assert isinstance(check_freshness(root, now=NOW), UnreadableBundle) + + def test_unknown_format_version_is_unreadable(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW, format_version=BUNDLE_FORMAT_VERSION + 1) + freshness = check_freshness(root, now=NOW) + assert isinstance(freshness, UnreadableBundle) + assert f"format_version {BUNDLE_FORMAT_VERSION + 1}" in freshness.reason + + +class TestPrepareBundle: + def test_fresh_directory_gets_a_fresh_manifest(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + prepared(root) + freshness = check_freshness(root, now=datetime.now(timezone.utc)) + assert isinstance(freshness, FreshBundle) + assert freshness.manifest.format_version == BUNDLE_FORMAT_VERSION + assert freshness.manifest.harness_version + + def test_record_wipes_the_previous_bundle_instead_of_reading_it(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + prepared(root).record( + test_key="old.py::test_old", + request=plain_request("/stale"), + response=RecordedResult(kind="unauthorized"), + ) + assert any(entry.is_dir() for entry in root.iterdir()) + prepared(root) + assert {entry.name for entry in root.iterdir()} == {MANIFEST_FILENAME} + + def test_refuses_to_wipe_a_directory_that_is_not_a_bundle(self, tmp_path: Path) -> None: + root = tmp_path / "precious" + root.mkdir() + (root / "notes.txt").write_text("keep me", encoding="utf-8") + outcome = prepare_bundle(root) + assert isinstance(outcome, UnsafeBundleDir) + assert MANIFEST_FILENAME in outcome.reason + assert (root / "notes.txt").read_text(encoding="utf-8") == "keep me" + + def test_refuses_a_path_that_is_a_file(self, tmp_path: Path) -> None: + target = tmp_path / "not-a-dir" + target.write_text("x", encoding="utf-8") + outcome = prepare_bundle(target) + assert isinstance(outcome, UnsafeBundleDir) + assert "not a directory" in outcome.reason + + +class TestSlugs: + def test_slug_for_test_is_deterministic(self) -> None: + key = "tests/e2e/suite/test_mod.py::TestX::test_case" + assert slug_for_test(key) == slug_for_test(key) + + def test_same_tail_in_different_files_never_collides(self) -> None: + first = slug_for_test("tests/e2e/a/test_a.py::test_case") + second = slug_for_test("tests/e2e/b/test_b.py::test_case") + assert first != second + assert first.startswith("test_case-") + assert second.startswith("test_case-") + + def test_interaction_filename_orders_and_slugs(self) -> None: + request = RecordedRequest(method="post", path="/chat/completions", headers={}) + assert interaction_filename(3, request) == "0003-post-chat-completions.json" + + +class TestRecordAndLoad: + def test_load_returns_interactions_in_recorded_order(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + recorder = prepared(root) + key = "suite/test_mod.py::test_ordered" + for path in ("/first", "/second", "/third"): + recorder.record( + test_key=key, + request=plain_request(path), + response=RecordedResult(kind="unauthorized"), + ) + loaded = load_bundle(root) + assert isinstance(loaded, LoadedBundle) + assert [ + interaction.request.path for interaction in loaded.interactions[slug_for_test(key)] + ] == ["/first", "/second", "/third"] + + def test_interactions_group_per_test(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + recorder = prepared(root) + for key in ("suite/test_a.py::test_one", "suite/test_b.py::test_two"): + recorder.record( + test_key=key, + request=plain_request(f"/{key[-3:]}"), + response=RecordedResult(kind="unauthorized"), + ) + loaded = load_bundle(root) + assert isinstance(loaded, LoadedBundle) + assert set(loaded.interactions) == { + slug_for_test("suite/test_a.py::test_one"), + slug_for_test("suite/test_b.py::test_two"), + } diff --git a/tests/e2e/test_fixture_canonical.py b/tests/e2e/test_fixture_canonical.py new file mode 100644 index 00000000000..30c57dc3ac6 --- /dev/null +++ b/tests/e2e/test_fixture_canonical.py @@ -0,0 +1,163 @@ +"""Harness coverage for canonical request identity (LIT-5741). + +No proxy and no ``e2e`` marker: pure functions over ``RecordedRequest``. Pins +the two failure modes match keys must avoid: keying on volatile material so +nothing ever matches (markers, virtual keys, ids, timestamps, volatile +headers), and keying on too little so different requests collide and a test +silently asserts against another request's response. +""" + +from __future__ import annotations + +import pytest +from pydantic import JsonValue + +from fixture_bundle import RecordedRequest +from fixture_canonical import CanonicalRequest, canonical_string, canonicalize, is_secret_field + + +def request( + method: str = "post", + path: str = "/chat/completions", + *, + headers: dict[str, str] | None = None, + params: dict[str, str] | None = None, + body: JsonValue | None = None, + form: dict[str, str] | None = None, + file_name: str | None = None, + file_sha256: str | None = None, + file_bytes: int | None = None, +) -> RecordedRequest: + return RecordedRequest( + method=method, + path=path, + headers=headers or {}, + params=params or {}, + body=body, + form=form, + file_name=file_name, + file_sha256=file_sha256, + file_bytes=file_bytes, + ) + + +class TestPlaceholders: + @pytest.mark.parametrize( + ("raw", "expected"), + [ + ("Reply ok. 4d5152a995b7", "Reply ok. "), + ("e2e-chat-stream-4d5152a995b7", "e2e-chat-stream-"), + ("sk-3mCXCTGmYuEEIU2i2qmVE3Xq6tSK1O0X6ZIRP1Lpw8ZlbNjt", ""), + ("9f1c8a2e-4b3d-4f6a-8f2f-0a1b2c3d4e5f", ""), + ("z" * 64, "z" * 64), + ("0123456789abcdef" * 4, ""), + ("2026-08-19T20:57:13.363499+00:00", ""), + ("2026-08-19", ""), + ("chatcmpl-C0LO6rRkfJlpJ2mqW9BHYo4Sm8FWl", ""), + ("batch_688a8b7f9a08819096e0f7c88fcd07c5", ""), + ("file-XyZ12345abc", ""), + ("gpt-4o-mini", "gpt-4o-mini"), + ("max_tokens", "max_tokens"), + ("sk-1234", "sk-1234"), + ], + ) + def test_rewrites_exactly_the_volatile_shapes(self, raw: str, expected: str) -> None: + assert canonical_string(raw) == expected + + +class TestSecretFields: + @pytest.mark.parametrize( + ("name", "secret"), + [ + ("api_key", True), + ("openai_api_key", True), + ("aws_secret_access_key", True), + ("aws_session_token", True), + ("vertex_credentials", True), + ("static_headers", True), + ("langfuse_secret_key", True), + ("model", False), + ("max_completion_tokens", False), + ("api_base", False), + ], + ) + def test_names_that_carry_credentials(self, name: str, secret: bool) -> None: + assert is_secret_field(name) is secret + + +class TestKeyStability: + def test_volatile_material_does_not_change_the_key(self) -> None: + """Acceptance: a suite recorded on one machine (fresh keys, that day's + dates, that run's markers) replays on another with no misses.""" + first = request( + headers={"authorization": "Bearer sk-run-one-aaaaaaaaaaaaaaaa", "x-request-id": "req-1"}, + params={"start_date": "2026-08-18"}, + body={ + "model": "e2e-chat-4d5152a995b7", + "messages": [{"role": "user", "content": "Reply ok. 4d5152a995b7"}], + "api_key": "sk-live-one-aaaaaaaaaaaaaaaa", + }, + ) + second = request( + headers={"authorization": "Bearer sk-run-two-bbbbbbbbbbbbbbbb", "x-request-id": "req-2"}, + params={"start_date": "2026-08-19"}, + body={ + "model": "e2e-chat-1a2b3c4d5e6f", + "messages": [{"role": "user", "content": "Reply ok. 1a2b3c4d5e6f"}], + "api_key": "os.environ/OPENAI_API_KEY", + }, + ) + assert canonicalize(first).key == canonicalize(second).key + + def test_serialization_order_is_not_identity(self) -> None: + ordered = request(body={"model": "m", "stream": True}) + reversed_order = request(body={"stream": True, "model": "m"}) + assert canonicalize(ordered).key == canonicalize(reversed_order).key + + def test_generated_ids_in_the_path_do_not_change_the_key(self) -> None: + first = request("get", "/v1/batches/batch_688a8b7f9a08819096e0f7c88fcd07c5") + second = request("get", "/v1/batches/batch_770b9c8f0b19920107f1f8d99fde18d6") + assert canonicalize(first).key == canonicalize(second).key + + +class TestKeyDistinctness: + def test_requests_differing_only_inside_canonicalized_fields_stay_distinct(self) -> None: + """Acceptance: a naive verb+path hash collides these; the content key + must not, or one test silently asserts against the other's response.""" + first = request(body={"messages": [{"content": "Reply ok. 4d5152a995b7"}]}) + second = request(body={"messages": [{"content": "Count to three. 4d5152a995b7"}]}) + naive = (first.method, first.path) + assert naive == (second.method, second.path) + assert canonicalize(first).key != canonicalize(second).key + + def test_a_kept_header_is_identity(self) -> None: + first = request(headers={"x-litellm-tags": "prod"}) + second = request(headers={"x-litellm-tags": "shadow"}) + assert canonicalize(first).key != canonicalize(second).key + + def test_a_volatile_header_is_not_identity(self) -> None: + first = request(headers={"traceparent": "00-aa-bb-01", "x-api-key": "one"}) + second = request(headers={"traceparent": "00-cc-dd-01", "x-api-key": "two"}) + assert canonicalize(first).key == canonicalize(second).key + + def test_secret_set_versus_unset_stays_distinct(self) -> None: + with_key = request(body={"api_key": "sk-live-aaaaaaaaaaaaaaaa"}) + without_key = request(body={"api_key": None}) + assert canonicalize(with_key).key != canonicalize(without_key).key + + def test_file_content_is_identity(self) -> None: + first = request( + "upload", "/v1/files", file_name="batch.jsonl", file_sha256="a" * 64, file_bytes=10 + ) + second = request( + "upload", "/v1/files", file_name="batch.jsonl", file_sha256="b" * 64, file_bytes=10 + ) + assert canonicalize(first).key != canonicalize(second).key + + +class TestKeyShape: + def test_key_names_method_path_and_digest(self) -> None: + canonical = canonicalize(request("post", "/model/new", body={"model_name": "m"})) + assert isinstance(canonical, CanonicalRequest) + assert canonical.key.startswith("post /model/new #") + assert len(canonical.key.rsplit("#", 1)[1]) == 16 diff --git a/tests/e2e/test_fixture_transport.py b/tests/e2e/test_fixture_transport.py new file mode 100644 index 00000000000..e61088d841c --- /dev/null +++ b/tests/e2e/test_fixture_transport.py @@ -0,0 +1,676 @@ +"""Harness coverage for the record/replay transports (LIT-5729). + +No proxy and no ``e2e`` marker. A fake in-memory ``Transport`` stands in for +the live one (dependency injection, no monkeypatching): recording must pass +every value through unchanged while writing one redacted interaction file per +call, and replay must serve identical values from the bundle alone - the +fake's call log proves nothing reaches the inner transport - failing hard +(``ReplayMiss``) on any content drift, printing the computed canonical key and +the closest recorded key (LIT-5741; the pure canonicalizer is pinned in +test_fixture_canonical.py). The collection-time gate and report header are +pinned here too, including the stale message that names the bundle's age. +""" + +from __future__ import annotations + +import hashlib +import sys +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from uuid import uuid4 + +import pytest +from pydantic import BaseModel + +from e2e_http import ( + AuthHeaders, + BinaryStream, + ProbeResult, + Result, + StreamingResponse, + Success, +) +from fixture_bundle import ( + BUNDLE_FORMAT_VERSION, + MANIFEST_FILENAME, + BundleRecorder, + Interaction, + LoadedBundle, + Manifest, + RecordedResult, + load_bundle, + prepare_bundle, + slug_for_test, +) +from fixture_canonical import canonicalize +from fixture_transport import ( + InvalidFixtureMode, + RecordingTransport, + ReplayMiss, + ReplaySource, + ReplayTransport, + current_test_key, + deterministic_marker, + fixture_mode_collection_error, + fixture_report_lines, + parse_fixture_mode, + recorded_request, + replay_leftover_error, + select_transport, +) +from transport import Transport + +NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc) + + +class Payload(BaseModel): + value: str + + +class Body(BaseModel): + prompt: str + + +class Query(BaseModel): + q: str + + +class DeployParams(BaseModel): + model: str + api_key: str | None = None + aws_secret_access_key: str | None = None + + +class DeployBody(BaseModel): + model_name: str + litellm_params: DeployParams + + +STREAMING = StreamingResponse( + status_code=200, + body="", + content_type="text/event-stream", + chunks=2, + stream_events=["one", "two"], + stream_done=True, +) +BINARY = BinaryStream(status_code=200, content_type="audio/mpeg", chunk_count=3, total_bytes=42) +PROBE = ProbeResult(status_code=200, body="alive") + + +@dataclass +class FakeTransport: + calls: list[str] = field(default_factory=list) + + def bearer(self, key: str) -> AuthHeaders: + return AuthHeaders(authorization=f"Bearer {key}") + + @property + def master(self) -> AuthHeaders: + return self.bearer("sk-fake-master") + + def _success[R: BaseModel](self, response_type: type[R]) -> Result[R]: + return Success(status_code=200, data=response_type.model_validate({"value": "live"})) + + def post[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + self.calls.append(f"post {path}") + return self._success(response_type) + + def get[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + params: BaseModel, + response_type: type[R], + timeout: float | None = None, + ) -> Result[R]: + self.calls.append(f"get {path}") + return self._success(response_type) + + def delete[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + response_type: type[R], + params: BaseModel | None = None, + ) -> Result[R]: + self.calls.append(f"delete {path}") + return self._success(response_type) + + def patch[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + self.calls.append(f"patch {path}") + return self._success(response_type) + + def put[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + self.calls.append(f"put {path}") + return self._success(response_type) + + def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: + self.calls.append(f"stream {path}") + return STREAMING + + def stream_binary( + self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 + ) -> BinaryStream: + self.calls.append(f"stream_binary {path}") + return BINARY + + def send( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + params: BaseModel | None = None, + stream: bool = False, + ) -> StreamingResponse: + self.calls.append(f"send {path}") + return STREAMING + + def probe(self, path: str, *, params: BaseModel) -> ProbeResult: + self.calls.append(f"probe {path}") + return PROBE + + def upload[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + form: BaseModel, + filename: str, + content: bytes, + file_content_type: str = "application/jsonl", + file_field: str = "file", + params: BaseModel | None = None, + response_type: type[R], + ) -> Result[R]: + self.calls.append(f"upload {path}") + return self._success(response_type) + + def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: + self.calls.append(f"download {path}") + return STREAMING + + +def make_recorder(root: Path) -> BundleRecorder: + recorder = prepare_bundle(root) + assert isinstance(recorder, BundleRecorder) + return recorder + + +def replay_source(root: Path) -> ReplaySource: + loaded = load_bundle(root) + assert isinstance(loaded, LoadedBundle) + return ReplaySource(bundle=loaded) + + +def this_tests_files(root: Path) -> list[Path]: + slug_dir = root / slug_for_test(current_test_key()) + return sorted(slug_dir.glob("*.json")) if slug_dir.is_dir() else [] + + +def write_manifest(root: Path, recorded_at: datetime) -> None: + root.mkdir(parents=True, exist_ok=True) + manifest = Manifest( + format_version=BUNDLE_FORMAT_VERSION, recorded_at=recorded_at, harness_version="abc1234" + ) + (root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(), encoding="utf-8") + + +class TestParseFixtureMode: + @pytest.mark.parametrize( + ("raw", "expected"), + [("live", "live"), ("record", "record"), ("replay", "replay"), ("", "live"), (" REPLAY ", "replay")], + ) + def test_known_values_normalize(self, raw: str, expected: str) -> None: + assert parse_fixture_mode(raw) == expected + + def test_unknown_value_is_invalid_with_the_original_spelling(self) -> None: + assert parse_fixture_mode("cached") == InvalidFixtureMode(value="cached") + + +class TestDeterministicMarker: + def test_sequence_is_a_pure_function_of_test_and_ordinal(self) -> None: + """A replay process must regenerate exactly the markers the record + process generated, so the Nth marker of a test is pinned to a pure + function of the node id and N.""" + key = current_test_key() + assert deterministic_marker() == hashlib.sha1(f"{key}#0".encode()).hexdigest()[:12] + assert deterministic_marker() == hashlib.sha1(f"{key}#1".encode()).hexdigest()[:12] + + +class TestCurrentTestKey: + def test_names_this_test_and_strips_the_phase(self) -> None: + key = current_test_key() + assert key.endswith("TestCurrentTestKey::test_names_this_test_and_strips_the_phase") + assert "(call)" not in key + + +class TestRecordingTransport: + def test_passes_the_result_through_and_writes_one_file_per_call(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + result = recording.post( + "/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload + ) + assert result == Success(status_code=200, data=Payload(value="live")) + assert fake.calls == ["post /model/new"] + files = this_tests_files(root) + assert [file.name for file in files] == ["0000-post-model-new.json"] + interaction = Interaction.model_validate_json(files[0].read_text(encoding="utf-8")) + assert interaction.request.method == "post" + assert interaction.request.path == "/model/new" + + def test_redacts_auth_header_values_in_the_recorded_request(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + headers = AuthHeaders.model_validate( + {"authorization": "Bearer sk-secret", "x-litellm-api-key": "sk-other"} + ) + recording.post("/key/generate", headers=headers, json=Body(prompt="x"), response_type=Payload) + interaction = Interaction.model_validate_json( + this_tests_files(root)[0].read_text(encoding="utf-8") + ) + assert interaction.request.headers == { + "authorization": "", + "x-litellm-api-key": "", + } + assert "sk-secret" not in this_tests_files(root)[0].read_text(encoding="utf-8") + + def test_redacts_credential_body_fields_in_the_recorded_request(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post( + "/model/new", + headers=fake.master, + json=DeployBody( + model_name="m", + litellm_params=DeployParams(model="openai/gpt", api_key="sk-live-provider-secret-123456"), + ), + response_type=Payload, + ) + raw = this_tests_files(root)[0].read_text(encoding="utf-8") + interaction = Interaction.model_validate_json(raw) + assert "sk-live-provider-secret-123456" not in raw + assert isinstance(interaction.request.body, dict) + params = interaction.request.body["litellm_params"] + assert isinstance(params, dict) + assert params["api_key"] == "" + assert params["aws_secret_access_key"] is None + + def test_upload_records_a_content_digest_not_the_bytes(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.upload( + "/v1/files", + headers=fake.master, + form=Query(q="batch"), + filename="batch.jsonl", + content=b'{"custom_id": "1"}', + response_type=Payload, + ) + interaction = Interaction.model_validate_json( + this_tests_files(root)[0].read_text(encoding="utf-8") + ) + assert interaction.request.file_name == "batch.jsonl" + assert interaction.request.file_bytes == len(b'{"custom_id": "1"}') + assert interaction.request.file_sha256 is not None + assert "custom_id" not in interaction.request.model_dump_json() + + +class TestReplayTransport: + def test_serves_recorded_values_without_touching_the_inner_transport( + self, tmp_path: Path + ) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recorded_post = recording.post( + "/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload + ) + recorded_get = recording.get( + "/v1/models", headers=fake.master, params=Query(q="all"), response_type=Payload + ) + recorded_stream = recording.stream( + "/chat/completions", headers=fake.master, json=Body(prompt="hi") + ) + recorded_probe = recording.probe("/health/liveliness", params=Query(q="1")) + recorded_binary = recording.stream_binary( + "/v1/audio/speech", headers=fake.master, json=Body(prompt="say") + ) + calls_after_record = list(fake.calls) + + replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") + assert ( + replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) + == recorded_post + ) + assert ( + replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) + == recorded_get + ) + assert ( + replay.stream("/chat/completions", headers=replay.master, json=Body(prompt="hi")) + == recorded_stream + ) + assert replay.probe("/health/liveliness", params=Query(q="1")) == recorded_probe + assert ( + replay.stream_binary("/v1/audio/speech", headers=replay.master, json=Body(prompt="say")) + == recorded_binary + ) + assert fake.calls == calls_after_record + + def test_miss_names_the_computed_key_and_the_closest_recorded_key(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") + with pytest.raises(ReplayMiss) as excinfo: + replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) + message = str(excinfo.value) + assert "no recorded interaction matches key get /v1/models #" in message + assert "closest recorded key is post /model/new #" in message + assert "0000-post-model-new.json" in message + assert "re-record with E2E_FIXTURE_MODE=record" in message + + def test_content_drift_on_the_same_route_misses_with_no_live_call(self, tmp_path: Path) -> None: + """The naive verb+path match replayed a stale response for a request + whose content had changed, silently passing; a content key must miss, + print both canonical forms' diff, and never reach the inner transport.""" + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + calls_after_record = list(fake.calls) + replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") + with pytest.raises(ReplayMiss) as excinfo: + replay.post("/model/new", headers=replay.master, json=Body(prompt="y"), response_type=Payload) + message = str(excinfo.value) + assert "no recorded interaction matches key post /model/new #" in message + assert "closest recorded key is post /model/new #" in message + assert '- "prompt": "x"' in message + assert '+ "prompt": "y"' in message + assert fake.calls == calls_after_record + + def test_exhausted_key_names_the_key(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") + replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) + with pytest.raises( + ReplayMiss, match=r"every recorded interaction for key post /model/new #\w{16} is already consumed" + ): + replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) + + def test_replays_out_of_recorded_order_across_distinct_keys(self, tmp_path: Path) -> None: + """Concurrent tests interleave independent calls nondeterministically + (e.g. a burst of parallel chat calls), so replay matches by content, + never by recorded position.""" + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + recording.post("/key/generate", headers=fake.master, json=Body(prompt="k"), response_type=Payload) + source = replay_source(root) + replay: Transport = ReplayTransport(source=source, master_key="sk-1234") + replay.post("/key/generate", headers=replay.master, json=Body(prompt="k"), response_type=Payload) + replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) + assert source.leftover_error(current_test_key()) is None + + def test_identical_requests_replay_their_responses_in_recorded_order(self, tmp_path: Path) -> None: + """A poll loop makes the same request repeatedly and asserts on the + progression, so duplicates under one key stay FIFO.""" + root = tmp_path / "bundle" + recorder = make_recorder(root) + recorder.record( + test_key=current_test_key(), + request=recorded_request( + "get", "/v1/models", headers=AuthHeaders(authorization="Bearer sk-x"), params=Query(q="all") + ), + response=RecordedResult(kind="success", status_code=200, data={"value": "first"}), + ) + recorder.record( + test_key=current_test_key(), + request=recorded_request( + "get", "/v1/models", headers=AuthHeaders(authorization="Bearer sk-x"), params=Query(q="all") + ), + response=RecordedResult(kind="success", status_code=200, data={"value": "second"}), + ) + replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") + first = replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) + second = replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) + assert first == Success(status_code=200, data=Payload(value="first")) + assert second == Success(status_code=200, data=Payload(value="second")) + + def test_concurrent_replays_of_one_key_serve_each_recording_exactly_once(self, tmp_path: Path) -> None: + """A burst of parallel identical calls consumes one shared pool: no + response duplicated, none forgotten, nothing left over at teardown. + The tiny switch interval forces thread preemption inside pool setup + and consumption, so a non-atomic pool build or pop fails this test.""" + root = tmp_path / "bundle" + recorder = make_recorder(root) + for ordinal in range(32): + recorder.record( + test_key=current_test_key(), + request=recorded_request( + "get", "/v1/models", headers=AuthHeaders(authorization="Bearer sk-x"), params=Query(q="all") + ), + response=RecordedResult(kind="success", status_code=200, data={"value": f"v{ordinal:02d}"}), + ) + source = replay_source(root) + replay: Transport = ReplayTransport(source=source, master_key="sk-1234") + barrier = threading.Barrier(8) + + def consume_one() -> str: + result = replay.get( + "/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload + ) + assert isinstance(result, Success) + return result.data.value + + def consume(_: int) -> tuple[str, ...]: + barrier.wait() + return tuple(consume_one() for _call in range(4)) + + previous_interval = sys.getswitchinterval() + sys.setswitchinterval(1e-6) + try: + with ThreadPoolExecutor(max_workers=8) as executor: + served = sorted(value for values in executor.map(consume, range(8)) for value in values) + finally: + sys.setswitchinterval(previous_interval) + assert served == [f"v{ordinal:02d}" for ordinal in range(32)] + assert source.leftover_error(current_test_key()) is None + + +class TestRecordedKeySets: + def test_two_separate_recordings_of_one_flow_produce_identical_key_sets( + self, tmp_path: Path + ) -> None: + """Everything a run randomizes (markers, virtual keys, dates) must + canonicalize out, so separately recorded runs of the same suite agree + on every match key and a bundle recorded elsewhere replays here.""" + + def record_flow(root: Path, run_date: str) -> list[str]: + fake = FakeTransport() + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + marker = deterministic_marker() + recording.post( + "/model/new", + headers=fake.master, + json=DeployBody( + model_name=f"e2e-chat-{marker}", + litellm_params=DeployParams(model="openai/gpt", api_key=f"sk-live-{uuid4().hex}"), + ), + response_type=Payload, + ) + recording.post( + "/chat/completions", + headers=recording.bearer(f"sk-{uuid4().hex}"), + json=Body(prompt=f"Reply with the single word ok. {marker}"), + response_type=Payload, + ) + recording.get( + "/spend/logs", headers=fake.master, params=Query(q=run_date), response_type=Payload + ) + loaded = load_bundle(root) + assert isinstance(loaded, LoadedBundle) + return sorted( + canonicalize(interaction.request).key + for interactions in loaded.interactions.values() + for interaction in interactions + ) + + first_keys = record_flow(tmp_path / "one", "2026-08-18") + second_keys = record_flow(tmp_path / "two", "2026-08-19") + assert first_keys == second_keys + assert len(first_keys) == 3 + + +class TestReplayLeftover: + def test_fully_consumed_recording_leaves_nothing(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + source = replay_source(root) + replay: Transport = ReplayTransport(source=source, master_key="sk-1234") + replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) + assert source.leftover_error(current_test_key()) is None + + def test_unconsumed_trailing_interactions_name_the_next_call(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + recording.probe("/health/liveliness", params=Query(q="1")) + source = replay_source(root) + replay: Transport = ReplayTransport(source=source, master_key="sk-1234") + replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) + error = source.leftover_error(current_test_key()) + assert error is not None + assert "1 of 2 recorded interactions never consumed" in error + assert "e.g. probe /health/liveliness #" in error + assert "re-record with E2E_FIXTURE_MODE=record" in error + + def test_test_without_recordings_has_no_leftover(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + make_recorder(root) + assert replay_source(root).leftover_error("suite.py::test_never_recorded") is None + + def test_inert_outside_replay_mode(self, tmp_path: Path) -> None: + missing = tmp_path / "missing" + assert replay_leftover_error(mode_raw="", bundle_dir=missing, test_key="k") is None + assert replay_leftover_error(mode_raw="record", bundle_dir=missing, test_key="k") is None + + def test_replay_mode_reads_the_shared_bundle(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + error = replay_leftover_error(mode_raw="replay", bundle_dir=root, test_key=current_test_key()) + assert error is not None + assert "1 of 1 recorded interactions never consumed" in error + + +class TestSelectTransport: + def test_live_returns_the_live_transport_untouched(self, tmp_path: Path) -> None: + fake = FakeTransport() + for mode_raw in ("live", ""): + assert ( + select_transport(fake, mode_raw=mode_raw, bundle_dir=tmp_path / "b", master_key="sk") + is fake + ) + + def test_record_wraps_live_and_starts_a_fresh_bundle(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=30)) + (root / "old-test-slug").mkdir() + (root / "old-test-slug" / "0000-post-old.json").write_text("{}", encoding="utf-8") + selected = select_transport(fake, mode_raw="record", bundle_dir=root, master_key="sk") + assert isinstance(selected, RecordingTransport) + assert selected.inner is fake + assert {entry.name for entry in root.iterdir()} == {MANIFEST_FILENAME} + + def test_replay_builds_a_transport_from_the_bundle_alone(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + make_recorder(root) + selected = select_transport(fake, mode_raw="replay", bundle_dir=root, master_key="sk-master") + assert isinstance(selected, ReplayTransport) + assert selected.master == AuthHeaders(authorization="Bearer sk-master") + + def test_invalid_mode_raises_naming_the_value(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="cached"): + select_transport( + FakeTransport(), mode_raw="cached", bundle_dir=tmp_path / "b", master_key="sk" + ) + + +class TestCollectionGate: + def test_invalid_mode_names_the_value_and_the_choices(self, tmp_path: Path) -> None: + assert ( + fixture_mode_collection_error("cached", tmp_path, now=NOW) + == "E2E_FIXTURE_MODE='cached' is not one of live, record, replay" + ) + + @pytest.mark.parametrize("mode_raw", ["live", "", "record"]) + def test_live_and_record_never_block_collection(self, mode_raw: str, tmp_path: Path) -> None: + assert fixture_mode_collection_error(mode_raw, tmp_path / "missing", now=NOW) is None + + def test_replay_with_no_bundle_says_how_to_record_one(self, tmp_path: Path) -> None: + reason = fixture_mode_collection_error("replay", tmp_path / "missing", now=NOW) + assert reason is not None + assert f"no {MANIFEST_FILENAME}" in reason + assert "E2E_FIXTURE_MODE=record" in reason + + def test_stale_replay_bundle_fails_naming_its_age(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=9, hours=5)) + reason = fixture_mode_collection_error("replay", root, now=NOW) + assert reason is not None + assert "age 9d5h exceeds the 7-day limit" in reason + assert "re-record with E2E_FIXTURE_MODE=record" in reason + + def test_fresh_replay_bundle_collects(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=2)) + assert fixture_mode_collection_error("replay", root, now=NOW) is None + + +class TestReportHeader: + def test_live_mode_prints_nothing(self, tmp_path: Path) -> None: + assert fixture_report_lines("live", tmp_path, now=NOW) == [] + assert fixture_report_lines("", tmp_path, now=NOW) == [] + + def test_record_and_replay_name_the_bundle(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + recorded_at = NOW - timedelta(days=1) + write_manifest(root, recorded_at) + assert fixture_report_lines("record", root, now=NOW) == [ + f"e2e fixture mode: record -> {root}" + ] + replay_lines = fixture_report_lines("replay", root, now=NOW) + assert len(replay_lines) == 1 + assert "replay" in replay_lines[0] + assert recorded_at.isoformat() in replay_lines[0] diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py index 50f1380a4f8..ebe093c591c 100644 --- a/tests/test_litellm/batches/test_batch_utils.py +++ b/tests/test_litellm/batches/test_batch_utils.py @@ -150,13 +150,13 @@ def test_parse_jsonl_empty_content_is_empty_list(): assert bu._get_file_content_as_dictionary(b"") == [] -def test_parse_jsonl_malformed_raises(): - with pytest.raises(Exception): - bu._get_file_content_as_dictionary(b"not valid json") +def test_parse_jsonl_malformed_lines_skipped(): + content = b'{"a": 1}\nnot valid json\n{"b": 2}\n' + assert bu._get_file_content_as_dictionary(content) == [{"a": 1}, {"b": 2}] # =========================================================================== # -# _iter_batch_input_lines / _iter_batch_input_entries (JSONL parsing) +# _iter_batch_input_lines / _iter_batch_output_entries (JSONL parsing) # =========================================================================== # @@ -173,19 +173,22 @@ def test_iter_input_lines_empty(): assert list(bu._iter_batch_input_lines(b"")) == [] -def test_iter_input_entries_parses_each_row(): +def test_iter_output_entries_parses_each_row(): content = b'{"body": {"model": "gpt-4o"}}\n{"body": {"model": "claude-3"}}\n' - assert list(bu._iter_batch_input_entries(content)) == [ + assert list(bu._iter_batch_output_entries(content)) == [ {"body": {"model": "gpt-4o"}}, {"body": {"model": "claude-3"}}, ] -def test_iter_input_entries_raises_on_malformed_line(): - # _iter_batch_input_entries raises on a bad row; callers that must survive - # bad rows iterate _iter_batch_input_lines and parse per-row instead. - with pytest.raises(Exception): - list(bu._iter_batch_input_entries(b'{"ok":1}\nnot-json\n')) +def test_iter_output_entries_skips_malformed_and_non_object_lines(): + content = b'{"ok": 1}\nnot-json\n[1, 2]\n{"ok": 2}\n' + assert list(bu._iter_batch_output_entries(content)) == [{"ok": 1}, {"ok": 2}] + + +def test_iter_output_entries_skips_undecodable_line(): + content = b'{"ok": 1}\n{"note": "\xff-bad"}\n{"ok": 2}\n' + assert list(bu._iter_batch_output_entries(content)) == [{"ok": 1}, {"ok": 2}] # =========================================================================== # @@ -471,6 +474,25 @@ def test_cost_from_content_completion_cost_path(monkeypatch): assert len(calls) == 2 # failed row not costed +def test_empty_body_line_does_not_zero_whole_batch(): + """A status-200 row with an empty body makes litellm.completion_cost raise; + that line must be skipped instead of zeroing the whole batch.""" + rows = [ + _success_row(usage=_usage(10, 5)), + { + "custom_id": "request-poison-empty", + "response": {"status_code": 200, "request_id": "inject-empty-body", "body": {}}, + }, + _success_row(usage=_usage(20, 10)), + ] + + cost, usage, models = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai") + + assert cost > 0.0 + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (30, 15, 45) + assert models == ["gpt-4o", "gpt-4o"] + + def test_cost_from_content_model_info_path(monkeypatch): # model_info set -> batch_cost_calculator(prompt_cost, completion_cost). import litellm.cost_calculator as cc diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 28d2cb22e7c..a5ad3d771e3 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -6213,6 +6213,32 @@ class TestDynamicTracerProviderCache(unittest.TestCase): self.assertTrue(entry.owns_exporter) self.assertIsNotNone(entry.provider._atexit_handler) + + def test_dynamic_providers_share_one_resource(self): + """Building the Resource scans every installed distribution's entry points, and the + dynamic providers reach it from the async logging path, so one logger builds it once.""" + logger = self._logger(cap=8) + + for i in range(4): + logger._get_tracer_with_dynamic_headers({"authorization": f"Basic tenant-{i}"}) + + entries = list(logger._tracer_provider_cache.values()) + self.assertEqual(len(entries), 4) + self.assertEqual(len({id(entry.provider.resource) for entry in entries}), 1) + self.assertIs(entries[0].provider.resource, logger._litellm_resource()) + + def test_resource_is_memoized_per_logger_not_shared(self): + """Two loggers must not share a Resource; the second's service.name would be wrong.""" + first = self._logger() + second = OpenTelemetry( + config=OpenTelemetryConfig(exporter="console", skip_set_global=True, service_name="svc-second") + ) + self.addCleanup(second._tracer_provider.shutdown) + + self.assertIsNot(first._litellm_resource(), second._litellm_resource()) + self.assertEqual(second._litellm_resource().attributes.get("service.name"), "svc-second") + + class TestOpenTelemetryDatabaseSemconvAttributes(unittest.TestCase): """A Postgres service span must name the PostgreSQL server it reached. diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 1826f56d667..06be96fefdf 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2324,6 +2324,87 @@ def test_data_residency_composes_with_service_tier(_local_model_cost_map): assert priority_eu_total == pytest.approx(priority_base_total * 1.10, rel=1e-9) +@pytest.mark.parametrize("model", ["gemini-3.5-flash", "claude-haiku-4-5@20251001"]) +@pytest.mark.parametrize("vertex_location", ["us-central1", "us-east5", "europe-west1", "asia-southeast1"]) +def test_vertex_regional_location_applies_uplift(vertex_location, model, _local_model_cost_map): + """Google bills every non-global Vertex endpoint at 1.1x the global rate for GA + Gemini 3+ and regional-pricing Claude models, so a request served from a regional + location must cost 1.1x what the same usage costs on the global endpoint.""" + from litellm.types.utils import Usage + + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + base = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="vertex_ai") + regional = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider="vertex_ai", + vertex_location=vertex_location, + ) + + base_total = base[0] + base[1] + regional_total = regional[0] + regional[1] + + assert base_total > 0 + assert regional_total == pytest.approx(base_total * 1.10, rel=1e-9) + assert regional[0] == pytest.approx(base[0] * 1.10, rel=1e-9) + assert regional[1] == pytest.approx(base[1] * 1.10, rel=1e-9) + + +@pytest.mark.parametrize("vertex_location", [None, "global", "GLOBAL"]) +def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_model_cost_map): + """The global endpoint prices at the base rate, whatever the casing, and an + unresolved location must never uplift.""" + from litellm.types.utils import Usage + + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + base = generic_cost_per_token( + model="claude-haiku-4-5@20251001", usage=usage, custom_llm_provider="vertex_ai" + ) + located = generic_cost_per_token( + model="claude-haiku-4-5@20251001", + usage=usage, + custom_llm_provider="vertex_ai", + vertex_location=vertex_location, + ) + + assert base == located + + +@pytest.mark.parametrize("model", ["claude-opus-4-1", "gemini-2.0-flash-001"]) +def test_vertex_location_no_uplift_for_uniformly_priced_model(model, _local_model_cost_map): + """Models Google prices uniformly across endpoints (Gemini 2.x, Claude Opus 4.1 + and older) carry no multiplier and must not move with the location.""" + from litellm.types.utils import Usage + + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + base = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="vertex_ai") + regional = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider="vertex_ai", + vertex_location="us-east5", + ) + + assert base == regional, f"{model} should not have a regional-endpoint uplift" + + +def test_vertex_uplift_invalid_multiplier_defaults_to_one(): + """A malformed multiplier in the cost map degrades to base pricing, never raises.""" + from litellm.litellm_core_utils.llm_cost_calc.utils import ( + get_vertex_regional_endpoint_uplift, + ) + + assert ( + get_vertex_regional_endpoint_uplift( + {"regional_endpoint_uplift_multiplier": "not-a-number"}, "us-east5" + ) + == 1.0 + ) + + def test_priority_service_tier_above_threshold_uses_priority_tier_rates_for_cached_tokens( _local_model_cost_map, ): @@ -2877,6 +2958,57 @@ def test_token_type_cost_breakdown_applies_regional_uplift(): assert text_input_cost + eu.cache_read_cost == pytest.approx(prompt_cost) +def test_token_type_cost_breakdown_applies_vertex_regional_uplift(): + """ + Non-global Vertex endpoints apply a flat 1.1x uplift to every token cost. The + per-type breakdown must apply the same uplift via vertex_location so it stays + reconciled with the uplifted input_cost/output_cost totals, instead of being + logged at the global rate. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "claude-haiku-4-5@20251001" + custom_llm_provider = "vertex_ai" + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=400, text_tokens=600 + ), + ) + + model_info = litellm.get_model_info( + model=model, custom_llm_provider=custom_llm_provider + ) + uplift = model_info["regional_endpoint_uplift_multiplier"] + assert uplift > 1.0 + + base = get_token_type_cost_breakdown( + model=model, custom_llm_provider=custom_llm_provider, usage=usage + ) + regional = get_token_type_cost_breakdown( + model=model, + custom_llm_provider=custom_llm_provider, + usage=usage, + vertex_location="us-east5", + ) + + assert base.cache_read_cost > 0 + assert regional.cache_read_cost == pytest.approx(base.cache_read_cost * uplift) + + # The uplifted breakdown must still reconcile with the uplifted totals. + prompt_cost, _completion_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + vertex_location="us-east5", + ) + text_input_cost = 600 * model_info["input_cost_per_token"] * uplift + assert text_input_cost + regional.cache_read_cost == pytest.approx(prompt_cost) + + def test_token_type_cost_breakdown_applies_anthropic_geo_multiplier(monkeypatch): """ Anthropic's regional (geo) uplift lives in provider_specific_entry and is diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 54016470f8b..c2d73ea467d 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -4958,3 +4958,150 @@ def test_pre_call_redacts_and_masks_raw_request(logging_obj): raw_api_base = logging_obj.model_call_details["raw_request_typed_dict"]["raw_request_api_base"] assert _GEMINI_KEY not in raw_api_base assert "key=*****" in raw_api_base + + +def _resolve(custom_llm_provider, litellm_params, optional_params, model): + from litellm.litellm_core_utils.litellm_logging import ( + _resolve_vertex_location_for_cost, + ) + + return _resolve_vertex_location_for_cost( + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + optional_params=optional_params, + model=model, + ) + + +def test_resolve_vertex_location_for_cost(): + """Vertex requests resolve the serving location the way dispatch does; other providers get None.""" + assert _resolve("openai", {"vertex_location": "us-east5"}, None, "gpt-4o") is None + assert _resolve(None, {}, None, "gemini-3.5-flash") is None + assert _resolve("vertex_ai", {"vertex_location": "us-east5"}, None, "gemini-3.5-flash") == "us-east5" + assert _resolve("vertex_ai", {"vertex_location": "global"}, None, "gemini-3.5-flash") == "global" + assert ( + _resolve("vertex_ai_beta", {"vertex_ai_location": "europe-west1"}, None, "claude-haiku-4-5@20251001") + == "europe-west1" + ) + + +def test_resolve_vertex_location_for_cost_reads_optional_params(monkeypatch): + """ + On the proxy the logging object predates deployment selection, so the deployment's + configured location only reaches it through optional_params. A configured global + location must beat the environment fallback, or every proxy call gets the regional uplift. + """ + monkeypatch.setenv("VERTEXAI_LOCATION", "us-east5") + monkeypatch.setattr(litellm, "vertex_location", None) + + assert _resolve("vertex_ai", {}, {"vertex_location": "global"}, "gemini-3.5-flash") == "global" + assert _resolve("vertex_ai", None, {"vertex_location": "europe-west1"}, "gemini-3.5-flash") == "europe-west1" + assert ( + _resolve( + "vertex_ai", + {"vertex_location": "us-east5"}, + {"vertex_location": "global"}, + "gemini-3.5-flash", + ) + == "global" + ) + assert _resolve("vertex_ai", {"vertex_location": "global"}, {}, "gemini-3.5-flash") == "global" + assert _resolve("vertex_ai", {}, {}, "gemini-3.5-flash") == "us-east5" + + +def test_resolve_vertex_location_for_cost_default_region(monkeypatch): + """With no location configured anywhere, resolution lands on the dispatch default us-central1.""" + monkeypatch.delenv("VERTEXAI_LOCATION", raising=False) + monkeypatch.delenv("VERTEX_LOCATION", raising=False) + monkeypatch.setattr(litellm, "vertex_location", None) + + assert _resolve("vertex_ai", {}, None, "gemini-3.5-flash") == "us-central1" + assert _resolve("vertex_ai", None, None, "gemini-3.5-flash") == "us-central1" + + +def test_response_cost_calculator_prices_proxy_vertex_calls_on_the_configured_location(monkeypatch): + """ + Proxy-shaped logging objects (created before the router picks a deployment) carry the + deployment's vertex_location only in optional_params. A global deployment must price at + base rates even when the environment points at a regional location, and a regional one + must price with the uplift. + """ + from datetime import datetime + + from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map + + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url="")) + monkeypatch.setenv("VERTEXAI_LOCATION", "us-east5") + monkeypatch.setattr(litellm, "vertex_location", None) + + def cost_at(location): + logging_obj = LitellmLogging( + model="gemini-3.5-flash", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id=f"vertex-loc-{location}", + function_id="f", + ) + logging_obj.update_environment_variables( + model="gemini-3.5-flash", + user="", + optional_params={"vertex_location": location}, + litellm_params={"api_base": ""}, + custom_llm_provider="vertex_ai", + ) + response = ModelResponse( + id="resp-1", + model="gemini-3.5-flash", + choices=[{"message": {"role": "assistant", "content": "hello"}, "index": 0, "finish_reason": "stop"}], + usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + ) + return logging_obj._response_cost_calculator(result=response) + + info = litellm.model_cost["vertex_ai/gemini-3.5-flash"] + expected_global = 10 * info["input_cost_per_token"] + 5 * info["output_cost_per_token"] + + assert cost_at("global") == pytest.approx(expected_global) + assert cost_at("us-east5") == pytest.approx(info["regional_endpoint_uplift_multiplier"] * expected_global) + + +def test_set_cost_breakdown_stores_vertex_location(): + """vertex_location is recorded in the pricing basis, None for non-vertex requests.""" + from datetime import datetime + + logging_obj = LitellmLogging( + model="vertex_ai/claude-haiku-4-5@20251001", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="vertex-location-set", + function_id="f", + ) + logging_obj.set_cost_breakdown( + input_cost=0.001, + output_cost=0.002, + total_cost=0.003, + cost_for_built_in_tools_cost_usd_dollar=0.0, + vertex_location="us-east5", + ) + assert logging_obj.cost_breakdown["vertex_location"] == "us-east5" + + no_location = LitellmLogging( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="vertex-location-absent", + function_id="f", + ) + no_location.set_cost_breakdown( + input_cost=0.001, + output_cost=0.002, + total_cost=0.003, + cost_for_built_in_tools_cost_usd_dollar=0.0, + ) + assert no_location.cost_breakdown.get("vertex_location") is None diff --git a/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py new file mode 100644 index 00000000000..270c59f595f --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py @@ -0,0 +1,163 @@ +"""Tests for the shared PTU rules: which deployments accrue flat cost, and what that zeroes.""" + +import os +from datetime import datetime, timezone +from unittest.mock import patch + +import pytest + +from litellm.litellm_core_utils.ptu_pricing import ( + CUSTOM_PRICING_FIELDS, + PTU_EMPTIED_PRICING_FIELDS, + PTU_ZEROED_PRICING_FIELDS, + PTU_ZEROED_TABLE_FIELDS, + SEARCH_CONTEXT_SIZES, + ptu_terms, + zeroed_ptu_pricing, +) +from litellm.types.router import ModelInfo + +_VALID = { + "team_id": "team-alpha", + "ptu_count": 100, + "cost_per_ptu_per_hour": 0.02, + "ptu_effective_from": "2026-01-01T00:00:00Z", +} + + +def _with_flag(model_info, declared=None, enabled=True): + with patch.dict(os.environ, {"LITELLM_ENABLE_PTU_COST_ATTRIBUTION": "True" if enabled else ""}, clear=False): + return zeroed_ptu_pricing(model_info, declared or {}) + + +def test_a_complete_reservation_is_accepted(): + terms = ptu_terms(_VALID) + + assert terms is not None + assert terms.team_id == "team-alpha" + assert terms.ptu_count == 100 + assert terms.effective_from == datetime(2026, 1, 1, tzinfo=timezone.utc) + assert terms.effective_to is None + + +@pytest.mark.parametrize( + "override", + [ + {"team_id": None}, + {"team_id": ""}, + {"ptu_count": None}, + {"cost_per_ptu_per_hour": None}, + {"ptu_count": 0}, + {"ptu_count": -1}, + {"ptu_count": ModelInfo.MAX_PTU_COUNT + 1}, + {"cost_per_ptu_per_hour": -0.01}, + {"cost_per_ptu_per_hour": ModelInfo.MAX_COST_PER_PTU_PER_HOUR + 1}, + {"ptu_count": "not-a-number"}, + {"ptu_effective_from": None}, + {"ptu_effective_from": "not-a-date"}, + {"ptu_effective_to": "not-a-date"}, + {"ptu_effective_to": "2025-01-01T00:00:00Z"}, + {"ptu_effective_to": "2026-01-01T00:00:00Z"}, + ], + ids=[ + "no team", + "blank team", + "no count", + "no rate", + "zero count", + "negative count", + "count over the cap", + "negative rate", + "rate over the cap", + "count not a number", + "no start", + "unparseable start", + "unparseable end", + "end before start", + "end equal to start", + ], +) +def test_an_incomplete_reservation_accrues_nothing(override): + """Anything the rollup declines to charge must also decline to be zeroed, or the + deployment serves its traffic for free with nothing charged in its place.""" + assert ptu_terms({**_VALID, **override}) is None + assert _with_flag({**_VALID, **override}) is None + + +def test_a_naive_start_is_read_as_utc(): + """config.yaml is hand-typed, and pydantic hands back a naive datetime for a date with + no offset.""" + terms = ptu_terms({**_VALID, "ptu_effective_from": datetime(2026, 5, 1, 12, 0)}) + + assert terms is not None + assert terms.effective_from == datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc) + + +def test_an_offset_start_is_converted_rather_than_relabelled(): + terms = ptu_terms({**_VALID, "ptu_effective_from": "2026-05-01T12:00:00-05:00"}) + + assert terms is not None + assert terms.effective_from == datetime(2026, 5, 1, 17, 0, tzinfo=timezone.utc) + + +def test_nothing_is_zeroed_while_the_feature_is_off(): + """No flat cost accrues with the flag off, so zeroing would serve the traffic free.""" + assert _with_flag(_VALID, enabled=False) is None + + +def test_the_standing_rates_are_all_zeroed(): + override = _with_flag(_VALID) + + assert override is not None + assert [field for field in PTU_ZEROED_PRICING_FIELDS if override[field] != 0.0] == [] + + +def test_tiered_pricing_is_emptied_rather_than_zeroed(): + """A tier outranks the flat rates written beside it, so a zero there would leave the + cost map's tiers billing the traffic the reserved capacity already covers.""" + override = _with_flag(_VALID, declared={"tiered_pricing": [{"range": [0, 1000], "input_cost_per_token": 0.003}]}) + + assert override is not None + for field in PTU_EMPTIED_PRICING_FIELDS: + assert override[field] == () + + +def test_the_search_context_table_is_zeroed_in_place_on_every_deployment(): + """An absent table means the provider's own default rather than free, so it is written + even when the deployment never declared one.""" + override = _with_flag(_VALID) + + assert override is not None + for field in PTU_ZEROED_TABLE_FIELDS: + assert dict(override[field]) == dict.fromkeys(SEARCH_CONTEXT_SIZES, 0.0) + + +def test_a_declared_table_does_not_become_a_scalar(): + """Zeroing it as a plain 0.0 would leave the provider's reader without a table to + consult, which is the same as absent.""" + override = _with_flag(_VALID, declared={"search_context_cost_per_query": {"search_context_size_medium": 0.05}}) + + assert override is not None + assert dict(override["search_context_cost_per_query"]) == dict.fromkeys(SEARCH_CONTEXT_SIZES, 0.0) + + +def test_a_rate_the_deployment_declares_itself_is_zeroed_too(): + """The standing set covers the mirrored rates. Anything else the operator wrote would + otherwise survive and bill the traffic the hourly charge already paid for.""" + extra = "input_cost_per_token_above_200k_tokens" + assert extra in CUSTOM_PRICING_FIELDS + assert extra not in PTU_ZEROED_PRICING_FIELDS + + override = _with_flag(_VALID, declared={extra: 9e-06}) + + assert override is not None + assert override[extra] == 0.0 + + +def test_a_setting_that_is_not_a_charge_is_left_alone(): + """CustomPricingLiteLLMParams also carries configuration, and zeroing one of those + would break the deployment rather than stop a charge.""" + override = _with_flag(_VALID, declared={"output_vector_size": 1536}) + + assert override is not None + assert "output_vector_size" not in override diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 10bd22689d0..0f21cce476b 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1262,3 +1262,83 @@ def test_get_combined_tool_content_joins_many_custom_tool_input_fragments_in_ord assert isinstance(combined[1], ChatCompletionMessageCustomToolCall) assert combined[1].custom.name == "run_script" assert combined[1].custom.input == "".join(object_fragments) + + +def _reasoning_stream_chunk() -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-reasoning", + model="claude-opus-4-8", + choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="10", role="assistant"))], + ) + + +def test_count_reasoning_tokens_returns_none_for_signature_only_thinking(): + from litellm.types.utils import Choices, Message, ModelResponse + + processor = ChunkProcessor(chunks=[_reasoning_stream_chunk()]) + response = ModelResponse( + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(content="10", role="assistant", reasoning_content=""), + ) + ] + ) + + assert processor.count_reasoning_tokens(response) is None + + +def test_count_reasoning_tokens_counts_visible_reasoning(): + from litellm.types.utils import Choices, Message, ModelResponse + + processor = ChunkProcessor(chunks=[_reasoning_stream_chunk()]) + response = ModelResponse( + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content="10", + role="assistant", + reasoning_content="let me count the primes under thirty", + ), + ) + ] + ) + + assert processor.count_reasoning_tokens(response) > 0 + + +@pytest.mark.parametrize( + "estimated_reasoning_tokens, expected_reasoning_tokens, expected_text_tokens", + [(40, 40, 60), (250, 100, 0)], +) +def test_calculate_usage_fills_unknown_split_from_reasoning_estimate( + estimated_reasoning_tokens, expected_reasoning_tokens, expected_text_tokens +): + from litellm.types.utils import CompletionTokensDetailsWrapper + + chunk = ModelResponseStream( + id="chatcmpl-unknown-split", + model="claude-opus-4-8", + choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=None, role=None))], + usage=Usage( + prompt_tokens=50, + completion_tokens=100, + total_tokens=150, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=None, text_tokens=None), + ), + ) + processor = ChunkProcessor(chunks=[chunk]) + + usage = processor.calculate_usage( + chunks=[chunk], + model="claude-opus-4-8", + completion_output="10", + reasoning_tokens=estimated_reasoning_tokens, + ) + + assert usage.completion_tokens == 100 + assert usage.completion_tokens_details.reasoning_tokens == expected_reasoning_tokens + assert usage.completion_tokens_details.text_tokens == expected_text_tokens diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index c1612aff3d3..05b44fffbc5 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1774,6 +1774,80 @@ def test_openrouter_streaming_cost_propagates_to_hidden_params(): assert provider_cost == 0.00025 +def test_perplexity_streaming_dict_cost_propagates_to_hidden_params(): + """ + Regression: Perplexity reports usage.cost as a breakdown object, which used to + blow up the end of the stream with + `float() argument must be a string or a real number, not 'dict'`. + """ + import litellm + from litellm.cost_calculator import get_response_cost_from_hidden_params + + chunks = [ + ModelResponseStream( + id="chatcmpl-pplx", + created=1742056047, + model="perplexity/sonar", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="Hi", role="assistant"), + ) + ], + usage=None, + ), + ModelResponseStream( + id="chatcmpl-pplx", + created=1742056048, + model="perplexity/sonar", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=None, + ), + ModelResponseStream( + id="chatcmpl-pplx", + created=1742056049, + model="perplexity/sonar", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="")) + ], + usage=Usage( + completion_tokens=18, + prompt_tokens=12, + total_tokens=30, + cost={ + "input_tokens_cost": 0.000012, + "output_tokens_cost": 0.000018, + "request_cost": 0.005, + "total_cost": 0.00503, + }, + ), + ), + ] + + complete_response = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "test"}] + ) + + assert complete_response is not None + + CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response) + + assert ( + get_response_cost_from_hidden_params(complete_response._hidden_params) + == 0.00503 + ) + + +def test_provider_reported_cost_ignores_unusable_shapes(): + assert CustomStreamWrapper._resolve_provider_reported_cost(None) is None + assert CustomStreamWrapper._resolve_provider_reported_cost({}) is None + assert CustomStreamWrapper._resolve_provider_reported_cost({"total_cost": None}) is None + assert CustomStreamWrapper._resolve_provider_reported_cost(0.5) == 0.5 + + def test_handle_special_delta_attributes( initialized_custom_stream_wrapper: CustomStreamWrapper, ): diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 867b148bfc3..391bd8566a2 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -221,6 +221,162 @@ def test_calculate_usage_clamps_text_tokens_when_reasoning_estimate_exceeds_outp assert usage.completion_tokens_details.text_tokens == 0 +def test_calculate_usage_prefers_provider_reported_thinking_tokens(): + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={ + "input_tokens": 32, + "output_tokens": 421, + "output_tokens_details": {"thinking_tokens": 372}, + }, + reasoning_content="", + completion_response={ + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig"}, + {"type": "text", "text": "10"}, + ] + }, + ) + + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 372 + assert usage.completion_tokens_details.text_tokens == 49 + + +def test_calculate_usage_provider_thinking_tokens_win_over_visible_reasoning_estimate(): + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={ + "input_tokens": 50, + "output_tokens": 811, + "output_tokens_details": {"thinking_tokens": 747}, + }, + reasoning_content="short visible reasoning that tokenizes to far fewer than 747 tokens", + ) + + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 747 + assert usage.completion_tokens_details.text_tokens == 64 + + +def test_calculate_usage_sums_provider_thinking_tokens_across_iterations(): + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={ + "input_tokens": 10, + "output_tokens": 300, + "iterations": [ + {"input_tokens": 5, "output_tokens": 100, "output_tokens_details": {"thinking_tokens": 60}}, + {"input_tokens": 5, "output_tokens": 200, "output_tokens_details": {"thinking_tokens": 90}}, + ], + }, + reasoning_content=None, + ) + + assert usage.completion_tokens == 300 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 150 + assert usage.completion_tokens_details.text_tokens == 150 + + +def test_calculate_usage_falls_back_when_only_some_iterations_report_thinking_tokens(): + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={ + "input_tokens": 10, + "output_tokens": 300, + "output_tokens_details": {"thinking_tokens": 240}, + "iterations": [ + {"input_tokens": 5, "output_tokens": 100, "output_tokens_details": {"thinking_tokens": 60}}, + {"input_tokens": 5, "output_tokens": 200}, + ], + }, + reasoning_content=None, + ) + + assert usage.completion_tokens == 300 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 240 + assert usage.completion_tokens_details.text_tokens == 60 + + +def test_calculate_usage_reports_unknown_split_when_only_some_iterations_report_thinking_tokens(): + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={ + "input_tokens": 10, + "output_tokens": 300, + "iterations": [ + {"input_tokens": 5, "output_tokens": 100, "output_tokens_details": {"thinking_tokens": 60}}, + {"input_tokens": 5, "output_tokens": 200}, + ], + }, + reasoning_content="", + completion_response={"content": [{"type": "thinking", "thinking": "", "signature": "sig"}]}, + ) + + assert usage.completion_tokens == 300 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens is None + assert usage.completion_tokens_details.text_tokens is None + + +def test_calculate_usage_reports_unknown_split_when_thinking_ran_without_a_count(): + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={"input_tokens": 32, "output_tokens": 580}, + reasoning_content="", + completion_response={ + "content": [ + {"type": "redacted_thinking", "data": "encrypted"}, + {"type": "text", "text": "10"}, + ] + }, + ) + + assert usage.completion_tokens == 580 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens is None + assert usage.completion_tokens_details.text_tokens is None + + +def test_calculate_usage_without_thinking_reports_all_output_as_text(): + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={"input_tokens": 32, "output_tokens": 171}, + reasoning_content=None, + completion_response={"content": [{"type": "text", "text": "10"}]}, + ) + + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 0 + assert usage.completion_tokens_details.text_tokens == 171 + + +def test_calculate_usage_ignores_malformed_provider_thinking_tokens(): + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={ + "input_tokens": 32, + "output_tokens": 100, + "output_tokens_details": {"thinking_tokens": "not-a-number"}, + }, + reasoning_content=None, + ) + + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 0 + assert usage.completion_tokens_details.text_tokens == 100 + + def test_calculate_usage_handles_mocked_output_tokens_with_reasoning_content(): config = AnthropicConfig() diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py index 6ea9098c228..5c1cd88835f 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py @@ -1,3 +1,4 @@ +import asyncio import json import os import sys @@ -20,9 +21,11 @@ class _RecordingLoggingIterator(BaseAnthropicMessagesStreamingIterator): def __init__(self, litellm_logging_obj: LiteLLMLoggingObj, request_body: dict): super().__init__(litellm_logging_obj=litellm_logging_obj, request_body=request_body) self.logged_chunks: list = [] + self.logging_call_count: int = 0 async def _handle_streaming_logging(self, collected_chunks): self.logged_chunks = list(collected_chunks) + self.logging_call_count += 1 def _make_logging_obj(test_name: str) -> LiteLLMLoggingObj: @@ -233,6 +236,70 @@ async def test_async_sse_wrapper_excludes_synthetic_error_event_from_logged_chun assert not any(chunk.startswith(b"event: error\n") for chunk in iterator.logged_chunks) +async def _events_then_hang(events): + for event in events: + yield event + await asyncio.Event().wait() + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_logs_partial_chunks_on_client_disconnect(): + """ + Regression test for LIT-5839: a client disconnect tears the generator + down with GeneratorExit at the yield, which used to skip the post-loop + logging dispatch entirely, so the partial output tokens the provider + already generated (and billed) never reached spend tracking. + """ + iterator = _RecordingLoggingIterator( + litellm_logging_obj=_make_logging_obj("test_disconnect_logs_partial_chunks"), + request_body={}, + ) + wrapped = iterator.async_sse_wrapper(_events_then_hang(TRUNCATED_TOOL_USE_EVENTS)) + streamed = [await wrapped.__anext__() for _ in range(len(TRUNCATED_TOOL_USE_EVENTS))] + assert iterator.logging_call_count == 0 + + await wrapped.aclose() + + assert iterator.logging_call_count == 1 + assert iterator.logged_chunks == streamed + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_logs_partial_chunks_on_cancellation(): + iterator = _RecordingLoggingIterator( + litellm_logging_obj=_make_logging_obj("test_cancellation_logs_partial_chunks"), + request_body={}, + ) + wrapped = iterator.async_sse_wrapper(_events_then_hang(TRUNCATED_TOOL_USE_EVENTS)) + streamed = [await wrapped.__anext__() for _ in range(len(TRUNCATED_TOOL_USE_EVENTS))] + + consume_task = asyncio.ensure_future(wrapped.__anext__()) + await asyncio.sleep(0.01) + consume_task.cancel() + with pytest.raises(asyncio.CancelledError): + await consume_task + + assert iterator.logging_call_count == 1 + assert iterator.logged_chunks == streamed + + +@pytest.mark.asyncio +async def test_async_sse_wrapper_skips_logging_on_disconnect_before_first_chunk(): + iterator = _RecordingLoggingIterator( + litellm_logging_obj=_make_logging_obj("test_disconnect_before_first_chunk"), + request_body={}, + ) + wrapped = iterator.async_sse_wrapper(_events_then_hang(())) + + consume_task = asyncio.ensure_future(wrapped.__anext__()) + await asyncio.sleep(0.01) + consume_task.cancel() + with pytest.raises(asyncio.CancelledError): + await consume_task + + assert iterator.logging_call_count == 0 + + def test_incomplete_stream_error_sse_event_is_valid_anthropic_error(): event = _incomplete_stream_error_sse_event().decode() lines = event.split("\n") diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 298360789eb..a3be3ebcfc7 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -6005,6 +6005,87 @@ def test_adaptive_thinking_dropped_when_max_tokens_too_small_converse(): assert "thinking" not in optional_params +def test_converse_usage_reports_unknown_split_for_signature_only_thinking(): + config = AmazonConverseConfig() + + usage = config.transform_usage( + ConverseTokenUsageBlock(inputTokens=32, outputTokens=581, totalTokens=613), + reasoning_content="", + thinking_ran=True, + ) + + assert usage.completion_tokens == 581 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens is None + assert usage.completion_tokens_details.text_tokens is None + + +def test_converse_usage_estimates_split_for_visible_thinking(): + config = AmazonConverseConfig() + + usage = config.transform_usage( + ConverseTokenUsageBlock(inputTokens=32, outputTokens=581, totalTokens=613), + reasoning_content="Let me think about how many primes there are under thirty.", + thinking_ran=True, + ) + + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens > 0 + assert ( + usage.completion_tokens_details.reasoning_tokens + usage.completion_tokens_details.text_tokens + == usage.completion_tokens + ) + + +def test_converse_usage_without_thinking_reports_all_output_as_text(): + config = AmazonConverseConfig() + + usage = config.transform_usage(ConverseTokenUsageBlock(inputTokens=32, outputTokens=171, totalTokens=203)) + + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 0 + assert usage.completion_tokens_details.text_tokens == 171 + + +def test_converse_transform_response_signature_only_thinking_reports_unknown_split(): + config = AmazonConverseConfig() + raw_response = MagicMock(status_code=200) + raw_response.text = json.dumps( + { + "output": { + "message": { + "role": "assistant", + "content": [ + {"reasoningContent": {"reasoningText": {"text": "", "signature": "sig"}}}, + {"text": "10"}, + ], + } + }, + "stopReason": "end_turn", + "usage": {"inputTokens": 32, "outputTokens": 581, "totalTokens": 613}, + } + ) + raw_response.json.return_value = json.loads(raw_response.text) + + response = config._transform_response( + model="bedrock/global.anthropic.claude-opus-4-8", + response=raw_response, + model_response=ModelResponse(), + stream=False, + logging_obj=None, + optional_params={}, + api_key=None, + data={}, + messages=[], + encoding=None, + ) + + assert response.choices[0].message.reasoning_content == "" + + assert response.usage.completion_tokens_details.reasoning_tokens is None + assert response.usage.completion_tokens_details.text_tokens is None + + def test_is_converse_usage_shape_distinguishes_camel_case_from_anthropic(): config = AmazonConverseConfig() assert config.is_converse_usage_shape({"inputTokens": 1, "outputTokens": 2}) is True diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index ce81edf3101..1a3fdb9f652 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -2948,3 +2948,38 @@ def test_bedrock_invoke_messages_allows_converted_websearch_function_tool(): headers={}, ) assert result["tools"][0]["name"] == "litellm_web_search" + + +@pytest.mark.asyncio +async def test_bedrock_sse_wrapper_dispatches_logging_on_client_disconnect(): + """ + Regression test for LIT-5839: closing the outer bedrock_sse_wrapper + mid-stream (what the proxy does on a client disconnect) must close the + inner async_sse_wrapper deterministically so the partial-stream logging + fires. `completion_start_time` is only stamped on the logging object by + that dispatch, so it observing a value proves the whole chain ran. + """ + cfg = AmazonAnthropicClaudeMessagesConfig() + + async def _hanging_stream(): + yield {"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 25, "output_tokens": 1}}} + yield {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "partial"}} + await asyncio.Event().wait() + + logging_obj = LiteLLMLoggingObj( + model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="chat", + start_time=datetime.now(), + litellm_call_id="test_bedrock_sse_wrapper_disconnect_logging", + function_id="test_bedrock_sse_wrapper_disconnect_logging", + ) + wrapped = cfg.bedrock_sse_wrapper(_hanging_stream(), litellm_logging_obj=logging_obj, request_body={}) + await wrapped.__anext__() + await wrapped.__anext__() + assert logging_obj.completion_start_time is None + + await wrapped.aclose() + + assert logging_obj.completion_start_time is not None diff --git a/tests/test_litellm/llms/bedrock/search/test_agentcore_search_transformation.py b/tests/test_litellm/llms/bedrock/search/test_agentcore_search_transformation.py new file mode 100644 index 00000000000..950336c7ad0 --- /dev/null +++ b/tests/test_litellm/llms/bedrock/search/test_agentcore_search_transformation.py @@ -0,0 +1,637 @@ +""" +Tests for Amazon Bedrock AgentCore Web Search integration. + +Mirror of tests/search_tests/test_agentcore_search.py placed in the +test_litellm tree so the AgentCoreSearchConfig transformation is exercised by +the sharded CI (coverage collection runs against this tree). +""" + +import json +import os + +import pytest +from unittest.mock import AsyncMock, patch, MagicMock + +import litellm +from litellm.llms.bedrock.search.transformation import ( + AGENTCORE_DEFAULT_MCP_PROTOCOL_VERSION, + AgentCoreSearchConfig, +) + +GATEWAY_URL = "https://testgateway-abc123.gateway.bedrock-agentcore.us-east-1.amazonaws.com/mcp" + +MCP_RESULTS = [ + { + "title": "Test Result 1", + "url": "https://example.com/1", + "text": "Snippet for result 1", + "publishedDate": "2026-06-16", + }, + { + "title": "Test Result 2", + "url": "https://example.com/2", + "text": "Snippet for result 2", + }, +] + + +def _mcp_response_body() -> dict: + return { + "jsonrpc": "2.0", + "id": 1, + "result": {"content": [{"type": "text", "text": json.dumps(MCP_RESULTS)}]}, + } + + +def _make_mock_response(json_body: dict = None, text: str = None) -> MagicMock: + mock_response = MagicMock() + mock_response.status_code = 200 + if text is not None: + mock_response.text = text + else: + mock_response.text = json.dumps(json_body) + mock_response.json.return_value = json_body + return mock_response + + +class TestAgentCoreSearch: + """ + Tests for AgentCore Web Search functionality with mocked network/signing. + """ + + @pytest.mark.asyncio + async def test_agentcore_search_request_payload(self): + """Validates the MCP tools/call payload and SigV4 signing without real AWS calls.""" + os.environ["AGENTCORE_GATEWAY_URL"] = GATEWAY_URL + + mock_response = _make_mock_response(_mcp_response_body()) + + with ( + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post, + patch.object( + AgentCoreSearchConfig, + "_sign_request", + return_value=( + {"Authorization": "AWS4-HMAC-SHA256 test", "Content-Type": "application/json"}, + json.dumps({"signed": True}).encode(), + ), + ) as mock_sign, + ): + mock_post.return_value = mock_response + + response = await litellm.asearch( + query="latest developments in AI", + search_provider="agentcore", + max_results=5, + ) + + mock_post.assert_called_once() + call_kwargs = mock_post.call_args.kwargs + assert call_kwargs["url"] == GATEWAY_URL + # Signed body must be sent verbatim + assert call_kwargs["data"] == json.dumps({"signed": True}).encode() + assert call_kwargs["json"] is None + + # Signing was invoked with the MCP request + mock_sign.assert_called_once() + sign_kwargs = mock_sign.call_args.kwargs + request_data = sign_kwargs["request_data"] + assert request_data["method"] == "tools/call" + assert request_data["params"]["name"] == "web-search-tool___WebSearch" + assert request_data["params"]["arguments"]["query"] == "latest developments in AI" + assert request_data["params"]["arguments"]["maxResults"] == 5 + assert sign_kwargs["service_name"] == "bedrock-agentcore" + + assert len(response.results) == 2 + assert response.results[0].title == "Test Result 1" + assert response.results[0].url == "https://example.com/1" + assert response.results[0].snippet == "Snippet for result 1" + assert response.results[0].date == "2026-06-16" + + def test_transform_search_request_query_truncation(self): + """AgentCore rejects queries > 200 chars; the request must truncate.""" + config = AgentCoreSearchConfig() + long_query = "a" * 300 + data = config.transform_search_request(query=long_query, optional_params={}) + assert len(data["params"]["arguments"]["query"]) == 200 + + def test_transform_search_request_joins_list_queries(self): + config = AgentCoreSearchConfig() + data = config.transform_search_request(query=["foo", "bar"], optional_params={}) + assert data["params"]["arguments"]["query"] == "foo bar" + + def test_transform_search_request_custom_tool_name(self): + config = AgentCoreSearchConfig() + data = config.transform_search_request(query="q", optional_params={"tool_name": "my-target___WebSearch"}) + assert data["params"]["name"] == "my-target___WebSearch" + + def test_transform_search_request_rejects_non_websearch_tool_name(self): + """A caller-supplied tool_name must not reach other tools on the gateway.""" + config = AgentCoreSearchConfig() + with pytest.raises(ValueError, match="must end with"): + config.transform_search_request(query="q", optional_params={"tool_name": "admin-target___DeleteUser"}) + + def test_transform_search_request_sends_documented_default_max_results(self): + """The documented default of 10 is sent explicitly, not left to the gateway.""" + config = AgentCoreSearchConfig() + data = config.transform_search_request(query="q", optional_params={}) + assert data["params"]["arguments"]["maxResults"] == 10 + + def test_get_complete_url_requires_gateway_url(self): + config = AgentCoreSearchConfig() + os.environ.pop("AGENTCORE_GATEWAY_URL", None) + with pytest.raises(ValueError, match="AGENTCORE_GATEWAY_URL"): + config.get_complete_url(api_base=None, optional_params={}) + + def test_get_complete_url_prefers_api_base(self): + config = AgentCoreSearchConfig() + assert config.get_complete_url(api_base=GATEWAY_URL, optional_params={}) == GATEWAY_URL + + def test_validate_environment_sets_mcp_headers(self): + """MCP Streamable HTTP requires accepting both JSON and SSE, and declaring + the protocol revision the client speaks.""" + config = AgentCoreSearchConfig() + headers = config.validate_environment(headers={}) + assert headers["Accept"] == "application/json, text/event-stream" + assert headers["Content-Type"] == "application/json" + assert headers["MCP-Protocol-Version"] == AGENTCORE_DEFAULT_MCP_PROTOCOL_VERSION + + def test_default_protocol_version_is_the_agentcore_gateway_default(self): + """A default AgentCore gateway supports only 2025-03-26 and answers + -32600 to anything newer, so that exact revision must be the default.""" + assert AGENTCORE_DEFAULT_MCP_PROTOCOL_VERSION == "2025-03-26" + + def test_protocol_version_env_override_wins(self): + """A gateway pinned to a newer supportedVersions list needs the header + to match, so AGENTCORE_MCP_PROTOCOL_VERSION must override the default.""" + config = AgentCoreSearchConfig() + with patch.dict(os.environ, {"AGENTCORE_MCP_PROTOCOL_VERSION": "2025-06-18"}): + headers = config.validate_environment(headers={}) + assert headers["MCP-Protocol-Version"] == "2025-06-18" + + def test_protocol_version_header_survives_signing(self): + """Both auth paths must keep the MCP-Protocol-Version header on the wire.""" + config = AgentCoreSearchConfig() + headers = config.validate_environment(headers={}) + + bearer_headers, _ = config.sign_request( + headers=headers, + optional_params={}, + request_data={"jsonrpc": "2.0"}, + api_base=GATEWAY_URL, + api_key="test-jwt-token", + ) + assert bearer_headers["MCP-Protocol-Version"] == AGENTCORE_DEFAULT_MCP_PROTOCOL_VERSION + + with patch.dict( + os.environ, + { + "AWS_ACCESS_KEY_ID": "AKIAIOSFODNN7EXAMPLE", + "AWS_SECRET_ACCESS_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + ): + signed_headers, _ = config.sign_request( + headers=headers, + optional_params={"aws_region_name": "us-east-1"}, + request_data={"jsonrpc": "2.0"}, + api_base=GATEWAY_URL, + ) + assert signed_headers["Authorization"].startswith("AWS4-HMAC-SHA256") + assert signed_headers["MCP-Protocol-Version"] == AGENTCORE_DEFAULT_MCP_PROTOCOL_VERSION + + def test_transform_search_response_parses_sse_frame(self): + """Gateway may answer with an SSE-framed JSON-RPC message.""" + config = AgentCoreSearchConfig() + body = _mcp_response_body() + sse_text = f"event: message\ndata: {json.dumps(body)}\n\n" + mock_response = _make_mock_response(text=sse_text) + + response = config.transform_search_response(raw_response=mock_response, logging_obj=MagicMock()) + assert len(response.results) == 2 + assert response.results[1].url == "https://example.com/2" + + def test_transform_search_response_parses_multiline_sse_data(self): + """SSE data may be split across several data: lines (joined per spec).""" + config = AgentCoreSearchConfig() + pretty = json.dumps(_mcp_response_body(), indent=2) + sse_text = "event: message\n" + "\n".join(f"data: {line}" for line in pretty.splitlines()) + "\n\n" + mock_response = _make_mock_response(text=sse_text) + + response = config.transform_search_response(raw_response=mock_response, logging_obj=MagicMock()) + assert len(response.results) == 2 + + def test_transform_search_response_skips_progress_events(self): + """A progress notification before the JSON-RPC result must not shadow it.""" + config = AgentCoreSearchConfig() + progress = {"jsonrpc": "2.0", "method": "notifications/progress", "params": {"progress": 1}} + sse_text = ( + f"event: message\ndata: {json.dumps(progress)}\n\n" + f"event: message\ndata: {json.dumps(_mcp_response_body())}\n\n" + ) + mock_response = _make_mock_response(text=sse_text) + + response = config.transform_search_response(raw_response=mock_response, logging_obj=MagicMock()) + assert len(response.results) == 2 + + def test_transform_search_response_raises_on_mcp_error(self): + config = AgentCoreSearchConfig() + mock_response = _make_mock_response( + {"jsonrpc": "2.0", "id": 1, "error": {"code": -32601, "message": "tool not found"}} + ) + with pytest.raises(Exception, match="tool not found"): + config.transform_search_response(raw_response=mock_response, logging_obj=MagicMock()) + + def test_transform_search_response_raises_on_tool_error(self): + """A failed tools/call comes back as HTTP 200 with result.isError; it must not be + reported to the caller as a successful search with zero results.""" + config = AgentCoreSearchConfig() + mock_response = _make_mock_response( + { + "jsonrpc": "2.0", + "id": 1, + "result": { + "isError": True, + "content": [{"type": "text", "text": "AccessDeniedException: not authorized"}], + }, + } + ) + with pytest.raises(Exception, match="AccessDeniedException"): + config.transform_search_response(raw_response=mock_response, logging_obj=MagicMock()) + + def test_transform_search_response_reads_structured_content(self): + """Connector 1.1.0+ puts the machine-readable results in structuredContent and may + leave the text block as prose, which must not come back as an empty result list.""" + config = AgentCoreSearchConfig() + mock_response = _make_mock_response( + { + "jsonrpc": "2.0", + "id": 1, + "result": { + "content": [{"type": "text", "text": "Here is a prose summary of what I found."}], + "structuredContent": {"id": "824f89d0", "results": MCP_RESULTS}, + }, + } + ) + + response = config.transform_search_response(raw_response=mock_response, logging_obj=MagicMock()) + assert [result.title for result in response.results] == ["Test Result 1", "Test Result 2"] + assert response.results[0].url == "https://example.com/1" + assert response.results[0].snippet == "Snippet for result 1" + assert response.results[0].date == "2026-06-16" + + def test_transform_search_response_does_not_duplicate_structured_content(self): + """1.1.0+ repeats the same results in both places, so parsing both would double them.""" + config = AgentCoreSearchConfig() + body = _mcp_response_body() + body["result"]["structuredContent"] = {"results": MCP_RESULTS} + mock_response = _make_mock_response(body) + + response = config.transform_search_response(raw_response=mock_response, logging_obj=MagicMock()) + assert len(response.results) == 2 + + def test_transform_search_response_parses_crlf_framed_sse(self): + """SSE streams may be CRLF framed; events must still split into separate events.""" + config = AgentCoreSearchConfig() + progress = {"jsonrpc": "2.0", "method": "notifications/progress", "params": {"progress": 1}} + sse_text = ( + f"event: message\r\ndata: {json.dumps(progress)}\r\n\r\n" + f"event: message\r\ndata: {json.dumps(_mcp_response_body())}\r\n\r\n" + ) + mock_response = _make_mock_response(text=sse_text) + + response = config.transform_search_response(raw_response=mock_response, logging_obj=MagicMock()) + assert len(response.results) == 2 + assert response.results[0].title == "Test Result 1" + + def test_sign_request_uses_bearer_token_when_api_key_set(self): + """CUSTOM_JWT gateways: api_key is sent as a bearer token, no SigV4.""" + config = AgentCoreSearchConfig() + request_data = {"jsonrpc": "2.0", "id": 1} + + headers, signed_body = config.sign_request( + headers={"Content-Type": "application/json"}, + optional_params={}, + request_data=request_data, + api_base=GATEWAY_URL, + api_key="test-jwt-token", + ) + assert headers["Authorization"] == "Bearer test-jwt-token" + assert signed_body == json.dumps(request_data).encode() + + def test_sign_request_uses_bearer_token_from_env(self): + """Server token is attached when the request targets the configured gateway host.""" + config = AgentCoreSearchConfig() + os.environ["AGENTCORE_GATEWAY_TOKEN"] = "env-jwt-token" + os.environ["AGENTCORE_GATEWAY_URL"] = GATEWAY_URL + try: + headers, _ = config.sign_request( + headers={}, + optional_params={}, + request_data={"jsonrpc": "2.0"}, + api_base=GATEWAY_URL, + ) + assert headers["Authorization"] == "Bearer env-jwt-token" + finally: + os.environ.pop("AGENTCORE_GATEWAY_TOKEN", None) + os.environ.pop("AGENTCORE_GATEWAY_URL", None) + + def test_sign_request_refuses_server_token_to_untrusted_host(self): + """Server-managed token must not be sent to a caller-chosen api_base.""" + config = AgentCoreSearchConfig() + os.environ["AGENTCORE_GATEWAY_TOKEN"] = "env-jwt-token" + os.environ["AGENTCORE_GATEWAY_URL"] = GATEWAY_URL + try: + with pytest.raises(ValueError, match="Refusing to send"): + config.sign_request( + headers={}, + optional_params={}, + request_data={"jsonrpc": "2.0"}, + api_base="https://attacker.example.com/mcp", + ) + finally: + os.environ.pop("AGENTCORE_GATEWAY_TOKEN", None) + os.environ.pop("AGENTCORE_GATEWAY_URL", None) + + def test_sign_request_uses_env_token_for_gateway_api_base_without_gateway_url(self): + """api_base pointing at a real gateway is a trusted destination for the env token, + so operators configuring api_base in yaml don't also need AGENTCORE_GATEWAY_URL.""" + config = AgentCoreSearchConfig() + os.environ["AGENTCORE_GATEWAY_TOKEN"] = "env-jwt-token" + os.environ.pop("AGENTCORE_GATEWAY_URL", None) + try: + headers, _ = config.sign_request( + headers={}, + optional_params={}, + request_data={"jsonrpc": "2.0"}, + api_base=GATEWAY_URL, + ) + assert headers["Authorization"] == "Bearer env-jwt-token" + finally: + os.environ.pop("AGENTCORE_GATEWAY_TOKEN", None) + + @pytest.mark.parametrize( + "untrusted_api_base", + [ + "https://attacker.example.com/mcp", + # gateway hostname in the path/query must not pass for the host + "https://attacker.example.com/gw.gateway.bedrock-agentcore.us-east-1.amazonaws.com/mcp", + ], + ) + def test_sign_request_refuses_sigv4_to_untrusted_host(self, untrusted_api_base): + """A SigV4 signature carries the proxy's credential scope and session token, so it + must never be sent to a host that is not the operator's gateway.""" + config = AgentCoreSearchConfig() + os.environ.pop("AGENTCORE_GATEWAY_TOKEN", None) + os.environ["AGENTCORE_GATEWAY_URL"] = GATEWAY_URL + try: + with patch.object( + AgentCoreSearchConfig.__mro__[2], # BaseAWSLLM + "_sign_request", + return_value=({}, b"{}"), + ) as mock_base_sign: + with pytest.raises(ValueError, match="Refusing to send"): + config.sign_request( + headers={}, + optional_params={"aws_region_name": "us-east-1"}, + request_data={"jsonrpc": "2.0"}, + api_base=untrusted_api_base, + ) + mock_base_sign.assert_not_called() + finally: + os.environ.pop("AGENTCORE_GATEWAY_URL", None) + + @pytest.mark.parametrize( + "plaintext_api_base", + [ + "http://gw.gateway.bedrock-agentcore.us-east-1.amazonaws.com/mcp", + "http://internal-gateway.corp/mcp", + ], + ) + def test_sign_request_refuses_server_token_over_plaintext_http(self, plaintext_api_base): + """A trusted hostname over plain http would expose the bearer token to + network observers, so credentials only ride https (or localhost).""" + config = AgentCoreSearchConfig() + os.environ["AGENTCORE_GATEWAY_TOKEN"] = "env-jwt-token" + os.environ["AGENTCORE_GATEWAY_URL"] = plaintext_api_base + try: + with pytest.raises(ValueError, match="plaintext"): + config.sign_request( + headers={}, + optional_params={}, + request_data={"jsonrpc": "2.0"}, + api_base=plaintext_api_base, + ) + finally: + os.environ.pop("AGENTCORE_GATEWAY_TOKEN", None) + os.environ.pop("AGENTCORE_GATEWAY_URL", None) + + def test_sign_request_refuses_sigv4_over_plaintext_http(self): + """Same for SigV4: a signature over plain http is replayable by observers.""" + config = AgentCoreSearchConfig() + os.environ.pop("AGENTCORE_GATEWAY_TOKEN", None) + with patch.object( + AgentCoreSearchConfig.__mro__[2], # BaseAWSLLM + "_sign_request", + return_value=({}, b"{}"), + ) as mock_base_sign: + with pytest.raises(ValueError, match="plaintext"): + config.sign_request( + headers={}, + optional_params={"aws_region_name": "us-east-1"}, + request_data={"jsonrpc": "2.0"}, + api_base="http://gw.gateway.bedrock-agentcore.us-east-1.amazonaws.com/mcp", + ) + mock_base_sign.assert_not_called() + + def test_sign_request_allows_plain_http_for_localhost(self): + """Local development against an MCP stub on 127.0.0.1 keeps working.""" + config = AgentCoreSearchConfig() + os.environ["AGENTCORE_GATEWAY_TOKEN"] = "env-jwt-token" + os.environ["AGENTCORE_GATEWAY_URL"] = "http://127.0.0.1:8931/mcp" + try: + headers, _ = config.sign_request( + headers={}, + optional_params={}, + request_data={"jsonrpc": "2.0"}, + api_base="http://127.0.0.1:8931/mcp", + ) + assert headers["Authorization"] == "Bearer env-jwt-token" + finally: + os.environ.pop("AGENTCORE_GATEWAY_TOKEN", None) + os.environ.pop("AGENTCORE_GATEWAY_URL", None) + + def test_sign_request_does_not_leak_bedrock_bearer_token(self): + """AWS_BEARER_TOKEN_BEDROCK is a Bedrock Runtime credential — it must not + replace SigV4 on requests to an AgentCore gateway.""" + config = AgentCoreSearchConfig() + + with patch.object( + AgentCoreSearchConfig.__mro__[2], # BaseAWSLLM + "_sign_request", + return_value=({}, b"{}"), + ) as mock_base_sign: + config.sign_request( + headers={}, + optional_params={}, + request_data={"jsonrpc": "2.0"}, + api_base=GATEWAY_URL, + ) + # api_key="" (falsy, not None) disables the base class's + # AWS_BEARER_TOKEN_BEDROCK env fallback. + assert mock_base_sign.call_args.kwargs["api_key"] == "" + + def test_sign_request_custom_hostname_requires_region(self): + """Custom hostname + empty AWS config chain → clear error, no guessed region.""" + config = AgentCoreSearchConfig() + custom_url = "https://gateway.internal.example.com/mcp" + os.environ["AGENTCORE_GATEWAY_URL"] = custom_url + + mock_session = MagicMock() + mock_session.region_name = None # nothing configured anywhere + try: + with patch("boto3.Session", return_value=mock_session): + with pytest.raises(ValueError, match="signing region"): + config.sign_request( + headers={}, + optional_params={}, + request_data={"jsonrpc": "2.0"}, + api_base=custom_url, + ) + finally: + os.environ.pop("AGENTCORE_GATEWAY_URL", None) + + def test_sign_request_custom_hostname_uses_shared_config_region(self): + """Custom hostname + region from AWS shared config (profile) must be honored.""" + config = AgentCoreSearchConfig() + custom_url = "https://gateway.internal.example.com/mcp" + os.environ["AGENTCORE_GATEWAY_URL"] = custom_url + + mock_session = MagicMock() + mock_session.region_name = "eu-west-1" # e.g. from ~/.aws/config profile + try: + with ( + patch("boto3.Session", return_value=mock_session), + patch.object( + AgentCoreSearchConfig.__mro__[2], # BaseAWSLLM + "_sign_request", + return_value=({}, b"{}"), + ) as mock_base_sign, + ): + config.sign_request( + headers={}, + optional_params={}, + request_data={"jsonrpc": "2.0"}, + api_base=custom_url, + ) + assert mock_base_sign.call_args.kwargs["optional_params"]["aws_region_name"] == "eu-west-1" + finally: + os.environ.pop("AGENTCORE_GATEWAY_URL", None) + + def test_sign_request_passes_explicit_aws_credentials(self): + """Explicit aws_* params (e.g. from a proxy search_tools entry) reach the signer.""" + config = AgentCoreSearchConfig() + + with patch.object( + AgentCoreSearchConfig.__mro__[2], # BaseAWSLLM + "_sign_request", + return_value=({}, b"{}"), + ) as mock_base_sign: + config.sign_request( + headers={}, + optional_params={ + "aws_access_key_id": "AKIATEST", + "aws_secret_access_key": "secret", + "aws_session_token": "token", + }, + request_data={"jsonrpc": "2.0"}, + api_base=GATEWAY_URL, + ) + passed = mock_base_sign.call_args.kwargs["optional_params"] + assert passed["aws_access_key_id"] == "AKIATEST" + assert passed["aws_secret_access_key"] == "secret" + assert passed["aws_session_token"] == "token" + + def test_sign_request_derives_region_from_gateway_url(self): + """Signing region must come from the gateway URL, not the caller's default region.""" + config = AgentCoreSearchConfig() + eu_url = "https://gw-x.gateway.bedrock-agentcore.eu-central-1.amazonaws.com/mcp" + + with patch.object( + AgentCoreSearchConfig.__mro__[2], # BaseAWSLLM + "_sign_request", + return_value=({}, b"{}"), + ) as mock_base_sign: + config.sign_request( + headers={}, + optional_params={}, + request_data={"jsonrpc": "2.0"}, + api_base=eu_url, + ) + assert mock_base_sign.call_args.kwargs["optional_params"]["aws_region_name"] == "eu-central-1" + + +class TestAgentCoreSearchEdgeCases: + """Branch coverage for response parsing and error mapping.""" + + def test_transform_search_response_skips_non_text_and_bad_json_blocks(self): + """Non-text blocks and unparseable text blocks are skipped, not fatal.""" + config = AgentCoreSearchConfig() + body = { + "jsonrpc": "2.0", + "id": 1, + "result": { + "content": [ + {"type": "image", "data": "..."}, + {"type": "text", "text": "not-json"}, + {"type": "text", "text": json.dumps(["scalar", {"title": "T", "url": "u", "text": "s"}])}, + ] + }, + } + mock_response = _make_mock_response(body) + + response = config.transform_search_response(raw_response=mock_response, logging_obj=MagicMock()) + # only the one dict item survives; non-dict list entries are skipped + assert len(response.results) == 1 + assert response.results[0].title == "T" + + def test_parse_mcp_body_sse_without_json_frame_raises(self): + """An SSE stream carrying no parseable JSON object is a 502.""" + config = AgentCoreSearchConfig() + mock_response = _make_mock_response(text="event: ping\ndata: not-json\n\n") + with pytest.raises(Exception, match="SSE without a JSON data frame"): + config._parse_mcp_body(mock_response) + + def test_parse_mcp_body_returns_last_event_when_no_result_frame(self): + """A stream of only notifications returns the last parsed event.""" + config = AgentCoreSearchConfig() + note = {"jsonrpc": "2.0", "method": "notifications/progress"} + mock_response = _make_mock_response(text=f"data: {json.dumps(note)}\n\n") + assert config._parse_mcp_body(mock_response) == note + + def test_sign_request_rejects_list_request_body(self): + config = AgentCoreSearchConfig() + with pytest.raises(TypeError, match="single dict"): + config.sign_request( + headers={}, + optional_params={}, + request_data=[{"jsonrpc": "2.0"}], + api_base=GATEWAY_URL, + ) + + def test_get_error_class_maps_status_and_message(self): + config = AgentCoreSearchConfig() + err = config.get_error_class(error_message="boom", status_code=503, headers={}) + assert getattr(err, "status_code", None) == 503 + assert "boom" in str(err) + + def test_search_cost_lookup_is_mapped(self, monkeypatch): + """Assert against the map in this checkout: the remote cost map litellm loads by + default only carries providers already released.""" + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + from litellm.search.cost_calculator import search_provider_cost_per_query + + monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) + assert search_provider_cost_per_query(model="agentcore/search", custom_llm_provider="agentcore") == (0.0, 0.0) diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 69f2312f203..9e9242137e6 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -2226,3 +2226,78 @@ def test_direct_vector_store_search_debug_log_omits_stored_credentials(caplog, i logged = "\n".join(record.getMessage() for record in caplog.records) assert "sup3r-s3cret-valkey-pw" not in logged assert "sk-embedding-s3cret" not in logged + + +@pytest.mark.asyncio +async def test_async_anthropic_messages_handler_carries_deployment_vertex_location_for_pricing(monkeypatch): + """ + The proxy pre-creates the logging object before the router picks a deployment, so the + native /v1/messages path must copy the deployment's vertex_location into the logging + params it updates; otherwise cost resolution falls back to the environment and every + call on this surface prices with the regional uplift (#34393). + """ + import contextlib + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import ( + Logging, + _resolve_vertex_location_for_cost, + ) + + monkeypatch.setenv("VERTEXAI_LOCATION", "us-east5") + monkeypatch.setattr(litellm, "vertex_location", None) + + handler = BaseLLMHTTPHandler() + + async def logging_obj_after_handler(generic_params): + logging_obj = Logging( + model="vertex_ai/claude-haiku-4-5@20251001", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="vertex-messages-location", + function_id="f", + ) + logging_obj.update_environment_variables( + model="vertex_ai/claude-haiku-4-5@20251001", + user="", + optional_params={}, + litellm_params={"api_base": ""}, + custom_llm_provider="vertex_ai", + ) + mock_config = Mock() + mock_config.validate_anthropic_messages_environment = Mock( + return_value=({"authorization": "Bearer t"}, "https://us-east5-aiplatform.googleapis.com") + ) + mock_config.transform_anthropic_messages_request = Mock( + return_value={"model": "claude-haiku-4-5@20251001", "messages": []} + ) + with contextlib.suppress(Exception): + await handler.async_anthropic_messages_handler( + model="claude-haiku-4-5@20251001", + messages=[{"role": "user", "content": "hi"}], + anthropic_messages_provider_config=mock_config, + anthropic_messages_optional_request_params={"max_tokens": 10}, + custom_llm_provider="vertex_ai", + litellm_params=generic_params, + logging_obj=logging_obj, + client=AsyncMock(), + kwargs={}, + ) + return logging_obj + + global_deployment = await logging_obj_after_handler(GenericLiteLLMParams(vertex_location="global")) + assert global_deployment.litellm_params["vertex_location"] == "global" + assert ( + _resolve_vertex_location_for_cost( + custom_llm_provider="vertex_ai", + litellm_params=global_deployment.litellm_params, + optional_params=global_deployment.optional_params, + model="claude-haiku-4-5@20251001", + ) + == "global" + ) + + unconfigured_deployment = await logging_obj_after_handler(GenericLiteLLMParams()) + assert "vertex_location" not in unconfigured_deployment.litellm_params diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 9064312fd6d..5ee8143fb8e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -4395,8 +4395,12 @@ class TestMCPServerManager: captured: dict = {} - def fake_create_tool_function(path, method, operation, base_url, headers=None): + def fake_create_tool_function( + path, method, operation, base_url, headers=None, server_label=None, relays_upstream_auth=False + ): captured["headers"] = headers + captured["server_label"] = server_label + captured["relays_upstream_auth"] = relays_upstream_auth async def tool_func(**kwargs): return "ok" @@ -4425,6 +4429,11 @@ class TestMCPServerManager: assert captured["headers"] is not None assert captured["headers"]["Authorization"] == "STATIC token" + # The label names the server in an upstream-failure error, so registration must thread it; + # without this the fake would simply tolerate the argument and prove nothing about it. + assert captured["server_label"] == "openapi-server" + # auth_type is none here, so a 401 from this upstream must not be dressed up as a re-auth signal + assert captured["relays_upstream_auth"] is False @pytest.mark.asyncio async def test_pre_call_tool_check_allowed_tools_list_allows_tool(self): @@ -10765,3 +10774,61 @@ class TestResolveOpenapiToolAuth: ) assert "Authorization" not in (forwarded or {}) + + +class TestOpenApiHandlerRelaysUpstreamAuth: + """`_call_openapi_tool_handler` must not flatten a re-auth signal into a generic message. + + Its catch-all turned every exception into "Error calling OpenAPI tool ...", which is an isError + result but loses the status, so the REST surface could no longer relay a 401 with the upstream's + WWW-Authenticate and the streamable surface could not name the status the caller must act on. + """ + + @staticmethod + def _server() -> MCPServer: + return MCPServer( + server_id="srv-openapi", + name="report_api", + server_name="report_api", + url="https://api.example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + spec_path="https://api.example.com/openapi.json", + ) + + @pytest.mark.asyncio + async def test_upstream_auth_error_keeps_its_type(self): + from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError + from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + + manager = MCPServerManager() + server = self._server() + + async def raising_handler(**_kwargs): + raise MCPUpstreamAuthError(status_code=401, www_authenticate="Bearer realm=x", server_name="report_api") + + tool = MagicMock() + tool.handler = raising_handler + with patch.object(global_mcp_tool_registry, "get_tool", return_value=tool): + with pytest.raises(MCPUpstreamAuthError) as exc: + await manager._call_openapi_tool_handler(server, "list_reports", {}) + + assert exc.value.status_code == 401 + assert exc.value.www_authenticate == "Bearer realm=x" + + @pytest.mark.asyncio + async def test_other_failures_still_become_an_error_result(self): + from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + + manager = MCPServerManager() + + async def raising_handler(**_kwargs): + raise RuntimeError("upstream returned HTTP 503") + + tool = MagicMock() + tool.handler = raising_handler + with patch.object(global_mcp_tool_registry, "get_tool", return_value=tool): + result = await manager._call_openapi_tool_handler(self._server(), "list_reports", {}) + + assert result.isError is True + assert "upstream returned HTTP 503" in result.content[0].text diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py index 1f9316ee9c8..e59616e53c1 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py @@ -27,12 +27,21 @@ from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( resolve_operation_params, ) +from litellm.proxy._experimental.mcp_server.exceptions import ( + MCPOpenApiUpstreamError, + MCPUpstreamAuthError, +) + GET_ASYNC_CLIENT_TARGET = "litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator.get_async_httpx_client" -def _create_mock_client(method: str, response_text: str) -> AsyncMock: - """Utility to create a mocked async httpx client for the given method.""" - response = SimpleNamespace(text=response_text) +def _create_mock_client(method: str, response_text: str, status_code: int = 200) -> AsyncMock: + """Utility to create a mocked async httpx client for the given method. + + ``status_code`` and ``headers`` are part of the real response the tool function reads, so the + fake carries them too; a fake that omits them cannot observe whether the status is checked. + """ + response = SimpleNamespace(text=response_text, status_code=status_code, headers={}) client = AsyncMock() setattr(client, method, AsyncMock(return_value=response)) return client @@ -1259,3 +1268,113 @@ class TestRequestExtraHeaders: headers_sent = async_client.get.call_args[1]["headers"] assert "Authorization" not in headers_sent + + +class TestUpstreamStatusIsClassified: + """A non-2xx upstream must never be returned as tool output. + + The body used to be returned verbatim whatever the status, so an upstream rejection arrived as a + successful tool result and the request logged as a success. 401 is singled out because it is the + only status the caller can act on by re-authenticating, matching `_call_regular_mcp_tool` where a + 403 deliberately does not produce a challenge. + """ + + @staticmethod + def _tool(status_code: int, text: str = "body", headers: dict | None = None, relays_upstream_auth: bool = True): + response = SimpleNamespace(text=text, status_code=status_code, headers=headers or {}) + client = AsyncMock() + client.get = AsyncMock(return_value=response) + return create_tool_function( + "/reports", + "get", + {"operationId": "list_reports"}, + "https://api.example.com", + server_label="report_api", + relays_upstream_auth=relays_upstream_auth, + ), client + + @pytest.mark.asyncio + async def test_success_still_returns_the_body(self): + tool, client = self._tool(200, text='{"reports": []}') + with patch(GET_ASYNC_CLIENT_TARGET, return_value=client): + assert await tool() == '{"reports": []}' + + @pytest.mark.asyncio + async def test_401_raises_the_reauth_signal_carrying_the_challenge(self): + tool, client = self._tool(401, text='{"error":"invalid_token"}', headers={"www-authenticate": 'Bearer realm="x"'}) + with patch(GET_ASYNC_CLIENT_TARGET, return_value=client): + with pytest.raises(MCPUpstreamAuthError) as exc: + await tool() + + assert exc.value.status_code == 401 + assert exc.value.www_authenticate == 'Bearer realm="x"' + assert exc.value.server_name == "report_api" + + @pytest.mark.asyncio + async def test_401_on_a_non_forwarding_server_is_not_a_reauth_signal(self): + """Only the client-forwarded modes carry the caller's own upstream token, so only they can act + on a 401. `_call_regular_mcp_tool` gates its signal the same way, and without the gate an + api_key server rejecting a token would push clients into an OAuth flow that does not apply.""" + tool, client = self._tool(401, text='{"error":"bad key"}', relays_upstream_auth=False) + with patch(GET_ASYNC_CLIENT_TARGET, return_value=client): + with pytest.raises(MCPOpenApiUpstreamError) as exc: + await tool() + + assert exc.value.status_code == 401 + + @pytest.mark.parametrize("method", ["post", "put", "patch", "delete"]) + @pytest.mark.parametrize("status_code, expected", [(401, "auth"), (500, "other")]) + @pytest.mark.asyncio + async def test_non_get_methods_are_classified_too(self, method: str, status_code: int, expected: str): + """post/put/patch/delete call raise_for_status inside the HTTP handler. + + Only `get` hands a 4xx back to the caller; the others raise `MaskedHTTPStatusError` before any + status check the tool function could do, so classifying the returned response alone would + leave every non-GET tool still serving an upstream error body as successful tool output. A + fake client that simply returns a response cannot observe this, which is why this test builds + the error the real handler raises. + """ + import httpx + + from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError + + request = httpx.Request(method.upper(), "https://api.example.com/reports") + raw = httpx.Response( + status_code, + headers={"www-authenticate": 'Bearer realm="x"'}, + text="internal hostname db-prod-7.corp.example.com", + request=request, + ) + masked = MaskedHTTPStatusError(httpx.HTTPStatusError("boom", request=request, response=raw)) + + client = AsyncMock() + setattr(client, method, AsyncMock(side_effect=masked)) + tool = create_tool_function( + "/reports", + method, + {"operationId": "list_reports"}, + "https://api.example.com", + server_label="report_api", + relays_upstream_auth=True, + ) + + expected_type = MCPUpstreamAuthError if expected == "auth" else MCPOpenApiUpstreamError + with patch(GET_ASYNC_CLIENT_TARGET, return_value=client): + with pytest.raises(expected_type) as exc: + await tool() + + assert exc.value.status_code == status_code + assert "db-prod-7" not in str(exc.value) + + @pytest.mark.parametrize("status_code", [403, 404, 429, 500, 503]) + @pytest.mark.asyncio + async def test_other_failures_raise_without_leaking_the_upstream_body(self, status_code: int): + secret_body = "internal hostname db-prod-7.corp.example.com and a stack trace" + tool, client = self._tool(status_code, text=secret_body) + with patch(GET_ASYNC_CLIENT_TARGET, return_value=client): + with pytest.raises(MCPOpenApiUpstreamError) as exc: + await tool() + + assert exc.value.status_code == status_code + assert secret_body not in str(exc.value) + assert str(exc.value) == f"upstream returned HTTP {status_code}" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py index 7bd846aeda4..bd953dc55f3 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py @@ -652,3 +652,80 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc assert captured["resolver_credential"] == {"Authorization": OPENAPI_PER_SERVER_TOKEN} assert captured["injected"] == OPENAPI_PER_SERVER_TOKEN assert _request_auth_header.get() is None + + +@pytest.mark.parametrize("failure", ["auth", "other"]) +@pytest.mark.asyncio +async def test_local_dispatch_reports_the_outcome_instead_of_success(failure: str): + """A failing local handler must never be reported as a successful tool result, and only an auth + failure may propagate. + + `_handle_local_mcp_tool` used to catch every exception and return it as TextContent, and both of + its callers then stamped `isError=False`, so an upstream rejection was served as tool output and + `extract_mcp_tool_result_error_message` logged the request as a success. + + The two kinds are split by consequence. `MCPUpstreamAuthError` propagates because both renderers + know it: the streamable path names the status and the REST path relays a real 401 with the + upstream's WWW-Authenticate. Anything else is reported as `isError=True` right here, because + `call_tool_rest_api` turns an unrecognized exception into HTTP 500 and an upstream 403 or 429 is + not a gateway crash. + """ + from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy._experimental.mcp_server.exceptions import ( + MCPOpenApiUpstreamError, + MCPUpstreamAuthError, + ) + + error = ( + MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name="report_api") + if failure == "auth" + else MCPOpenApiUpstreamError(429, "report_api") + ) + + async def raising_handler(**_kwargs): + raise error + + fake_tool = MagicMock() + fake_tool.name = "list_reports" + fake_tool.handler = raising_handler + server = MCPServer( + server_id="srv-openapi", + name="report_api", + server_name="report_api", + url="https://api.example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + spec_path="https://api.example.com/openapi.json", + ) + user = UserAPIKeyAuth(api_key="sk-user", user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value) + + with ( + patch.object(mcp_module.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=server), + patch.object(mcp_module.global_mcp_server_manager, "pre_call_tool_check", new=AsyncMock(return_value={})), + patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool), + patch.object( + mcp_module.global_mcp_server_manager, + "resolve_openapi_upstream_auth", + new=AsyncMock(return_value=(None, None)), + ), + patch( + "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + return_value=True, + ), + ): + call = mcp_module.execute_mcp_tool( + name="list_reports", + arguments={}, + allowed_mcp_servers=[server], + start_time=datetime.now(timezone.utc), + user_api_key_auth=user, + ) + if failure == "auth": + with pytest.raises(MCPUpstreamAuthError): + await call + return + result = await call + + # A non-auth upstream failure stays a 200 with isError, so REST does not report it as a gateway 500 + assert result.isError is True + assert "upstream returned HTTP 429" in result.content[0].text diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 270f3eca0f9..1d1bd9ebf8a 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -6392,3 +6392,168 @@ def test_is_user_proxy_admin_rejects_view_only_admin(): assert _is_user_proxy_admin(user_obj=viewer) is False assert _is_user_proxy_admin(user_obj=admin) is True assert _is_user_proxy_admin(user_obj=None) is False + + +def _make_wildcard_access_group_router(): + """ + `openai/*` tagged into an access group, plus an untagged `azure/*`, mirroring a + proxy that fronts a whole provider behind one wildcard deployment. + """ + from litellm import Router + + return Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "fake"}, + "model_info": { + "id": "wildcard-openai", + "access_groups": ["default-models"], + }, + }, + { + "model_name": "azure/*", + "litellm_params": {"model": "azure/*", "api_key": "fake"}, + "model_info": {"id": "wildcard-azure"}, + }, + ] + ) + + +def test_can_object_call_model_access_group_wildcard_accepts_bare_model_name(): + """ + Regression: a key holding only the access group name was denied for `gpt-4o` + while `openai/gpt-4o` was allowed, because group membership resolved through the + pattern router's raw regex and skipped the `{provider}/{model}` retry that both + routing and the direct-wildcard grant already perform. + """ + from litellm.proxy.auth.auth_checks import _can_object_call_model + + router = _make_wildcard_access_group_router() + + assert ( + _can_object_call_model( + model="gpt-4o", + llm_router=router, + models=["default-models"], + object_type="key", + ) + is True + ) + + +def test_can_object_call_model_access_group_wildcard_accepts_prefixed_model_name(): + from litellm.proxy.auth.auth_checks import _can_object_call_model + + router = _make_wildcard_access_group_router() + + assert ( + _can_object_call_model( + model="openai/gpt-4o", + llm_router=router, + models=["default-models"], + object_type="key", + ) + is True + ) + + +@pytest.mark.parametrize( + "model", + [ + "totally-made-up-model-zzz", # no provider can be inferred + "azure/some-deployment", # wildcard exists but carries no access group + ], +) +def test_can_object_call_model_access_group_wildcard_does_not_over_grant(model): + """The bare-name retry must not turn an access group into a blanket grant.""" + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.auth_checks import _can_object_call_model + + router = _make_wildcard_access_group_router() + + with pytest.raises(ProxyException): + _can_object_call_model( + model=model, + llm_router=router, + models=["default-models"], + object_type="key", + ) + + +def test_can_object_call_model_access_group_rejects_unconsumed_namespace(): + """ + `bedrockz/...` infers provider `bedrock` from a fragment of the name, so + re-prefixing would smuggle an unrecognized namespace through a `bedrock/*` group. + """ + from litellm import Router + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.auth_checks import _can_object_call_model + + router = Router( + model_list=[ + { + "model_name": "bedrock/*", + "litellm_params": {"model": "bedrock/*"}, + "model_info": { + "id": "wildcard-bedrock", + "access_groups": ["bedrock-models"], + }, + } + ] + ) + + assert ( + _can_object_call_model( + model="anthropic.claude-3-5-sonnet-20240620-v1:0", + llm_router=router, + models=["bedrock-models"], + object_type="key", + ) + is True + ) + + with pytest.raises(ProxyException): + _can_object_call_model( + model="bedrockz/anthropic.claude-3-5-sonnet-20240620-v1:0", + llm_router=router, + models=["bedrock-models"], + object_type="key", + ) + + +def test_can_object_call_model_team_scoped_wildcard_accepts_bare_model_name(): + """ + Same regression as the proxy-wide wildcard, but for a team-scoped deployment + whose public name is a wildcard: those live in a separate per-team pattern + index that needed the same `{provider}/{model}` retry. + """ + from litellm import Router + from litellm.proxy.auth.auth_checks import _can_object_call_model + + router = Router( + model_list=[ + { + "model_name": "openai/*_team-a_abc", + "litellm_params": {"model": "openai/*", "api_key": "fake"}, + "model_info": { + "id": "team-byok-wildcard", + "team_id": "team-a", + "team_public_model_name": "openai/*", + "access_groups": ["team-models"], + }, + } + ] + ) + + for model in ("gpt-4o", "openai/gpt-4o"): + assert ( + _can_object_call_model( + model=model, + llm_router=router, + models=["team-models"], + object_type="team", + team_id="team-a", + ) + is True + ) diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index f0aa49ff123..59048067674 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -25,6 +25,7 @@ from litellm.proxy.client.cli.commands.auth import ( save_token, whoami, ) +from litellm.proxy.client.cli.commands.claude_settings import SettingsFileOwner def _mock_cli_sso_start_response( @@ -267,7 +268,7 @@ class TestTokenUtilities: def test_load_token_io_error(self): """Test loading token with IO error""" with ( - patch("builtins.open", side_effect=IOError("Permission denied")), + patch("builtins.open", side_effect=OSError("Permission denied")), patch("litellm.proxy.client.cli.commands.auth.get_token_file_path") as mock_path, patch("os.path.exists", return_value=True), ): @@ -1029,3 +1030,83 @@ class TestSaveTokenPrivateWrite: assert json.loads(token_file.read_text()) == {"key": "sk-original", "timestamp": 1234567890} assert list(token_file.parent.glob(".tmp-*")) == [] + + +class TestLoginConfigClaude: + """`lite login --config-claude` wiring into ~/.claude/settings.json""" + + def setup_method(self): + self.runner = CliRunner() + + def _run_login(self, tmp_path, args, base_url="https://test.example.com"): + settings_path = tmp_path / "claude" / "settings.json" + backup_path = tmp_path / "claude_settings_backup.json" + poll_response = Mock() + poll_response.status_code = 200 + poll_response.json.return_value = { + "status": "ready", + "key": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.jwt", + "user_id": "test-user-123", + "team_id": "team-1", + "teams": ["team-1"], + } + with ( + patch("webbrowser.open"), + patch("requests.post", return_value=_mock_cli_sso_start_response()), + patch("requests.get", return_value=poll_response), + patch("litellm.proxy.client.cli.commands.auth.save_token"), + patch("litellm.proxy.client.cli.interface.show_commands"), + patch("litellm.proxy.client.cli.commands.auth.CLAUDE_SETTINGS_PATH", settings_path), + patch( + "litellm.proxy.client.cli.commands.auth.SETTINGS_FILE_OWNERS", + (SettingsFileOwner(backup_path, "lite up", "lite down"),), + ), + patch( + "litellm.proxy.client.cli.commands.claude_settings.shutil.which", + return_value="/usr/local/bin/lite", + ), + ): + result = self.runner.invoke(login, args, obj={"base_url": base_url}) + return result, settings_path, backup_path + + def test_default_login_does_not_touch_claude_settings(self, tmp_path): + result, settings_path, _backup_path = self._run_login(tmp_path, []) + + assert result.exit_code == 0 + assert "Login successful!" in result.output + assert not settings_path.exists() + assert "Configured Claude Code" not in result.output + + def test_flag_writes_the_settings_file_and_reports_success(self, tmp_path): + result, settings_path, _backup_path = self._run_login(tmp_path, ["--config-claude"]) + + assert result.exit_code == 0 + written = json.loads(settings_path.read_text()) + assert written["env"]["ANTHROPIC_BASE_URL"] == "https://test.example.com" + assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://test.example.com auth print-token" + assert "Configured Claude Code" in result.output + + def test_flag_preserves_unrelated_settings_on_an_existing_file(self, tmp_path): + settings_path = tmp_path / "claude" / "settings.json" + settings_path.parent.mkdir(parents=True) + settings_path.write_text(json.dumps({"theme": "dark", "env": {"KEEP": "me"}})) + + result, _settings_path, _backup_path = self._run_login(tmp_path, ["--config-claude"]) + + assert result.exit_code == 0 + written = json.loads(settings_path.read_text()) + assert written["theme"] == "dark" + assert written["env"]["KEEP"] == "me" + + def test_settings_failure_is_reported_without_claiming_login_failed(self, tmp_path): + settings_path = tmp_path / "claude" / "settings.json" + settings_path.parent.mkdir(parents=True) + settings_path.write_text("not json at all {{{") + + result, _settings_path, _backup_path = self._run_login(tmp_path, ["--config-claude"]) + + assert result.exit_code != 0 + assert "Login successful!" in result.output + assert "could not configure Claude Code" in result.output + assert "invalid JSON" in result.output + assert "Authentication failed" not in result.output diff --git a/tests/test_litellm/proxy/client/cli/test_claude_settings.py b/tests/test_litellm/proxy/client/cli/test_claude_settings.py new file mode 100644 index 00000000000..bc9744eb410 --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/test_claude_settings.py @@ -0,0 +1,269 @@ +import json +import shlex +import stat +import time +from unittest.mock import patch + +import pytest +from click.testing import CliRunner + +from litellm.proxy.client.cli import cli +from litellm.proxy.client.cli.commands.claude_settings import ( + AUTOROUTE_BACKUP_PATH, + BACKUP_PATH, + SETTINGS_FILE_OWNERS, + ClaudeSettingsError, + SettingsFileOwner, + resolve_api_key_helper, + write_claude_settings, +) + + +def _owners(*backup_paths): + """Stand-in owners for the real `lite up` / `lite autoroute up` registry.""" + return tuple(SettingsFileOwner(path, "lite up", "lite down") for path in backup_paths) + +CLAUDE_SETTINGS_MODULE = "litellm.proxy.client.cli.commands.claude_settings" +AUTH_MODULE = "litellm.proxy.client.cli.commands.auth" + + +@pytest.fixture +def paths(tmp_path): + return tmp_path / "claude" / "settings.json", tmp_path / "backup.json" + + +@pytest.fixture +def lite_on_path(): + with patch(f"{CLAUDE_SETTINGS_MODULE}.shutil.which", return_value="/usr/local/bin/lite"): + yield + + +class TestWriteClaudeSettings: + def test_creates_the_file_and_its_parent_when_missing(self, paths, lite_on_path): + settings_path, backup_path = paths + assert not settings_path.parent.exists() + + write_claude_settings("https://proxy.example.com/", settings_path, _owners(backup_path)) + + written = json.loads(settings_path.read_text()) + assert written["env"]["ANTHROPIC_BASE_URL"] == "https://proxy.example.com" + assert written["apiKeyHelper"] == "/usr/local/bin/lite --base-url https://proxy.example.com auth print-token" + + def test_updates_an_existing_file_preserving_unrelated_settings(self, paths, lite_on_path): + settings_path, backup_path = paths + settings_path.parent.mkdir(parents=True) + settings_path.write_text( + json.dumps( + { + "theme": "dark", + "permissions": {"allow": ["Bash"]}, + "env": {"SOME_OTHER_VAR": "keep-me", "ANTHROPIC_BASE_URL": "https://old.example.com"}, + "apiKeyHelper": "old-helper", + } + ) + ) + + write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + + written = json.loads(settings_path.read_text()) + assert written["theme"] == "dark" + assert written["permissions"] == {"allow": ["Bash"]} + assert written["env"]["SOME_OTHER_VAR"] == "keep-me" + assert written["env"]["ANTHROPIC_BASE_URL"] == "https://proxy.example.com" + assert written["apiKeyHelper"] != "old-helper" + + def test_rerunning_against_a_new_proxy_refreshes_both_base_url_and_helper(self, paths, lite_on_path): + settings_path, backup_path = paths + + write_claude_settings("https://first.example.com", settings_path, _owners(backup_path)) + write_claude_settings("https://second.example.com", settings_path, _owners(backup_path)) + + written = json.loads(settings_path.read_text()) + assert written["env"]["ANTHROPIC_BASE_URL"] == "https://second.example.com" + assert "second.example.com" in written["apiKeyHelper"] + assert "first.example.com" not in written["apiKeyHelper"] + + def test_drops_a_stray_static_api_key_so_the_helper_token_wins(self, paths, lite_on_path): + settings_path, backup_path = paths + settings_path.parent.mkdir(parents=True) + settings_path.write_text(json.dumps({"env": {"ANTHROPIC_API_KEY": "sk-leaked"}})) + + write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + + assert "ANTHROPIC_API_KEY" not in json.loads(settings_path.read_text())["env"] + + def test_written_file_is_owner_only(self, paths, lite_on_path): + settings_path, backup_path = paths + write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + assert stat.S_IMODE(settings_path.stat().st_mode) == 0o600 + + def test_refuses_while_lite_up_holds_a_backup(self, paths, lite_on_path): + settings_path, backup_path = paths + backup_path.write_text("{}") + + with pytest.raises(ClaudeSettingsError, match="lite down"): + write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + + assert not settings_path.exists() + + def test_refuses_on_corrupt_existing_settings_without_touching_the_file(self, paths, lite_on_path): + settings_path, backup_path = paths + settings_path.parent.mkdir(parents=True) + settings_path.write_text("not json at all {{{") + + with pytest.raises(ClaudeSettingsError, match="invalid JSON"): + write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + + assert settings_path.read_text() == "not json at all {{{" + + def test_reports_an_actionable_error_when_lite_is_not_on_path(self, paths): + settings_path, backup_path = paths + with patch(f"{CLAUDE_SETTINGS_MODULE}.shutil.which", return_value=None): + with pytest.raises(ClaudeSettingsError, match="Could not find `lite`"): + write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + + assert not settings_path.exists() + + def test_reports_an_actionable_error_on_a_non_utf8_file(self, paths, lite_on_path): + """Bytes that are not valid UTF-8 must not escape as UnicodeDecodeError. + + UnicodeDecodeError is a ValueError, not an OSError, so a decode-side catch + is easy to miss; login's broad `except Exception` would then relabel it as + an authentication failure and exit 0. + """ + settings_path, backup_path = paths + settings_path.parent.mkdir(parents=True) + settings_path.write_bytes(b'{"theme": "\xff\xfe"}') + + with pytest.raises(ClaudeSettingsError, match="invalid JSON"): + write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + + def test_reports_an_actionable_error_when_the_file_cannot_be_read(self, paths, lite_on_path): + """An unreadable settings file must not surface as "Authentication failed". + + login wraps the whole flow in a broad `except Exception`, so any OSError + escaping this function gets relabelled as an auth failure and sends the + user looking at their SSO config instead of at file permissions. + """ + settings_path, backup_path = paths + settings_path.parent.mkdir(parents=True) + settings_path.mkdir() + + with pytest.raises(ClaudeSettingsError, match="Could not read"): + write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + + def test_reports_an_actionable_error_when_the_file_cannot_be_written(self, paths, lite_on_path): + settings_path, backup_path = paths + with patch( + f"{CLAUDE_SETTINGS_MODULE}.write_private_json", + side_effect=OSError("Read-only file system"), + ): + with pytest.raises(ClaudeSettingsError, match="Read-only file system"): + write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + + +class TestApiKeyHelperIsActuallyInvocable: + """The helper string is executed verbatim by Claude Code, so it has to parse. + + Asserting only on its text is what let a malformed command (`--base-url`, a + top-level group option, placed after the `print-token` subcommand) ship: click + rejects it with "No such option" and every Claude Code request loses its token. + """ + + def _helper_args(self, base_url): + with patch(f"{CLAUDE_SETTINGS_MODULE}.shutil.which", return_value="/usr/local/bin/lite"): + return shlex.split(resolve_api_key_helper(base_url))[1:] + + def test_the_generated_command_parses(self): + result = CliRunner().invoke(cli, self._helper_args("http://localhost:4000")) + + assert "No such option" not in result.output + assert result.exit_code != 2 + + def test_the_generated_command_reaches_print_token(self): + with patch(f"{AUTH_MODULE}.load_token", return_value=None): + result = CliRunner().invoke(cli, self._helper_args("http://localhost:4000")) + + assert "Not authenticated" in result.output + + def test_the_generated_command_carries_the_base_url_through(self): + stale = { + "base_url": "http://other-proxy.example.com", + "key": "sk-stale", + "timestamp": time.time(), + } + with patch(f"{AUTH_MODULE}.load_token", return_value=stale): + result = CliRunner().invoke(cli, self._helper_args("http://localhost:4000")) + + assert "Not authenticated for this server" in result.output + + +class TestConflictingOwnersOfTheSettingsFile: + """Both `lite up` and `lite autoroute up` restore a backup when they stop. + + Guarding only one of them leaves the other free to silently revert this + write, which is the exact hazard the guard exists to prevent. + """ + + def test_any_owner_holding_a_backup_blocks_the_write(self, tmp_path, lite_on_path): + settings_path = tmp_path / "claude" / "settings.json" + + for index, owner in enumerate(SETTINGS_FILE_OWNERS): + backup = tmp_path / f"backup-{index}.json" + backup.write_text("{}") + stand_in = SettingsFileOwner(backup, owner.start_command, owner.stop_command) + with pytest.raises(ClaudeSettingsError, match="currently managing"): + write_claude_settings("https://proxy.example.com", settings_path, (stand_in,)) + backup.unlink() + assert not settings_path.exists() + + def test_the_error_names_the_owner_that_actually_holds_the_file(self, tmp_path, lite_on_path): + settings_path = tmp_path / "claude" / "settings.json" + backup = tmp_path / "auto.json" + backup.write_text("{}") + autoroute = SettingsFileOwner(backup, "lite autoroute up", "lite autoroute down") + + with pytest.raises(ClaudeSettingsError, match="`lite autoroute up` is currently managing"): + write_claude_settings("https://proxy.example.com", settings_path, (autoroute,)) + with pytest.raises(ClaudeSettingsError, match="Run `lite autoroute down` first"): + write_claude_settings("https://proxy.example.com", settings_path, (autoroute,)) + + def test_the_registry_matches_the_paths_the_commands_actually_use(self): + """A second definition of the autoroute dir must not drift from this one.""" + from litellm.proxy.client.cli.commands.autoroute.process import AUTOROUTE_DIR + + assert AUTOROUTE_BACKUP_PATH == AUTOROUTE_DIR / "claude_settings_backup.json" + assert {o.backup_path for o in SETTINGS_FILE_OWNERS} == {BACKUP_PATH, AUTOROUTE_BACKUP_PATH} + assert {o.stop_command for o in SETTINGS_FILE_OWNERS} == {"lite down", "lite autoroute down"} + + +class TestDoesNotDestroyUserOwnedStructure: + def test_writes_through_a_symlinked_settings_file(self, tmp_path, lite_on_path): + """os.replace() swaps the symlink for a regular file, detaching a dotfiles repo. + + There is no backup here to undo that, so the link must survive and its + target must be the thing that gets updated. + """ + real = tmp_path / "dotfiles" / "settings.json" + real.parent.mkdir() + real.write_text(json.dumps({"theme": "dark"})) + link = tmp_path / "claude" / "settings.json" + link.parent.mkdir() + link.symlink_to(real) + + write_claude_settings("https://proxy.example.com", link, ()) + + assert link.is_symlink() + assert json.loads(real.read_text())["env"]["ANTHROPIC_BASE_URL"] == "https://proxy.example.com" + assert json.loads(real.read_text())["theme"] == "dark" + + def test_refuses_rather_than_discarding_a_non_object_env(self, paths, lite_on_path): + """merge coerces a non-dict env to {}; that is silent data loss on a persistent write.""" + settings_path, backup_path = paths + settings_path.parent.mkdir(parents=True) + settings_path.write_text(json.dumps({"theme": "dark", "env": "not-an-object"})) + + with pytest.raises(ClaudeSettingsError, match="non-object"): + write_claude_settings("https://proxy.example.com", settings_path, _owners(backup_path)) + + assert json.loads(settings_path.read_text())["env"] == "not-an-object" diff --git a/tests/test_litellm/proxy/client/cli/test_up_commands.py b/tests/test_litellm/proxy/client/cli/test_up_commands.py index 1b182553644..51de0dcf11d 100644 --- a/tests/test_litellm/proxy/client/cli/test_up_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_up_commands.py @@ -10,6 +10,7 @@ from click.testing import CliRunner from litellm.proxy.client.cli.commands import up as up_module from litellm.proxy.client.cli.commands.agents import AgentRunError +from litellm.proxy.client.cli.commands.claude_settings import ClaudeSettingsError from litellm.proxy.client.cli.commands.up import ( BackupRecord, UpError, @@ -25,6 +26,7 @@ from litellm.proxy.client.cli.commands.up import ( ) UP_MODULE = "litellm.proxy.client.cli.commands.up" +AUTH_MODULE = "litellm.proxy.client.cli.commands.auth" def _patch_paths(monkeypatch, tmp_path): @@ -92,13 +94,13 @@ class TestLoadJsonOrEmpty: def test_raises_clean_error_on_invalid_json(self, tmp_path): path = tmp_path / "settings.json" path.write_text("not json at all {{{") - with pytest.raises(UpError, match="invalid JSON"): + with pytest.raises(ClaudeSettingsError, match="invalid JSON"): load_json_or_empty(path) def test_raises_clean_error_on_non_object_root(self, tmp_path): path = tmp_path / "settings.json" path.write_text(json.dumps([1, 2, 3])) - with pytest.raises(UpError, match="invalid JSON"): + with pytest.raises(ClaudeSettingsError, match="invalid JSON"): load_json_or_empty(path) @@ -201,16 +203,16 @@ class TestResolveApiKeyHelper: def test_returns_helper_command_bound_to_the_selected_proxy(self, monkeypatch): monkeypatch.setattr(shutil, "which", lambda name: "/usr/local/bin/lite") helper = resolve_api_key_helper("http://localhost:4000") - assert helper == "/usr/local/bin/lite auth print-token --base-url http://localhost:4000" + assert helper == "/usr/local/bin/lite --base-url http://localhost:4000 auth print-token" def test_quotes_a_base_url_containing_shell_metacharacters(self, monkeypatch): monkeypatch.setattr(shutil, "which", lambda name: "/usr/local/bin/lite") helper = resolve_api_key_helper("http://example.com/path; rm -rf /") - assert helper == "/usr/local/bin/lite auth print-token --base-url 'http://example.com/path; rm -rf /'" + assert helper == "/usr/local/bin/lite --base-url 'http://example.com/path; rm -rf /' auth print-token" def test_raises_when_lite_not_on_path(self, monkeypatch): monkeypatch.setattr(shutil, "which", lambda name: None) - with pytest.raises(UpError, match="Could not find `lite`"): + with pytest.raises(ClaudeSettingsError, match="Could not find `lite`"): resolve_api_key_helper("http://localhost:4000") @@ -396,3 +398,50 @@ class TestDownCommand: assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) assert "invalid or unexpected JSON" in result.output + + +class TestUpCanInvokeTheRealLoginCommand: + """`lite up` calls ctx.invoke(login) on the real command object. + + Every other test in this file monkeypatches `up_module.login` with a fake, so + none of them would notice a login parameter that ctx.invoke cannot supply. + """ + + def test_ctx_invoke_supplies_every_login_parameter(self): + from litellm.proxy.client.cli.commands.auth import login as real_login + + reached = [] + + @click.command() + @click.pass_context + def driver(ctx): + ctx.obj = {"base_url": "http://127.0.0.1:9"} + ctx.invoke(real_login) + + with patch( + f"{AUTH_MODULE}._start_cli_sso_flow", + side_effect=lambda base_url: reached.append(base_url) or RuntimeError("stop"), + ): + result = CliRunner().invoke(driver, [], standalone_mode=False) + + assert not isinstance(result.exception, TypeError), result.exception + assert reached == ["http://127.0.0.1:9"] + + def test_ctx_invoke_leaves_claude_settings_alone(self, tmp_path): + from litellm.proxy.client.cli.commands.auth import login as real_login + + settings_path = tmp_path / "settings.json" + + @click.command() + @click.pass_context + def driver(ctx): + ctx.obj = {"base_url": "http://127.0.0.1:9"} + ctx.invoke(real_login) + + with ( + patch(f"{AUTH_MODULE}.CLAUDE_SETTINGS_PATH", settings_path), + patch(f"{AUTH_MODULE}._start_cli_sso_flow", side_effect=RuntimeError("stop")), + ): + CliRunner().invoke(driver, [], standalone_mode=False) + + assert not settings_path.exists() diff --git a/tests/test_litellm/proxy/common_utils/test_key_rotation_integration.py b/tests/test_litellm/proxy/common_utils/test_key_rotation_integration.py index bdea37f1358..3bc62d549b0 100644 --- a/tests/test_litellm/proxy/common_utils/test_key_rotation_integration.py +++ b/tests/test_litellm/proxy/common_utils/test_key_rotation_integration.py @@ -25,6 +25,11 @@ from litellm.proxy._types import ( from litellm.proxy.common_utils.key_rotation_manager import KeyRotationManager +@pytest.fixture +def disable_audit_logging_for_mocked_key(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr("litellm.store_audit_logs", False) + + class TestKeyRotationManagerPassesKeyAlias: """ Regression tests to ensure KeyRotationManager passes key_alias @@ -155,7 +160,10 @@ class TestKeyRotationSecretNamingStability: """ @pytest.mark.asyncio - async def test_rotation_hook_uses_initial_secret_name_fallback(self): + async def test_rotation_hook_uses_initial_secret_name_fallback( + self, + disable_audit_logging_for_mocked_key, + ): """ GIVEN: A key WITHOUT an alias (has an initial_secret_name based on token ID) WHEN: The key is rotated @@ -206,7 +214,10 @@ class TestKeyRotationSecretNamingStability: ), f"Secret name drift! Expected {initial_secret_name}, got {call_kwargs['new_secret_name']}. This causes secret sprawl." @pytest.mark.asyncio - async def test_rotation_hook_pre_rotation_alias_consistency(self): + async def test_rotation_hook_pre_rotation_alias_consistency( + self, + disable_audit_logging_for_mocked_key, + ): """ GIVEN: A key WITH an alias WHEN: The key is rotated diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 6cf41497404..f3c2ca65d02 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -2533,3 +2533,187 @@ async def test_daily_transaction_internal_call_keeps_spend_but_not_request_count assert internal["autorouter_savings_spend"] == 0.0 assert user_sent["api_requests"] == 1 assert user_sent["successful_requests"] == 1 + + +def _deadlock_error(): + from prisma.errors import RawQueryError + + return RawQueryError( + data={"user_facing_error": {"error_code": "P2034", "meta": {"table": "LiteLLM_VerificationToken"}}} + ) + + +def _empty_spend_transactions(**overrides): + base = { + "user_list_transactions": {}, + "end_user_list_transactions": {}, + "key_list_transactions": {}, + "team_list_transactions": {}, + "team_member_list_transactions": {}, + "org_list_transactions": {}, + "tag_list_transactions": {}, + "agent_list_transactions": {}, + } + return {**base, **overrides} + + +def _good_tx(mock_batcher): + tx = AsyncMock() + tx.__aenter__ = AsyncMock(return_value=tx) + tx.__aexit__ = AsyncMock(return_value=False) + tx.batch_ = MagicMock( + return_value=AsyncMock( + __aenter__=AsyncMock(return_value=mock_batcher), + __aexit__=AsyncMock(return_value=False), + ) + ) + return tx + + +def _failing_tx(error): + tx = MagicMock() + tx.__aenter__ = AsyncMock(side_effect=error) + tx.__aexit__ = AsyncMock(return_value=False) + return tx + + +@pytest.mark.asyncio +async def test_commit_spend_updates_retries_deadlock_then_commits(monkeypatch): + """Regression: a deadlock on the key-spend UPDATE is retried and commits the increment exactly once.""" + slept = [] + monkeypatch.setattr( + "litellm.proxy.db.db_spend_update_writer.asyncio.sleep", + AsyncMock(side_effect=lambda s: slept.append(s)), + ) + + mock_batcher = MagicMock() + mock_prisma_client = MagicMock() + mock_prisma_client.db.tx = MagicMock(side_effect=[_failing_tx(_deadlock_error()), _good_tx(mock_batcher)]) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + + await DBSpendUpdateWriter()._commit_spend_updates_to_db( + prisma_client=mock_prisma_client, + n_retry_times=3, + proxy_logging_obj=proxy_logging, + db_spend_update_transactions=_empty_spend_transactions(key_list_transactions={"sk-abc": 0.5}), + ) + + assert mock_prisma_client.db.tx.call_count == 2 + mock_batcher.litellm_verificationtoken.update_many.assert_called_once() + call_kwargs = mock_batcher.litellm_verificationtoken.update_many.call_args[1] + assert call_kwargs["where"] == {"token": "sk-abc"} + assert call_kwargs["data"]["spend"] == {"increment": 0.5} + assert len(slept) == 1 + proxy_logging.failure_handler.assert_not_called() + + +@pytest.mark.asyncio +async def test_commit_spend_updates_raises_after_exhausting_deadlock_retries(monkeypatch): + """A deadlock that never clears must surface after the retry budget is spent, not loop or swallow.""" + monkeypatch.setattr("litellm.proxy.db.db_spend_update_writer.asyncio.sleep", AsyncMock(return_value=None)) + + mock_prisma_client = MagicMock() + mock_prisma_client.db.tx = MagicMock(side_effect=lambda *a, **k: _failing_tx(_deadlock_error())) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + + from prisma.errors import RawQueryError + + with pytest.raises(RawQueryError): + await DBSpendUpdateWriter()._commit_spend_updates_to_db( + prisma_client=mock_prisma_client, + n_retry_times=2, + proxy_logging_obj=proxy_logging, + db_spend_update_transactions=_empty_spend_transactions(key_list_transactions={"sk-abc": 0.5}), + ) + + assert mock_prisma_client.db.tx.call_count == 3 + + +@pytest.mark.asyncio +async def test_commit_spend_updates_does_not_retry_non_deadlock_data_error(monkeypatch): + """A non-retryable data-layer error raises on the first attempt, never retried against the increment.""" + monkeypatch.setattr("litellm.proxy.db.db_spend_update_writer.asyncio.sleep", AsyncMock(return_value=None)) + + from prisma.errors import UniqueViolationError + + non_deadlock = UniqueViolationError( + data={"user_facing_error": {"error_code": "P2002", "meta": {"table": "LiteLLM_VerificationToken"}}} + ) + mock_prisma_client = MagicMock() + mock_prisma_client.db.tx = MagicMock(side_effect=lambda *a, **k: _failing_tx(non_deadlock)) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + + with pytest.raises(UniqueViolationError): + await DBSpendUpdateWriter()._commit_spend_updates_to_db( + prisma_client=mock_prisma_client, + n_retry_times=3, + proxy_logging_obj=proxy_logging, + db_spend_update_transactions=_empty_spend_transactions(key_list_transactions={"sk-abc": 0.5}), + ) + + mock_prisma_client.db.tx.assert_called_once() + + +@pytest.mark.asyncio +async def test_update_daily_spend_retries_deadlock(monkeypatch): + """The daily-spend upsert path retries a deadlock on the bulk upsert and then drains successfully.""" + mock_prisma_client = MagicMock() + mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[_deadlock_error(), None]) + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + + monkeypatch.setattr("litellm.proxy.db.db_spend_update_writer.asyncio.sleep", AsyncMock(return_value=None)) + daily_spend_transactions = {"k1": _daily_txn()} + await DBSpendUpdateWriter._update_daily_spend( + n_retry_times=3, + prisma_client=mock_prisma_client, + proxy_logging_obj=proxy_logging, + daily_spend_transactions=daily_spend_transactions, + entity_type="user", + entity_id_field="user_id", + ) + + assert mock_prisma_client.db.execute_raw.call_count == 2 + assert daily_spend_transactions == {} + proxy_logging.failure_handler.assert_not_called() + + +@pytest.mark.parametrize( + "transactions_key, sample_key", + [ + ("user_list_transactions", "user-1"), + ("team_list_transactions", "team-1"), + ("team_member_list_transactions", "team_id::team-1::user_id::user-1"), + ("org_list_transactions", "org-1"), + ("tag_list_transactions", "tag-1"), + ("agent_list_transactions", "agent-1"), + ], +) +@pytest.mark.asyncio +async def test_commit_spend_updates_retries_deadlock_on_every_entity_path(monkeypatch, transactions_key, sample_key): + """Every per-entity spend path, not just keys, retries a deadlock instead of dropping the increment.""" + monkeypatch.setattr("litellm.proxy.db.db_spend_update_writer.asyncio.sleep", AsyncMock(return_value=None)) + + mock_batcher = MagicMock() + mock_prisma_client = MagicMock() + mock_prisma_client.db.tx = MagicMock(side_effect=[_failing_tx(_deadlock_error()), _good_tx(mock_batcher)]) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + proxy_logging.call_details = {} + + await DBSpendUpdateWriter()._commit_spend_updates_to_db( + prisma_client=mock_prisma_client, + n_retry_times=3, + proxy_logging_obj=proxy_logging, + db_spend_update_transactions=_empty_spend_transactions(**{transactions_key: {sample_key: 0.5}}), + ) + + assert mock_prisma_client.db.tx.call_count == 2 + proxy_logging.failure_handler.assert_not_called() diff --git a/tests/test_litellm/proxy/db/test_exception_handler.py b/tests/test_litellm/proxy/db/test_exception_handler.py index a188289bfce..474e571e592 100644 --- a/tests/test_litellm/proxy/db/test_exception_handler.py +++ b/tests/test_litellm/proxy/db/test_exception_handler.py @@ -549,3 +549,36 @@ def test_handle_db_exception_surfaces_a_permanent_fault_even_when_degraded_mode_ with pytest.raises(BinaryNotFoundError): PrismaDBExceptionHandler.handle_db_exception(BinaryNotFoundError("query engine binary not found")) + + +@pytest.mark.parametrize( + "error", + [ + RawQueryError(data={"user_facing_error": {"error_code": "P2034", "meta": {"table": "t"}}}), + PrismaError("Transaction failed due to a write conflict or a deadlock. Please retry your transaction"), + RawQueryError(data={"user_facing_error": {"message": "deadlock detected", "meta": {"table": "t"}}}), + RawQueryError( + data={"user_facing_error": {"message": "ERROR: 40P01: deadlock detected", "meta": {"table": "t"}}} + ), + ], +) +def test_is_deadlock_error_matches_postgres_deadlock(error): + """A Postgres deadlock surfaced through prisma (P2034 or 40P01 / "deadlock detected" text) is recognized.""" + assert PrismaDBExceptionHandler.is_deadlock_error(error) is True + + +@pytest.mark.parametrize( + "error", + [ + UniqueViolationError(data={"user_facing_error": {"error_code": "P2002", "meta": {"table": "t"}}}), + RecordNotFoundError(data={"user_facing_error": {"meta": {"table": "t"}}}), + PrismaError("validation failed on query"), + PrismaError("can't reach database server"), + httpx.ConnectError("connection refused"), + RuntimeError("deadlock detected"), + ValueError("40P01"), + ], +) +def test_is_deadlock_error_excludes_non_deadlocks(error): + """Non-deadlock prisma errors, connectivity failures, and non-prisma exceptions are not treated as deadlocks.""" + assert PrismaDBExceptionHandler.is_deadlock_error(error) is False diff --git a/tests/test_litellm/proxy/db/test_proxy_worker_heartbeat.py b/tests/test_litellm/proxy/db/test_proxy_worker_heartbeat.py new file mode 100644 index 00000000000..33ae6190411 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_proxy_worker_heartbeat.py @@ -0,0 +1,94 @@ +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.proxy.db.proxy_worker_heartbeat import ( + BEAT_SQL, + COUNT_SQL, + DEREGISTER_SQL, + PROXY_WORKER_LIVENESS_WINDOW_SECONDS, + PRUNE_SQL, + STALE_ROW_RETENTION_SECONDS, + ProxyWorkerHeartbeat, + count_live_proxy_workers, +) +from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + +def _prisma(): + prisma = MagicMock() + prisma.db.execute_raw = AsyncMock() + prisma.db.query_raw = AsyncMock() + return prisma + + +@pytest.mark.asyncio +async def test_beat_upserts_own_row_then_prunes_stale_rows(): + prisma = _prisma() + heartbeat = ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="worker-1") + await heartbeat.beat() + calls = prisma.db.execute_raw.call_args_list + assert calls[0].args == (BEAT_SQL, "worker-1", heartbeat.hostname) + assert calls[1].args == (PRUNE_SQL, STALE_ROW_RETENTION_SECONDS) + + +@pytest.mark.asyncio +async def test_beat_survives_a_database_error(): + prisma = _prisma() + prisma.db.execute_raw = AsyncMock(side_effect=RuntimeError("db down")) + await ProxyWorkerHeartbeat(prisma_client=prisma).beat() + + +def test_each_worker_process_gets_its_own_id(): + prisma = _prisma() + first = ProxyWorkerHeartbeat(prisma_client=prisma) + second = ProxyWorkerHeartbeat(prisma_client=prisma) + assert first.worker_id != second.worker_id + + +@pytest.mark.asyncio +async def test_deregister_deletes_only_its_own_row(): + prisma = _prisma() + await ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="worker-1").deregister() + assert prisma.db.execute_raw.call_args.args == (DEREGISTER_SQL, "worker-1") + + +@pytest.mark.asyncio +async def test_deregister_survives_a_database_error(): + prisma = _prisma() + prisma.db.execute_raw = AsyncMock(side_effect=RuntimeError("db down")) + await ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="worker-1").deregister() + + +@pytest.mark.asyncio +async def test_count_reads_workers_within_the_liveness_window(): + prisma = _prisma() + prisma.db.query_raw.return_value = [{"live_workers": 3}] + assert await count_live_proxy_workers(prisma) == 3 + assert prisma.db.query_raw.call_args.args == (COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS) + + +@pytest.mark.asyncio +async def test_count_reads_from_the_primary_when_reads_route_to_a_replica(): + writer = MagicMock() + writer.query_raw = AsyncMock(return_value=[{"live_workers": 2}]) + reader = MagicMock() + reader.query_raw = AsyncMock(return_value=[{"live_workers": 1}]) + prisma = MagicMock() + prisma.db = RoutingPrismaWrapper(writer=writer, reader=reader) + assert await count_live_proxy_workers(prisma) == 2 + reader.query_raw.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_count_returns_unknown_when_the_query_fails(): + prisma = _prisma() + prisma.db.query_raw.side_effect = RuntimeError("db down") + assert await count_live_proxy_workers(prisma) is None + + +@pytest.mark.asyncio +async def test_count_returns_unknown_for_a_malformed_row(): + prisma = _prisma() + prisma.db.query_raw.return_value = [{"unexpected": "shape"}] + assert await count_live_proxy_workers(prisma) is None diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index e2705bd5fec..831f659051c 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -2467,61 +2467,140 @@ class TestNoRedisWarning: def _router(redis_cache): return SimpleNamespace(cache=SimpleNamespace(redis_cache=redis_cache)) - def test_warns_when_no_redis_is_configured(self, monkeypatch): + @staticmethod + def _prisma_with_workers(live_workers=None, error=None): + prisma = MagicMock() + if error is not None: + prisma.db.query_raw = AsyncMock(side_effect=error) + else: + prisma.db.query_raw = AsyncMock(return_value=[{"live_workers": live_workers}]) + return prisma + + @pytest.mark.asyncio + async def test_warns_when_no_redis_and_no_db_to_count_workers(self, monkeypatch): monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) with ( patch("litellm.proxy.proxy_server.redis_usage_cache", None), patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + patch("litellm.proxy.proxy_server.prisma_client", None), ): - assert _show_no_redis_warning() is True + assert await _show_no_redis_warning() is True - def test_warns_when_there_is_no_router_at_all(self, monkeypatch): + @pytest.mark.asyncio + async def test_warns_when_there_is_no_router_at_all(self, monkeypatch): monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) with ( patch("litellm.proxy.proxy_server.redis_usage_cache", None), patch("litellm.proxy.proxy_server.llm_router", None), + patch("litellm.proxy.proxy_server.prisma_client", None), ): - assert _show_no_redis_warning() is True + assert await _show_no_redis_warning() is True - def test_stays_quiet_when_a_coordination_redis_is_configured(self, monkeypatch): + @pytest.mark.asyncio + async def test_stays_quiet_for_a_confirmed_single_worker(self, monkeypatch): + """One live worker needs no cross-worker coordination, so no env var is needed.""" monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) + with ( + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + patch("litellm.proxy.proxy_server.prisma_client", self._prisma_with_workers(1)), + ): + assert await _show_no_redis_warning() is False + + @pytest.mark.asyncio + @pytest.mark.parametrize("live_workers", [2, 5]) + async def test_warns_when_multiple_workers_share_the_db(self, monkeypatch, live_workers): + monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) + with ( + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + patch("litellm.proxy.proxy_server.prisma_client", self._prisma_with_workers(live_workers)), + ): + assert await _show_no_redis_warning() is True + + @pytest.mark.asyncio + async def test_warns_when_the_worker_census_is_empty(self, monkeypatch): + """Zero rows means the census cannot CONFIRM a single worker, so warn.""" + monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) + with ( + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + patch("litellm.proxy.proxy_server.prisma_client", self._prisma_with_workers(0)), + ): + assert await _show_no_redis_warning() is True + + @pytest.mark.asyncio + async def test_warns_when_the_worker_census_query_fails(self, monkeypatch): + monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) + with ( + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + patch( + "litellm.proxy.proxy_server.prisma_client", + self._prisma_with_workers(error=RuntimeError("db down")), + ), + ): + assert await _show_no_redis_warning() is True + + @pytest.mark.asyncio + async def test_stays_quiet_when_a_coordination_redis_is_configured(self, monkeypatch): + monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) + prisma = self._prisma_with_workers(5) with ( patch("litellm.proxy.proxy_server.redis_usage_cache", MagicMock()), patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + patch("litellm.proxy.proxy_server.prisma_client", prisma), ): - assert _show_no_redis_warning() is False + assert await _show_no_redis_warning() is False + prisma.db.query_raw.assert_not_called() - def test_stays_quiet_when_only_the_router_has_redis(self, monkeypatch): + @pytest.mark.asyncio + async def test_stays_quiet_when_only_the_router_has_redis(self, monkeypatch): """router_settings.redis_host alone backs cooldowns and usage-based routing.""" monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) with ( patch("litellm.proxy.proxy_server.redis_usage_cache", None), patch("litellm.proxy.proxy_server.llm_router", self._router(MagicMock())), + patch("litellm.proxy.proxy_server.prisma_client", self._prisma_with_workers(5)), ): - assert _show_no_redis_warning() is False + assert await _show_no_redis_warning() is False + @pytest.mark.asyncio @pytest.mark.parametrize("value", ["true", "True"]) - def test_env_var_suppresses_the_warning(self, monkeypatch, value): + async def test_env_var_suppresses_the_warning_despite_multiple_workers(self, monkeypatch, value): monkeypatch.setenv("LITELLM_DISABLE_NO_REDIS_WARNING", value) with ( patch("litellm.proxy.proxy_server.redis_usage_cache", None), patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + patch("litellm.proxy.proxy_server.prisma_client", self._prisma_with_workers(5)), ): - assert _show_no_redis_warning() is False + assert await _show_no_redis_warning() is False - def test_env_var_set_false_keeps_the_warning(self, monkeypatch): + @pytest.mark.asyncio + async def test_env_var_set_false_keeps_the_warning_for_multiple_workers(self, monkeypatch): monkeypatch.setenv("LITELLM_DISABLE_NO_REDIS_WARNING", "false") with ( patch("litellm.proxy.proxy_server.redis_usage_cache", None), patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + patch("litellm.proxy.proxy_server.prisma_client", self._prisma_with_workers(2)), ): - assert _show_no_redis_warning() is True + assert await _show_no_redis_warning() is True + + @pytest.mark.asyncio + async def test_env_var_set_false_does_not_force_the_warning_for_a_single_worker(self, monkeypatch): + monkeypatch.setenv("LITELLM_DISABLE_NO_REDIS_WARNING", "false") + with ( + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + patch("litellm.proxy.proxy_server.prisma_client", self._prisma_with_workers(1)), + ): + assert await _show_no_redis_warning() is False @pytest.mark.asyncio @pytest.mark.parametrize("has_prisma_client", [True, False]) async def test_readiness_details_carries_the_flag(self, monkeypatch, has_prisma_client): monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) - prisma_client = MagicMock() if has_prisma_client else None + prisma_client = self._prisma_with_workers(2) if has_prisma_client else None with ( patch("litellm.proxy.proxy_server.prisma_client", prisma_client), patch("litellm.proxy.proxy_server.redis_usage_cache", None), diff --git a/tests/test_litellm/proxy/hooks/test_batch_file_validation.py b/tests/test_litellm/proxy/hooks/test_batch_file_validation.py index 8698ee8ba12..4ef94f0965b 100644 --- a/tests/test_litellm/proxy/hooks/test_batch_file_validation.py +++ b/tests/test_litellm/proxy/hooks/test_batch_file_validation.py @@ -1739,18 +1739,18 @@ def _make_batch_input_bytes(n_rows: int, padding: int = 200) -> bytes: return ("\n".join(rows)).encode("utf-8") -def test_iter_batch_input_entries_matches_dict_list(): +def test_iter_batch_output_entries_matches_dict_list(): from litellm.batches.batch_utils import ( _get_file_content_as_dictionary, - _iter_batch_input_entries, + _iter_batch_output_entries, ) raw = _make_batch_input_bytes(50) - streamed = list(_iter_batch_input_entries(raw)) + streamed = list(_iter_batch_output_entries(raw)) assert streamed == _get_file_content_as_dictionary(raw) assert streamed[0]["custom_id"] == "request-0" # tolerant of blank lines and a missing trailing newline - assert list(_iter_batch_input_entries(raw + b"\n\n")) == streamed + assert list(_iter_batch_output_entries(raw + b"\n\n")) == streamed def test_streaming_count_peak_below_dict_list(): @@ -1759,7 +1759,7 @@ def test_streaming_count_peak_below_dict_list(): from litellm.batches.batch_utils import ( _get_file_content_as_dictionary, - _iter_batch_input_entries, + _iter_batch_output_entries, ) raw = _make_batch_input_bytes(8000) @@ -1777,7 +1777,7 @@ def test_streaming_count_peak_below_dict_list(): def _stream(): count = 0 models: set = set() - for entry in _iter_batch_input_entries(raw): + for entry in _iter_batch_output_entries(raw): count += 1 model = (entry.get("body") or {}).get("model") if model: diff --git a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py index 787e5776897..fa7320b2bc6 100644 --- a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py +++ b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py @@ -4,6 +4,7 @@ Tests for KeyManagementEventHooks. Validates that email and secret manager operations are independent and non-blocking. """ +import asyncio import os import sys from unittest.mock import AsyncMock, MagicMock, patch @@ -155,6 +156,44 @@ class TestKeyManagementEventHooksIndependentOperations: assert email_called["called"] is True +@pytest.mark.parametrize( + ("premium_user", "expected_audit_log_calls"), + ((True, 1), (False, 0)), +) +@pytest.mark.asyncio +async def test_key_generated_audit_log_uses_license_default( + monkeypatch: pytest.MonkeyPatch, + premium_user: bool, + expected_audit_log_calls: int, +): + from litellm.proxy._types import GenerateKeyRequest, GenerateKeyResponse, UserAPIKeyAuth + + monkeypatch.setattr("litellm.store_audit_logs", None) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", premium_user) + monkeypatch.delenv("LITELLM_STORE_AUDIT_LOGS", raising=False) + + response = GenerateKeyResponse(key="sk-test-key", token_id="token-123") + with ( + patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new_callable=AsyncMock, + ) as mock_create_audit_log, + patch.object( + KeyManagementEventHooks, + "_store_virtual_key_in_secret_manager", + new_callable=AsyncMock, + ), + ): + await KeyManagementEventHooks.async_key_generated_hook( + data=GenerateKeyRequest(), + response=response, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-admin-key", user_id="admin"), + ) + await asyncio.sleep(0.01) + + assert mock_create_audit_log.await_count == expected_audit_log_calls + + class TestRotateVirtualKeyInSecretManager: """Tests for _rotate_virtual_key_in_secret_manager with team_id support.""" diff --git a/tests/test_litellm/proxy/hooks/test_send_invite_email.py b/tests/test_litellm/proxy/hooks/test_send_invite_email.py index c916af5c128..83fb1f5faba 100644 --- a/tests/test_litellm/proxy/hooks/test_send_invite_email.py +++ b/tests/test_litellm/proxy/hooks/test_send_invite_email.py @@ -212,8 +212,9 @@ async def test_v1_key_generation_sends_email_when_send_invite_email_true(): mock_proxy_logging_obj = MagicMock() mock_proxy_logging_obj.slack_alerting_instance = mock_slack_alerting - with patch.object( - KeyManagementEventHooks, "_send_key_created_email", mock_send_key_created_email + with ( + patch("litellm.store_audit_logs", False), + patch.object(KeyManagementEventHooks, "_send_key_created_email", mock_send_key_created_email), ): with patch( "litellm.logging_callback_manager.get_custom_loggers_for_type", @@ -257,8 +258,9 @@ async def test_v1_key_generation_no_email_when_send_invite_email_false(): mock_proxy_logging_obj = MagicMock() mock_proxy_logging_obj.slack_alerting_instance = mock_slack_alerting - with patch.object( - KeyManagementEventHooks, "_send_key_created_email", mock_send_key_created_email + with ( + patch("litellm.store_audit_logs", False), + patch.object(KeyManagementEventHooks, "_send_key_created_email", mock_send_key_created_email), ): with patch( "litellm.logging_callback_manager.get_custom_loggers_for_type", diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 77149457e82..3fd023552f5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -4,6 +4,7 @@ Unit tests for auto router management endpoints import os import sys +from pathlib import Path import pytest from fastapi import HTTPException @@ -325,9 +326,7 @@ class TestAutoRouterBenchmarks: from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals totals = _benchmark_totals(self.ROW) - bucket_hits = ( - totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits - ) + bucket_hits = totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits assert bucket_hits == 27 assert totals.cache.hit_rate_pct == pytest.approx(100.0 * 28 / 38, abs=0.1) @@ -490,7 +489,7 @@ from litellm.proxy.management_endpoints.auto_router_endpoints import ( start_shadow_eval, stop_shadow_eval_job, ) -from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalJobResponse, StartShadowEvalRequest +from litellm.types.management_endpoints.auto_router_endpoints import StartShadowEvalRequest VIEWER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, api_key="sk-view", user_id="viewer") NON_ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-user", user_id="user") @@ -507,19 +506,23 @@ def _shadow_router() -> MagicMock: return router -def _job_record(**overrides: object) -> MagicMock: - """Spec'd like a real prisma row: only the table's columns exist as attributes, so - from_attributes validation falls back to model defaults for everything else.""" +def _leg_record(**overrides: object) -> MagicMock: + """Spec'd like a real prisma row: only the table's columns exist as attributes. One + row is one key's leg of a job; legs sharing group_id are one job.""" defaults = { - "id": "job-1", + "id": "leg-1", + "group_id": "job-1", "api_key_id": "key-hash", "router_name": "my-router", + "direction": "forward", + "baseline_model": None, "judge_model": "anthropic/claude-sonnet-5", "shadow_percentage": 10.0, "max_turns": 200, "created_at": datetime(2026, 8, 11, tzinfo=timezone.utc), "ends_at": datetime.now(timezone.utc) + timedelta(days=7), "stopped_at": None, + "stopped_by": None, } fields = {**defaults, **overrides} record = MagicMock(spec=list(fields)) @@ -538,23 +541,90 @@ def _key_record( return record -def _shadow_prisma(active_job=None, agg_rows=None) -> MagicMock: +def _shadow_prisma(legs=(), agg_rows=None, by_leg_rows=None, known_keys=("key-hash", "key-hash-2")) -> MagicMock: + """The job-table fake honours the filters it is handed, so a read that forgets + stopped_at sees rows the partial index would have released, one that forgets + direction sees the opposite-direction legs a key may hold at the same time, and a + group read that matched on a leg id would come back empty.""" prisma = MagicMock() - prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=_key_record()) - prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[_key_record()]) - prisma.db.execute_raw = AsyncMock(return_value=0) - prisma.db.litellm_shadowevaljob.find_first = AsyncMock(return_value=active_job) - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=None) - prisma.db.litellm_shadowevaljob.find_many = AsyncMock(return_value=[]) - prisma.db.litellm_shadowevaljob.create = AsyncMock(return_value=_job_record()) - prisma.db.litellm_shadowevaljob.update = AsyncMock( - return_value=_job_record(stopped_at=datetime.now(timezone.utc)) - ) + prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[_key_record(token) for token in known_keys]) + async def execute_raw(sql: str, *params: object): + if "SET stopped_by" in sql: + group = [row for row in stored if row.group_id == params[0]] + counts = {row["job_id"]: row["attempt_count"] for row in prisma.attempt_rows} + sampling = any(row.stopped_at is None and counts.get(row.id, 0) < row.max_turns for row in group) + window_open = bool(group) and group[0].ends_at > datetime.now(timezone.utc) + claimable = [row for row in group if row.stopped_by is None] + if not (claimable and sampling and window_open): + return 0 + for row in claimable: + row.stopped_by = params[1] + if row.stopped_at is None: + row.stopped_at = datetime.fromisoformat(str(params[2])).replace(tzinfo=timezone.utc) + return len(claimable) + return 0 + + prisma.db.execute_raw = AsyncMock(side_effect=execute_raw) + stored = legs if isinstance(legs, list) else list(legs) + + async def find_many_legs(where=None, **_: object): + current = list(stored) + w = dict(where or {}) + if "api_key_id" in w: + wanted = w["api_key_id"]["in"] if isinstance(w["api_key_id"], dict) else [w["api_key_id"]] + current = [row for row in current if row.api_key_id in wanted] + if "direction" in w: + current = [row for row in current if row.direction == w["direction"]] + if "stopped_at" in w: + current = [row for row in current if row.stopped_at is w["stopped_at"]] + if "group_id" in w: + wanted = w["group_id"]["in"] if isinstance(w["group_id"], dict) else [w["group_id"]] + current = [row for row in current if row.group_id in wanted] + return current + + def newest_groups(rows, limit): + latest: dict = {} + for row in rows: + if row.group_id not in latest or row.created_at > latest[row.group_id]: + latest[row.group_id] = row.created_at + ordered = sorted(latest, key=lambda group_id: latest[group_id], reverse=True) + return ordered[: int(limit)] + + def leg_dict(row): + fields = ( + "id", + "group_id", + "api_key_id", + "router_name", + "direction", + "baseline_model", + "judge_model", + "shadow_percentage", + "max_turns", + "created_at", + "ends_at", + "stopped_at", + "stopped_by", + ) + return {field: getattr(row, field) for field in fields} + + prisma.db.litellm_shadowevaljob.find_many = AsyncMock(side_effect=find_many_legs) + prisma.db.litellm_shadowevaljob.create_many = AsyncMock(return_value=1) + prisma.db.litellm_shadowevaljob.update_many = AsyncMock(return_value=1) prisma.db.litellm_shadowevalattempt.find_first = AsyncMock(return_value=None) + prisma.attempt_rows = [] async def query_raw(sql: str, *params: object): + if "AS attempt_count" in sql: + return prisma.attempt_rows + if "GROUP BY group_id" in sql: + scoped = [row for row in stored if "api_key_id = $2" not in sql or row.api_key_id == params[1]] + keep = set(newest_groups(scoped, params[0])) + return [leg_dict(row) for row in stored if row.group_id in keep] if "FILTER (WHERE outcome != 'error')::int AS judged_count" in sql: return [{"judged_count": 10, "error_count": 2, "judge_spend": 0.031}] + if "SELECT job_id AS grp" in sql: + return by_leg_rows if by_leg_rows is not None else [] return agg_rows if agg_rows is not None else [] prisma.db.query_raw = AsyncMock(side_effect=query_raw) @@ -563,7 +633,7 @@ def _shadow_prisma(active_job=None, agg_rows=None) -> MagicMock: def _start_request(**overrides: object) -> StartShadowEvalRequest: payload = { - "api_key_id": "key-hash", + "api_key_ids": ("key-hash",), "router_name": "my-router", "shadow_percentage": 10.0, "judge_model": "anthropic/claude-sonnet-5", @@ -575,44 +645,55 @@ def _start_request(**overrides: object) -> StartShadowEvalRequest: @pytest.mark.asyncio -async def test_start_shadow_eval_creates_job_and_frees_expired_or_exhausted_ones(monkeypatch: pytest.MonkeyPatch): - """Expiry and turn-budget exhaustion both end sampling on their own; either must - release the key's slot in the active-job index so a new eval can start.""" +async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeypatch: pytest.MonkeyPatch): + """N keys become N sibling rows sharing group_id and identical config, written by a + single create_many so a unique-index loser rolls back the whole claim, and expiry or + budget exhaustion frees every requested key's slot first.""" import litellm.proxy.proxy_server as proxy_server prisma = _shadow_prisma() monkeypatch.setattr(proxy_server, "prisma_client", prisma) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - response = await start_shadow_eval(_start_request(), ADMIN) + response = await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN) - assert response.status == "running" - assert response.max_turns == 200 - assert response.judged_count is None - sweep_sql, sweep_key = prisma.db.execute_raw.call_args.args + sweep_sql, sweep_keys = prisma.db.execute_raw.call_args.args assert "stopped_at IS NULL" in sweep_sql - assert "ends_at <= NOW()" in sweep_sql + assert "j.ends_at <= (NOW() AT TIME ZONE 'utc')" in sweep_sql + assert "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in sweep_sql assert ">= j.max_turns" in sweep_sql - assert sweep_key == "key-hash" - create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"] - assert create_data["api_key_id"] == "key-hash" - assert create_data["created_by"] == "admin" - assert "status" not in create_data + assert "j.api_key_id = ANY($1::text[])" in sweep_sql + assert sweep_keys == ["key-hash", "key-hash-2"] + prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert [row["api_key_id"] for row in rows] == ["key-hash", "key-hash-2"] + assert len({frozenset((k, v) for k, v in row.items() if k != "api_key_id") for row in rows}) == 1 + assert len({row["group_id"] for row in rows}) == 1 + assert all(row["max_turns"] == 200 and row["created_by"] == "admin" for row in rows) + assert all("status" not in row and "id" not in row for row in rows) + assert response.job_id == rows[0]["group_id"] + assert response.status == "running" + assert response.judged_count is None + assert [(key.api_key_id, key.max_turns, key.key_alias) for key in response.keys] == [ + ("key-hash", 200, "prod-alpha"), + ("key-hash-2", 200, "prod-alpha"), + ] @pytest.mark.asyncio @pytest.mark.parametrize( - "caller,request_overrides,active,expected_status", + "caller,request_overrides,claimed,expected_status", [ - (NON_ADMIN, {}, None, 403), - (VIEWER, {}, None, 403), - (ADMIN, {"router_name": "not-a-router"}, None, 400), - (ADMIN, {"judge_model": "not/a real model!"}, None, 400), - (ADMIN, {"judge_model": "my-router"}, None, 400), - (ADMIN, {}, "active", 409), - (ADMIN, {"direction": "reverse", "baseline_model": "my-router"}, None, 400), - (ADMIN, {"direction": "reverse", "baseline_model": "not/a real model!"}, None, 400), - (ADMIN, {"direction": "reverse", "baseline_model": "openai/gpt-4o", "router_name": "not-a-router"}, None, 400), + (NON_ADMIN, {}, (), 403), + (VIEWER, {}, (), 403), + (ADMIN, {"router_name": "not-a-router"}, (), 400), + (ADMIN, {"judge_model": "not/a real model!"}, (), 400), + (ADMIN, {"judge_model": "my-router"}, (), 400), + (ADMIN, {}, ("key-hash",), 409), + (ADMIN, {"api_key_ids": ("key-hash", "key-hash-2")}, ("key-hash-2",), 409), + (ADMIN, {"direction": "reverse", "baseline_model": "my-router"}, (), 400), + (ADMIN, {"direction": "reverse", "baseline_model": "not/a real model!"}, (), 400), + (ADMIN, {"direction": "reverse", "baseline_model": "openai/gpt-4o", "router_name": "not-a-router"}, (), 400), ], ids=[ "non-admin", @@ -621,23 +702,143 @@ async def test_start_shadow_eval_creates_job_and_frees_expired_or_exhausted_ones "unresolvable-judge", "router-as-judge", "already-active", + "one-of-several-keys-already-active", "router-as-baseline", "unresolvable-baseline", "reverse-still-needs-an-auto-router", ], ) async def test_start_shadow_eval_rejections( - monkeypatch: pytest.MonkeyPatch, caller, request_overrides, active, expected_status + monkeypatch: pytest.MonkeyPatch, caller, request_overrides, claimed, expected_status ): import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma(active_job=_job_record() if active else None) + prisma = _shadow_prisma(legs=[_leg_record(id=f"leg-{key}", group_id="job-7", api_key_id=key) for key in claimed]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) with pytest.raises(HTTPException) as exc: await start_shadow_eval(_start_request(**request_overrides), caller) assert exc.value.status_code == expected_status + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_start_shadow_eval_names_the_busy_key_and_its_job(monkeypatch: pytest.MonkeyPatch): + """A key busy elsewhere blocks the whole start rather than being silently dropped from + it, and the 409 names which key and which job so the caller can stop or drop it.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(id="leg-b", group_id="job-7", api_key_id="key-hash-2")]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN) + assert exc.value.status_code == 409 + assert "key-hash-2 (job job-7)" in exc.value.detail + + +@pytest.mark.asyncio +async def test_start_shadow_eval_reuses_a_key_whose_previous_job_already_stopped(monkeypatch: pytest.MonkeyPatch): + """The claim is held by unstopped legs only, matching the partial unique index. A read + that forgets that would strand every key that has ever finished a job.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(group_id="job-7", stopped_at=datetime.now(timezone.utc))]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + job = await start_shadow_eval(_start_request(), ADMIN) + + assert job.status == "running" + prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_start_shadow_eval_reverse_records_its_arms_and_holds_its_own_slot(monkeypatch: pytest.MonkeyPatch): + """The two directions ask opposite questions of the same key, so a forward job holding + the slot must not block a reverse one. The second reverse start still 409s.""" + import litellm.proxy.proxy_server as proxy_server + + legs = [_leg_record(group_id="job-fwd")] + prisma = _shadow_prisma(legs=legs) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + reverse = _start_request(direction="reverse", baseline_model="openai/gpt-4o") + response = await start_shadow_eval(reverse, ADMIN) + + assert (response.direction, response.baseline_model) == ("reverse", "openai/gpt-4o") + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert rows[0]["direction"] == "reverse" + assert rows[0]["baseline_model"] == "openai/gpt-4o" + + legs.append(_leg_record(id="leg-2", group_id="job-rev", direction="reverse")) + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(reverse, ADMIN) + assert exc.value.status_code == 409 + + +@pytest.mark.asyncio +async def test_start_shadow_eval_forward_leaves_the_baseline_column_empty(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + await start_shadow_eval(_start_request(), ADMIN) + + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert rows[0]["direction"] == "forward" + assert rows[0]["baseline_model"] is None + + +@pytest.mark.asyncio +async def test_start_shadow_eval_rejects_keys_this_proxy_does_not_know(monkeypatch: pytest.MonkeyPatch): + """A typo'd api_key_id would otherwise create a leg no traffic can ever match. Every + unknown key is named at once, so a caller passing several fixes them in one round.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(known_keys=("key-hash",)) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(api_key_ids=("key-hash", "typo-a", "typo-b")), ADMIN) + assert exc.value.status_code == 400 + assert "typo-a, typo-b" in exc.value.detail + assert "key-hash," not in exc.value.detail + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +def test_start_shadow_eval_request_dedupes_and_bounds_the_key_set(): + """A key named twice would collide with itself on the one-active-per-key index, a job + scoping no key samples nothing, and the key-count cap bounds every downstream read.""" + assert _start_request(api_key_ids=("a", "b", "a")).api_key_ids == ("a", "b") + assert len(_start_request(api_key_ids=tuple(f"k{i}" for i in range(100))).api_key_ids) == 100 + with pytest.raises(ValidationError): + _start_request(api_key_ids=()) + with pytest.raises(ValidationError): + _start_request(api_key_ids=tuple(f"k{i}" for i in range(101))) + + +@pytest.mark.asyncio +async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + from prisma.errors import UniqueViolationError + + prisma = _shadow_prisma() + prisma.db.litellm_shadowevaljob.create_many = AsyncMock( + side_effect=UniqueViolationError(MagicMock(message="unique constraint")) + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(), ADMIN) + assert exc.value.status_code == 409 @pytest.mark.parametrize( @@ -657,97 +858,25 @@ def test_start_request_pins_baseline_model_to_reverse(overrides): @pytest.mark.asyncio -async def test_start_shadow_eval_reverse_records_its_arms_and_holds_its_own_slot(monkeypatch: pytest.MonkeyPatch): - """The two directions ask opposite questions of the same key, so a forward job holding - the slot must not block a reverse one. The second reverse start still 409s.""" - import litellm.proxy.proxy_server as proxy_server - - prisma = _shadow_prisma() - active = {"forward": _job_record()} - prisma.db.litellm_shadowevaljob.find_first = AsyncMock( - side_effect=lambda where, **_: active.get(str(where.get("direction"))) - ) - prisma.db.litellm_shadowevaljob.create = AsyncMock( - return_value=_job_record(direction="reverse", baseline_model="openai/gpt-4o") - ) - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - reverse = _start_request(direction="reverse", baseline_model="openai/gpt-4o") - response = await start_shadow_eval(reverse, ADMIN) - - assert (response.direction, response.baseline_model) == ("reverse", "openai/gpt-4o") - create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"] - assert create_data["direction"] == "reverse" - assert create_data["baseline_model"] == "openai/gpt-4o" - - active["reverse"] = _job_record(id="job-2", direction="reverse") - with pytest.raises(HTTPException) as exc: - await start_shadow_eval(reverse, ADMIN) - assert exc.value.status_code == 409 - - -@pytest.mark.asyncio -async def test_start_shadow_eval_forward_leaves_the_baseline_column_empty(monkeypatch: pytest.MonkeyPatch): - import litellm.proxy.proxy_server as proxy_server - - prisma = _shadow_prisma() - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - await start_shadow_eval(_start_request(), ADMIN) - - create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"] - assert create_data["direction"] == "forward" - assert create_data["baseline_model"] is None - - -@pytest.mark.asyncio -async def test_start_shadow_eval_rejects_a_key_this_proxy_does_not_know(monkeypatch: pytest.MonkeyPatch): - """A typo'd api_key_id would otherwise create a job no traffic can ever match.""" - import litellm.proxy.proxy_server as proxy_server - - prisma = _shadow_prisma() - prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - with pytest.raises(HTTPException) as exc: - await start_shadow_eval(_start_request(), ADMIN) - assert exc.value.status_code == 400 - assert "not a key on this proxy" in exc.value.detail - - -@pytest.mark.asyncio -async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch): - import litellm.proxy.proxy_server as proxy_server - from prisma.errors import UniqueViolationError - - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.create = AsyncMock( - side_effect=UniqueViolationError(MagicMock(message="unique constraint")) - ) - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - with pytest.raises(HTTPException) as exc: - await start_shadow_eval(_start_request(), ADMIN) - assert exc.value.status_code == 409 - - -@pytest.mark.asyncio -async def test_get_shadow_eval_job_derives_counts_spend_and_stratified_results(monkeypatch: pytest.MonkeyPatch): +async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monkeypatch: pytest.MonkeyPatch): + """One read answers for every leg: totals and stratifications aggregate over the + group's leg ids, and the by-key slice maps each leg id back to its key hash.""" import litellm.proxy.proxy_server as proxy_server tier_rows = [ {"grp": "SIMPLE", "turn_count": 8, "real_wins": 2, "shadow_wins": 4, "ties": 2, "avg_confidence": 0.8}, {"grp": "REASONING", "turn_count": 2, "real_wins": 2, "shadow_wins": 0, "ties": 0, "avg_confidence": 0.9}, ] - prisma = _shadow_prisma(agg_rows=tier_rows) - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record()) - prisma.db.litellm_shadowevalattempt.find_first = AsyncMock( - return_value=MagicMock(error="judge call failed: boom") + leg_rows = [ + {"grp": "leg-1", "turn_count": 6, "real_wins": 1, "shadow_wins": 4, "ties": 1, "avg_confidence": 0.7}, + {"grp": "leg-2", "turn_count": 4, "real_wins": 3, "shadow_wins": 0, "ties": 1, "avg_confidence": 0.6}, + ] + prisma = _shadow_prisma( + legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=50)], + agg_rows=tier_rows, + by_leg_rows=leg_rows, ) + prisma.db.litellm_shadowevalattempt.find_first = AsyncMock(return_value=MagicMock(error="judge call failed: boom")) monkeypatch.setattr(proxy_server, "prisma_client", prisma) response = await get_shadow_eval_job("job-1", VIEWER) @@ -762,6 +891,13 @@ async def test_get_shadow_eval_job_derives_counts_spend_and_stratified_results(m assert response.results.by_tier[0].shadow_win_rate_pct == 50.0 assert response.results.overall_shadow_win_rate_pct == 40.0 assert response.results.overall_tie_rate_pct == 20.0 + assert [(s.group, s.turn_count) for s in response.results.by_key] == [("key-hash", 6), ("key-hash-2", 4)] + assert response.results.by_key[0].shadow_win_rate_pct == 66.7 + assert [(key.api_key_id, key.max_turns) for key in response.keys] == [("key-hash", 200), ("key-hash-2", 50)] + totals_args = [call.args for call in prisma.db.query_raw.await_args_list if "judged_count" in call.args[0]] + assert totals_args == [(totals_args[0][0], ["leg-1", "leg-2"])] + error_where = prisma.db.litellm_shadowevalattempt.find_first.call_args.kwargs["where"] + assert error_where == {"job_id": {"in": ["leg-1", "leg-2"]}, "outcome": "error"} @pytest.mark.asyncio @@ -780,79 +916,326 @@ async def test_get_shadow_eval_job_404s_and_gates_on_role(monkeypatch: pytest.Mo @pytest.mark.asyncio -async def test_list_shadow_eval_jobs_returns_derived_status_without_aggregates(monkeypatch: pytest.MonkeyPatch): +async def test_list_shadow_eval_jobs_collapses_legs_into_jobs_newest_first(monkeypatch: pytest.MonkeyPatch): + """A job over two keys is one list entry with both keys, not two entries, and a job + whose keys all stopped reads stopped while a half-stopped one still runs.""" import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.find_many = AsyncMock( - return_value=[ - _job_record(), - _job_record(id="job-2", ends_at=datetime.now(timezone.utc) - timedelta(days=1)), - _job_record(id="job-3", stopped_at=datetime.now(timezone.utc)), + stamp = datetime.now(timezone.utc) + prisma = _shadow_prisma( + legs=[ + _leg_record(created_at=datetime(2026, 8, 13, tzinfo=timezone.utc)), + _leg_record( + id="leg-2", + api_key_id="key-hash-2", + stopped_at=stamp, + created_at=datetime(2026, 8, 13, tzinfo=timezone.utc), + ), + _leg_record( + id="leg-3", + group_id="job-2", + stopped_at=stamp, + created_at=datetime(2026, 8, 12, tzinfo=timezone.utc), + ), + _leg_record( + id="leg-4", + group_id="job-3", + ends_at=datetime.now(timezone.utc) - timedelta(days=1), + created_at=datetime(2026, 8, 11, tzinfo=timezone.utc), + ), ] ) monkeypatch.setattr(proxy_server, "prisma_client", prisma) jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) - assert [job.status for job in jobs] == ["running", "completed", "stopped"] - swept = ShadowEvalJobResponse.model_validate( - _job_record( - id="job-4", - ends_at=datetime.now(timezone.utc) - timedelta(days=1), - stopped_at=datetime.now(timezone.utc), - ), - from_attributes=True, - ) - assert swept.status == "completed" + assert [(job.job_id, job.status) for job in jobs] == [ + ("job-1", "running"), + ("job-2", "stopped"), + ("job-3", "completed"), + ] + assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"] assert all(job.judged_count is None and job.results is None for job in jobs) - assert prisma.db.query_raw.await_count == 0 + legs_sql, legs_limit = prisma.db.query_raw.await_args_list[0].args + assert "GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int" in legs_sql + assert legs_limit == 50 + counts_sql, _ = prisma.db.query_raw.await_args_list[1].args + assert "AS attempt_count" in counts_sql + assert "j.stopped_at IS NULL OR a.created_at <= j.stopped_at" in counts_sql + assert prisma.db.query_raw.await_count == 2 + prisma.db.litellm_shadowevaljob.find_many.assert_not_called() @pytest.mark.asyncio -async def test_shadow_eval_responses_name_the_shadowed_key(monkeypatch: pytest.MonkeyPatch): +async def test_list_shadow_eval_jobs_filters_to_jobs_containing_the_key(monkeypatch: pytest.MonkeyPatch): + """The filter matches a key anywhere in a job's key set and still returns the whole + job, sibling keys included.""" import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.find_many = AsyncMock( - return_value=[_job_record(), _job_record(id="job-2", api_key_id="deleted-key-hash")] + prisma = _shadow_prisma( + legs=[ + _leg_record(), + _leg_record(id="leg-2", api_key_id="key-hash-2"), + _leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash-2"), + _leg_record(id="leg-4", group_id="job-3"), + ] + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id="key-hash-2", limit=50) + + assert [job.job_id for job in jobs] == ["job-1", "job-2"] + assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"] + + +@pytest.mark.parametrize( + ("stopped_flags", "days_left", "expected"), + [ + ((False, False), 7, "running"), + ((True, False), 7, "running"), + ((True, True), 7, "stopped"), + ((True, True), -1, "completed"), + ((False, False), -1, "completed"), + ], +) +@pytest.mark.asyncio +async def test_job_status_runs_until_every_key_stops_and_completed_outranks_stopped( + monkeypatch: pytest.MonkeyPatch, stopped_flags: tuple[bool, ...], days_left: int, expected: str +): + import litellm.proxy.proxy_server as proxy_server + + stamp = datetime.now(timezone.utc) + prisma = _shadow_prisma( + legs=[ + _leg_record( + id=f"leg-{index}", + api_key_id=f"key-{index}", + stopped_at=stamp if stopped else None, + ends_at=datetime.now(timezone.utc) + timedelta(days=days_left), + ) + for index, stopped in enumerate(stopped_flags) + ] + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + + assert [job.status for job in jobs] == [expected] + + +@pytest.mark.asyncio +async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch: pytest.MonkeyPatch): + """A job whose keys all exhausted their turn budgets stopped sampling on its own, so + it must read completed on the very next list, before any sweep stamps its legs; one + key under budget keeps the whole job running. An operator starting an unrelated eval + must never look like it terminated a finished one.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma( + legs=[ + _leg_record(max_turns=5), + _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=5), + _leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=5), + _leg_record(id="leg-4", group_id="job-2", api_key_id="key-hash-2", max_turns=5), + ] + ) + prisma.attempt_rows = [ + {"job_id": "leg-1", "attempt_count": 5}, + {"job_id": "leg-2", "attempt_count": 6}, + {"job_id": "leg-3", "attempt_count": 5}, + {"job_id": "leg-4", "attempt_count": 3}, + ] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + + by_id = {job.job_id: job for job in jobs} + assert by_id["job-1"].status == "completed" + assert all(key.stopped_at is None for key in by_id["job-1"].keys) + assert by_id["job-2"].status == "running" + assert {key.api_key_id: key.attempt_count for key in by_id["job-2"].keys} == {"key-hash": 5, "key-hash-2": 3} + + +@pytest.mark.asyncio +async def test_recorded_operator_stop_outranks_budget_arithmetic(monkeypatch: pytest.MonkeyPatch): + """A detached attempt can land around the stop and push the raw count past the + budget; the recorded stopped_by must keep the job reading stopped regardless.""" + import litellm.proxy.proxy_server as proxy_server + + stamp = datetime.now(timezone.utc) + prisma = _shadow_prisma(legs=[_leg_record(max_turns=5, stopped_at=stamp, stopped_by="admin")]) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + assert jobs[0].status == "stopped" + assert jobs[0].stopped_by == "admin" + + detail = await get_shadow_eval_job("job-1", VIEWER) + assert detail.status == "stopped" + + +@pytest.mark.asyncio +async def test_backfilled_legacy_stop_never_reads_as_completion(monkeypatch: pytest.MonkeyPatch): + """Jobs stopped before stopped_by existed are backfilled with 'unknown' by the + migration, so even one whose stray attempts crossed the budget stays stopped.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma( + legs=[_leg_record(max_turns=5, stopped_at=datetime.now(timezone.utc), stopped_by="unknown")] + ) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + assert jobs[0].status == "stopped" + + +def test_stopped_by_migration_backfills_every_job_that_displayed_stopped(): + """The migration must close the pre-column population: without the backfill, a + legacy stop whose stray attempts crossed the budget would read completed.""" + import litellm_proxy_extras + + sql = ( + Path(litellm_proxy_extras.__file__).parent + / "migrations" + / "20260818224500_add_shadow_eval_stopped_by" + / "migration.sql" + ).read_text() + assert 'ADD COLUMN "stopped_by" TEXT' in sql + assert "SET stopped_by = 'unknown'" in sql + assert "WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc')" in sql + + +@pytest.mark.asyncio +async def test_stop_rejects_a_job_that_already_spent_its_budget(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(max_turns=3)]) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 3}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + with pytest.raises(HTTPException) as exhausted: + await stop_shadow_eval_job("job-1", ADMIN) + assert exhausted.value.status_code == 400 + assert "completed" in exhausted.value.detail + prisma.db.litellm_shadowevaljob.update_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma( + legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="deleted-key-hash")], + known_keys=("key-hash", "key-hash-2"), ) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) monkeypatch.setattr(proxy_server, "prisma_client", prisma) - started = await start_shadow_eval(_start_request(), ADMIN) - assert (started.key_alias, started.key_name) == ("prod-alpha", "sk-...lpha") - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) - assert [(job.key_alias, job.key_name) for job in jobs] == [("prod-alpha", "sk-...lpha"), (None, None)] + assert [(key.key_alias, key.key_name) for key in jobs[0].keys] == [ + (None, None), + ("prod-alpha", "sk-...lpha"), + ] batched_where = prisma.db.litellm_verificationtoken.find_many.call_args.kwargs["where"] assert batched_where == {"token": {"in": ["deleted-key-hash", "key-hash"]}} - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record()) detail = await get_shadow_eval_job("job-1", VIEWER) - assert detail.key_alias == "prod-alpha" + assert [key.key_alias for key in detail.keys] == [None, "prod-alpha"] @pytest.mark.asyncio -async def test_stop_shadow_eval_sets_stopped_at_and_rejects_non_running(monkeypatch: pytest.MonkeyPatch): +async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_running( + monkeypatch: pytest.MonkeyPatch, +): + """One stop ends sampling for the whole job, while a leg that already stopped on its + own budget keeps the stopped_at it earned.""" import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record()) + earned = datetime.now(timezone.utc) - timedelta(hours=1) + prisma = _shadow_prisma(legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", stopped_at=earned)]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) stopped = await stop_shadow_eval_job("job-1", ADMIN) - assert stopped.status == "stopped" - update = prisma.db.litellm_shadowevaljob.update.call_args.kwargs - assert set(update["data"]) == {"stopped_at"} - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock( - return_value=_job_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1)) - ) + assert stopped.status == "stopped" + assert stopped.stopped_by == "admin" + stop_sql, stop_group, stop_operator, stop_stamp = prisma.db.execute_raw.call_args.args + assert "SET stopped_by = $2, stopped_at = COALESCE(stopped_at, $3::timestamp)" in stop_sql + assert "WHERE group_id = $1 AND stopped_by IS NULL" in stop_sql + assert "ends_at > (NOW() AT TIME ZONE 'utc')" in stop_sql + assert ") < k.max_turns" in stop_sql + assert (stop_group, stop_operator) == ("job-1", "admin") + assert datetime.fromisoformat(stop_stamp).tzinfo is None + assert prisma.db.execute_raw.await_count == 1 + prisma.db.litellm_shadowevaljob.update_many.assert_not_called() + by_key = {key.api_key_id: key.stopped_at for key in stopped.keys} + assert by_key["key-hash-2"] == earned + assert by_key["key-hash"] is not None and by_key["key-hash"] != earned + + done_leg = _leg_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1)) + prisma_done = _shadow_prisma(legs=[done_leg]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_done) with pytest.raises(HTTPException) as exc: await stop_shadow_eval_job("job-1", ADMIN) assert exc.value.status_code == 400 + assert "already completed" in exc.value.detail + assert done_leg.stopped_by is None with pytest.raises(HTTPException) as forbidden: await stop_shadow_eval_job("job-1", VIEWER) assert forbidden.value.status_code == 403 + + +def test_every_shadow_eval_sql_constant_speaks_naive_utc(): + """The tables store naive UTC wall time (prisma's convention), so SQL-side time must be + NOW() AT TIME ZONE 'utc' and python-side params must cast ::timestamp; a bare NOW() or a + timestamptz cast writes session-local wall time into the naive column and skews every + comparison against prisma-written stamps.""" + import litellm.proxy.management_endpoints.auto_router_endpoints as module + + sql_constants = {name: value for name, value in vars(module).items() if name.endswith("_SQL")} + assert sql_constants + for name, sql in sql_constants.items(): + assert "::timestamptz" not in sql, name + for occurrence in sql.split("NOW()")[1:]: + assert occurrence.startswith(" AT TIME ZONE 'utc'"), name + + +@pytest.mark.asyncio +async def test_a_stop_racing_the_last_budgeted_attempt_reports_completed_not_stopped( + monkeypatch: pytest.MonkeyPatch, +): + """The statement claims the job only while a leg still samples, so a stop landing in + the same instant the budget spends records nothing and the job keeps reading + completed; stamping it would misreport a self-ended job as operator-stopped forever.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(max_turns=2)]) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 2}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + with pytest.raises(HTTPException) as exc: + await stop_shadow_eval_job("job-1", ADMIN) + assert exc.value.status_code == 400 + assert "already completed" in exc.value.detail + assert prisma.db.litellm_shadowevaljob.find_many.await_args.kwargs["where"] == {"group_id": "job-1"} + + +@pytest.mark.asyncio +async def test_two_racing_stops_produce_exactly_one_winner(monkeypatch: pytest.MonkeyPatch): + """The statement's stopped_by IS NULL predicate lets only one racer claim rows; the + loser reads the stamped state and gets the same answer a late caller gets.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record()]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + first = await stop_shadow_eval_job("job-1", ADMIN) + assert first.status == "stopped" + + with pytest.raises(HTTPException) as exc: + await stop_shadow_eval_job("job-1", ADMIN) + assert exc.value.status_code == 400 + assert "already stopped" in exc.value.detail diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 93a4788caf8..8e661af8daa 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -22,6 +22,7 @@ from litellm.proxy._types import ( NewUserRequest, LiteLLM_BudgetTable, LiteLLM_OrganizationTable, + LiteLLM_ProjectTableCachedObj, LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, LiteLLM_VerificationToken, @@ -31,9 +32,12 @@ from litellm.proxy._types import ( ResetSpendRequest, UpdateKeyRequest, ) +from litellm.proxy.auth.auth_checks import _project_cache_key from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_org_key_limits, + _check_project_key_limits, _check_team_key_limits, _common_key_generation_helper, _enforce_upperbound_key_params, @@ -16645,3 +16649,49 @@ async def test_regenerate_key_repoints_live_membership_not_the_key_row_it_read( assert await _authorized_models_for_key( access_groups, new_token_hash, ["ag-revoked-since", "ag-attached-since"] ) == ["attached-model"] + + +async def _cache_with_project(project_id: str, project_models: list[str]) -> UserApiKeyCache: + user_api_key_cache = UserApiKeyCache() + await user_api_key_cache.async_set_cache( + key=_project_cache_key(project_id), + value=LiteLLM_ProjectTableCachedObj(project_id=project_id, team_id="team-lit-5823", models=project_models), + model_type=LiteLLM_ProjectTableCachedObj, + ) + return user_api_key_cache + + +@pytest.mark.parametrize("request_cls", [GenerateKeyRequest, UpdateKeyRequest]) +@pytest.mark.parametrize("sentinel", ["all-team-models", "all-proxy-models"]) +@pytest.mark.asyncio +async def test_check_project_key_limits_accepts_inherited_model_sentinels(request_cls, sentinel): + """LIT-5823: the sentinels inherit a parent scope, so a project allowlist must not treat them as model names.""" + user_api_key_cache = await _cache_with_project("proj-lit-5823", ["gpt-5.4-nano"]) + + await _check_project_key_limits( + project_id="proj-lit-5823", + data=request_cls(key="sk-lit-5823", models=[sentinel]), + prisma_client=MagicMock(), + user_api_key_cache=user_api_key_cache, + ) + + +@pytest.mark.parametrize("request_cls", [GenerateKeyRequest, UpdateKeyRequest]) +@pytest.mark.parametrize( + "key_models", + [["gpt-5.4-mini"], ["all-team-models", "gpt-5.4-mini"], ["gpt-5.4-nano", "all-proxy-models", "gpt-5.4-mini"]], +) +@pytest.mark.asyncio +async def test_check_project_key_limits_still_rejects_real_model_outside_project(request_cls, key_models): + user_api_key_cache = await _cache_with_project("proj-lit-5823", ["gpt-5.4-nano"]) + + with pytest.raises(HTTPException) as exc_info: + await _check_project_key_limits( + project_id="proj-lit-5823", + data=request_cls(key="sk-lit-5823", models=key_models), + prisma_client=MagicMock(), + user_api_key_cache=user_api_key_cache, + ) + + assert exc_info.value.status_code == 400 + assert "Model 'gpt-5.4-mini' not in project's allowed models" in exc_info.value.detail["error"] diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 05072a0a6d7..2d54a391cf0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -91,6 +91,8 @@ mock_prisma_client = MagicMock() mock_prisma_client.db = MagicMock() mock_prisma_client.db.litellm_teamtable = MagicMock() mock_prisma_client.db.litellm_teamtable.update = AsyncMock() +mock_prisma_client.db.litellm_auditlog = MagicMock() +mock_prisma_client.db.litellm_auditlog.create = AsyncMock() # Fixture to provide the mock prisma client @@ -103,6 +105,11 @@ def mock_db_client(): mock_prisma_client.reset_mock() +@pytest.fixture +def disable_audit_logging_for_mocked_team(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr("litellm.store_audit_logs", False) + + # Fixture to provide a mock admin user auth object @pytest.fixture def mock_admin_auth(): @@ -2060,7 +2067,9 @@ async def test_team_model_add_delete_refresh_team_cache(endpoint_name): @pytest.mark.asyncio -async def test_update_team_team_member_budget_not_passed_to_db(): +async def test_update_team_team_member_budget_not_passed_to_db( + disable_audit_logging_for_mocked_team, +): """ Test that 'team_member_budget' is never passed to prisma_client.db.litellm_teamtable.update regardless of whether the value is set or None. @@ -2498,7 +2507,9 @@ async def test_upsert_team_member_budget_table_no_existing_budget(): @pytest.mark.asyncio -async def test_update_team_with_team_member_budget_duration(): +async def test_update_team_with_team_member_budget_duration( + disable_audit_logging_for_mocked_team, +): """ Test that team/update endpoint properly handles team_member_budget_duration. """ @@ -5171,7 +5182,9 @@ async def test_update_team_standalone_budget_raise_blocked_for_team_admin(): @pytest.mark.asyncio -async def test_update_team_standalone_budget_raise_allowed_for_proxy_admin(): +async def test_update_team_standalone_budget_raise_allowed_for_proxy_admin( + disable_audit_logging_for_mocked_team, +): """ Test that a proxy admin CAN raise a standalone team's budget on /team/update. @@ -5325,7 +5338,9 @@ async def test_update_team_standalone_budget_removal_blocked_for_team_admin(): @pytest.mark.asyncio -async def test_update_team_standalone_uncapped_team_admin_sets_finite_allowed(): +async def test_update_team_standalone_uncapped_team_admin_sets_finite_allowed( + disable_audit_logging_for_mocked_team, +): """ When a team currently has NO cap (max_budget=None / unlimited), a team admin setting a finite max_budget is a RESTRICTION, not a raise, and is @@ -5407,7 +5422,9 @@ async def test_update_team_standalone_uncapped_team_admin_sets_finite_allowed(): @pytest.mark.asyncio -async def test_update_team_standalone_unchanged_budget_allowed(): +async def test_update_team_standalone_unchanged_budget_allowed( + disable_audit_logging_for_mocked_team, +): """ Test that /team/update for a standalone team does NOT compare against the caller's personal max_budget when the budget is unchanged. @@ -5508,7 +5525,9 @@ async def test_update_team_standalone_unchanged_budget_allowed(): @pytest.mark.asyncio -async def test_update_team_standalone_lower_budget_allowed(): +async def test_update_team_standalone_lower_budget_allowed( + disable_audit_logging_for_mocked_team, +): """ Test that /team/update for a standalone team allows lowering the budget below the team's current value even when the new value still exceeds the @@ -5691,7 +5710,9 @@ async def test_update_team_org_scoped_budget_exceeds_org_limit(): @pytest.mark.asyncio -async def test_update_team_standalone_models_not_gated_by_user_limit(): +async def test_update_team_standalone_models_not_gated_by_user_limit( + disable_audit_logging_for_mocked_team, +): """ Test that /team/update for a standalone team does NOT gate the team's models by the caller's personal allowed models. @@ -5775,7 +5796,9 @@ async def test_update_team_standalone_models_not_gated_by_user_limit(): @pytest.mark.asyncio -async def test_update_team_org_scoped_budget_bypasses_user_limit(): +async def test_update_team_org_scoped_budget_bypasses_user_limit( + disable_audit_logging_for_mocked_team, +): """ Test that /team/update for an org-scoped team does NOT validate budget against user's personal max_budget. @@ -5890,7 +5913,9 @@ async def test_update_team_org_scoped_budget_bypasses_user_limit(): @pytest.mark.asyncio -async def test_update_team_org_scoped_models_bypasses_user_limit(): +async def test_update_team_org_scoped_models_bypasses_user_limit( + disable_audit_logging_for_mocked_team, +): """ Test that /team/update for an org-scoped team does NOT validate models against user's personal models. @@ -6080,7 +6105,9 @@ async def test_update_team_org_scoped_models_not_in_org_models(): @pytest.mark.asyncio -async def test_update_team_org_scoped_models_with_all_proxy_models(): +async def test_update_team_org_scoped_models_with_all_proxy_models( + disable_audit_logging_for_mocked_team, +): """ Test that /team/update for an org-scoped team succeeds when organization has 'all-proxy-models'. @@ -6196,7 +6223,9 @@ async def test_update_team_org_scoped_models_with_all_proxy_models(): @pytest.mark.asyncio -async def test_update_team_tpm_limit_not_gated_by_user_limit(): +async def test_update_team_tpm_limit_not_gated_by_user_limit( + disable_audit_logging_for_mocked_team, +): """ Test that /team/update does NOT gate the team's tpm_limit by the caller's personal tpm_limit. @@ -6279,7 +6308,9 @@ async def test_update_team_tpm_limit_not_gated_by_user_limit(): @pytest.mark.asyncio -async def test_update_team_rpm_limit_not_gated_by_user_limit(): +async def test_update_team_rpm_limit_not_gated_by_user_limit( + disable_audit_logging_for_mocked_team, +): """ Test that /team/update does NOT gate the team's rpm_limit by the caller's personal rpm_limit. @@ -6795,7 +6826,9 @@ async def test_update_team_org_scoped_rpm_exceeds_org_limit(): @pytest.mark.asyncio -async def test_update_team_org_scoped_tpm_rpm_bypasses_user_limit(): +async def test_update_team_org_scoped_tpm_rpm_bypasses_user_limit( + disable_audit_logging_for_mocked_team, +): """ Test that /team/update for an org-scoped team bypasses user's TPM/RPM limits. @@ -6905,7 +6938,9 @@ async def test_update_team_org_scoped_tpm_rpm_bypasses_user_limit(): @pytest.mark.asyncio -async def test_update_team_guardrails_with_org_id(): +async def test_update_team_guardrails_with_org_id( + disable_audit_logging_for_mocked_team, +): """ Test that updating team guardrails works when team has an organization_id. The fix ensures 'teams' field is included when fetching organization data. @@ -7242,7 +7277,10 @@ async def test_persist_deleted_team_records(): @pytest.mark.asyncio -async def test_delete_team_persists_deleted_teams(monkeypatch): +async def test_delete_team_persists_deleted_teams( + monkeypatch, + disable_audit_logging_for_mocked_team, +): from litellm.proxy._types import DeleteTeamRequest mock_prisma_client = AsyncMock() @@ -7325,7 +7363,10 @@ async def test_delete_team_persists_deleted_teams(monkeypatch): @pytest.mark.asyncio -async def test_delete_team_sweeps_references_outside_members_with_roles(monkeypatch): +async def test_delete_team_sweeps_references_outside_members_with_roles( + monkeypatch, + disable_audit_logging_for_mocked_team, +): """ Regression pin for LIT-5511: a deleted team stayed visible on user records. @@ -7431,7 +7472,10 @@ async def test_delete_team_sweeps_references_outside_members_with_roles(monkeypa @pytest.mark.asyncio -async def test_delete_team_evicts_the_auth_cache_of_the_keys_it_deletes(monkeypatch): +async def test_delete_team_evicts_the_auth_cache_of_the_keys_it_deletes( + monkeypatch, + disable_audit_logging_for_mocked_team, +): """ A virtual key scoped to the team is deleted from the db with the team, but auth resolves a cached key object without re-reading the team, so leaving the cache entry behind lets that key @@ -7493,7 +7537,10 @@ async def test_delete_team_evicts_the_auth_cache_of_the_keys_it_deletes(monkeypa @pytest.mark.asyncio -async def test_delete_team_failing_reconcile_sweep_cannot_strand_the_team_in_cache(monkeypatch): +async def test_delete_team_failing_reconcile_sweep_cannot_strand_the_team_in_cache( + monkeypatch, + disable_audit_logging_for_mocked_team, +): """ The reconcile sweep runs after the team row is committed deleted. If it ran before cache eviction, a sweep failure would return an error with the team gone from the db but still @@ -7555,7 +7602,10 @@ async def test_delete_team_failing_reconcile_sweep_cannot_strand_the_team_in_cac @pytest.mark.asyncio -async def test_delete_team_broadcasts_cache_invalidation_to_other_workers(monkeypatch): +async def test_delete_team_broadcasts_cache_invalidation_to_other_workers( + monkeypatch, + disable_audit_logging_for_mocked_team, +): """ Evicting locally only reaches the worker that handled the delete. Without the broadcast, every other worker keeps serving the deleted team, and the deleted team's keys, out of its own @@ -7619,7 +7669,10 @@ async def test_delete_team_broadcasts_cache_invalidation_to_other_workers(monkey @pytest.mark.asyncio -async def test_delete_team_survives_a_failing_cache_backend(monkeypatch): +async def test_delete_team_survives_a_failing_cache_backend( + monkeypatch, + disable_audit_logging_for_mocked_team, +): """ Cache eviction runs after the reference sweep has already committed, so a cache backend that is unreachable must not abort the delete. If it did, `/team/delete` would fail with the team @@ -8088,6 +8141,7 @@ async def test_update_team_soft_budget_validation( expected_soft_budget, expected_max_budget, error_message, + disable_audit_logging_for_mocked_team, ): """ Test soft_budget validation in /team/update endpoint. @@ -8498,7 +8552,11 @@ async def test_get_team_daily_activity_member_without_permission_filters_by_keys @pytest.mark.asyncio -async def test_update_team_with_router_settings(mock_db_client, mock_admin_auth): +async def test_update_team_with_router_settings( + mock_db_client, + mock_admin_auth, + disable_audit_logging_for_mocked_team, +): """ Test that /team/update correctly handles router_settings by: 1. Accepting router_settings as a dict parameter @@ -11594,7 +11652,9 @@ async def test_update_team_output_token_estimate_lowered_rejected_for_team_admin @pytest.mark.asyncio -async def test_update_team_output_token_estimate_unchanged_allows_team_admin_edit(): +async def test_update_team_output_token_estimate_unchanged_allows_team_admin_edit( + disable_audit_logging_for_mocked_team, +): """The team settings form resends every field it renders, so gating on presence would break a team admin editing an unrelated setting.""" import contextlib @@ -12014,7 +12074,9 @@ class _FakeMirrorDb: @pytest.mark.asyncio -async def test_update_team_syncs_access_group_assigned_team_ids_in_both_directions(): +async def test_update_team_syncs_access_group_assigned_team_ids_in_both_directions( + disable_audit_logging_for_mocked_team, +): """ A team-side edit of `access_group_ids` must be mirrored onto every affected access group's `assigned_team_ids`, in one transaction, in both directions. @@ -12170,7 +12232,9 @@ async def test_sync_reads_the_committed_team_row_rather_than_the_callers_snapsho @pytest.mark.asyncio -async def test_new_team_and_delete_team_both_drive_the_mirror(): +async def test_new_team_and_delete_team_both_drive_the_mirror( + disable_audit_logging_for_mocked_team, +): """Every writer of `team.access_group_ids` has to reach the mirror, not just update. These pin the wiring on the other two paths; the mirror's own behavior is covered above. diff --git a/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py b/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py index 48e4353966b..2d54d249713 100644 --- a/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py +++ b/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py @@ -19,6 +19,7 @@ from litellm.proxy.management_helpers.audit_logs import ( _build_audit_log_payload, _dispatch_audit_log_to_callbacks, create_audit_log_for_update, + is_audit_logging_enabled, ) from litellm.types.utils import StandardAuditLogPayload @@ -49,6 +50,34 @@ def _make_audit_log( ) +@pytest.mark.parametrize( + ("premium_user", "configured_value", "environment_value", "expected"), + ( + (True, None, None, True), + (True, False, None, False), + (True, None, "false", False), + (False, None, None, False), + (False, True, None, True), + (True, True, "false", True), + ), +) +def test_is_audit_logging_enabled_precedence( + monkeypatch: pytest.MonkeyPatch, + premium_user: bool, + configured_value: bool | None, + environment_value: str | None, + expected: bool, +): + monkeypatch.setattr(litellm, "store_audit_logs", configured_value) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", premium_user) + if environment_value is None: + monkeypatch.delenv("LITELLM_STORE_AUDIT_LOGS", raising=False) + else: + monkeypatch.setenv("LITELLM_STORE_AUDIT_LOGS", environment_value) + + assert is_audit_logging_enabled() is expected + + class TestBuildAuditLogPayload: def test_builds_correct_payload(self): audit_log = _make_audit_log() @@ -185,12 +214,14 @@ class TestCreateAuditLogForUpdateWithCallbacks: with ( patch("litellm.proxy.proxy_server.premium_user", False), patch("litellm.store_audit_logs", True), + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, ): audit_log = _make_audit_log() await create_audit_log_for_update(audit_log) await asyncio.sleep(0.1) mock_logger.async_log_audit_log_event.assert_not_called() + mock_prisma.db.litellm_auditlog.create.assert_not_called() @pytest.mark.asyncio async def test_no_dispatch_when_store_audit_logs_false(self): diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_batch_file_validation.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_batch_file_validation.py new file mode 100644 index 00000000000..f5542fc0446 --- /dev/null +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_batch_file_validation.py @@ -0,0 +1,176 @@ +import io + +import pytest + +from litellm.proxy._types import ProxyException +from litellm.proxy.openai_files_endpoints.batch_file_validation import ( + BATCH_LINE_REQUIRED_KEYS, + BatchFileEmpty, + BatchFileInvalidJsonLine, + BatchFileLineNotObject, + BatchFileMissingLineKey, + BatchFileTooLarge, + BatchFileWrongExtension, + check_batch_file_upload, + raise_batch_file_validation_failure, +) + +VALID_LINE = ( + b'{"custom_id": "req-1", "method": "POST", "url": "/v1/chat/completions",' + b' "body": {"model": "gpt-4.1-nano", "messages": [{"role": "user", "content": "hi"}]}}' +) + + +def test_valid_bytes_pass(): + assert check_batch_file_upload("batch.jsonl", VALID_LINE + b"\n" + VALID_LINE + b"\n", 10) is None + + +def test_valid_binaryio_passes_and_resets_position(): + handle = io.BytesIO(VALID_LINE + b"\n" + VALID_LINE + b"\n") + handle.seek(17) + assert check_batch_file_upload("batch.jsonl", handle, 10) is None + assert handle.tell() == 0 + + +def test_uppercase_extension_accepted(): + assert check_batch_file_upload("BATCH.JSONL", VALID_LINE, None) is None + + +@pytest.mark.parametrize("filename", ["batch.csv", "batch.json", "batch", None]) +def test_wrong_extension_rejected(filename): + assert check_batch_file_upload(filename, VALID_LINE, None) == BatchFileWrongExtension(filename=filename or "") + + +def test_size_over_cap_rejected_for_bytes(): + content = b"x" * (2 * 1024 * 1024) + assert check_batch_file_upload("batch.jsonl", content, 1) == BatchFileTooLarge( + size_bytes=len(content), limit_mb=1 + ) + + +def test_size_over_cap_rejected_for_binaryio(): + content = b"x" * (2 * 1024 * 1024) + assert check_batch_file_upload("batch.jsonl", io.BytesIO(content), 1) == BatchFileTooLarge( + size_bytes=len(content), limit_mb=1 + ) + + +def test_size_exactly_at_cap_allowed(): + line = VALID_LINE + b"\n" + padding_key = b'{"custom_id": "pad", "method": "POST", "url": "/v1/chat/completions", "body": {"note": "' + pad_line = padding_key + b"a" * (1024 * 1024 - len(line) - len(padding_key) - len(b'"}}\n')) + b'"}}\n' + content = line + pad_line + assert len(content) == 1024 * 1024 + assert check_batch_file_upload("batch.jsonl", content, 1) is None + + +def test_no_cap_skips_size_check(): + content = (VALID_LINE + b"\n") * 5000 + assert check_batch_file_upload("batch.jsonl", content, None) is None + + +@pytest.mark.parametrize("cap", [0, -3]) +def test_nonpositive_cap_disables_size_check(cap): + content = (VALID_LINE + b"\n") * 5000 + assert check_batch_file_upload("batch.jsonl", content, cap) is None + + +@pytest.mark.parametrize("content", [b"", b"\n\n", b" \n\t\n"]) +def test_empty_file_rejected(content): + assert check_batch_file_upload("batch.jsonl", content, None) == BatchFileEmpty() + + +def test_invalid_json_line_rejected_with_line_number(): + content = VALID_LINE + b"\n" + b"not json at all\n" + VALID_LINE + b"\n" + assert check_batch_file_upload("batch.jsonl", content, None) == BatchFileInvalidJsonLine(line_number=2) + + +def test_non_utf8_line_rejected_as_invalid_json(): + assert check_batch_file_upload("batch.jsonl", b"\xff\xfe\x00\x01\n", None) == BatchFileInvalidJsonLine( + line_number=1 + ) + + +def test_non_object_line_rejected(): + content = VALID_LINE + b"\n" + b'["custom_id", "method"]\n' + assert check_batch_file_upload("batch.jsonl", content, None) == BatchFileLineNotObject(line_number=2) + + +@pytest.mark.parametrize("missing_key", BATCH_LINE_REQUIRED_KEYS) +def test_missing_required_key_rejected(missing_key): + import json + + line_dict = { + "custom_id": "req-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": "gpt-4.1-nano"}, + } + del line_dict[missing_key] + content = VALID_LINE + b"\n" + json.dumps(line_dict).encode() + b"\n" + assert check_batch_file_upload("batch.jsonl", content, None) == BatchFileMissingLineKey( + line_number=2, key=missing_key + ) + + +def test_blank_lines_do_not_shift_line_numbers(): + content = b"\n" + VALID_LINE + b"\n\n" + b"broken\n" + assert check_batch_file_upload("batch.jsonl", content, None) == BatchFileInvalidJsonLine(line_number=4) + + +def test_failed_scan_leaves_handle_open_and_reset(): + handle = io.BytesIO(b"not json\n" + VALID_LINE + b"\n") + assert check_batch_file_upload("batch.jsonl", handle, None) == BatchFileInvalidJsonLine(line_number=1) + assert not handle.closed + assert handle.tell() == 0 + + +def test_scan_stops_at_first_failure(): + class ExplodingLines(io.BytesIO): + def __init__(self): + super().__init__(b"not json\n" + VALID_LINE + b"\n") + self.lines_read = 0 + + def __next__(self): + self.lines_read += 1 + return super().__next__() + + handle = ExplodingLines() + assert check_batch_file_upload("batch.jsonl", handle, None) == BatchFileInvalidJsonLine(line_number=1) + assert handle.lines_read == 1 + + +@pytest.mark.parametrize( + "failure, expected_code, expected_param, expected_fragments", + [ + ( + BatchFileTooLarge(size_bytes=220200960, limit_mb=10), + "413", + "file", + ("210.0 MB", "max_batch_file_size_mb", "10 MB", "not forwarded"), + ), + ( + BatchFileWrongExtension(filename="batch.csv"), + "400", + "file", + ("batch.csv", ".jsonl", "not forwarded"), + ), + (BatchFileEmpty(), "400", "file", ("no request lines", "not forwarded")), + (BatchFileInvalidJsonLine(line_number=3), "400", "file", ("line 3", "not valid JSON")), + (BatchFileLineNotObject(line_number=2), "400", "file", ("line 2", "JSON object")), + ( + BatchFileMissingLineKey(line_number=5, key="method"), + "400", + "method", + ("'method'", "line 5", "custom_id, method, url, body"), + ), + ], +) +def test_failures_map_to_openai_shaped_proxy_exceptions(failure, expected_code, expected_param, expected_fragments): + with pytest.raises(ProxyException) as exc_info: + raise_batch_file_validation_failure(failure) + assert exc_info.value.code == expected_code + assert exc_info.value.type == "invalid_request_error" + assert exc_info.value.param == expected_param + for fragment in expected_fragments: + assert fragment in exc_info.value.message diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index e363a266688..99fb19f0d60 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -31,6 +31,11 @@ from litellm.caching.caching import DualCache from litellm.proxy.proxy_server import hash_token from litellm.proxy.utils import ProxyLogging +VALID_BATCH_LINE = ( + b'{"custom_id": "req-1", "method": "POST", "url": "/v1/chat/completions",' + b' "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "hi"}]}}\n' +) + @pytest.fixture def llm_router() -> Router: @@ -225,7 +230,10 @@ def test_invalid_purpose(mocker: MockerFixture, monkeypatch, llm_router: Router) assert response.status_code == 400 print(f"response: {response.json()}") - assert "Invalid purpose: my-bad-purpose" in response.json()["error"]["message"] + error = response.json()["error"] + assert "Invalid purpose: my-bad-purpose" in error["message"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "purpose" def test_get_file_content_rejects_raw_cloud_storage_uri(llm_router: Router): @@ -1599,7 +1607,7 @@ def _post_file_with_team_metadata( user_key = UserAPIKeyAuth(api_key="test-key", team_metadata=team_metadata) app.dependency_overrides[user_api_key_auth] = lambda: user_key - test_file = ("mydata.jsonl", b'{"prompt": "Hello"}', "application/json") + test_file = ("mydata.jsonl", VALID_BATCH_LINE, "application/jsonl") try: response = client.post( "/v1/files", @@ -1703,7 +1711,7 @@ def _post_file_raw( user_key = UserAPIKeyAuth(api_key="test-key", team_metadata=team_metadata) app.dependency_overrides[user_api_key_auth] = lambda: user_key - test_file = ("mydata.jsonl", b'{"prompt": "Hello"}', "application/json") + test_file = ("mydata.jsonl", VALID_BATCH_LINE, "application/jsonl") try: response = client.post( "/v1/files", @@ -2749,7 +2757,7 @@ def test_create_file_provider_only_resolves_named_vertex_credentials( try: response = client.post( "/v1/files", - files={"file": ("batch.jsonl", b"{}", "application/jsonl")}, + files={"file": ("batch.jsonl", VALID_BATCH_LINE, "application/jsonl")}, data={"purpose": "batch"}, headers={ "Authorization": "Bearer test-key", @@ -2991,7 +2999,7 @@ def test_create_file_provider_only_skips_other_team_vertex_deployment( try: response = client.post( "/v1/files", - files={"file": ("batch.jsonl", b"{}", "application/jsonl")}, + files={"file": ("batch.jsonl", VALID_BATCH_LINE, "application/jsonl")}, data={"purpose": "batch"}, headers={ "Authorization": "Bearer test-key", @@ -3341,3 +3349,170 @@ def test_raw_provider_file_id_retrieve_allowed_when_managed_files_not_required( assert response.status_code == 200, response.text mock_retrieve.assert_called_once() + + +def _setup_batch_upload_endpoint(monkeypatch, llm_router: Router) -> list: + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.openai_files_endpoints import files_endpoints as fe + + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", llm_router) + setup_proxy_logging_object(monkeypatch, llm_router) + + forwarded_calls: list = [] + + async def fake_route_create_file(**kwargs): + forwarded_calls.append(kwargs) + return OpenAIFileObject( + id="dummy-id", + object="file", + bytes=0, + created_at=1234567890, + filename="batch.jsonl", + purpose="batch", + status="uploaded", + ) + + monkeypatch.setattr(fe, "route_create_file", fake_route_create_file) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test-user" + ) + return forwarded_calls + + +def _teardown_batch_upload_endpoint(): + import litellm.proxy.proxy_server as ps + + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +def test_create_file_batch_over_max_batch_file_size_mb_rejected_before_forwarding( + monkeypatch, llm_router: Router +): + import litellm.proxy.proxy_server as ps + + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + monkeypatch.setitem(ps.general_settings, "max_batch_file_size_mb", 1) + + oversized = VALID_BATCH_LINE * (2 * 1024 * 1024 // len(VALID_BATCH_LINE) + 1) + try: + response = client.post( + "/v1/files", + files={"file": ("batch.jsonl", oversized, "application/jsonl")}, + data={"purpose": "batch"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 413, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "file" + assert "max_batch_file_size_mb" in error["message"] + assert "1 MB" in error["message"] + assert forwarded_calls == [] + + +def test_create_file_batch_under_max_batch_file_size_mb_forwards(monkeypatch, llm_router: Router): + import litellm.proxy.proxy_server as ps + + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + monkeypatch.setitem(ps.general_settings, "max_batch_file_size_mb", 1) + + try: + response = client.post( + "/v1/files", + files={"file": ("batch.jsonl", VALID_BATCH_LINE, "application/jsonl")}, + data={"purpose": "batch"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 200, response.text + assert len(forwarded_calls) == 1 + + +def test_create_file_batch_wrong_extension_rejected_before_forwarding(monkeypatch, llm_router: Router): + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + + try: + response = client.post( + "/v1/files", + files={"file": ("batch.csv", VALID_BATCH_LINE, "text/csv")}, + data={"purpose": "batch"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "file" + assert "batch.csv" in error["message"] + assert ".jsonl" in error["message"] + assert forwarded_calls == [] + + +def test_create_file_batch_missing_line_key_rejected_before_forwarding(monkeypatch, llm_router: Router): + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + + bad_line = b'{"custom_id": "req-1", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo"}}\n' + try: + response = client.post( + "/v1/files", + files={"file": ("batch.jsonl", VALID_BATCH_LINE + bad_line, "application/jsonl")}, + data={"purpose": "batch"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "method" + assert "line 2" in error["message"] + assert forwarded_calls == [] + + +def test_create_file_batch_invalid_json_line_rejected_before_forwarding(monkeypatch, llm_router: Router): + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + + try: + response = client.post( + "/v1/files", + files={"file": ("batch.jsonl", b"this is not jsonl\n", "application/jsonl")}, + data={"purpose": "batch"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 400, response.text + error = response.json()["error"] + assert error["param"] == "file" + assert "line 1" in error["message"] + assert "not valid JSON" in error["message"] + assert forwarded_calls == [] + + +def test_create_file_non_batch_purpose_skips_batch_validation(monkeypatch, llm_router: Router): + forwarded_calls = _setup_batch_upload_endpoint(monkeypatch, llm_router) + + try: + response = client.post( + "/v1/files", + files={"file": ("notes.txt", b"plain text, not jsonl", "text/plain")}, + data={"purpose": "user_data"}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + _teardown_batch_upload_endpoint() + + assert response.status_code == 200, response.text + assert len(forwarded_calls) == 1 diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index b56a8da7c66..a9454854948 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -719,6 +719,7 @@ class TestVertexAIPassThroughHandler: # Create mock logging object mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.optional_params = {} mock_logging_obj.litellm_call_id = "test-call-id-123" mock_logging_obj.model_call_details = {} @@ -895,6 +896,7 @@ class TestVertexAIPassThroughHandler: # Create mock logging object mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.optional_params = {} mock_logging_obj.litellm_call_id = "test-call-id-123" mock_logging_obj.model_call_details = {} @@ -965,6 +967,7 @@ class TestVertexAIPassThroughHandler: mock_httpx_response.status_code = 200 mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.optional_params = {} mock_logging_obj.litellm_call_id = "test-call-id-embed" mock_logging_obj.model_call_details = {} @@ -1023,6 +1026,7 @@ class TestVertexAIPassThroughHandler: mock_httpx_response.status_code = 200 mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.optional_params = {} mock_logging_obj.litellm_call_id = "test-call-id-batch" mock_logging_obj.model_call_details = {} @@ -1079,6 +1083,7 @@ class TestVertexAIPassThroughHandler: mock_httpx_response.status_code = 200 mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.optional_params = {} mock_logging_obj.litellm_call_id = "test-call-id-gemini-studio" mock_logging_obj.model_call_details = {} @@ -1109,6 +1114,117 @@ class TestVertexAIPassThroughHandler: assert result["kwargs"].get("model") == "gemini-embedding-2-preview" mock_completion_cost.assert_called_once() + @pytest.mark.parametrize("streaming", [False, True]) + def test_vertex_passthrough_handler_prices_regional_endpoint_with_uplift(self, monkeypatch, streaming): + """ + Both cost computations for a passthrough call must price on the URL's serving location: + the handler-computed cost, and the async success recompute, which re-resolves the + location from the logging object and previously fell through empty optional_params to + the us-central1 default, billing the regional uplift on global traffic too (#34393). + """ + import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( + VertexPassthroughLoggingHandler, + ) + + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.get_model_cost_map(url=""), + "vertex_ai/gemini-fake-regional": { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "regional_endpoint_uplift_multiplier": 1.1, + }, + }, + ) + + response_body: Final = { + "candidates": [ + { + "content": {"parts": [{"text": "hello"}], "role": "model"}, + "finishReason": "STOP", + } + ], + "usageMetadata": { + "promptTokenCount": 10, + "candidatesTokenCount": 20, + "totalTokenCount": 30, + }, + } + + def costs_for(location: str) -> tuple[float, float]: + url_route: Final = ( + f"https://{location}-aiplatform.googleapis.com/v1/projects/p/locations/{location}" + "/publishers/google/models/gemini-fake-regional:" + f"{'streamGenerateContent' if streaming else 'generateContent'}" + ) + start_time: Final = datetime.datetime.now() + end_time: Final = datetime.datetime.now() + logging_obj: Final = Logging( + model="gemini-fake-regional", + messages=[{"role": "user", "content": "hi"}], + stream=streaming, + call_type="pass_through_endpoint", + start_time=start_time, + litellm_call_id="call-id", + function_id="fn-id", + ) + logging_obj.update_environment_variables( + model="gemini-fake-regional", + user="unknown", + optional_params={}, + litellm_params={}, + call_type="pass_through_endpoint", + ) + if streaming: + result = VertexPassthroughLoggingHandler._handle_logging_vertex_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=Mock(), + url_route=url_route, + request_body={}, + endpoint_type="vertex_ai", + start_time=start_time, + all_chunks=[json.dumps(response_body)], + model=None, + end_time=end_time, + ) + else: + mock_httpx_response: Final = Mock() + mock_httpx_response.json.return_value = response_body + mock_httpx_response.headers = {} + mock_httpx_response.status_code = 200 + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=mock_httpx_response, + logging_obj=logging_obj, + url_route=url_route, + result="test-result", + start_time=start_time, + end_time=end_time, + cache_hit=False, + ) + recomputed: Final = logging_obj._response_cost_calculator(result=result["result"]) + return result["kwargs"]["response_cost"], recomputed + + global_handler_cost, global_recomputed_cost = costs_for("global") + regional_handler_cost, regional_recomputed_cost = costs_for("us-east5") + + plain_cost: Final = 10 * 1e-06 + 20 * 2e-06 + assert global_handler_cost == pytest.approx(plain_cost, rel=1e-9) + assert regional_handler_cost == pytest.approx(plain_cost * 1.10, rel=1e-9), ( + "regional Vertex passthrough traffic must bill at 1.1x the global rate" + ) + assert global_recomputed_cost == pytest.approx(plain_cost, rel=1e-9), ( + "the logging recompute must not price global passthrough traffic as regional" + ) + assert regional_recomputed_cost == pytest.approx(plain_cost * 1.10, rel=1e-9) + class TestVertexAIDiscoveryPassThroughHandler: """ diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index f31f67c317a..58465772b3b 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -769,6 +769,214 @@ async def test_ProxyConfig_get_config_missing_file_raises(monkeypatch): await pc.get_config(config_file_path="/no/such/path.yaml") +# --------------------------------------------------------------------------- +# ProxyConfig._initialize_secret_manager_from_raw_config +# --------------------------------------------------------------------------- + +VAULT_SECRET_MANAGER_MODULE = ''' +import os + +from litellm.integrations.custom_secret_manager import CustomSecretManager + +VAULT = {"LITELLM_MASTER_KEY": "master-from-vault", "MY_PROVIDER_KEY": "provider-from-vault"} + + +class VaultSecretManager(CustomSecretManager): + def __init__(self): + super().__init__() + # The loader re-executes this module on every construction, so an in-module counter + # would reset. Append to a file instead, to count constructions across the whole load. + with open(os.environ["VAULT_CONSTRUCTION_LOG"], "a") as f: + f.write("constructed\\n") + + def sync_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + return VAULT.get(secret_name) + + async def async_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + return VAULT.get(secret_name) +''' + +VAULT_BACKED_CONFIG = """ +model_list: + - model_name: my-model + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/MY_PROVIDER_KEY + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + key_management_system: custom + key_management_settings: + custom_secret_manager: vault_secret_manager.VaultSecretManager + hosted_keys: + - LITELLM_MASTER_KEY + - MY_PROVIDER_KEY +""" + + +def _write_vault_backed_config(tmp_path, monkeypatch, config_yaml: str) -> str: + """Write a config whose secrets live only in a custom secret manager, never in the env.""" + (tmp_path / "vault_secret_manager.py").write_text(VAULT_SECRET_MANAGER_MODULE) + config_file = tmp_path / "c.yaml" + config_file.write_text(config_yaml) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + monkeypatch.delenv("LITELLM_MASTER_KEY", raising=False) + monkeypatch.delenv("MY_PROVIDER_KEY", raising=False) + monkeypatch.setenv("VAULT_CONSTRUCTION_LOG", str(tmp_path / "constructions.log")) + monkeypatch.setattr(litellm, "secret_manager_client", None) + return str(config_file) + + +def _construction_count(tmp_path) -> int: + log = tmp_path / "constructions.log" + return len(log.read_text().splitlines()) if log.exists() else 0 + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_resolves_keys_held_only_by_the_secret_manager(tmp_path, monkeypatch): + """Regression for GH #35239. + + get_config() used to resolve every ``os.environ/`` reference and write the result + back into the config before the secret manager was initialized, so any key that lived + only in the manager became a permanent ``None``. + """ + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, VAULT_BACKED_CONFIG) + + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "master_key": cfg["general_settings"]["master_key"], + "api_key": cfg["model_list"][0]["litellm_params"]["api_key"], + "hosted_keys": litellm._key_management_settings.hosted_keys, + } == { + "master_key": "master-from-vault", + "api_key": "provider-from-vault", + "hosted_keys": ["LITELLM_MASTER_KEY", "MY_PROVIDER_KEY"], + } + + +@pytest.mark.asyncio +async def test_ProxyConfig_load_config_builds_the_secret_manager_exactly_once(tmp_path, monkeypatch): + """The full startup path must not build the manager, then throw it away and build another. + + A discarded client costs a Vault/CyberArk re-auth and leaks a gRPC channel on Google KMS. + """ + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, VAULT_BACKED_CONFIG) + + _router, _model_list, general_settings = await ProxyConfig().load_config( + router=None, config_file_path=config_file_path + ) + + assert { + "constructions": _construction_count(tmp_path), + "master_key": general_settings["master_key"], + } == {"constructions": 1, "master_key": "master-from-vault"} + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_reuses_an_already_initialized_secret_manager(tmp_path, monkeypatch): + """get_config() also runs on management-endpoint request paths. + + Rebuilding the client on every call would re-execute the custom manager module, drop the + Vault/CyberArk token caches, and leak a gRPC channel per request on Google KMS. + """ + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, VAULT_BACKED_CONFIG) + + await ProxyConfig().get_config(config_file_path=config_file_path) + first_client = litellm.secret_manager_client + second = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "client_reused": litellm.secret_manager_client is first_client, + "master_key": second["general_settings"]["master_key"], + } == {"client_reused": True, "master_key": "master-from-vault"} + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_without_key_management_system_leaves_secret_manager_unset( + tmp_path, monkeypatch +): + """No ``key_management_system`` means no manager, an unresolvable reference stays None, and + nothing is warned about: with no manager there is nothing to have been absent from.""" + config_yaml = VAULT_BACKED_CONFIG.replace(" key_management_system: custom\n", "") + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, config_yaml) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "master_key": cfg["general_settings"]["master_key"], + "api_key": cfg["model_list"][0]["litellm_params"]["api_key"], + "client": litellm.secret_manager_client, + "warned_about": [call.args[1] for call in warn.call_args_list], + } == {"master_key": None, "api_key": None, "client": None, "warned_about": []} + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_warns_when_a_reference_is_missing_from_the_secret_manager( + tmp_path, monkeypatch +): + """A reference the manager cannot resolve is logged, instead of silently becoming None.""" + config_yaml = VAULT_BACKED_CONFIG.replace("MY_PROVIDER_KEY", "NOT_IN_VAULT") + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, config_yaml) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "api_key": cfg["model_list"][0]["litellm_params"]["api_key"], + "warned_about": [call.args[1] for call in warn.call_args_list], + } == {"api_key": None, "warned_about": ["os.environ/NOT_IN_VAULT"]} + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_does_not_warn_for_a_name_outside_hosted_keys(tmp_path, monkeypatch): + """``hosted_keys`` is an allowlist, so a name outside it is never looked up in the manager. + + Warning about it would claim a lookup that never happened, on every optional env-only + reference, on every config reload. + """ + config_yaml = VAULT_BACKED_CONFIG.replace("api_key: os.environ/MY_PROVIDER_KEY", "api_key: os.environ/ENV_ONLY") + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, config_yaml) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "api_key": cfg["model_list"][0]["litellm_params"]["api_key"], + "client_is_up": litellm.secret_manager_client is not None, + "warned_about": [call.args[1] for call in warn.call_args_list], + } == {"api_key": None, "client_is_up": True, "warned_about": []} + + +@pytest.mark.asyncio +async def test_ProxyConfig_get_config_does_not_warn_under_write_only_access_mode(tmp_path, monkeypatch): + """``write_only`` means reads never reach the manager, so an absent name is not its fault. + + That mode exists so the manager can store virtual keys while config secrets stay in the + environment, which makes env-only references the expected state rather than an error. + """ + config_yaml = VAULT_BACKED_CONFIG.replace( + " key_management_settings:\n", " key_management_settings:\n access_mode: write_only\n" + ) + config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, config_yaml) + warn = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.verbose_proxy_logger.warning", warn) + + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + + assert { + "master_key": cfg["general_settings"]["master_key"], + "client_is_up": litellm.secret_manager_client is not None, + "warned_about": [call.args[1] for call in warn.call_args_list], + } == {"master_key": None, "client_is_up": True, "warned_about": []} + + # --------------------------------------------------------------------------- # ProxyConfig.update_config_state / get_config_state # --------------------------------------------------------------------------- @@ -2442,6 +2650,43 @@ async def test_ProxyConfig__update_general_settings_updates_max_parallel(monkeyp } +@pytest.mark.asyncio +async def test_ProxyConfig__update_general_settings_applies_db_max_batch_file_size_mb(monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + pc = ProxyConfig() + await pc._update_general_settings({"max_batch_file_size_mb": 5}) + from litellm.proxy import proxy_server as ps + + assert ps.general_settings.get("max_batch_file_size_mb") == 5 + + +@pytest.mark.asyncio +async def test_ProxyConfig__update_general_settings_yaml_max_batch_file_size_mb_wins_over_db(monkeypatch): + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"max_batch_file_size_mb": 3}, + ) + pc = ProxyConfig() + pc._yaml_general_settings_keys = {"max_batch_file_size_mb"} + await pc._update_general_settings({"max_batch_file_size_mb": 5}) + from litellm.proxy import proxy_server as ps + + assert ps.general_settings.get("max_batch_file_size_mb") == 3 + + +@pytest.mark.asyncio +async def test_ProxyConfig__update_general_settings_cleared_db_max_batch_file_size_mb_lifts_cap(monkeypatch): + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"max_batch_file_size_mb": 8}, + ) + pc = ProxyConfig() + await pc._update_general_settings({"max_parallel_requests": 1}) + from litellm.proxy import proxy_server as ps + + assert ps.general_settings.get("max_batch_file_size_mb") is None + + @pytest.mark.asyncio async def test_ProxyConfig__update_general_settings_none_input_noop(): pc = ProxyConfig() diff --git a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py index f41756d2b87..333fd597b49 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py +++ b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py @@ -210,8 +210,10 @@ async def test_rollup_prunes_stale_row_when_config_is_gone(): where = table.delete_many.await_args.kwargs["where"] assert where["date"] == DAY.isoformat() assert where["api_key"] == PTU_SENTINEL_API_KEY - # the row is garbage because this run did not refresh it, not because of a key list + # the row is garbage because this run did not refresh it, and it is reachable at all + # because the run scanned the deployment it belongs to assert "lt" in where["updated_at"] + assert "model" not in where, "a database-only run has no reason to bound the sweep" @pytest.mark.asyncio @@ -705,13 +707,20 @@ class _FakeSentinelTable: async def delete_many(self, where): self.delete_many_calls.append(where) cutoff = where["updated_at"]["lt"] + # honouring "model" matters: a fake that ignored an unknown clause would delete + # the row the prune-scoping test exists to protect and still report a pass + allowed = where.get("model", {}).get("in") doomed = [ k for k, v in self.rows.items() - if k[1] == where["date"] and k[2] == where["api_key"] and v["updated_at"] < cutoff + if k[1] == where["date"] + and k[2] == where["api_key"] + and v["updated_at"] < cutoff + and (allowed is None or k[3] in allowed) ] for k in doomed: del self.rows[k] + return len(doomed) async def find_many(self, where=None): """Read back sentinel rows the way prisma would, honouring api_key and a date range.""" @@ -785,11 +794,11 @@ async def test_an_older_run_cannot_delete_a_newer_runs_row(): @pytest.mark.asyncio async def test_a_later_clean_run_clears_the_row_the_race_left_behind(): - """The race can leave a charge for a since-removed deployment in place for a day; the - next run, seeing only the current config, must sweep it.""" + """The race can leave a charge for a no-longer-priced deployment in place for a day; + the next run, seeing only the current config, must sweep it.""" table = _FakeSentinelTable() ptu = {"ptu_count": 10, "cost_per_ptu_per_hour": 2.0, "team_id": "t"} - stale_key = ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-removed") + stale_key = ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-retired") table.rows[stale_key] = { "ptu_flat_cost": 480.0, "model_group": "retired", @@ -797,7 +806,14 @@ async def test_a_later_clean_run_clears_the_row_the_race_left_behind(): } await run_ptu_flat_cost_rollup( - _prisma_for([_model_row(model_id="dep-live", model_info=ptu)], table), target_date=DAY + _prisma_for( + [ + _model_row(model_id="dep-live", model_info=ptu), + _model_row(model_id="dep-retired", model_info={"team_id": "t"}), + ], + table, + ), + target_date=DAY, ) assert stale_key not in table.rows @@ -1715,16 +1731,19 @@ async def test_a_run_holding_the_lock_still_prunes(): """Losing the sweep entirely would leave stale charges forever, so the guarded path, which is the normal one, keeps it.""" table = _FakeSentinelTable() - table.seed("t", DAY, "dep-gone", 480.0, updated_at=datetime(2020, 1, 1, tzinfo=timezone.utc)) + table.seed("t", DAY, "dep-unpriced", 480.0, updated_at=datetime(2020, 1, 1, tzinfo=timezone.utc)) prisma = _prisma_for( - [_model_row(model_id="dep-live", model_info={"ptu_count": 5, "cost_per_ptu_per_hour": 2.0, "team_id": "t"})], + [ + _model_row(model_id="dep-live", model_info={"ptu_count": 5, "cost_per_ptu_per_hour": 2.0, "team_id": "t"}), + _model_row(model_id="dep-unpriced", model_info={"team_id": "t"}), + ], table, ) await run_scheduled_ptu_rollup(prisma, pod_lock_manager=_pod_lock(acquired=True), target_date=DAY) assert table.delete_many_calls != [] - assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-gone") not in table.rows + assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-unpriced") not in table.rows assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-live") in table.rows @@ -1737,7 +1756,7 @@ async def test_the_prune_cutoff_allows_for_clock_skew_between_hosts(): just_written = datetime.now(timezone.utc) - timedelta(seconds=30) table.seed("t", DAY, "dep-concurrent", 480.0, updated_at=just_written) table.seed("t", DAY, "dep-stale", 480.0, updated_at=datetime.now(timezone.utc) - timedelta(hours=6)) - prisma = _prisma_for([], table) + prisma = _prisma_for([_model_row(model_id="dep-concurrent"), _model_row(model_id="dep-stale")], table) await run_ptu_flat_cost_rollup(prisma, target_date=DAY) @@ -1747,6 +1766,127 @@ async def test_the_prune_cutoff_allows_for_clock_skew_between_hosts(): assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-stale") not in table.rows +@pytest.mark.asyncio +async def test_a_run_pricing_config_cannot_prune_a_row_it_did_not_scan(monkeypatch): + """Staleness alone stops being evidence once two hosts hold different configuration: a + row this run never considered belongs to a deployment another host is pricing from its + own file, and sweeping it drops that charge.""" + table = _FakeSentinelTable() + table.seed("t", DAY, "dep-elsewhere", 480.0, updated_at=datetime(2020, 1, 1, tzinfo=timezone.utc)) + entry = _router_entry(model_id="cfg-here", model_info=dict(_VALID_PTU)) + monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry)) + + await run_scheduled_ptu_rollup(_prisma_for([], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY) + + assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-elsewhere") in table.rows + assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "cfg-here") in table.rows + assert table.delete_many_calls[-1]["model"]["in"] == ("cfg-here",) + + +@pytest.mark.asyncio +async def test_a_deployment_deleted_from_the_table_keeps_the_day_it_was_charged(monkeypatch): + """The accepted cost of bounding the prune, driven through the sequence that produces + it: charge the day while the deployment exists, remove it, run the day again. Nothing + scans it now, so nothing may judge its row, and the amount it was billed stands.""" + table = _FakeSentinelTable() + ptu = {"ptu_count": 5, "cost_per_ptu_per_hour": 2.0, "team_id": "t"} + live_row = _model_row(model_id="dep-live", model_info=ptu) + doomed_row = _model_row(model_id="dep-doomed", model_info=ptu) + charged_key = ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-doomed") + monkeypatch.setattr( + ptu_rollup, "_running_router", lambda: _router_holding(_router_entry(model_id="cfg", model_info=dict(ptu))) + ) + + await run_scheduled_ptu_rollup( + _prisma_for([live_row, doomed_row], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY + ) + billed = table.rows[charged_key]["ptu_flat_cost"] + table.rows[charged_key]["updated_at"] = datetime(2020, 1, 1, tzinfo=timezone.utc) + + await run_scheduled_ptu_rollup( + _prisma_for([live_row], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY + ) + + assert table.rows[charged_key]["ptu_flat_cost"] == billed + assert "dep-doomed" not in table.delete_many_calls[-1]["model"]["in"] + + +@pytest.mark.asyncio +async def test_a_database_only_run_sweeps_exactly_as_it_did_before(): + """The bound exists for charges another host declares. A deployment nobody declares any + more still has its leftover row swept, which is what the table-only sweep always did.""" + table = _FakeSentinelTable() + table.seed("t", DAY, "dep-gone", 480.0, updated_at=datetime(2020, 1, 1, tzinfo=timezone.utc)) + prisma = _prisma_for( + [_model_row(model_id="dep-live", model_info={"ptu_count": 5, "cost_per_ptu_per_hour": 2.0, "team_id": "t"})], + table, + ) + + await run_scheduled_ptu_rollup(prisma, pod_lock_manager=_pod_lock(acquired=True), target_date=DAY) + + assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-gone") not in table.rows + assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "dep-live") in table.rows + + +@pytest.mark.asyncio +async def test_every_deployment_that_prices_is_inside_the_set_that_bounds_the_prune(): + """The bound has to be a superset of what the same run wrote, or a run's own charge + could fall outside its own delete filter and never be reconciled.""" + table = _FakeSentinelTable() + prisma = _prisma_for( + [ + _model_row(model_id="dep-a", model_info={"ptu_count": 5, "cost_per_ptu_per_hour": 2.0, "team_id": "t"}), + _model_row(model_id="dep-b", model_info={"ptu_count": 9, "cost_per_ptu_per_hour": 1.0, "team_id": "u"}), + _model_row(model_id="dep-unpriced", model_info={"team_id": "t"}), + ], + table, + ) + + loaded = await ptu_rollup._load_ptu_models(prisma) + + assert {model.model_id for model in loaded.models} <= loaded.scanned_ids + assert loaded.scanned_ids == {"dep-a", "dep-b", "dep-unpriced"} + + +@pytest.mark.asyncio +async def test_a_priced_deployment_is_in_the_bound_even_with_an_id_the_scan_skips(): + """The bound is built by construction rather than by coincidence. The row scan drops a + falsy id while the parser still prices one, and a charge outside its own run's delete + filter could never be reconciled by any later run.""" + prisma = _prisma_for( + [_model_row(model_id="", model_info={"ptu_count": 5, "cost_per_ptu_per_hour": 2.0, "team_id": "t"})], + _FakeSentinelTable(), + ) + + loaded = await ptu_rollup._load_ptu_models(prisma) + + assert {model.model_id for model in loaded.models} <= loaded.scanned_ids + + +@pytest.mark.asyncio +async def test_the_prune_splits_the_id_set_across_statements(monkeypatch): + """Every id is one bind variable and the server refuses a statement carrying more than + 32767, so a proxy with that many deployments would fail the prune outright, and with it + the rest of the scheduled run.""" + monkeypatch.setattr(ptu_rollup, "_PRUNE_ID_CHUNK_SIZE", 2) + table = _FakeSentinelTable() + ptu = {"ptu_count": 5, "cost_per_ptu_per_hour": 2.0, "team_id": "t"} + deployments = [_model_row(model_id=f"dep-{n}", model_info=ptu) for n in range(4)] + monkeypatch.setattr( + ptu_rollup, "_running_router", lambda: _router_holding(_router_entry(model_id="dep-4", model_info=dict(ptu))) + ) + table.seed("t", DAY, "dep-3", 480.0, updated_at=datetime(2020, 1, 1, tzinfo=timezone.utc)) + + await run_scheduled_ptu_rollup( + _prisma_for(deployments, table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY + ) + + chunks = [call["model"]["in"] for call in table.delete_many_calls] + assert len(chunks) == 3 + assert all(len(chunk) <= 2 for chunk in chunks) + assert sorted(i for chunk in chunks for i in chunk) == [f"dep-{n}" for n in range(5)] + + @pytest.mark.asyncio async def test_scheduled_rollup_writes_nothing_when_ptu_attribution_is_disabled(monkeypatch): """Startup already skips scheduling the cron, so this guards the function itself: a @@ -1760,3 +1900,148 @@ async def test_scheduled_rollup_writes_nothing_when_ptu_attribution_is_disabled( assert result is None assert table.rows == {} assert table.upsert_keys == [] + + +# --- config.yaml deployments reach the rollup through the router ---------------- + + +def _router_holding(*entries): + """A stand-in for the proxy's router, carrying whatever model_list is passed.""" + return types.SimpleNamespace(model_list=list(entries)) + + +@pytest.mark.asyncio +async def test_a_config_declared_deployment_is_priced(monkeypatch): + """The whole point. A PTU deployment the proxy only knows from config.yaml is not in + LiteLLM_ProxyModelTable, so a DB-only scan bills the provider's reservation to nobody.""" + entry = _router_entry(model_id="cfg-1", model_name="gpt-4o-ptu", model_info=dict(_VALID_PTU)) + monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry)) + + loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable())) + + assert [(m.model_id, m.model_name, m.team_id) for m in loaded.models] == [("cfg-1", "gpt-4o-ptu", "t")] + assert "cfg-1" in loaded.scanned_ids + + +@pytest.mark.asyncio +async def test_a_database_backed_router_entry_is_not_counted_twice(monkeypatch): + """Every deployment loaded from the table is also in the router, flagged db_model. Pricing + both copies would write two charges for one reservation.""" + row = _model_row(model_id="db-1", model_info=dict(_VALID_PTU)) + mirrored = _router_entry(model_id="db-1", model_info={**_VALID_PTU, "db_model": True}) + monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(mirrored)) + + loaded = await ptu_rollup._load_ptu_models(_prisma_for([row], _FakeSentinelTable())) + + assert [m.model_id for m in loaded.models] == ["db-1"] + + +@pytest.mark.asyncio +async def test_a_router_entry_sharing_an_id_with_the_table_is_priced_once(monkeypatch): + """db_model is data the router carries rather than something this module controls, so the + id anti-join is what actually maps onto the failure: two charges under one id.""" + row = _model_row(model_id="both-1", model_info=dict(_VALID_PTU)) + unflagged = _router_entry(model_id="both-1", model_info=dict(_VALID_PTU)) + monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(unflagged)) + + loaded = await ptu_rollup._load_ptu_models(_prisma_for([row], _FakeSentinelTable())) + + assert [m.model_id for m in loaded.models] == ["both-1"] + + +@pytest.mark.asyncio +async def test_a_client_credential_clone_is_not_priced(monkeypatch): + """Supplying an api_key on a request mints a clone of the deployment under a fresh id, + carrying the source's PTU config. Pricing it bills one reservation per distinct caller key.""" + source = _router_entry(model_id="cfg-1", model_info=dict(_VALID_PTU)) + clone = _router_entry(model_id="cfg-1-clone", model_info={**_VALID_PTU, "original_model_id": "cfg-1"}) + monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(source, clone)) + + loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable())) + + assert [m.model_id for m in loaded.models] == ["cfg-1"] + + +@pytest.mark.asyncio +async def test_a_config_deployment_without_ptu_config_is_scanned_but_not_priced(monkeypatch): + """It has to stay in the scanned set or its leftover sentinel rows become unprunable.""" + entry = _router_entry(model_id="cfg-plain", model_info={"team_id": "t"}) + monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry)) + + loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable())) + + assert loaded.models == () + assert "cfg-plain" in loaded.scanned_ids + + +@pytest.mark.asyncio +async def test_no_router_in_the_process_prices_the_database_alone(monkeypatch): + """The rollup is importable and callable outside a running proxy.""" + monkeypatch.setattr(ptu_rollup, "_running_router", lambda: None) + + loaded = await ptu_rollup._load_ptu_models( + _prisma_for([_model_row(model_id="db-1", model_info=dict(_VALID_PTU))], _FakeSentinelTable()) + ) + + assert [m.model_id for m in loaded.models] == ["db-1"] + + +@pytest.mark.asyncio +async def test_a_config_deployment_is_charged_end_to_end(monkeypatch): + """Through the scheduled entry point, so the charge lands in a sentinel row rather than + stopping at the loader.""" + table = _FakeSentinelTable() + entry = _router_entry(model_id="cfg-1", model_name="gpt-4o-ptu", model_info=dict(_VALID_PTU)) + monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(entry)) + + await run_scheduled_ptu_rollup(_prisma_for([], table), pod_lock_manager=_pod_lock(acquired=True), target_date=DAY) + + assert ("t", DAY.isoformat(), PTU_SENTINEL_API_KEY, "cfg-1") in table.rows + + +@pytest.mark.asyncio +async def test_a_stale_database_backed_router_entry_is_not_treated_as_config(monkeypatch): + """The reconcile can leave a deployment on the router after its row is gone. The id + anti-join cannot see that one, so the flag is what keeps it from being priced as though + config.yaml had declared it.""" + stale = _router_entry(model_id="db-gone", model_info={**_VALID_PTU, "db_model": True}) + monkeypatch.setattr(ptu_rollup, "_running_router", lambda: _router_holding(stale)) + + loaded = await ptu_rollup._load_ptu_models(_prisma_for([], _FakeSentinelTable())) + + assert loaded.models == () + + +def test_the_router_lookup_reads_the_proxys_own_global(): + """Every other config test replaces this helper, so without one test driving the real + body a typo in the module path or the attribute name leaves the whole feature dead in + production with the suite still green.""" + import sys + import types as _types + + assert ptu_rollup._running_router() is None or "litellm.proxy.proxy_server" in sys.modules + + sentinel = object() + stub = _types.SimpleNamespace(llm_router=sentinel) + real = sys.modules.get("litellm.proxy.proxy_server") + sys.modules["litellm.proxy.proxy_server"] = stub + try: + assert ptu_rollup._running_router() is sentinel + del stub.llm_router + assert ptu_rollup._running_router() is None + finally: + if real is None: + del sys.modules["litellm.proxy.proxy_server"] + else: + sys.modules["litellm.proxy.proxy_server"] = real + + +def test_the_router_lookup_returns_none_outside_a_proxy(): + import sys + + real = sys.modules.pop("litellm.proxy.proxy_server", None) + try: + assert ptu_rollup._running_router() is None + finally: + if real is not None: + sys.modules["litellm.proxy.proxy_server"] = real diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 1435547c434..9006288bdae 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -841,6 +841,44 @@ def test_the_baseline_is_priced_on_the_basis_the_request_was_billed_at(basis, ex assert reported == pytest.approx(expected_multiplier * baseline - served) +def test_the_baseline_is_priced_on_the_vertex_location_the_request_was_billed_at(monkeypatch): + """A request served from a regional Vertex endpoint was billed with the + regional-endpoint uplift, so the counterfactual single-model operator would + have paid it too. The served model carries no uplift field, so only the + baseline moves with the recorded location.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + gemini = litellm.get_model_info("gemini-3.5-flash", "vertex_ai") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + assert gemini.get("regional_endpoint_uplift_multiplier") == 1.1 + assert haiku.get("regional_endpoint_uplift_multiplier") is None, "served model must not move with the basis" + + usage = _usage(fresh=20_000, cached=0, written=0, out=1_000) + served = 20_000 * haiku["input_cost_per_token"] + 1_000 * haiku["output_cost_per_token"] + baseline = 20_000 * gemini["input_cost_per_token"] + 1_000 * gemini["output_cost_per_token"] + + regional = compute_autorouter_savings( + baseline_model="vertex_ai/gemini-3.5-flash", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + conversation_continuing=False, + cost_breakdown=_breakdown(served, vertex_location="us-east5"), + ) + global_endpoint = compute_autorouter_savings( + baseline_model="vertex_ai/gemini-3.5-flash", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + conversation_continuing=False, + cost_breakdown=_breakdown(served, vertex_location="global"), + ) + + assert regional == pytest.approx(1.1 * baseline - served) + assert global_endpoint == pytest.approx(baseline - served) + + def test_a_baseline_recorded_on_the_decision_turns_the_driver_on(): """An operator who configures nothing still sees the driver work.""" result = compute_savings_spend( diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index e5add059260..9710dc44e99 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -3164,3 +3164,80 @@ def test_batch_cost_row_id_is_stable_across_repeated_accounting(): ] assert ids[0] == ids[1] == "batch_same_batch_cost" + + +def _make_failed_request_standard_logging_payload() -> StandardLoggingPayload: + base: Final = _make_standard_logging_payload_with_usage_object(usage_object={}) + return cast( + StandardLoggingPayload, + { + **base, + "status": "failure", + "call_type": "aresponses", + "model_id": "mid-123", + "model_group": "group-x", + "api_base": "https://api.openai.com/v1/responses", + "custom_llm_provider": "openai", + }, + ) + + +def test_get_logging_payload_failed_request_falls_back_to_standard_logging_payload(): + """Failed-request kwargs from the proxy failure hook carry no deployment info + (LIT-5795), so the attribution columns must come from the failure-time + standard_logging_object.""" + payload = get_logging_payload( + kwargs={ + "model": "group-x", + "litellm_params": {"metadata": {"user_api_key": "test-key", "status": "failure"}}, + "standard_logging_object": _make_failed_request_standard_logging_payload(), + }, + response_obj={}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + assert payload["model_id"] == "mid-123" + assert payload["model_group"] == "group-x" + assert payload["api_base"] == "https://api.openai.com/v1/responses" + assert payload["custom_llm_provider"] == "openai" + + +def test_get_logging_payload_request_kwargs_win_over_standard_logging_payload(): + payload = get_logging_payload( + kwargs={ + "model": "group-y", + "custom_llm_provider": "anthropic", + "litellm_params": { + "api_base": "https://kwargs.example.com", + "metadata": { + "user_api_key": "test-key", + "model_group": "kwargs-group", + "model_info": {"id": "kwargs-mid"}, + }, + }, + "standard_logging_object": _make_failed_request_standard_logging_payload(), + }, + response_obj={}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + assert payload["model_id"] == "kwargs-mid" + assert payload["model_group"] == "kwargs-group" + assert payload["api_base"] == "https://kwargs.example.com" + assert payload["custom_llm_provider"] == "anthropic" + + +def test_get_logging_payload_failed_request_without_standard_logging_payload_leaves_fields_empty(): + payload = get_logging_payload( + kwargs={ + "model": "group-x", + "litellm_params": {"metadata": {"user_api_key": "test-key", "status": "failure"}}, + }, + response_obj={}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + assert payload["model_id"] == "" + assert payload["model_group"] == "" + assert payload["api_base"] == "" + assert payload["custom_llm_provider"] == "" diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index d6cf0e30139..07877514b69 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -468,6 +468,23 @@ class TestPostCallFailureHookLiftsRecoveredPartialSpend: assert request_data["response_cost"] == 3.5e-05 assert "litellm_logging_obj" not in request_data + @pytest.mark.asyncio + async def test_recovered_usage_without_cost_clobbers_client_cost_with_zero(self): + from litellm.types.utils import Usage + + recovered_usage = Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31) + logging_obj = MagicMock() + logging_obj.model_call_details = {"combined_usage_object": recovered_usage} + request_data = { + "litellm_logging_obj": logging_obj, + "response_cost": 999.0, + "metadata": {}, + } + await self._run(request_data) + + assert request_data["combined_usage_object"] is recovered_usage + assert request_data["response_cost"] == 0.0 + @pytest.mark.asyncio async def test_no_recovered_usage_is_noop(self): logging_obj = MagicMock() @@ -478,6 +495,111 @@ class TestPostCallFailureHookLiftsRecoveredPartialSpend: assert "response_cost" not in request_data +class TestPostCallFailureHookLiftsStandardLoggingObject: + """Failure callbacks read standard_logging_object from request_data, but + post_call_failure_hook pops litellm_logging_obj before they run. The hook + must lift the logging obj's standard_logging_object onto request_data so + failed-request spend logs keep deployment attribution (LIT-5795). + """ + + async def _run(self, request_data): + from unittest.mock import AsyncMock, patch + + from litellm.proxy._types import UserAPIKeyAuth + + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [] + with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): + await proxy_logging_obj.post_call_failure_hook( + request_data=request_data, + original_exception=Exception("boom"), + user_api_key_dict=UserAPIKeyAuth(), + ) + + @pytest.mark.asyncio + async def test_lifts_standard_logging_object(self): + sl_object = {"model_id": "mid-123", "model_group": "group-x"} + logging_obj = MagicMock() + logging_obj.model_call_details = {"standard_logging_object": sl_object} + request_data = {"litellm_logging_obj": logging_obj, "metadata": {}} + await self._run(request_data) + assert request_data["standard_logging_object"] is sl_object + assert "litellm_logging_obj" not in request_data + + @pytest.mark.asyncio + async def test_logging_obj_value_overwrites_preexisting_key(self): + authoritative = {"model_id": "from-logging-obj"} + logging_obj = MagicMock() + logging_obj.model_call_details = {"standard_logging_object": authoritative} + request_data = { + "litellm_logging_obj": logging_obj, + "standard_logging_object": {"model_id": "client-injected"}, + "metadata": {}, + } + await self._run(request_data) + assert request_data["standard_logging_object"] is authoritative + + @pytest.mark.asyncio + async def test_client_supplied_key_is_stripped_when_logging_obj_supplies_none(self): + spoofed = {"model_id": "client-injected"} + request_data = {"standard_logging_object": spoofed, "metadata": {}} + await self._run(request_data) + assert "standard_logging_object" not in request_data + + logging_obj = MagicMock() + logging_obj.model_call_details = {} + request_data_with_obj = { + "litellm_logging_obj": logging_obj, + "standard_logging_object": spoofed, + "metadata": {}, + } + await self._run(request_data_with_obj) + assert "standard_logging_object" not in request_data_with_obj + + @pytest.mark.asyncio + async def test_pass_through_failure_never_relifts_client_supplied_key(self): + from datetime import datetime + from unittest.mock import AsyncMock, patch + + from fastapi import HTTPException + + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy._types import UserAPIKeyAuth + + logging_obj = Logging( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="test-function-id", + ) + request_data = { + "litellm_logging_obj": logging_obj, + "standard_logging_object": {"model_id": "client-injected"}, + "metadata": {}, + } + proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging_obj.alert_types = [] + with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): + await proxy_logging_obj.post_call_failure_hook( + request_data=request_data, + original_exception=HTTPException(status_code=401, detail="unauthorized"), + user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"), + ) + assert "standard_logging_object" not in request_data + assert "standard_logging_object" not in logging_obj.model_call_details + + @pytest.mark.asyncio + async def test_no_standard_logging_object_is_noop(self): + logging_obj = MagicMock() + logging_obj.model_call_details = {} + request_data = {"litellm_logging_obj": logging_obj, "metadata": {}} + await self._run(request_data) + assert "standard_logging_object" not in request_data + + class TestPostCallFailureHookEstimatesDispatchedInputTokens: """A non-stream request that failed after dispatch (timeout, provider error) consumed provider-billed input tokens but recovered no usage. diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py index dd21bbc9e8a..7057a112c83 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py @@ -83,8 +83,8 @@ async def test_update_end_user_spend_retries_on_connect_error( mock_prisma_client: Any, monkeypatch: pytest.MonkeyPatch ) -> None: """``DB_RETRY_SAFE_ERROR_TYPES`` (ConnectError, statements provably never - sent) retries with backoff; once retries are exhausted the original - exception bubbles up via ``_raise_failed_update_spend_exception``. + sent) retries with jittered backoff; once retries are exhausted the + original exception bubbles up via ``_raise_failed_update_spend_exception``. """ import httpx import litellm.proxy.utils as utils_mod @@ -107,7 +107,8 @@ async def test_update_end_user_spend_retries_on_connect_error( proxy_logging_obj=proxy_logging, end_user_list_transactions={"u": 1.0}, ) - assert sleeps == [1.0] + assert len(sleeps) == 1 + assert 1.0 <= sleeps[0] <= 2.0 @pytest.mark.asyncio @@ -149,6 +150,79 @@ async def test_update_end_user_spend_non_connection_error_raises_immediately( ) +def _end_user_deadlock_error() -> Exception: + from prisma.errors import RawQueryError + + return RawQueryError(data={"user_facing_error": {"error_code": "P2034", "meta": {"table": "LiteLLM_EndUserTable"}}}) + + +def _failing_tx(error: Exception) -> Any: + tx = MagicMock() + tx.__aenter__ = AsyncMock(side_effect=error) + tx.__aexit__ = AsyncMock(return_value=False) + return tx + + +@pytest.mark.asyncio +async def test_update_end_user_spend_retries_on_deadlock_then_commits( + mock_prisma_client: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """Regression for #27989: a Postgres deadlock (P2034/40P01) on the end-user + spend batch is retried with jittered backoff and the increments land, + instead of raising immediately and dropping the flushed spend.""" + sleeps: list[float] = [] + + async def _fake_sleep(seconds: float) -> None: + sleeps.append(seconds) + + monkeypatch.setattr(asyncio, "sleep", _fake_sleep) + + batcher = MagicMock() + batcher.litellm_endusertable.upsert = MagicMock() + transaction = MagicMock() + transaction.batch_ = lambda: _AsyncCM(batcher) + mock_prisma_client.db.tx = MagicMock(side_effect=[_failing_tx(_end_user_deadlock_error()), _AsyncCM(transaction)]) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + + await ProxyUpdateSpend.update_end_user_spend( + n_retry_times=3, + prisma_client=mock_prisma_client, + proxy_logging_obj=proxy_logging, + end_user_list_transactions={"end-user-1": 0.25}, + ) + + assert mock_prisma_client.db.tx.call_count == 2 + batcher.litellm_endusertable.upsert.assert_called_once() + assert batcher.litellm_endusertable.upsert.call_args.kwargs["where"] == {"user_id": "end-user-1"} + assert len(sleeps) == 1 + assert 1.0 <= sleeps[0] <= 2.0 + proxy_logging.failure_handler.assert_not_called() + + +@pytest.mark.asyncio +async def test_update_end_user_spend_raises_after_exhausting_deadlock_retries( + mock_prisma_client: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + from prisma.errors import RawQueryError + + monkeypatch.setattr(asyncio, "sleep", AsyncMock(return_value=None)) + mock_prisma_client.db.tx = MagicMock(side_effect=lambda timeout: _failing_tx(_end_user_deadlock_error())) + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + + with pytest.raises(RawQueryError): + await ProxyUpdateSpend.update_end_user_spend( + n_retry_times=2, + prisma_client=mock_prisma_client, + proxy_logging_obj=proxy_logging, + end_user_list_transactions={"end-user-1": 0.25}, + ) + + assert mock_prisma_client.db.tx.call_count == 3 + + @pytest.mark.asyncio async def test_update_spend_logs_writes_batches_via_create_many( mock_prisma_client: Any, make_spend_log_row: Any diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index fef8c2d1349..aae053c2e8e 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -419,6 +419,107 @@ class TestLiteLLMCompletionResponsesConfig: ] assert len(message_items) == 2, "Should have two message items" + def test_signature_only_thinking_block_still_emits_reasoning_item(self): + response = ModelResponse( + id="test-id", + created=1234567890, + model="test-model", + object="chat.completion", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content="10", + role="assistant", + reasoning_content="", + thinking_blocks=[ + {"type": "thinking", "thinking": "", "signature": "signature-payload"} + ], + ), + ) + ], + ) + + responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Test input", + responses_api_request={}, + chat_completion_response=response, + ) + + reasoning_items = [ + item for item in responses_api_response.output if item.type == "reasoning" + ] + assert len(reasoning_items) == 1, "Signature-only thinking should still surface a reasoning item" + assert reasoning_items[0].content == [] + assert "signature-payload" in reasoning_items[0].encrypted_content + + def test_redacted_thinking_block_preserved_as_encrypted_content(self): + response = ModelResponse( + id="test-id", + created=1234567890, + model="test-model", + object="chat.completion", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content="10", + role="assistant", + thinking_blocks=[{"type": "redacted_thinking", "data": "redacted-payload"}], + ), + ) + ], + ) + + responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Test input", + responses_api_request={}, + chat_completion_response=response, + ) + + reasoning_items = [ + item for item in responses_api_response.output if item.type == "reasoning" + ] + assert len(reasoning_items) == 1 + assert "redacted-payload" in reasoning_items[0].encrypted_content + + def test_visible_thinking_keeps_text_and_signature(self): + response = ModelResponse( + id="test-id", + created=1234567890, + model="test-model", + object="chat.completion", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message( + content="10", + role="assistant", + reasoning_content="counting the primes", + thinking_blocks=[ + {"type": "thinking", "thinking": "counting the primes", "signature": "sig"} + ], + ), + ) + ], + ) + + responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="Test input", + responses_api_request={}, + chat_completion_response=response, + ) + + reasoning_items = [ + item for item in responses_api_response.output if item.type == "reasoning" + ] + assert len(reasoning_items) == 1 + assert reasoning_items[0].content[0].text == "counting the primes" + assert "sig" in reasoning_items[0].encrypted_content + def test_transform_chat_completion_response_status_with_stop(self): """ Test that transforming a chat completion response with 'stop' finish_reason @@ -2537,10 +2638,10 @@ class TestUsageTransformation: assert response_usage.output_tokens_details.text_tokens == 50 assert response_usage.output_tokens_details.image_tokens == 100 - def test_reasoning_tokens_not_forced_to_zero_when_absent(self): - # Regression: previously the else branch wrote reasoning_tokens=0 even when - # completion_tokens_details had no reasoning (reasoning_tokens=None). That caused - # the proxy to always report reasoning_tokens=0 for non-thinking responses. + def test_reasoning_tokens_fall_back_to_zero_when_absent(self): + # The OpenAI SDK's ResponseUsage requires output_tokens_details.reasoning_tokens + # as an int, so an absent count degrades to 0 on the responses wire instead of + # dropping output_tokens_details and breaking SDK clients. usage = Usage( prompt_tokens=10, completion_tokens=50, @@ -2571,7 +2672,8 @@ class TestUsageTransformation: ) assert response_usage.output_tokens_details is not None - assert response_usage.output_tokens_details.reasoning_tokens is None + assert response_usage.output_tokens_details.reasoning_tokens == 0 + assert response_usage.output_tokens_details.text_tokens == 50 def test_reasoning_tokens_preserved_when_thinking_occurred(self): # Regression: reasoning_tokens must survive the chat->responses translation diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index e1e8d9553b3..006fb79b08f 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -267,6 +267,91 @@ class TestReasoningMarkerScoring: # 2+ reasoning markers should force REASONING tier assert tier == ComplexityTier.REASONING + def test_reasoning_override_does_not_rescue_a_simple_score(self, complexity_router): + """Reasoning markers on an otherwise trivial prompt must not reach REASONING.""" + prompt = "hi, step by step, pros and cons" + tier, score, signals = complexity_router.classify(prompt) + assert score < complexity_router.config.tier_boundaries["simple_medium"] + assert any("step by step" in s and "pros and cons" in s for s in signals) + assert tier == ComplexityTier.SIMPLE + + def test_reasoning_override_applies_at_the_simple_medium_boundary(self, complexity_router): + """A score sitting exactly on simple_medium is not SIMPLE, so the override still promotes it.""" + prompt = ( + "Give me the pros and cons, step by step, of moving our checkout service " + "to an event-driven architecture." + ) + tier, score, signals = complexity_router.classify(prompt) + assert score == complexity_router.config.tier_boundaries["simple_medium"] + assert tier == ComplexityTier.REASONING + + def test_explicit_zero_floor_restores_the_unconditional_override(self, mock_router_instance, basic_config): + """0 is a real floor, not an absent one, so the markers alone promote again.""" + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "reasoning_override_min_score": 0.0}, + ) + tier, score, _ = router.classify("hi, step by step, pros and cons") + assert score < router.config.tier_boundaries["simple_medium"] + assert tier == ComplexityTier.REASONING + + def test_floor_defaults_to_simple_medium_and_follows_it(self, mock_router_instance, basic_config): + """Unset tracks simple_medium, so moving that boundary moves the floor with it.""" + prompt = ( + "Give me the pros and cons, step by step, of moving our checkout service " + "to an event-driven architecture." + ) + low = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "tier_boundaries": {"simple_medium": 0.20}}, + ) + high = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "tier_boundaries": {"simple_medium": 0.30}}, + ) + assert low._effective_reasoning_override_min_score() == 0.20 + assert high._effective_reasoning_override_min_score() == 0.30 + assert low.classify(prompt)[0] == ComplexityTier.REASONING + assert high.classify(prompt)[0] != ComplexityTier.REASONING + + def test_explicit_floor_overrides_the_boundary(self, mock_router_instance, basic_config): + """A configured floor decides the override, not simple_medium.""" + prompt = ( + "Give me the pros and cons, step by step, of moving our checkout service " + "to an event-driven architecture." + ) + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **basic_config, + "tier_boundaries": {"simple_medium": 0.10}, + "reasoning_override_min_score": 0.90, + }, + ) + tier, score, _ = router.classify(prompt) + assert score > router.config.tier_boundaries["simple_medium"] + assert router._effective_reasoning_override_min_score() == 0.90 + assert tier != ComplexityTier.REASONING + + def test_configured_floor_is_applied_with_greater_or_equal(self, mock_router_instance, basic_config): + """A score landing exactly on the configured floor still promotes.""" + prompt = ( + "Give me the pros and cons, step by step, of moving our checkout service " + "to an event-driven architecture." + ) + router = ComplexityRouter( + model_name="test-complexity-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "reasoning_override_min_score": 0.25}, + ) + tier, score, _ = router.classify(prompt) + assert score == 0.25 + assert tier == ComplexityTier.REASONING + def test_system_prompt_reasoning_not_counted(self, complexity_router): """Reasoning markers in system prompt should not count for override.""" user_prompt = "What is 2+2?" @@ -2081,6 +2166,38 @@ class TestRouterPreRoutingAliasOverrides: assert request_kwargs["drop_params"] is True assert request_kwargs["cache_control_injection_points"] == [{"location": "message", "role": "system"}] + @pytest.mark.asyncio + async def test_tier_litellm_params_are_applied_before_deployment_selection(self): + router = Router( + model_list=[ + { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": { + "SIMPLE": { + "model_name": "gpt-4o-mini", + "litellm_params": {"reasoning_effort": "xhigh"}, + } + } + }, + }, + }, + {"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}}, + ] + ) + request_kwargs: Dict = {"reasoning_effort": "low"} + + deployment = await router.async_get_available_deployment( + model="smart-router", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + + assert deployment["model_name"] == "gpt-4o-mini" + assert request_kwargs["reasoning_effort"] == "xhigh" + @pytest.mark.asyncio async def test_alias_custom_pricing_is_not_applied_to_request_kwargs(self): """Custom pricing on the alias prices the alias, not the tier deployment @@ -3766,7 +3883,7 @@ class TestSessionAffinity: cache.async_set_cache.assert_called_once() call_kwargs = cache.async_set_cache.call_args.kwargs assert call_kwargs["ttl"] == 120 - assert call_kwargs["value"] == "gpt-4o-mini" + assert call_kwargs["value"] == {"model": "gpt-4o-mini", "tier": "SIMPLE"} @pytest.mark.asyncio async def test_ttl_refreshed_on_cache_hit(self, mock_router_instance, basic_config): @@ -3790,7 +3907,7 @@ class TestSessionAffinity: assert result.model == "o1-preview" cache.async_set_cache.assert_called_once() call_kwargs = cache.async_set_cache.call_args.kwargs - assert call_kwargs["value"] == "o1-preview" + assert call_kwargs["value"] == {"model": "o1-preview", "tier": "REASONING"} assert call_kwargs["ttl"] == 90 @pytest.mark.asyncio @@ -5460,12 +5577,14 @@ class TestRedactedLoggingDropsPromptText: "tier_boundaries": {"simple_medium": 0.15, "medium_complex": 0.35, "complex_reasoning": 0.6}, "classifier_model": "claude-haiku", "escalated": True, + "tier_litellm_params": {"reasoning_effort": "xhigh"}, "signals": ["code (python)"], "matched_keyword": "deploy to k8s", "escalation_keyword": "LITELLM ESCALATE", } kept = Router._redact_prompt_text_if_needed(request_kwargs={}, routing_decision=full) assert set(full) - set(kept) == {"signals", "matched_keyword", "escalation_keyword"} + assert kept["tier_litellm_params"] == {"reasoning_effort": "xhigh"} @pytest.mark.asyncio async def test_redaction_via_request_header_is_honored(self): @@ -7188,8 +7307,6 @@ class TestClassificationRubrics: }, ) assert config.classifier_llm_config.system_prompt == "Grade the data sensitivity of the request." - - def _custom_tier_config(**overrides) -> Dict: """A valid operator-defined tier set: two built-in names plus one custom tier.""" return { @@ -8038,3 +8155,243 @@ class TestPlanModeTierFloor: assert result.model == "gpt-4o" assert result.routing_decision is not None assert result.routing_decision["tier"] == "MEDIUM" +def test_tier_model_params_are_normalized_without_changing_model_pools(): + config = ComplexityRouterConfig( + tiers={ + "SIMPLE": "mini", + "REASONING": [ + {"model_name": "opus", "litellm_params": {"reasoning_effort": "xhigh"}}, + "abc", + ], + } + ) + + assert config.tiers == {"SIMPLE": "mini", "REASONING": ["opus", "abc"]} + assert config.tier_model_configs["REASONING"][0].litellm_params == {"reasoning_effort": "xhigh"} + rebuilt = ComplexityRouterConfig.model_validate(config.model_dump()) + assert rebuilt.tier_model_configs["REASONING"][0].litellm_params == {"reasoning_effort": "xhigh"} + + +def test_tier_model_params_accept_a_single_object(): + config = ComplexityRouterConfig( + tiers={"REASONING": {"model_name": "opus", "litellm_params": {"thinking": {"type": "enabled"}}}} + ) + + assert config.tiers == {"REASONING": "opus"} + assert config.tier_model_configs["REASONING"][0].model_name == "opus" + + +@pytest.mark.parametrize( + "tiers", + [ + {"REASONING": [{"litellm_params": {"reasoning_effort": "xhigh"}}]}, + ], +) +def test_tier_model_params_reject_malformed_entries(tiers): + with pytest.raises(ValidationError): + ComplexityRouterConfig(tiers=tiers) + + +def test_tier_model_params_reject_duplicate_models(): + with pytest.raises(ValidationError, match="duplicate model_name"): + ComplexityRouterConfig( + tiers={ + "REASONING": [ + {"model_name": "opus", "litellm_params": {"reasoning_effort": "xhigh"}}, + {"model_name": "opus", "litellm_params": {"reasoning_effort": "low"}}, + ] + } + ) + + +def test_non_adaptive_empty_tier_pool_remains_valid(): + config = ComplexityRouterConfig(tiers={"SIMPLE": []}) + assert config.tiers == {"SIMPLE": []} + + +def test_adaptive_empty_tier_pool_is_rejected(): + with pytest.raises(ValidationError, match="adaptive=True"): + ComplexityRouterConfig(adaptive=True, tiers={"SIMPLE": []}) + + +def test_tier_model_params_are_used_by_pools_and_savings_baseline(mock_router_instance): + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": { + "SIMPLE": "mini", + "REASONING": [{"model_name": "opus", "litellm_params": {"reasoning_effort": "xhigh"}}, "abc"], + } + }, + ) + + assert router._tier_pools() == {"SIMPLE": ["mini"], "REASONING": ["opus", "abc"]} + assert router._hardest_tier_models() == ("opus", "abc") + assert router._litellm_params_for_model(ComplexityTier.REASONING, "opus") == {"reasoning_effort": "xhigh"} + + +@pytest.mark.asyncio +async def test_tier_model_params_reach_the_hook_response_and_override_client_values(mock_router_instance): + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": { + "REASONING": { + "model_name": "opus", + "litellm_params": {"reasoning_effort": "xhigh", "max_tokens": 512}, + } + }, + "keyword_tier_rules": [{"keywords": ["reason carefully"], "tier": "REASONING"}], + }, + ) + request_kwargs = {"reasoning_effort": "low", "metadata": {}} + + response = await router.async_pre_routing_hook( + model="test-router", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "reason carefully about this"}], + ) + + assert response is not None + assert response.litellm_params == {"reasoning_effort": "xhigh", "max_tokens": 512} + assert response.routing_decision is not None + assert response.routing_decision["tier_litellm_params"] == response.litellm_params + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["classification", "keyword", "session"]) +async def test_tier_params_mask_credentials_in_routing_decision(route, mock_router_instance): + params = {"reasoning_effort": "xhigh", "api_key": "secret-tier-key"} + config = { + "tiers": { + tier.value: {"model_name": "opus", "litellm_params": params} + for tier in ComplexityTier + }, + "keyword_tier_rules": [{"keywords": ["reason carefully"], "tier": "REASONING"}] + if route == "keyword" + else None, + "session_affinity": route == "session", + } + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + request_kwargs = {"metadata": {"session_id": "masked-params-session"}} + if route == "session": + mock_router_instance.cache = DualCache() + await mock_router_instance.cache.async_set_cache( + key=router._get_session_affinity_cache_key("masked-params-session", request_kwargs), + value={"model": "opus", "tier": "REASONING"}, + ) + message = "reason carefully about this" if route == "keyword" else "hello" + + response = await router.async_pre_routing_hook( + model="test-router", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": message}], + ) + + assert response is not None + assert response.litellm_params == params + assert response.routing_decision is not None + assert response.routing_decision["tier_litellm_params"] == { + "reasoning_effort": "xhigh", + "api_key": "secr*******-key", + } + + +@pytest.mark.asyncio +async def test_session_pin_outside_tiers_does_not_inherit_medium_params(mock_router_instance): + mock_router_instance.cache = DualCache() + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": { + "SIMPLE": "mini", + "MEDIUM": {"model_name": "medium", "litellm_params": {"reasoning_effort": "low"}}, + }, + "session_affinity": True, + "default_model": "orphan", + }, + ) + request_kwargs = {"metadata": {"session_id": "orphan-session"}} + await mock_router_instance.cache.async_set_cache( + key=router._get_session_affinity_cache_key("orphan-session", request_kwargs), + value="orphan", + ) + + response = await router.async_pre_routing_hook( + model="test-router", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hello"}], + ) + + assert response is not None + assert response.model == "orphan" + assert response.litellm_params == {} + + +@pytest.mark.asyncio +async def test_session_pin_uses_recorded_tier_when_model_is_in_multiple_tiers(mock_router_instance): + mock_router_instance.cache = DualCache() + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": { + "SIMPLE": {"model_name": "shared", "litellm_params": {"reasoning_effort": "low"}}, + "REASONING": {"model_name": "shared", "litellm_params": {"reasoning_effort": "xhigh"}}, + }, + "session_affinity": True, + }, + ) + request_kwargs = {"metadata": {"session_id": "shared-session"}} + await mock_router_instance.cache.async_set_cache( + key=router._get_session_affinity_cache_key("shared-session", request_kwargs), + value={"model": "shared", "tier": "SIMPLE"}, + ) + + response = await router.async_pre_routing_hook( + model="test-router", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hello"}], + ) + + assert response is not None + assert response.litellm_params == {"reasoning_effort": "low"} + assert response.routing_decision is not None + assert response.routing_decision["tier"] == "SIMPLE" + + +@pytest.mark.asyncio +async def test_session_pin_survives_json_list_round_trip(mock_router_instance): + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value=["shared", "SIMPLE"]) + mock_router_instance.cache = cache + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": { + "SIMPLE": {"model_name": "shared", "litellm_params": {"reasoning_effort": "low"}}, + "REASONING": {"model_name": "shared", "litellm_params": {"reasoning_effort": "xhigh"}}, + }, + "session_affinity": True, + }, + ) + request_kwargs = {"metadata": {"session_id": "json-round-trip-session"}} + + response = await router.async_pre_routing_hook( + model="test-router", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hello"}], + ) + + assert response is not None + assert response.model == "shared" + assert response.litellm_params == {"reasoning_effort": "low"} + assert cache.async_set_cache.call_args.kwargs["value"] == {"model": "shared", "tier": "SIMPLE"} diff --git a/tests/test_litellm/secret_managers/test_secret_managers_main.py b/tests/test_litellm/secret_managers/test_secret_managers_main.py index 3631f640136..acc91b691d4 100644 --- a/tests/test_litellm/secret_managers/test_secret_managers_main.py +++ b/tests/test_litellm/secret_managers/test_secret_managers_main.py @@ -7,7 +7,14 @@ from unittest.mock import Mock, patch import pytest -from litellm.secret_managers.main import get_secret, normalize_nonempty_secret_str +import litellm +from litellm.integrations.custom_secret_manager import CustomSecretManager +from litellm.secret_managers.main import ( + get_secret, + normalize_nonempty_secret_str, + secret_manager_would_be_consulted, +) +from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem # Set up logging for debugging logging.basicConfig(level=logging.DEBUG) @@ -364,3 +371,63 @@ def test_unsupported_oidc_provider(): ) def test_normalize_nonempty_secret_str(raw, expected): assert normalize_nonempty_secret_str(raw) == expected + + +class _SpySecretManager(CustomSecretManager): + """Records every name the manager is actually asked for.""" + + def __init__(self, asked): + self.asked = asked + + def sync_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + self.asked.append(secret_name) + return "a-value" + + async def async_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + self.asked.append(secret_name) + return "a-value" + + +@pytest.mark.parametrize( + ("access_mode", "hosted_keys", "secret_name", "expected"), + [ + ("read_only", None, "ANY_NAME", True), + ("read_only", ["ALLOWED"], "ALLOWED", True), + ("read_only", ["ALLOWED"], "NOT_ALLOWED", False), + ("read_and_write", ["ALLOWED"], "ALLOWED", True), + ("write_only", None, "ANY_NAME", False), + ("write_only", ["ALLOWED"], "ALLOWED", False), + ], +) +def test_secret_manager_would_be_consulted_matches_get_secret( + monkeypatch, access_mode, hosted_keys, secret_name, expected +): + """The predicate must agree with what get_secret actually does, not with a reading of it. + + Callers use it to tell "the manager does not have this key" apart from "the manager was + never asked", so a predicate that drifts from get_secret's gating makes them state a + lookup that never happened. + """ + asked = [] + monkeypatch.setattr(litellm, "secret_manager_client", _SpySecretManager(asked)) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr( + litellm, + "_key_management_settings", + KeyManagementSettings(access_mode=access_mode, hosted_keys=hosted_keys), + ) + monkeypatch.delenv(secret_name, raising=False) + + predicted = secret_manager_would_be_consulted(f"os.environ/{secret_name}") + get_secret(f"os.environ/{secret_name}") + + assert {"predicted": predicted, "actually_consulted": bool(asked)} == { + "predicted": expected, + "actually_consulted": expected, + } + + +def test_secret_manager_would_be_consulted_is_false_without_a_client(monkeypatch): + monkeypatch.setattr(litellm, "secret_manager_client", None) + + assert secret_manager_would_be_consulted("os.environ/ANY_NAME") is False diff --git a/tests/test_litellm/test_circleci_path_filter.py b/tests/test_litellm/test_circleci_path_filter.py index b8e763b4979..b427a1a3bd8 100644 --- a/tests/test_litellm/test_circleci_path_filter.py +++ b/tests/test_litellm/test_circleci_path_filter.py @@ -7,6 +7,9 @@ category, it prints `run` or `skip`. The gating contract we lock in here: * docs-only changes (``*.md``, ``*.mdx``, ``docs/``) run nothing * client-only changes (``ui/``) run client jobs but skip backend jobs * any backend change runs both client and backend jobs + * ``ui`` tracks ``ui/`` plus CI config, so a backend-only change skips it + where ``client`` would still run, while a change to the workflows that + define the dashboard jobs still exercises them If this logic silently regresses, real test jobs get skipped, so these cases are the guardrail against that. @@ -40,6 +43,7 @@ def classify(category: str, changed: list[str]) -> str: DOCS = ["README.md", "docs/my_website/index.mdx", "litellm/anywhere.md"] CLIENT = ["ui/litellm-dashboard/src/App.tsx"] BACKEND = ["litellm/main.py"] +CI = [".github/workflows/test-litellm-ui-unit.yml"] @pytest.mark.parametrize( @@ -48,20 +52,27 @@ BACKEND = ["litellm/main.py"] # docs-only: skip everything ("backend", DOCS, "skip"), ("client", DOCS, "skip"), + ("ui", DOCS, "skip"), ("backend", [], "skip"), ("client", [], "skip"), + ("ui", [], "skip"), # client-only: backend skips, client runs ("backend", CLIENT, "skip"), ("client", CLIENT, "run"), + ("ui", CLIENT, "run"), ("backend", CLIENT + DOCS, "skip"), ("client", CLIENT + DOCS, "run"), + ("ui", CLIENT + DOCS, "run"), # any backend change: both run ("backend runs both") ("backend", BACKEND, "run"), ("client", BACKEND, "run"), + ("ui", BACKEND, "skip"), ("backend", BACKEND + DOCS, "run"), ("client", BACKEND + DOCS, "run"), + ("ui", BACKEND + DOCS, "skip"), ("backend", BACKEND + CLIENT, "run"), ("client", BACKEND + CLIENT, "run"), + ("ui", BACKEND + CLIENT, "run"), ], ) def test_classify_decisions(category: str, changed: list[str], expected: str) -> None: @@ -71,6 +82,31 @@ def test_classify_decisions(category: str, changed: list[str], expected: str) -> def test_markdown_under_ui_counts_as_client_not_docs() -> None: assert classify("client", ["ui/litellm-dashboard/README.md"]) == "run" assert classify("backend", ["ui/litellm-dashboard/README.md"]) == "skip" + assert classify("ui", ["ui/litellm-dashboard/README.md"]) == "run" + + +def test_ci_config_changes_reach_every_category() -> None: + """A workflow edit has to exercise the jobs it defines, otherwise the change + ships unvalidated: the dashboard jobs would skip on the very pull request + that rewrites them.""" + assert classify("ui", CI) == "run" + assert classify("backend", CI) == "run" + assert classify("client", CI) == "run" + + +def test_markdown_under_dot_github_is_still_docs() -> None: + """`.github/**` counting as CI config must not drag the pull request template + and other markdown back into running the full suite.""" + assert classify("ui", [".github/pull_request_template.md"]) == "skip" + assert classify("backend", [".github/pull_request_template.md"]) == "skip" + + +def test_ui_and_client_diverge_on_a_backend_only_change() -> None: + """`client` gates CircleCI's dashboard end-to-end jobs, which drive a real + proxy and so must run on backend changes. `ui` gates the dashboard build and + its unit tests, which cannot see the backend at all.""" + assert classify("client", BACKEND) == "run" + assert classify("ui", BACKEND) == "skip" def test_non_docs_directory_with_docs_in_name_is_backend() -> None: diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 75c90d793fe..98938dee62e 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1742,6 +1742,81 @@ def test_azure_ai_cache_cost_calculation(): ), f"Output cost mismatch: got {output_cost}, expected {expected_output_cost}" +def test_vertex_regional_deployment_costs_uplift_over_global(monkeypatch): + """ + Regression for https://github.com/BerriAI/litellm/issues/34393: two Vertex + deployments differing only in vertex_location must not price identically. + Google bills non-global endpoints at 1.1x for regional-pricing models, so the + regional request costs 1.1x the global one for the exact same usage, through + both vertex cost routes (Claude via cost_per_token, Gemini via + cost_per_character's token fallback). + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + usage = Usage(prompt_tokens=15, completion_tokens=5, total_tokens=20) + for model in ("claude-haiku-4-5@20251001", "gemini-3.5-flash"): + global_prompt, global_completion = cost_per_token( + model=model, + custom_llm_provider="vertex_ai", + usage_object=usage, + vertex_location="global", + ) + regional_prompt, regional_completion = cost_per_token( + model=model, + custom_llm_provider="vertex_ai", + usage_object=usage, + vertex_location="us-east5", + ) + global_total = global_prompt + global_completion + regional_total = regional_prompt + regional_completion + assert global_total > 0 + assert regional_total == pytest.approx(global_total * 1.10, rel=1e-9), ( + f"{model}: regional Vertex request must cost 1.1x the global one" + ) + + +def test_vertex_uplift_composes_with_above_128k_pricing(monkeypatch): + """The regional-endpoint uplift multiplies whatever rate the request priced at, + including the above-128k dynamic rates, so a synthetic model carrying both keys + prices regional above-128k usage at 1.1x the above-128k rate.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.get_model_cost_map(url=""), + "vertex_ai/fake-regional-128k-model": { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "input_cost_per_token_above_128k_tokens": 2e-06, + "output_cost_per_token_above_128k_tokens": 4e-06, + "regional_endpoint_uplift_multiplier": 1.1, + }, + }, + ) + + usage = Usage(prompt_tokens=200_000, completion_tokens=10, total_tokens=200_010) + global_prompt, global_completion = cost_per_token( + model="fake-regional-128k-model", + custom_llm_provider="vertex_ai", + usage_object=usage, + vertex_location="global", + ) + regional_prompt, regional_completion = cost_per_token( + model="fake-regional-128k-model", + custom_llm_provider="vertex_ai", + usage_object=usage, + vertex_location="europe-west1", + ) + + assert global_prompt == pytest.approx(200_000 * 2e-06, rel=1e-9) + assert regional_prompt == pytest.approx(global_prompt * 1.10, rel=1e-9) + assert regional_completion == pytest.approx(global_completion * 1.10, rel=1e-9) + + def test_cost_discount_vertex_ai(): """ Test that cost discount is applied correctly for Vertex AI provider diff --git a/tests/test_litellm/test_detect_changes.py b/tests/test_litellm/test_detect_changes.py new file mode 100644 index 00000000000..d8feb2371a3 --- /dev/null +++ b/tests/test_litellm/test_detect_changes.py @@ -0,0 +1,235 @@ +"""Regression tests for the GitHub Actions change-based job gating. + +`.github/scripts/detect_changes.sh` decides whether a pull request's jobs do +real work. It asks the API which files the pull request touches and hands them +to `classify_changes.sh` under one category. The contract locked in here: + + * a UI-only pull request skips backend jobs even when the checked-out merge + ref carries backend commits from the base branch + * the ui category is the mirror image: it skips when only backend files + changed, so a backend-only PR stops building and unit-testing the dashboard + * anything the classification cannot resolve (no pull request, an API + failure, a truncated file list, a broken classifier) runs the job +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / ".github" / "scripts" / "detect_changes.sh" +CLASSIFIER = REPO_ROOT / ".circleci" / "scripts" / "classify_changes.sh" + +UI_FILE = "ui/litellm-dashboard/src/components/Teams.tsx" +BACKEND_FILE = "litellm/proxy/proxy_server.py" + + +def _git(cwd: Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True) + + +def _merge_ref_checkout(tmp_path: Path) -> Path: + """A checkout shaped like `refs/pull/N/merge`: a UI-only branch merged into a + base tip that has moved ahead by a backend commit since the branch was cut.""" + work = tmp_path / "work" + work.mkdir() + _git(work, "init", "-q", "-b", "main") + _git(work, "config", "user.email", "t@t") + _git(work, "config", "user.name", "t") + (work / "seed.txt").write_text("seed\n") + _git(work, "add", "-A") + _git(work, "commit", "-qm", "base") + + _git(work, "checkout", "-q", "-b", "feature") + ui = work / UI_FILE + ui.parent.mkdir(parents=True, exist_ok=True) + ui.write_text("export const Teams = () => null\n") + _git(work, "add", "-A") + _git(work, "commit", "-qm", "ui change") + + _git(work, "checkout", "-q", "main") + backend = work / BACKEND_FILE + backend.parent.mkdir(parents=True, exist_ok=True) + backend.write_text("x = 1\n") + _git(work, "add", "-A") + _git(work, "commit", "-qm", "someone else's backend change") + + _git(work, "merge", "-q", "--no-ff", "-m", "Merge feature into main", "feature") + return work + + +def _scripts_tree(tmp_path: Path, classifier_body: str | None = None) -> Path: + """Copy the scripts into a throwaway tree, preserving their relative layout.""" + root = tmp_path / "tree" + (root / ".github" / "scripts").mkdir(parents=True) + (root / ".circleci" / "scripts").mkdir(parents=True) + shutil.copy(SCRIPT, root / ".github" / "scripts" / SCRIPT.name) + target = root / ".circleci" / "scripts" / CLASSIFIER.name + if classifier_body is None: + shutil.copy(CLASSIFIER, target) + else: + target.write_text(classifier_body) + target.chmod(0o755) + return root + + +def _run( + tmp_path: Path, + *, + files: list[str], + cwd: Path | None = None, + pr_number: str = "37540", + changed_file_count: str | None = None, + gh_exit_code: int = 0, + classifier_body: str | None = None, + category: str | None = None, +) -> tuple[str, str]: + """Run the script against a stubbed `gh`; returns (decision, stdout).""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + listing = "".join(f"echo {f}\n" for f in files) + stub = bin_dir / "gh" + stub.write_text(f"#!/usr/bin/env bash\n{listing}exit {gh_exit_code}\n") + stub.chmod(0o755) + + output_file = tmp_path / "github_output" + output_file.write_text("") + + env = {k: v for k, v in os.environ.items() if k not in {"GH_TOKEN", "GITHUB_TOKEN"}} + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + env["GITHUB_OUTPUT"] = str(output_file) + env["REPO"] = "BerriAI/litellm" + env["PR_NUMBER"] = pr_number + env["CHANGED_FILE_COUNT"] = changed_file_count if changed_file_count is not None else str(len(files)) + if category is not None: + env["CATEGORY"] = category + else: + env.pop("CATEGORY", None) + + tree = _scripts_tree(tmp_path, classifier_body) + result = subprocess.run( + ["bash", str(tree / ".github" / "scripts" / SCRIPT.name)], + cwd=cwd or tmp_path, + capture_output=True, + text=True, + env=env, + check=True, + ) + return output_file.read_text().strip(), result.stdout + + +def test_ui_only_pr_skips_even_when_the_merge_ref_carries_backend_commits(tmp_path: Path) -> None: + """The bug this replaces: diffing the checked-out merge ref against the event's + base sha attributed the base branch's own backend commits to the pull request, + so every UI-only PR ran the full backend suite.""" + work = _merge_ref_checkout(tmp_path) + tracked = subprocess.run( + ["git", "diff", "--name-only", "HEAD~2", "HEAD"], + cwd=work, + capture_output=True, + text=True, + check=True, + ).stdout.split() + assert BACKEND_FILE in tracked, "the checkout must contain the base branch's backend commit" + + decision, _ = _run(tmp_path, files=[UI_FILE], cwd=work) + assert decision == "decision=skip" + + +def test_backend_file_in_the_pr_runs(tmp_path: Path) -> None: + decision, _ = _run(tmp_path, files=[UI_FILE, BACKEND_FILE]) + assert decision == "decision=run" + + +def test_docs_only_pr_skips(tmp_path: Path) -> None: + decision, _ = _run(tmp_path, files=["README.md", "docs/my-website/index.mdx"]) + assert decision == "decision=skip" + + +def test_non_pull_request_event_runs(tmp_path: Path) -> None: + decision, stdout = _run(tmp_path, files=[UI_FILE], pr_number="") + assert decision == "decision=run" + assert "not a pull_request event" in stdout + + +def test_api_failure_runs(tmp_path: Path) -> None: + decision, stdout = _run(tmp_path, files=[], gh_exit_code=1) + assert decision == "decision=run" + assert "could not list the files" in stdout + + +def test_empty_file_list_runs(tmp_path: Path) -> None: + decision, stdout = _run(tmp_path, files=[], changed_file_count="0") + assert decision == "decision=run" + assert "listed no files" in stdout + + +def test_pr_past_the_listing_ceiling_runs(tmp_path: Path) -> None: + """The API caps its file listing, so a larger PR would be classified from a + truncated set and could skip backend jobs it needs.""" + decision, stdout = _run(tmp_path, files=[UI_FILE], changed_file_count="3001") + assert decision == "decision=run" + assert "past the 3000-file listing ceiling" in stdout + + +def test_broken_classifier_runs(tmp_path: Path) -> None: + decision, stdout = _run( + tmp_path, + files=[UI_FILE], + classifier_body="#!/usr/bin/env bash\nexit 1\n", + ) + assert decision == "decision=run" + assert "classify_changes.sh failed" in stdout + + +def test_unexpected_classifier_output_runs(tmp_path: Path) -> None: + decision, stdout = _run( + tmp_path, + files=[UI_FILE], + classifier_body="#!/usr/bin/env bash\ncat >/dev/null\necho maybe\n", + ) + assert decision == "decision=run" + assert "unexpected decision: maybe" in stdout + + +def test_ui_category_skips_a_backend_only_pr(tmp_path: Path) -> None: + """The dashboard build and its unit tests cannot be affected by a pull request + that touches no `ui/` file, and the `client` category cannot express that + because it deliberately runs whenever the backend changes.""" + decision, _ = _run(tmp_path, files=[BACKEND_FILE], category="ui") + assert decision == "decision=skip" + + +def test_ui_category_runs_a_ui_only_pr(tmp_path: Path) -> None: + decision, _ = _run(tmp_path, files=[UI_FILE], category="ui") + assert decision == "decision=run" + + +def test_ui_category_runs_a_mixed_pr(tmp_path: Path) -> None: + decision, _ = _run(tmp_path, files=[UI_FILE, BACKEND_FILE], category="ui") + assert decision == "decision=run" + + +def test_absent_category_still_runs_a_backend_pr(tmp_path: Path) -> None: + """Callers that pass no category keep the pre-existing backend behaviour.""" + assert _run(tmp_path, files=[BACKEND_FILE])[0] == "decision=run" + + +def test_absent_category_still_skips_a_ui_pr(tmp_path: Path) -> None: + assert _run(tmp_path, files=[UI_FILE])[0] == "decision=skip" + + +def test_ui_category_fails_open_when_the_api_fails(tmp_path: Path) -> None: + decision, stdout = _run(tmp_path, files=[], gh_exit_code=1, category="ui") + assert decision == "decision=run" + assert "detect-changes[ui]" in stdout + + +def test_ui_category_runs_when_the_ui_workflows_themselves_change(tmp_path: Path) -> None: + """Without this the dashboard jobs would skip on the pull request that edits + them, shipping a workflow change nothing ever exercised.""" + decision, _ = _run(tmp_path, files=[".github/workflows/test-litellm-ui-unit.yml"], category="ui") + assert decision == "decision=run" diff --git a/tests/test_litellm/test_responses_api_bridge_non_stream.py b/tests/test_litellm/test_responses_api_bridge_non_stream.py index c272b151865..08d55ee8290 100644 --- a/tests/test_litellm/test_responses_api_bridge_non_stream.py +++ b/tests/test_litellm/test_responses_api_bridge_non_stream.py @@ -297,7 +297,8 @@ def test_transform_usage_with_zero_values(): cached_tokens=0 is preserved (cache was available; nothing was cached). reasoning_tokens=0 is preserved the same way: an explicit provider-reported - zero passes through, while an absent value (None) is omitted. + zero passes through, while an absent value (None) falls back to 0 because the + Responses API wire contract requires reasoning_tokens as an int. """ completion_response = create_mock_completion_response( model="gpt-4", @@ -321,6 +322,32 @@ def test_transform_usage_with_zero_values(): print("✓ Transformation preserves explicit reasoning_tokens=0 and omits absent values") +def test_transform_usage_unknown_reasoning_split_keeps_output_tokens_details(): + """ + An unknown reasoning split (reasoning_tokens=None, text_tokens=None) must still + emit output_tokens_details with an integer reasoning_tokens: the OpenAI SDK's + ResponseUsage requires the field, so omitting it breaks /v1/responses clients. + """ + from openai.types.responses.response_usage import ( + OutputTokensDetails as OpenAISDKOutputTokensDetails, + ) + + from litellm.types.utils import CompletionTokensDetailsWrapper + + usage = Usage( + prompt_tokens=100, + completion_tokens=500, + total_tokens=600, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=None, text_tokens=None), + ) + + responses_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(usage) + + assert responses_usage.output_tokens_details is not None + assert responses_usage.output_tokens_details.reasoning_tokens == 0 + OpenAISDKOutputTokensDetails.model_validate(responses_usage.output_tokens_details.model_dump(exclude_none=True)) + + def test_input_tokens_details_requires_cached_tokens(): """ Test that InputTokensDetails has cached_tokens as an int with default value 0. diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index dfe46d54ab8..4674a8b1dfa 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -1584,3 +1584,116 @@ def test_inherit_builtin_tiered_output_rate_leaves_a_user_rate_alone(): ) assert model_info["output_cost_per_token"] == 9e-07 + + +# --- a config.yaml PTU deployment must not also bill per token ------------------ + +_PTU_MODEL_INFO = { + "team_id": "team-alpha", + "ptu_count": 100, + "cost_per_ptu_per_hour": 0.02, + "ptu_effective_from": "2026-01-01T00:00:00Z", +} + + +def _ptu_router(model_info=None, litellm_params=None, ptu_enabled=True): + """A router built the way loading config.yaml builds one.""" + with patch.dict(os.environ, {"LITELLM_ENABLE_PTU_COST_ATTRIBUTION": "True" if ptu_enabled else ""}, clear=False): + return Router( + model_list=[ + { + "model_name": "gpt-4o-ptu", + "litellm_params": { + "model": "anthropic/claude-sonnet-4-5-20250929", + "api_key": "sk-not-used", + **(litellm_params or {}), + }, + "model_info": dict(_PTU_MODEL_INFO if model_info is None else model_info), + } + ] + ) + + +def test_a_config_ptu_deployment_bills_nothing_per_token(): + """Reserved capacity is already billed by the hour, so charging its traffic bills the + same tokens twice. Left unset the rate falls back to the public cost map, which makes + the double charge the default rather than an opt-in.""" + router = _ptu_router(litellm_params={"input_cost_per_token": 5e-06, "output_cost_per_token": 1.5e-05}) + entry = router.model_list[0] + + assert entry["litellm_params"]["input_cost_per_token"] == 0.0 + assert entry["litellm_params"]["output_cost_per_token"] == 0.0 + assert entry["model_info"]["input_cost_per_token"] == 0.0 + assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token"] == 0.0 + + +@pytest.mark.parametrize( + "backend", + ["anthropic/claude-sonnet-4-5-20250929", "azure/gpt-4o", "gemini/gemini-2.5-flash"], +) +def test_a_config_ptu_deployment_imports_no_cache_rate_from_its_backend(backend): + """The cache back-fill runs whenever input_cost_per_token is set, and 0.0 is set, so a + partially zeroed deployment would silently inherit the backend model's real cache rates. + Every backend here publishes non-zero ones, which is what makes the assertion mean + something.""" + cache_fields = ( + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr", + "cache_creation_input_token_cost_above_200k_tokens", + "cache_read_input_token_cost", + "cache_read_input_token_cost_above_200k_tokens", + ) + builtin = litellm.get_model_info(model=backend) + assert any(builtin.get(field) for field in cache_fields), "backend publishes no cache pricing to leak" + + router = _ptu_router(litellm_params={"model": backend}) + priced = litellm.model_cost[router.model_list[0]["model_info"]["id"]] + + assert [field for field in cache_fields if priced.get(field)] == [] + + +def test_zeroing_a_ptu_deployment_leaves_its_backend_model_priced(): + """A sibling deployment on the same backend must keep billing normally.""" + backend = "anthropic/claude-sonnet-4-5-20250929" + builtin = litellm.get_model_info(model=backend)["input_cost_per_token"] + assert builtin > 0 + + _ptu_router(litellm_params={"model": backend}) + + assert litellm.get_model_info(model=backend)["input_cost_per_token"] == builtin + + +def test_zeroing_does_not_change_the_deployment_id(): + """The id is a hash of the deployment's params and keys its cooldowns, its budget, and + every spend row already written against it.""" + params = {"input_cost_per_token": 5e-06} + priced = _ptu_router(litellm_params=params, ptu_enabled=False).model_list[0]["model_info"]["id"] + zeroed = _ptu_router(litellm_params=params).model_list[0]["model_info"]["id"] + + assert priced == zeroed + + +def test_a_database_backed_deployment_is_left_alone(): + """The write endpoints already zero those, and they answer 400 rather than silently + rewriting a rate the caller sent.""" + entry = _ptu_router(model_info={**_PTU_MODEL_INFO, "db_model": True}).model_list[0] + + assert entry["litellm_params"].get("input_cost_per_token") is None + + +def test_nothing_is_zeroed_while_the_feature_is_off(): + """No flat cost accrues with the flag off, so zeroing would serve the traffic free.""" + entry = _ptu_router(litellm_params={"input_cost_per_token": 5e-06}, ptu_enabled=False).model_list[0] + + assert entry["litellm_params"]["input_cost_per_token"] == 5e-06 + + +@pytest.mark.parametrize("dropped", ["team_id", "ptu_effective_from"], ids=["no team_id", "no ptu_effective_from"]) +def test_a_deployment_the_rollup_will_not_charge_is_not_zeroed(dropped): + """The rollup refuses to price a reservation missing either field, so zeroing on the + looser count-and-rate test alone would leave the deployment serving for free with + nothing charged in its place.""" + incomplete = {k: v for k, v in _PTU_MODEL_INFO.items() if k != dropped} + entry = _ptu_router(model_info=incomplete, litellm_params={"input_cost_per_token": 5e-06}).model_list[0] + + assert entry["litellm_params"]["input_cost_per_token"] == 5e-06 diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index efccdc4a986..60453931595 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2,6 +2,7 @@ import json import logging import os import sys +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -831,6 +832,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "output_cost_per_token_above_200k_tokens_priority": {"type": "number"}, "output_cost_per_token_above_272k_tokens_priority": {"type": "number"}, "output_cost_per_token_above_272k_tokens_flex": {"type": "number"}, + "regional_endpoint_uplift_multiplier": {"type": "number"}, "regional_processing_uplift_multiplier_eu": {"type": "number"}, "regional_processing_uplift_multiplier_us": {"type": "number"}, "input_cost_per_pixel": {"type": "number"}, @@ -3787,8 +3789,8 @@ def test_deepseek_v4_models_in_cost_map(): configured in model_prices_and_context_window.json. Prices sourced from https://api-docs.deepseek.com/quick_start/pricing: - - deepseek-v4-flash: $0.14/M input, $0.28/M output - - deepseek-v4-pro: $0.435/M input, $0.87/M output (75% discounted active price) + - deepseek-v4-flash: $0.44/M input, $1.32/M output + - deepseek-v4-pro: $1.32/M input, $3.96/M output Closes https://github.com/BerriAI/litellm/issues/26709 """ @@ -3801,8 +3803,8 @@ def test_deepseek_v4_models_in_cost_map(): # --- bare model names --- for key, expected_input, expected_output, expected_cache in [ - ("deepseek-v4-flash", 1.4e-07, 2.8e-07, 2.8e-09), - ("deepseek-v4-pro", 4.35e-07, 8.7e-07, 3.625e-09), + ("deepseek-v4-flash", 4.4e-07, 1.32e-06, 1.4e-08), + ("deepseek-v4-pro", 1.32e-06, 3.96e-06, 4.4e-08), ]: info = model_cost.get(key) assert info is not None, f"{key} missing from model_prices_and_context_window.json" @@ -3817,8 +3819,8 @@ def test_deepseek_v4_models_in_cost_map(): # --- provider-prefixed names --- for key, expected_input, expected_output, expected_cache in [ - ("deepseek/deepseek-v4-flash", 1.4e-07, 2.8e-07, 2.8e-09), - ("deepseek/deepseek-v4-pro", 4.35e-07, 8.7e-07, 3.625e-09), + ("deepseek/deepseek-v4-flash", 4.4e-07, 1.32e-06, 1.4e-08), + ("deepseek/deepseek-v4-pro", 1.32e-06, 3.96e-06, 4.4e-08), ]: info = model_cost.get(key) assert info is not None, f"{key} missing from model_prices_and_context_window.json" @@ -3845,8 +3847,8 @@ def test_deepseek_v4_models_in_backup_cost_map(): # --- bare model names --- for key, expected_input, expected_output, expected_cache in [ - ("deepseek-v4-flash", 1.4e-07, 2.8e-07, 2.8e-09), - ("deepseek-v4-pro", 4.35e-07, 8.7e-07, 3.625e-09), + ("deepseek-v4-flash", 4.4e-07, 1.32e-06, 1.4e-08), + ("deepseek-v4-pro", 1.32e-06, 3.96e-06, 4.4e-08), ]: info = model_cost.get(key) assert info is not None, f"{key} missing from backup JSON" @@ -3859,8 +3861,8 @@ def test_deepseek_v4_models_in_backup_cost_map(): # --- provider-prefixed names --- for key, expected_input, expected_output, expected_cache in [ - ("deepseek/deepseek-v4-flash", 1.4e-07, 2.8e-07, 2.8e-09), - ("deepseek/deepseek-v4-pro", 4.35e-07, 8.7e-07, 3.625e-09), + ("deepseek/deepseek-v4-flash", 4.4e-07, 1.32e-06, 1.4e-08), + ("deepseek/deepseek-v4-pro", 1.32e-06, 3.96e-06, 4.4e-08), ]: info = model_cost.get(key) assert info is not None, f"{key} missing from backup JSON" @@ -4384,6 +4386,45 @@ def test_get_prompt_cache_min_tokens_differs_per_platform_for_same_model(local_m ) +GEMINI_4096_CACHE_MIN_MODELS: Final = tuple( + prefix + base + for base in ( + "gemini-3.5-flash", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + ) + for prefix in ("", "gemini/", "vertex_ai/") +) + + +def test_gemini_3_flash_and_31_pro_preview_resolve_4096_cache_minimum(local_model_cost_map: None) -> None: + """Regression for the cost map missing prompt_cache_min_tokens on these models: Google rejects + explicit caching below 4,096 tokens for them (https://ai.google.dev/gemini-api/docs/caching), so + the 1024 default sent cachedContents creates Vertex answered with a hard 400.""" + wrong: Final = { + model: get_prompt_cache_min_tokens(model=model) + for model in GEMINI_4096_CACHE_MIN_MODELS + if get_prompt_cache_min_tokens(model=model) != 4096 + } + assert not wrong, f"prompt_cache_min_tokens must be 4096: {wrong}" + + +def test_gemini_4096_cache_minimum_present_in_root_cost_map() -> None: + """The root map ships to the CDN independently of the bundled backup, so both must carry the + minimum or proxies reading one of them regress to the 1024 default.""" + root_map_path: Final = os.path.join(os.path.dirname(__file__), "..", "..", "model_prices_and_context_window.json") + with open(root_map_path) as f: + root_map: Final = json.load(f) + wrong: Final = { + model: root_map[model].get("prompt_cache_min_tokens") + for model in GEMINI_4096_CACHE_MIN_MODELS + if root_map[model].get("prompt_cache_min_tokens") != 4096 + } + assert not wrong, f"prompt_cache_min_tokens must be 4096: {wrong}" + + def test_get_prompt_cache_min_tokens_unmapped_model_falls_back_to_default(local_model_cost_map: None) -> None: """get_model_info raises for a model it has no entry for. The resolver must swallow that and fall back to the default, otherwise the raise reaches callers that would read it as diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 6f77a621a9e..20cd1165577 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22809 + "limit": 22805 }, "LIT002": { "limit": 26878 diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 98225de5d8b..2d68ea2aa68 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -4,11 +4,6 @@ "count": 1 } }, - "src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupEditModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx": { "no-restricted-imports": { "count": 1 @@ -62,9 +57,6 @@ "local/no-complex-jsx-arrow": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/immutability": { "count": 1 } @@ -90,9 +82,6 @@ "src/app/(dashboard)/budgets/_components/budget_modal.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 } }, "src/app/(dashboard)/budgets/_components/budget_panel.tsx": { @@ -103,9 +92,6 @@ "src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 } }, "src/app/(dashboard)/caching/_components/cache_dashboard.tsx": { @@ -158,9 +144,6 @@ }, "no-nested-ternary": { "count": 2 - }, - "no-restricted-imports": { - "count": 1 } }, "src/app/(dashboard)/cost-tracking/_components/how_it_works.tsx": { @@ -213,11 +196,6 @@ "count": 1 } }, - "src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx": { "no-nested-ternary": { "count": 3 @@ -248,9 +226,6 @@ "no-nested-ternary": { "count": 2 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -262,9 +237,6 @@ "max-lines": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 }, @@ -541,38 +513,12 @@ "count": 1 } }, - "src/app/(dashboard)/mcp-servers/_components/AwsSigV4Fields.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/mcp-servers/_components/CreateMCPServer.tsx": { "max-lines": { "count": 1 }, "no-nested-ternary": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 - }, - "react-hooks/set-state-in-effect": { - "count": 4 - } - }, - "src/app/(dashboard)/mcp-servers/_components/DcrBridgeToggle.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/mcp-servers/_components/IdJagFormFields.tsx": { - "no-restricted-imports": { - "count": 1 } }, "src/app/(dashboard)/mcp-servers/_components/MCPNetworkSettings.tsx": { @@ -580,16 +526,6 @@ "count": 2 } }, - "src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/mcp-servers/_components/MCPSubmissionsTab.tsx": { "react-hooks/set-state-in-effect": { "count": 1 @@ -606,25 +542,14 @@ "count": 1 } }, - "src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/mcp-servers/_components/OAuthFormFields.tsx": { "no-nested-ternary": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 } }, "src/app/(dashboard)/mcp-servers/_components/OpenAPIFormSection.tsx": { "local/no-complex-jsx-arrow": { "count": 1 - }, - "no-restricted-imports": { - "count": 2 } }, "src/app/(dashboard)/mcp-servers/_components/OpenAPIQuickPicker.tsx": { @@ -632,36 +557,6 @@ "count": 1 } }, - "src/app/(dashboard)/mcp-servers/_components/OpenApiByokFields.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/mcp-servers/_components/StdioConfiguration.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/mcp-servers/_components/TokenEndpointAuthMethodField.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/mcp-servers/_components/TokenExchangeFormFields.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/mcp-servers/_components/ToolTestPanel.tsx": { "react-hooks/set-state-in-effect": { "count": 1 @@ -670,9 +565,6 @@ "src/app/(dashboard)/mcp-servers/_components/UserEnvVarsModal.tsx": { "no-nested-ternary": { "count": 2 - }, - "no-restricted-imports": { - "count": 1 } }, "src/app/(dashboard)/mcp-servers/_components/index.tsx": { @@ -684,9 +576,6 @@ "local/filename-pascal-case": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/static-components": { "count": 4 } @@ -726,15 +615,6 @@ }, "max-lines": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 - }, - "react-hooks/immutability": { - "count": 1 - }, - "react-hooks/set-state-in-effect": { - "count": 5 } }, "src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx": { @@ -780,11 +660,6 @@ "count": 1 } }, - "src/app/(dashboard)/memory/_components/MemoryEditModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx": { "no-nested-ternary": { "count": 1 @@ -889,11 +764,6 @@ "count": 1 } }, - "src/app/(dashboard)/playground/components/compareUI/components/ModelSelector.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/playground/components/complianceUI/ComplianceUI.tsx": { "local/no-complex-jsx-arrow": { "count": 2 @@ -993,9 +863,6 @@ "local/filename-pascal-case": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/immutability": { "count": 1 } @@ -1097,9 +964,6 @@ "local/filename-pascal-case": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/immutability": { "count": 1 } @@ -1125,16 +989,6 @@ "count": 1 } }, - "src/app/(dashboard)/projects/_components/ProjectModals/CreateProjectModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/projects/_components/ProjectModals/EditProjectModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/projects/_components/ProjectModals/ProjectBaseForm.tsx": { "react-hooks/set-state-in-effect": { "count": 2 @@ -1205,9 +1059,6 @@ "no-nested-ternary": { "count": 3 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 2 } @@ -1238,18 +1089,10 @@ "count": 2 } }, - "src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/search-tools/_components/SearchTools.tsx": { "local/no-complex-jsx-arrow": { "count": 2 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/static-components": { "count": 1 } @@ -1272,14 +1115,6 @@ "src/app/(dashboard)/skills/_components/add_plugin_form.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 - } - }, - "src/app/(dashboard)/tag-management/_components/components/CreateTagModal.tsx": { - "no-restricted-imports": { - "count": 1 } }, "src/app/(dashboard)/tag-management/_components/index.tsx": { @@ -1392,29 +1227,13 @@ "local/filename-pascal-case": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } }, - "src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx": { - "no-restricted-imports": { - "count": 2 - } - }, - "src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx": { "no-nested-ternary": { "count": 2 - }, - "no-restricted-imports": { - "count": 1 } }, "src/app/(dashboard)/vector-stores/_components/index.tsx": { @@ -1453,9 +1272,6 @@ "local/no-complex-jsx-arrow": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 2 } @@ -1470,11 +1286,6 @@ "count": 1 } }, - "src/app/onboarding/OnboardingFormBody.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/AIHub/ModelHubTable.test.tsx": { "max-params": { "count": 1 @@ -1511,16 +1322,6 @@ "count": 1 } }, - "src/components/CloudZeroCostTracking/CloudZeroCreateModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/CloudZeroCostTracking/CloudZeroUpdateModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/CreateUserButton.tsx": { "no-restricted-imports": { "count": 1 @@ -1559,33 +1360,15 @@ "count": 1 } }, - "src/components/SSOModals.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/Settings/AdminSettings/HashicorpVault/EditHashicorpVaultModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/Settings/AdminSettings/HashicorpVault/HashicorpVault.tsx": { "no-nested-ternary": { "count": 1 } }, - "src/components/Settings/AdminSettings/LoggingSettings/LoggingSettings.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx": { "no-nested-ternary": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -1595,31 +1378,16 @@ "count": 1 } }, - "src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.test.tsx": { "max-nested-callbacks": { "count": 1 } }, - "src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/Settings/AdminSettings/SSOSettings/RedactableField.tsx": { "no-nested-ternary": { "count": 1 } }, - "src/components/Settings/AdminSettings/SSOSettings/RoleMappings.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/Settings/AdminSettings/UISettings/PageVisibilitySettings.tsx": { "react-hooks/set-state-in-render": { "count": 1 @@ -1635,23 +1403,7 @@ "count": 1 } }, - "src/components/Settings/RouterSettings/Fallbacks/AddFallbacksModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx": { - "local/no-complex-jsx-arrow": { - "count": 1 - }, - "no-restricted-imports": { - "count": 1 - } - }, "src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx": { - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -1668,7 +1420,7 @@ }, "src/components/Teams.tsx": { "local/no-complex-jsx-arrow": { - "count": 4 + "count": 3 }, "max-lines": { "count": 1 @@ -1691,11 +1443,6 @@ "count": 1 } }, - "src/components/UsagePage/components/EntityUsage/TopKeyView.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/UsagePage/utils/value_formatters.tsx": { "local/filename-pascal-case": { "count": 1 @@ -1704,9 +1451,6 @@ "src/components/VirtualKeysPage/keyTableColumns.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 } }, "src/components/activity_metrics.tsx": { @@ -1717,16 +1461,6 @@ "count": 1 } }, - "src/components/add_model/AdaptiveRoutingConfig.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/add_model/AddModelForm.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/add_model/AddModelForm.tsx": { "local/no-complex-jsx-arrow": { "count": 1 @@ -1734,26 +1468,6 @@ "no-nested-ternary": { "count": 1 }, - "no-restricted-imports": { - "count": 2 - } - }, - "src/components/add_model/ClassificationMethodConfig.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/add_model/ComplexityRouterConfig.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/add_model/EscalationKeywords.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/add_model/KeywordTierRules.tsx": { "no-restricted-imports": { "count": 1 } @@ -1763,11 +1477,6 @@ "count": 1 } }, - "src/components/add_model/SemanticKeywordMatching.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/add_model/add_auto_router_tab.tsx": { "local/filename-pascal-case": { "count": 1 @@ -1784,9 +1493,6 @@ "src/components/add_model/advanced_settings.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 3 } }, "src/components/add_model/auto_router_connection_test.tsx": { @@ -1806,9 +1512,6 @@ "local/no-complex-jsx-arrow": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 2 } @@ -1829,9 +1532,6 @@ }, "no-nested-ternary": { "count": 1 - }, - "no-restricted-imports": { - "count": 2 } }, "src/components/add_model/model_connection_test.tsx": { @@ -1849,9 +1549,6 @@ "no-nested-ternary": { "count": 3 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/immutability": { "count": 3 } @@ -1859,20 +1556,9 @@ "src/components/add_pass_through.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/agent_management/AgentSelector.test.tsx": { - "react/display-name": { - "count": 1 } }, "src/components/agent_management/AgentSelector.tsx": { - "no-restricted-imports": { - "count": 1 - }, "prefer-const": { "count": 1 } @@ -1963,11 +1649,6 @@ "count": 1 } }, - "src/components/common_components/AccessGroupSelector.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/common_components/DeleteResourceModal.tsx": { "react-hooks/set-state-in-effect": { "count": 1 @@ -1978,16 +1659,6 @@ "count": 1 } }, - "src/components/common_components/MetadataKeyValueFields.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/common_components/MetadataKeyValueFields.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/common_components/ModelAliasManager.tsx": { "react-hooks/set-state-in-effect": { "count": 1 @@ -1998,16 +1669,6 @@ "count": 1 } }, - "src/components/common_components/PassThroughGuardrailsSection.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/common_components/RateLimitTypeFormItem.test.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/common_components/budget_duration_dropdown.tsx": { "local/filename-pascal-case": { "count": 1 @@ -2016,9 +1677,6 @@ "src/components/common_components/check_openapi_schema.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 2 } }, "src/components/common_components/fetch_teams.tsx": { @@ -2050,9 +1708,6 @@ "src/components/common_components/user_search_modal.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 } }, "src/components/constants.tsx": { @@ -2092,16 +1747,6 @@ "count": 1 } }, - "src/components/key_team_helpers/BudgetFallbacksEditor.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/key_team_helpers/BudgetWindowsEditor.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/key_team_helpers/fetch_available_models_team_key.tsx": { "local/filename-pascal-case": { "count": 1 @@ -2165,14 +1810,6 @@ "count": 1 } }, - "src/components/mcp_server_management/MCPServerSelector.tsx": { - "local/no-complex-jsx-arrow": { - "count": 1 - }, - "no-restricted-imports": { - "count": 1 - } - }, "src/components/mcp_server_management/MCPToolPermissions.tsx": { "local/no-complex-jsx-arrow": { "count": 1 @@ -2186,9 +1823,6 @@ "src/components/mcp_tools/MCPToolArgumentsForm.tsx": { "no-nested-ternary": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 } }, "src/components/mcp_tools/McpCrudPermissionPanel.tsx": { @@ -2203,20 +1837,12 @@ }, "src/components/model_add/CredentialModal.tsx": { "no-restricted-imports": { - "count": 2 + "count": 1 } }, "src/components/model_add/reuse_credentials.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/model_dashboard/ModelSettingsModal/ModelSettingsModal.tsx": { - "no-restricted-imports": { - "count": 1 } }, "src/components/model_filters.tsx": { @@ -2236,9 +1862,6 @@ "local/filename-pascal-case": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "prefer-const": { "count": 5 }, @@ -2289,11 +1912,6 @@ "count": 1 } }, - "src/components/organisms/RegenerateKeyModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/organisms/create_key_button.tsx": { "local/filename-pascal-case": { "count": 1 @@ -2380,11 +1998,6 @@ "count": 1 } }, - "src/components/router_settings/RoutingStrategySelector.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/router_settings/index.tsx": { "local/filename-pascal-case": { "count": 1 @@ -2393,11 +2006,6 @@ "count": 2 } }, - "src/components/routing_groups/RoutingGroupModal.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/routing_groups/index.tsx": { "local/filename-pascal-case": { "count": 1 @@ -2536,14 +2144,6 @@ "src/components/team/EditMembership.tsx": { "no-nested-ternary": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/team/LoggingSettings.tsx": { - "no-restricted-imports": { - "count": 1 } }, "src/components/team/TeamInfo.tsx": { @@ -2553,9 +2153,6 @@ "no-nested-ternary": { "count": 3 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -2891,11 +2488,6 @@ "count": 2 } }, - "src/contexts/AntdGlobalProvider.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/contexts/AuthContext.tsx": { "react-hooks/set-state-in-effect": { "count": 1 diff --git a/ui/litellm-dashboard/eslint.config.mjs b/ui/litellm-dashboard/eslint.config.mjs index 35603123736..3cbb93f9a48 100644 --- a/ui/litellm-dashboard/eslint.config.mjs +++ b/ui/litellm-dashboard/eslint.config.mjs @@ -62,6 +62,10 @@ const eslintConfig = [ message: "antd is being phased out; build new UI with shadcn/ui primitives instead of adding antd imports.", }, + { + group: ["@ant-design/icons", "@ant-design/icons/*"], + message: "@ant-design/icons is gone from the dashboard; use lucide-react instead.", + }, ], }, ], diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 186d38234d5..154a19da7f3 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -9,7 +9,6 @@ "version": "0.1.0", "dependencies": { "@ant-design/cssinjs": "1.24.0", - "@ant-design/icons": "5.6.1", "@anthropic-ai/sdk": "0.92.0", "@base-ui/react": "^1.6.0", "@headlessui/tailwindcss": "0.2.2", @@ -32,9 +31,9 @@ "openapi-fetch": "^0.17.0", "openapi-react-query": "^0.5.4", "papaparse": "5.5.3", - "react": "18.3.1", + "react": "19.2.8", "react-copy-to-clipboard": "5.1.1", - "react-dom": "18.3.1", + "react-dom": "19.2.8", "react-hook-form": "7.82.0", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", @@ -55,9 +54,9 @@ "@testing-library/react": "16.3.2", "@testing-library/user-event": "14.6.1", "@types/node": "20.19.37", - "@types/react": "18.2.48", + "@types/react": "19.2.18", "@types/react-copy-to-clipboard": "5.0.7", - "@types/react-dom": "18.3.7", + "@types/react-dom": "19.2.4", "@types/react-syntax-highlighter": "15.5.13", "@vitest/coverage-v8": "3.2.6", "@vitest/ui": "3.2.6", @@ -200,9 +199,9 @@ } }, "node_modules/@ant-design/icons-svg": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/@ant-design/icons-svg/-/icons-svg-4.4.2.tgz", - "integrity": "sha512-vHbT+zJEVzllwP+CM+ul7reTEfBR0vgxFe7+lREAsAA7YGsYpboiq2sQNeQeRvh09GfQgs/GyFEvZpJ9cLXpXA==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@ant-design/icons-svg/-/icons-svg-4.5.0.tgz", + "integrity": "sha512-1BTUFyKPTBZ53MuTP8s0k5SFEXL7o3VHEOwLgzaoWKwnBeqIcqUtVshc4SKzhI6uACfqhJqBwBUE9FsWR3uULA==", "license": "MIT" }, "node_modules/@ant-design/react-slick": { @@ -606,19 +605,6 @@ } } }, - "node_modules/@base-ui/react/node_modules/@floating-ui/react-dom": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.8.tgz", - "integrity": "sha512-cC52bHwM/n/CxS87FH0yWdngEZrjdtLW/qVruo68qg+prK7ZQ4YGdut2GyDVpoGeAYe/h899rVeOVm6Oi40k2A==", - "license": "MIT", - "dependencies": { - "@floating-ui/dom": "^1.7.6" - }, - "peerDependencies": { - "react": ">=16.8.0", - "react-dom": ">=16.8.0" - } - }, "node_modules/@base-ui/utils": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/@base-ui/utils/-/utils-0.3.1.tgz", @@ -1441,28 +1427,41 @@ } }, "node_modules/@floating-ui/core": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.5.tgz", - "integrity": "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", + "integrity": "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==", "license": "MIT", "dependencies": { - "@floating-ui/utils": "^0.2.11" + "@floating-ui/utils": "^0.2.12" } }, "node_modules/@floating-ui/dom": { - "version": "1.7.6", - "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.6.tgz", - "integrity": "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", + "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", "license": "MIT", "dependencies": { - "@floating-ui/core": "^1.7.5", - "@floating-ui/utils": "^0.2.11" + "@floating-ui/core": "^1.8.0", + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/react-dom": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.9.tgz", + "integrity": "sha512-JDjEFGCpImxDCA7JJKviA0M9+RtmJdj0m/NVU5IMgBK+AmZouAQQ7/+2GLH0GXXY0YMw9oXPB8hKdbPYg5QLYg==", + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.8.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" } }, "node_modules/@floating-ui/utils": { - "version": "0.2.11", - "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.11.tgz", - "integrity": "sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==", + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", "license": "MIT" }, "node_modules/@headlessui/tailwindcss": { @@ -2627,9 +2626,9 @@ "license": "MIT" }, "node_modules/@rc-component/async-validator": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@rc-component/async-validator/-/async-validator-5.1.0.tgz", - "integrity": "sha512-n4HcR5siNUXRX23nDizbZBQPO0ZM/5oTtmKZ6/eqL0L2bo747cklFdZGRN2f+c9qWGICwDzrhW0H7tE9PptdcA==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@rc-component/async-validator/-/async-validator-5.1.2.tgz", + "integrity": "sha512-WYbrZSjzznU1ekD0qFq2qRxt309VoS61MTG5npnFQlKYcoy9IzU8T+ZCIhq5bGAXRbXysABFWTspicMfmWFwow==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.24.4" @@ -2669,9 +2668,9 @@ } }, "node_modules/@rc-component/mini-decimal": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@rc-component/mini-decimal/-/mini-decimal-1.1.3.tgz", - "integrity": "sha512-bk/FJ09fLf+NLODMAFll6CfYrHPBioTedhW6lxDBuuWucJEqFUd4l/D/5JgIi3dina6sYahB8iuPAZTNz2pMxw==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rc-component/mini-decimal/-/mini-decimal-1.1.4.tgz", + "integrity": "sha512-xiuXcaCwyOWpD8a8scdExFl+bntNphAW8XeenL1ig2en0AAZY0Pcp4pC0dI22qJ+NvxKn9RoNIoRdqYU3BLH4w==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.18.0" @@ -2717,9 +2716,9 @@ } }, "node_modules/@rc-component/qrcode": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@rc-component/qrcode/-/qrcode-1.1.1.tgz", - "integrity": "sha512-LfLGNymzKdUPjXUbRP+xOhIWY4jQ+YMj5MmWAcgcAq1Ij8XP7tRmAXqyuv96XvLUBE/5cA8hLFl9eO1JQMujrA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rc-component/qrcode/-/qrcode-1.1.3.tgz", + "integrity": "sha512-aGv6alnn4HbDEsURzKP+jv13rbi1VxmAYfBNZr5GKF1iohMNWy5tAVoJ1E3cOvzMB1kbUPvCXchM6zSFlRGPhA==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.24.7" @@ -3661,12 +3660,12 @@ } }, "node_modules/@tanstack/react-store": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@tanstack/react-store/-/react-store-0.11.0.tgz", - "integrity": "sha512-tX4YXh3PDkmpvGQWkWqKpzs/MSqbtuwY9dWdWhtV9Q50PmO+jOkUKIWIX4G85dwt7lxdHLXsiaEKPdKmC8F41w==", + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/@tanstack/react-store/-/react-store-0.11.1.tgz", + "integrity": "sha512-HaIGKI3YLmjBYIvy5DFDY23oNaYZIsTZfngey07Uh5iLVJgM3bIGCnZeOFOqzjFld9JHWcaHJnasD/bKoGKwJQ==", "license": "MIT", "dependencies": { - "@tanstack/store": "0.11.0", + "@tanstack/store": "0.11.1", "use-sync-external-store": "^1.6.0" }, "funding": { @@ -3699,9 +3698,9 @@ } }, "node_modules/@tanstack/store": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@tanstack/store/-/store-0.11.0.tgz", - "integrity": "sha512-WlzzCt3xi0G6pCAJu1U+2jiECwabETDpQDi3hfkFZvJii9AuZqEKbOiVarX1/bWhTNjU486yQtJCCasi/0q+Cw==", + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/@tanstack/store/-/store-0.11.1.tgz", + "integrity": "sha512-mzTOBhypOuDJAy/D8n2MfUZ1HFkXnmSETviRyhqEC8LUE7/IZQExOTxMANj3KjTofYTkFNpBY67qaVrT41YccA==", "license": "MIT", "funding": { "type": "github", @@ -3999,21 +3998,13 @@ "@types/node": "*" } }, - "node_modules/@types/prop-types": { - "version": "15.7.15", - "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", - "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", - "license": "MIT" - }, "node_modules/@types/react": { - "version": "18.2.48", - "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.48.tgz", - "integrity": "sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w==", + "version": "19.2.18", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz", + "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", "license": "MIT", "dependencies": { - "@types/prop-types": "*", - "@types/scheduler": "*", - "csstype": "^3.0.2" + "csstype": "^3.2.2" } }, "node_modules/@types/react-copy-to-clipboard": { @@ -4027,13 +4018,13 @@ } }, "node_modules/@types/react-dom": { - "version": "18.3.7", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", - "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "version": "19.2.4", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.4.tgz", + "integrity": "sha512-Bsc+QHgp+P/F02XDzNCY9jnZNCUuLki36KT7VKrTXXLdHf+vHMNZnW1rVu5DNW/rCK+fya3DATySbLM4yhtKUw==", "dev": true, "license": "MIT", "peerDependencies": { - "@types/react": "^18.0.0" + "@types/react": "^19.2.0" } }, "node_modules/@types/react-syntax-highlighter": { @@ -4046,12 +4037,6 @@ "@types/react": "*" } }, - "node_modules/@types/scheduler": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.26.0.tgz", - "integrity": "sha512-WFHp9YUJQ6CKshqoC37iOlHnQSmxNc795UhB26CyBBttrN9svdIrUjl/NjnNmfcwtncN0h/0PPAFWv9ovP8mLA==", - "license": "MIT" - }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -11565,13 +11550,10 @@ } }, "node_modules/react": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", - "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", + "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", "license": "MIT", - "dependencies": { - "loose-envify": "^1.1.0" - }, "engines": { "node": ">=0.10.0" } @@ -11590,16 +11572,15 @@ } }, "node_modules/react-dom": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", - "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", + "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", "license": "MIT", "dependencies": { - "loose-envify": "^1.1.0", - "scheduler": "^0.23.2" + "scheduler": "^0.27.0" }, "peerDependencies": { - "react": "^18.3.1" + "react": "^19.2.8" } }, "node_modules/react-hook-form": { @@ -12204,13 +12185,10 @@ } }, "node_modules/scheduler": { - "version": "0.23.2", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", - "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", - "license": "MIT", - "dependencies": { - "loose-envify": "^1.1.0" - } + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" }, "node_modules/scroll-into-view-if-needed": { "version": "3.1.0", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index ac13a12620d..3f9c29d2afe 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -25,7 +25,6 @@ }, "dependencies": { "@ant-design/cssinjs": "1.24.0", - "@ant-design/icons": "5.6.1", "@anthropic-ai/sdk": "0.92.0", "@base-ui/react": "^1.6.0", "@headlessui/tailwindcss": "0.2.2", @@ -48,9 +47,9 @@ "openapi-fetch": "^0.17.0", "openapi-react-query": "^0.5.4", "papaparse": "5.5.3", - "react": "18.3.1", + "react": "19.2.8", "react-copy-to-clipboard": "5.1.1", - "react-dom": "18.3.1", + "react-dom": "19.2.8", "react-hook-form": "7.82.0", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", @@ -71,9 +70,9 @@ "@testing-library/react": "16.3.2", "@testing-library/user-event": "14.6.1", "@types/node": "20.19.37", - "@types/react": "18.2.48", + "@types/react": "19.2.18", "@types/react-copy-to-clipboard": "5.0.7", - "@types/react-dom": "18.3.7", + "@types/react-dom": "19.2.4", "@types/react-syntax-highlighter": "15.5.13", "@vitest/coverage-v8": "3.2.6", "@vitest/ui": "3.2.6", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupEditModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupEditModal.tsx index 6514c7072bb..7370b3bc195 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupEditModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupEditModal.tsx @@ -1,7 +1,6 @@ "use client"; import React, { useState } from "react"; -import { Modal } from "antd"; import { toast } from "@/lib/toast"; import { useZodForm } from "@/lib/forms/useZodForm"; @@ -18,6 +17,7 @@ import { MODELS_TAB, type AccessGroupFormValues, } from "./AccessGroupBaseForm"; +import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; interface AccessGroupEditModalProps { visible: boolean; @@ -87,13 +87,18 @@ function AccessGroupEditForm({ accessGroup, onCancel, onSuccess }: Omit - - + !open && onCancel()}> + + + Edit Access Group + + + + ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx index 8f51177bafe..4fc51910161 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx @@ -3,7 +3,7 @@ import { useDeleteAccessGroup } from "@/app/(dashboard)/hooks/accessGroups/useDe import { Plus, SearchIcon, X } from "lucide-react"; import { useMemo, useState } from "react"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { LegacyPageHeader } from "@/components/shared/LegacyPageHeader"; import { Button } from "@/components/ui/button"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { AccessGroupDetail } from "./AccessGroupsDetailsPage"; @@ -61,7 +61,7 @@ export function AccessGroupsPage() { return (
- = ({ proxySettings }) => { <> ✨ Security Settings - + + + SSO Configuration Deprecated + + Editing SSO Settings on this page is deprecated and will be removed in a future version. Please use the + SSO Settings tab for SSO configuration. + +
= ({ proxySettings }) => { accessToken={accessToken} ssoConfigured={ssoConfigured} /> - setIsAllowedIPModalVisible(false)} - footer={[ - , - , - ]} - > - - - - IP Address - Action - - - - {allowedIPs.map((ip, index) => ( - - {ip} - - {ip !== all_ip_address_allowed && ( - - )} - + !open && setIsAllowedIPModalVisible(false)}> + + + Manage Allowed IP Addresses + +
+ + + IP Address + Action - ))} - -
-
+ + + {allowedIPs.map((ip, index) => ( + + {ip} + + {ip !== all_ip_address_allowed && ( + + )} + + + ))} + + + + + + + + - setIsAddIPModalVisible(false)} - footer={null} - > - - + !open && setIsAddIPModalVisible(false)}> + + + Add Allowed IP Address + + + + - setIsDeleteIPModalVisible(false)} - onOk={confirmDeleteIP} - footer={[ - , - , - ]} - > - Are you sure you want to delete the IP address: {ipToDelete}? - + !open && setIsDeleteIPModalVisible(false)}> + + + Confirm Delete + + Are you sure you want to delete the IP address: {ipToDelete}? + + + + + + {/* UI Access Control Modal */} - !open && handleUIAccessControlCancel()} > - { - handleUIAccessControlOk(); - toast.success("UI Access Control settings updated successfully"); - }} - /> - + + + UI Access Control Settings + + { + handleUIAccessControlOk(); + toast.success("UI Access Control settings updated successfully"); + }} + /> + +
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx index 5e828537475..09737c3151b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx @@ -1,9 +1,9 @@ import React, { useState, useEffect } from "react"; -import { Modal, Select, Steps, Tag } from "antd"; +import { Select, Steps, Tag } from "antd"; import { FormProvider, useForm, useWatch } from "react-hook-form"; import { toast } from "@/lib/toast"; import { Logo } from "@/components/molecules/logo/Logo"; -import { CheckCircleFilled, KeyOutlined, RobotOutlined, AppstoreOutlined } from "@ant-design/icons"; +import { Bot, CircleCheck, Key, LayoutGrid } from "lucide-react"; import CreatedKeyDisplay from "@/components/shared/CreatedKeyDisplay"; import { Button } from "@/components/ui/button"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; @@ -49,6 +49,7 @@ import { import MCPServerSelector from "@/components/mcp_server_management/MCPServerSelector"; import MCPToolPermissions from "@/components/mcp_server_management/MCPToolPermissions"; import GuardrailSelector from "@/components/guardrails/GuardrailSelector"; +import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; const { Step } = Steps; @@ -706,7 +707,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok }`} onClick={() => handleAgentTypeChange(CUSTOM_AGENT_TYPE)} > - +
Custom / Other @@ -845,7 +846,8 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok
{/* Agent name chip */}
- } color="purple" className="px-3 py-1 text-sm"> + + {agentName}
@@ -883,7 +885,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok
- + Create a new key for this agent

A dedicated key scoped to this agent.

@@ -919,7 +921,7 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok
- + Assign an existing key

Re-assign a key you already have to this agent.

@@ -957,10 +959,11 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok const renderReadyStep = () => (
- +

Agent Created!

- } color="purple" className="px-3 py-1 text-sm"> + + {createdAgentName}
@@ -983,72 +986,64 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok ); return ( - - {selectedLogo && currentStep < 1 && ( - - )} -

Add New Agent

-
- } - open={visible} - onCancel={handleClose} - footer={null} - width={900} - className="top-8" - styles={{ - body: { padding: "24px" }, - header: { padding: "24px 24px 0 24px", border: "none" }, - }} - > - -
- - - - - - - + !open && handleClose()}> + + +
+ {selectedLogo && currentStep < 1 && ( + + )} + Add New Agent +
+
+ +
+ + + + + + + - -
event.preventDefault()} className="space-y-4"> - {currentStep === 0 && renderConfigureStep()} - {currentStep === 1 && renderEntitlementsStep()} - {currentStep === 2 && renderObservabilityStep()} - {currentStep === 3 && renderAssignKeyStep()} - {currentStep === 4 && renderReadyStep()} -
-
+ +
event.preventDefault()} className="space-y-4"> + {currentStep === 0 && renderConfigureStep()} + {currentStep === 1 && renderEntitlementsStep()} + {currentStep === 2 && renderObservabilityStep()} + {currentStep === 3 && renderAssignKeyStep()} + {currentStep === 4 && renderReadyStep()} +
+
-
-
- {currentStep > 0 && currentStep < 4 && ( - - )} -
-
- {currentStep < 4 && ( - - )} - {currentStep < 3 && } - {currentStep === 3 && ( - - )} - {currentStep === 4 && } +
+
+ {currentStep > 0 && currentStep < 4 && ( + + )} +
+
+ {currentStep < 4 && ( + + )} + {currentStep < 3 && } + {currentStep === 3 && ( + + )} + {currentStep === 4 && } +
-
- - + + +
); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx index d37065317c2..c507d56a63b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx @@ -1,5 +1,5 @@ import React, { useState, useEffect, useMemo } from "react"; -import { Spin, Descriptions } from "antd"; +import { cx } from "@/lib/cva.config"; import { FormProvider, useForm, useWatch } from "react-hook-form"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; @@ -40,6 +40,26 @@ interface AgentInfoViewProps { isAdmin: boolean; } +const DetailList: React.FC<{ children: React.ReactNode; className?: string }> = ({ children, className }) => ( +
+ {children} +
+); + +const DetailItem: React.FC<{ label: React.ReactNode; children: React.ReactNode }> = ({ label, children }) => ( + <> +
+ {label} +
+
{children}
+ +); + const AgentInfoView: React.FC = ({ agentId, onClose, accessToken, isAdmin }) => { const [agent, setAgent] = useState(null); const [selectedKey, setSelectedKey] = useState(null); @@ -195,7 +215,7 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT return (
- +
); @@ -276,51 +296,41 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT
{/* Overview Panel */} - - {agent.agent_id} - {agent.agent_name} - {agent.agent_card_params?.name || "-"} - {agent.agent_card_params?.description || "-"} - {agent.agent_card_params?.url || "-"} - {agent.agent_card_params?.version || "-"} - - {agent.agent_card_params?.protocolVersion || "-"} - - + + {agent.agent_id} + {agent.agent_name} + {agent.agent_card_params?.name || "-"} + {agent.agent_card_params?.description || "-"} + {agent.agent_card_params?.url || "-"} + {agent.agent_card_params?.version || "-"} + {agent.agent_card_params?.protocolVersion || "-"} + {agent.agent_card_params?.capabilities?.streaming ? "Yes" : "No"} - + {agent.agent_card_params?.capabilities?.pushNotifications && ( - Yes + Yes )} {agent.agent_card_params?.capabilities?.stateTransitionHistory && ( - Yes - )} - - {agent.agent_card_params?.skills?.length || 0} configured - - {agent.litellm_params?.model && ( - {agent.litellm_params.model} + Yes )} + {agent.agent_card_params?.skills?.length || 0} configured + {agent.litellm_params?.model && {agent.litellm_params.model}} {agent.litellm_params?.make_public !== undefined && ( - - {agent.litellm_params.make_public ? "Yes" : "No"} - + {agent.litellm_params.make_public ? "Yes" : "No"} )} {agent.agent_card_params?.iconUrl && ( - {agent.agent_card_params.iconUrl} + {agent.agent_card_params.iconUrl} )} {agent.agent_card_params?.documentationUrl && ( - - {agent.agent_card_params.documentationUrl} - + {agent.agent_card_params.documentationUrl} )} - {agent.tpm_limit ?? "Unlimited"} - {agent.rpm_limit ?? "Unlimited"} - {agent.session_tpm_limit ?? "Unlimited"} - {agent.session_rpm_limit ?? "Unlimited"} - {formatDate(agent.created_at)} - {formatDate(agent.updated_at)} - + {agent.tpm_limit ?? "Unlimited"} + {agent.rpm_limit ?? "Unlimited"} + {agent.session_tpm_limit ?? "Unlimited"} + {agent.session_rpm_limit ?? "Unlimited"} + {formatDate(agent.created_at)} + {formatDate(agent.updated_at)} + @@ -331,21 +341,19 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT Object.keys(agent.object_permission.mcp_tool_permissions).length > 0)) && (

MCP Tool Permissions

- + {agent.object_permission.mcp_servers && agent.object_permission.mcp_servers.length > 0 && ( - - {agent.object_permission.mcp_servers.join(", ")} - + {agent.object_permission.mcp_servers.join(", ")} )} {agent.object_permission.mcp_access_groups && agent.object_permission.mcp_access_groups.length > 0 && ( - + {agent.object_permission.mcp_access_groups.join(", ")} - + )} {agent.object_permission.mcp_tool_permissions && Object.keys(agent.object_permission.mcp_tool_permissions).length > 0 && ( - +
{Object.entries(agent.object_permission.mcp_tool_permissions).map(([serverId, tools]) => (
@@ -354,9 +362,9 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT
))}
-
+ )} -
+
)} @@ -365,9 +373,9 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT {agent.agent_card_params?.skills && agent.agent_card_params.skills.length > 0 && (

Skills

- + {agent.agent_card_params.skills.map((skill: any, index: number) => ( - +
ID: {skill.id} @@ -385,9 +393,9 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT
)}
-
+ ))} -
+
)}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx index fd657376066..5761d7308cd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx @@ -1,6 +1,5 @@ import { ChevronRight } from "lucide-react"; import React from "react"; -import { Modal } from "antd"; import { z } from "zod/v4"; import { useCreateBudget } from "@/app/(dashboard)/hooks/budgets/useBudgets"; import { applyBudgetPrecision } from "./budgetPrecision"; @@ -12,6 +11,7 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/component import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { useZodForm } from "@/lib/forms/useZodForm"; +import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; const budgetShape = { budget_id: z.string().min(1, "Please input a human-friendly name for the budget"), @@ -40,11 +40,6 @@ const BudgetModal: React.FC = ({ isModalVisible, setIsModalVis const form = useZodForm(budgetSchema, { defaultValues: { budget_id: "" } }); const createBudget = useCreateBudget(); - const handleOk = () => { - setIsModalVisible(false); - form.reset(); - }; - const handleCancel = () => { setIsModalVisible(false); form.reset(); @@ -68,102 +63,100 @@ const BudgetModal: React.FC = ({ isModalVisible, setIsModalVis }; return ( - -
- - - {({ ref, ...field }) => } - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - + !open && handleCancel()}> + + + Create Budget + + + + + {({ ref, ...field }) => } + + + {({ ref, value, onChange, ...field }) => ( + onChange(event.target.value === "" ? null : event.target.valueAsNumber)} + /> + )} + + + {({ ref, value, onChange, ...field }) => ( + onChange(event.target.value === "" ? null : event.target.valueAsNumber)} + /> + )} + - - - Optional Settings - - - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( - - )} - - - - + + + Optional Settings + + + + + {({ ref, value, onChange, ...field }) => ( + onChange(event.target.value === "" ? null : event.target.valueAsNumber)} + /> + )} + + + {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( + + )} + + + + -
- -
-
-
+
+ +
+ + + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx index 9aeedb29ecc..ad0f04f77aa 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx @@ -6,7 +6,7 @@ import { Plus, Wallet } from "lucide-react"; import React, { useCallback, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { LegacyPageHeader } from "@/components/shared/LegacyPageHeader"; import { ToolbarSeparator } from "@/components/shared/ToolbarSeparator"; import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; @@ -76,7 +76,7 @@ const BudgetPanel: React.FC = ({ accessToken }) => { return (
- } title="Budgets" subtitle="Spend, TPM and RPM limits you can assign to customers." diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx index 7ebab06cba6..3f9de881710 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx @@ -1,6 +1,5 @@ import { ChevronRight } from "lucide-react"; import React, { useEffect } from "react"; -import { Modal } from "antd"; import { useForm } from "react-hook-form"; import { useUpdateBudget } from "@/app/(dashboard)/hooks/budgets/useBudgets"; import { budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; @@ -12,6 +11,7 @@ import { Button } from "@/components/ui/button"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; type EditBudgetFormValues = Pick< budgetItem, @@ -46,11 +46,6 @@ const EditBudgetModal: React.FC = ({ isModalVisible, setIs form.reset(toFormValues(existingBudget)); }, [existingBudget, form]); - const handleOk = () => { - setIsModalVisible(false); - form.reset(); - }; - const handleCancel = () => { setIsModalVisible(false); form.reset(); @@ -74,95 +69,100 @@ const EditBudgetModal: React.FC = ({ isModalVisible, setIs }; return ( - -
- - - {({ ref, ...field }) => } - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - + !open && handleCancel()}> + + + Edit Budget + + + + + {({ ref, ...field }) => } + + + {({ ref, value, onChange, ...field }) => ( + onChange(event.target.value === "" ? null : event.target.valueAsNumber)} + /> + )} + + + {({ ref, value, onChange, ...field }) => ( + onChange(event.target.value === "" ? null : event.target.valueAsNumber)} + /> + )} + - - - Optional Settings - - - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( - - )} - - - - + + + Optional Settings + + + + + {({ ref, value, onChange, ...field }) => ( + onChange(event.target.value === "" ? null : event.target.valueAsNumber)} + /> + )} + + + {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( + + )} + + + + -
- -
-
-
+
+ +
+ + + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx index 7bb550f729e..b307e3d0f2a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx @@ -77,7 +77,15 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ baseline_model: null, judge_model: "anthropic/claude-sonnet-5", shadow_percentage: 10, - max_turns: 200, + keys: [ + { + api_key_id: "hashed-key-abc", + max_turns: 200, + stopped_at: null, + key_alias: "prod-alpha", + key_name: "sk-...alpha", + }, + ], judged_count: 42, error_count: 1, judge_spend: 3.21, @@ -110,19 +118,28 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ avg_judge_confidence: 0.8, }, ], + by_key: [], overall_shadow_win_rate_pct: 48.0, overall_tie_rate_pct: 22.0, }, created_at: "2026-08-07T00:00:00Z", ends_at: "2026-09-07T00:00:00Z", - stopped_at: null, - api_key_id: "hashed-key-abc", - key_alias: "prod-alpha", - key_name: "sk-...alpha", last_error: null, ...overrides, }); +const keyEntry = ( + api_key_id: string, + overrides: Partial = {}, +): ShadowEvalJob["keys"][number] => ({ + api_key_id, + max_turns: 200, + stopped_at: null, + key_alias: null, + key_name: null, + ...overrides, +}); + const mockHooks = ({ jobs = [], detailsById = {}, @@ -199,8 +216,8 @@ describe("ShadowEvalSection", () => { it("gives every active job its own card with a stop button, with the form still offered", () => { mockHooks({ jobs: [ - job({ job_id: "job-a", status: "running", api_key_id: "key-a" }), - job({ job_id: "job-b", status: "running", api_key_id: "key-b" }), + job({ job_id: "job-a", status: "running", keys: [keyEntry("key-a")] }), + job({ job_id: "job-b", status: "running", keys: [keyEntry("key-b")] }), ], }); render(); @@ -342,7 +359,7 @@ describe("ShadowEvalSection", () => { expect(container).toBeEmptyDOMElement(); }); - it("keeps the start button disabled until key, router, and judge model are picked, then submits them", async () => { + it("keeps the start button disabled until key, router, and judge model are picked, then submits the key as a list", async () => { const user = userEvent.setup(); const { start } = mockHooks({}); render(); @@ -361,7 +378,7 @@ describe("ShadowEvalSection", () => { await user.click(screen.getByText("Start shadow eval")); const expectedBody = { - api_key_id: "hash-alpha", + api_key_ids: ["hash-alpha"], router_name: "gpt-auto", direction: "forward", shadow_percentage: 10, @@ -396,7 +413,7 @@ describe("ShadowEvalSection", () => { await user.click(screen.getByText("Start shadow eval")); const expectedBody = { - api_key_id: "hash-alpha", + api_key_ids: ["hash-alpha"], router_name: "gpt-auto", direction: "reverse", baseline_model: "prod-claude", @@ -429,9 +446,9 @@ describe("ShadowEvalSection", () => { }); it("labels the shadowed key by alias, then masked name, then truncated hash", () => { - expect(shadowedKeyLabel(job())).toBe("prod-alpha"); - expect(shadowedKeyLabel(job({ key_alias: null }))).toBe("sk-...alpha"); - expect(shadowedKeyLabel(job({ key_alias: null, key_name: null }))).toBe("hashed-key…"); + expect(shadowedKeyLabel(job().keys[0])).toBe("prod-alpha"); + expect(shadowedKeyLabel(keyEntry("hashed-key-abc", { key_name: "sk-...alpha" }))).toBe("sk-...alpha"); + expect(shadowedKeyLabel(keyEntry("hashed-key-abc"))).toBe("hashed-key…"); }); it("keeps an older job's verdicts reachable through the previous evaluations list", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx index 005636615b1..6d240a84c98 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx @@ -24,6 +24,7 @@ import { useStartShadowEval, useStopShadowEval, type ShadowEvalJob, + type ShadowEvalJobKey, type ShadowEvalSlice, } from "./useShadowEval"; @@ -50,19 +51,24 @@ const routerMatchedOrBeatPct = ( ? 100 - results.overall_shadow_win_rate_pct : results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct; -export const shadowedKeyLabel = (job: ShadowEvalJob): string => - job.key_alias || job.key_name || `${job.api_key_id.slice(0, 10)}…`; +export const shadowedKeyLabel = (key: ShadowEvalJobKey): string => + key.key_alias || key.key_name || `${key.api_key_id.slice(0, 10)}…`; + +const shadowedKeysLabel = (job: ShadowEvalJob): string => + job.keys.length === 1 ? shadowedKeyLabel(job.keys[0]) : `${job.keys.length} keys`; + +const totalBudget = (job: ShadowEvalJob): number => job.keys.reduce((sum, key) => sum + key.max_turns, 0); const jobHeadline = (job: ShadowEvalJob): React.ReactNode => job.direction === "reverse" ? ( <> Comparing {job.router_name} to{" "} {job.baseline_model} on {job.shadow_percentage}% of{" "} - {shadowedKeyLabel(job)} traffic + {shadowedKeysLabel(job)} traffic ) : ( <> - Shadowing {job.shadow_percentage}% of {shadowedKeyLabel(job)} traffic + Shadowing {job.shadow_percentage}% of {shadowedKeysLabel(job)} traffic via {job.router_name} ); @@ -178,7 +184,8 @@ const emptyResultsText = (job: ShadowEvalJob, resultsError: boolean): string => const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({ job, resultsError = false }) => { const results = job.results; - if (!results || (results.by_tier.length === 0 && results.by_current_model.length === 0)) { + const stratifications = results ? [results.by_tier, results.by_current_model, results.by_key] : []; + if (!results || stratifications.every((slices) => slices.length === 0)) { return

{emptyResultsText(job, resultsError)}

; } return ( @@ -224,7 +231,7 @@ const JobResults: React.FC<{

{jobHeadline(job)}

- {(job.judged_count ?? 0).toLocaleString()} of {job.max_turns.toLocaleString()} turns judged ·{" "} + {(job.judged_count ?? 0).toLocaleString()} of {totalBudget(job).toLocaleString()} turns judged ·{" "} {(job.error_count ?? 0).toLocaleString()} errored · {usd(job.judge_spend ?? 0)} judge spend {active && remaining ? ` · ${remaining}` : ""}

@@ -386,14 +393,13 @@ const StartForm: React.FC = () => { const percentageValid = parsedPct >= 0.1 && parsedPct <= 100; const parsedMaxTurns = Number.parseInt(maxTurns, 10); const maxTurnsValid = parsedMaxTurns >= 1 && parsedMaxTurns <= 2000; - const filled = - [apiKeyId, routerName, judgeModel].every((field) => field !== "") && - (direction === "forward" || baselineModel !== ""); + const baselinePicked = direction === "forward" || baselineModel !== ""; + const filled = [apiKeyId, routerName, judgeModel].every((field) => field !== "") && baselinePicked; const boundsValid = percentageValid && maxTurnsValid; const valid = Boolean(accessToken) && filled && boundsValid; const handleStart = () => { const startBody = { - api_key_id: apiKeyId, + api_key_ids: [apiKeyId], router_name: routerName, direction, ...(direction === "reverse" ? { baseline_model: baselineModel } : {}), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts index 7645fcc3346..eef98320e67 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts @@ -7,6 +7,7 @@ import { $api, fetchClient } from "@/lib/http/api"; import type { components } from "@/lib/http/schema"; export type ShadowEvalJob = components["schemas"]["ShadowEvalJobResponse"]; +export type ShadowEvalJobKey = components["schemas"]["ShadowEvalJobKeyResponse"]; export type ShadowEvalSlice = components["schemas"]["ShadowEvalSlice"]; export type StartShadowEvalRequest = components["schemas"]["StartShadowEvalRequest"]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.integration.test.tsx index 83875cfc676..4680a4d504a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/add_provider_form.integration.test.tsx @@ -1,5 +1,3 @@ -// eslint-disable-next-line no-restricted-imports -- the parent cost_tracking_settings still owns this antd Form, and the point of this test is that AddProviderForm registers nothing in it -import { Form } from "antd"; import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -10,42 +8,31 @@ import { DiscountConfig } from "./types"; const onAddProvider = vi.fn(); const onParentFinish = vi.fn(); -const readParentStore = vi.fn(); -const ParentOwnedForm = () => { - const [form] = Form.useForm(); - return ( -
- { - readParentStore(form.getFieldsValue()); - onAddProvider(); - }} - /> - - ); -}; +const ParentOwnedForm = () => ( +
{ + event.preventDefault(); + onParentFinish(); + }} + className="space-y-6" + > + + +); -describe("AddProviderForm inside the antd form its parent owns", () => { +describe("AddProviderForm inside the form its parent owns", () => { beforeEach(() => { vi.clearAllMocks(); }); - it("registers no field in the parent FormInstance, so the parent's resetFields is a no-op", async () => { - const user = userEvent.setup(); - renderWithProviders(); - - await user.click(screen.getByRole("button", { name: /add provider discount/i })); - - expect(readParentStore).toHaveBeenCalledTimes(1); - expect(readParentStore.mock.calls[0][0]).toEqual({}); - }); - it("drives both the onAddProvider prop and the parent form submit from one click", async () => { const user = userEvent.setup(); renderWithProviders(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.integration.test.tsx index f811b55bc86..d4fbb7e153e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.integration.test.tsx @@ -72,7 +72,7 @@ describe("CostTrackingSettings submit paths", () => { const header = screen.getByText("Provider Discounts").closest("button"); if (header) await user.click(header); await user.click(await screen.findByRole("button", { name: /add provider discount/i })); - await screen.findByText("Add Provider Discount", { selector: "h2" }); + await screen.findByRole("dialog", { name: "Add Provider Discount" }); }; const submitDiscount = () => @@ -101,7 +101,7 @@ describe("CostTrackingSettings submit paths", () => { const header = screen.getByText("Fee/Price Margin").closest("button"); if (header) await user.click(header); await user.click(await screen.findByRole("button", { name: /add provider margin/i })); - await screen.findByText("Add Provider Margin", { selector: "h2" }); + await screen.findByRole("dialog", { name: "Add Provider Margin" }); await user.click(screen.getAllByRole("combobox")[0]); await user.click((await screen.findAllByRole("option"))[0]); @@ -136,7 +136,7 @@ describe("CostTrackingSettings submit paths", () => { const header = screen.getByText("Fee/Price Margin").closest("button"); if (header) await user.click(header); await user.click(await screen.findByRole("button", { name: /add provider margin/i })); - await screen.findByText("Add Provider Margin", { selector: "h2" }); + await screen.findByRole("dialog", { name: "Add Provider Margin" }); await user.click(screen.getAllByRole("combobox")[0]); await user.click((await screen.findAllByRole("option"))[0]); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx index 9cdc8509fbf..a8609aef629 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx @@ -136,7 +136,7 @@ describe("CostTrackingSettings", () => { const addButton = await screen.findByRole("button", { name: /add provider discount/i }); await user.click(addButton); - expect(await screen.findByText("Add Provider Discount", { selector: "h2" })).toBeInTheDocument(); + expect(await screen.findByRole("dialog", { name: "Add Provider Discount" })).toBeInTheDocument(); }); }); @@ -153,7 +153,7 @@ describe("CostTrackingSettings", () => { const addButton = await screen.findByRole("button", { name: /add provider margin/i }); await user.click(addButton); - expect(await screen.findByText("Add Provider Margin", { selector: "h2" })).toBeInTheDocument(); + expect(await screen.findByRole("dialog", { name: "Add Provider Margin" })).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx index 85350a539c2..bf5ae27aee0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx @@ -1,6 +1,5 @@ import React, { useState, useEffect } from "react"; import { ChevronDown } from "lucide-react"; -import { Modal } from "antd"; import { AlertDialog, AlertDialogCancel, @@ -24,6 +23,7 @@ import HowItWorks from "./how_it_works"; import { useDiscountConfig } from "./use_discount_config"; import { useMarginConfig } from "./use_margin_config"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; +import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; const DOCS_LINKS = [ { label: "Custom pricing for models", href: "https://docs.litellm.ai/docs/proxy/custom_pricing" }, @@ -331,77 +331,61 @@ const CostTrackingSettings: React.FC = ({ userID, use )} - -

Add Provider Discount

+ !open && handleModalCancel()}> + + +
+ Add Provider Discount +
+
+
+

+ Select a provider and set its discount percentage. Enter a value between 0% and 100% (e.g., 5 for a 5% + discount). +

+
event.preventDefault()} className="space-y-6"> + +
- } - open={isModalVisible} - width={1000} - onCancel={handleModalCancel} - footer={null} - className="top-8" - styles={{ - body: { padding: "24px" }, - header: { padding: "24px 24px 0 24px", border: "none" }, - }} - > -
-

- Select a provider and set its discount percentage. Enter a value between 0% and 100% (e.g., 5 for a 5% - discount). -

-
event.preventDefault()} className="space-y-6"> - - -
-
+ + - -

Add Provider Margin

+ !open && handleMarginModalCancel()}> + + +
+ Add Provider Margin +
+
+
+

+ Select a provider (or "Global" for all providers) and configure the margin. You can use + percentage-based or fixed amount. +

+
event.preventDefault()} className="space-y-6"> + +
- } - open={isMarginModalVisible} - width={1000} - onCancel={handleMarginModalCancel} - footer={null} - className="top-8" - styles={{ - body: { padding: "24px" }, - header: { padding: "24px 24px 0 24px", border: "none" }, - }} - > -
-

- Select a provider (or "Global" for all providers) and configure the margin. You can use - percentage-based or fixed amount. -

-
event.preventDefault()} className="space-y-6"> - - -
-
+ +
); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.test.tsx index 78526251049..60bf235040f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.test.tsx @@ -45,19 +45,7 @@ describe("GuardrailConfig", () => { it("should show custom code textarea when custom code override is toggled on", async () => { const user = userEvent.setup(); render(); - // Walk up from "Custom Code Override" heading to find the enclosing section, - // then locate the switch within it - const heading = screen.getByText("Custom Code Override"); - let container = heading.parentElement; - let customCodeSwitch: Element | null = null; - while (container && !customCodeSwitch) { - customCodeSwitch = container.querySelector('[role="switch"]'); - container = container.parentElement; - } - if (!customCodeSwitch) { - throw new Error("Could not find the Custom Code Override switch via DOM traversal"); - } - await user.click(customCodeSwitch); + await user.click(screen.getByRole("switch", { name: "Custom Code Override" })); expect(screen.getByPlaceholderText(/async def evaluate/)).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx index 667c52087f1..271de78c272 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx @@ -1,13 +1,11 @@ -import { - CheckCircleOutlined, - CodeOutlined, - PlayCircleOutlined, - RollbackOutlined, - SaveOutlined, -} from "@ant-design/icons"; -import { Input, Select, Switch } from "antd"; +import { CircleCheck, CirclePlay, Code, Save, Undo2 } from "lucide-react"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Button } from "@/components/ui/button"; -import React, { useState } from "react"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { Textarea } from "@/components/ui/textarea"; +import React, { useId, useState } from "react"; interface GuardrailConfigProps { guardrailName: string; @@ -27,6 +25,28 @@ const versions = [ { id: "v1", label: "v1", date: "2026-01-28", author: "admin@company.com", changes: "Initial configuration" }, ]; +const ACTION_ITEMS = [ + { value: "block", label: "Block Request" }, + { value: "flag", label: "Flag for Review" }, + { value: "log", label: "Log Only" }, + { value: "fallback", label: "Use Fallback Response" }, +]; + +const PROVIDER_ITEMS = [ + { value: "bedrock", label: "AWS Bedrock Guardrails" }, + { value: "google", label: "Google Cloud AI Safety" }, + { value: "litellm", label: "LiteLLM Built-in" }, + { value: "custom", label: "Custom Code" }, +]; + +const GUARDRAIL_TYPE_ITEMS = [ + { value: "Content Safety", label: "Content Safety" }, + { value: "PII", label: "PII Detection" }, + { value: "Topic", label: "Topic Restriction" }, + { value: "prompt_injection", label: "Prompt Injection" }, + { value: "custom", label: "Custom" }, +]; + export function GuardrailConfig({ guardrailName, guardrailType, provider }: GuardrailConfigProps) { const [action, setAction] = useState("block"); const [enabled, setEnabled] = useState(true); @@ -35,6 +55,7 @@ export function GuardrailConfig({ guardrailName, guardrailType, provider }: Guar const [rerunStatus, setRerunStatus] = useState<"idle" | "running" | "success" | "error">("idle"); const [version, setVersion] = useState("v3"); const [showVersionHistory, setShowVersionHistory] = useState(false); + const enabledToggleId = useId(); const handleRerun = () => { setRerunStatus("running"); @@ -52,22 +73,32 @@ export function GuardrailConfig({ guardrailName, guardrailType, provider }: Guar
Version:
@@ -109,45 +140,53 @@ export function GuardrailConfig({ guardrailName, guardrailType, provider }: Guar
- + + + + + {PROVIDER_ITEMS.map((item) => ( + + {item.label} + + ))} + +
- + + + + + {GUARDRAIL_TYPE_ITEMS.map((item) => ( + + {item.label} + + ))} + +
@@ -156,8 +195,10 @@ export function GuardrailConfig({ guardrailName, guardrailType, provider }: Guar
- - Guardrail enabled in production + +
@@ -167,16 +208,16 @@ export function GuardrailConfig({ guardrailName, guardrailType, provider }: Guar

- + Custom Code Override

Replace the built-in guardrail with custom evaluation code

- +
{useCustomCode && ( - setCustomCode(e.target.value)} placeholder={`async def evaluate(input_text: str, context: dict) -> dict: @@ -200,13 +241,13 @@ export function GuardrailConfig({ guardrailName, guardrailType, provider }: Guar
{rerunStatus === "success" && ( - 7/10 would now pass with new config + 7/10 would now pass with new config )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.integration.test.tsx index 5c34eca9804..ef1746310ae 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.integration.test.tsx @@ -136,7 +136,7 @@ describe("TeamGuardrailsTab submit payload", () => { await openSubmitModal(user); await fillRequiredFields(user, "https://guard.example.com/v1/check"); - await user.click(screen.getAllByRole("combobox")[1]); + await user.click(screen.getByRole("combobox", { name: "Mode" })); const options = await screen.findAllByText("During Call"); await user.click(options[options.length - 1]); await submit(user); @@ -149,13 +149,13 @@ describe("TeamGuardrailsTab submit payload", () => { const user = userEvent.setup(); await openSubmitModal(user); - const mode = screen.getAllByRole("combobox")[1]; + const mode = screen.getByRole("combobox", { name: "Mode" }); expect(mode).toHaveTextContent("Pre Call"); await user.click(mode); await user.click(await screen.findByRole("option", { name: "During Call" })); - expect(screen.getAllByRole("combobox")[1]).toHaveTextContent("During Call"); + expect(screen.getByRole("combobox", { name: "Mode" })).toHaveTextContent("During Call"); }); it("blocks an empty submit and reports every required field", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx index fc77562e932..146e0fa86e6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/TeamGuardrailsTab.tsx @@ -17,7 +17,6 @@ import { InfoIcon, CircleHelp, } from "lucide-react"; -import { Modal } from "antd"; import { z } from "zod/v4"; import { listGuardrailSubmissions, @@ -39,6 +38,8 @@ import { Textarea } from "@/components/ui/textarea"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { isAntdUrl } from "@/lib/forms/antdUrl"; import { useZodForm } from "@/lib/forms/useZodForm"; +import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; const GUARDRAIL_MODES = [ { value: "pre_call", label: "Pre Call" }, @@ -1001,6 +1002,7 @@ export function TeamGuardrailsTab({ accessToken }: TeamGuardrailsTabProps) { />
} - - - {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( - - )} - - - {({ ref, ...field }) => ( - - )} - - - {({ ref, ...field }) => ( -