diff --git a/.cargo/config.toml b/.cargo/config.toml index e6afd7ff530..c3206b2662e 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -17,3 +17,6 @@ rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] [target.aarch64-apple-darwin] rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] + +[env] +SQLX_OFFLINE = "true" diff --git a/.circleci/config.yml b/.circleci/config.yml index 7d4e2e40769..7276da9877b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3419,7 +3419,7 @@ workflows: name: integration-<< matrix.suite >> matrix: parameters: - suite: [management, accounting, database, providers, mcp, sdk, cost, browser] + suite: [management, accounting, database, providers, mcp, sdk, cost, security, browser] - integration_contracts: name: integration-extensions suite: extensions diff --git a/.circleci/scripts/unit_selection.sh b/.circleci/scripts/unit_selection.sh index 3f4f5620176..3a207ca1778 100755 --- a/.circleci/scripts/unit_selection.sh +++ b/.circleci/scripts/unit_selection.sh @@ -46,6 +46,7 @@ legacy_paths() { echo tests/unit/google_genai echo tests/unit/router_strategy echo tests/unit/router_utils + echo tests/unit/proxy/common_utils/test_cache_aware_routing.py echo tests/unit/enterprise/enterprise_callbacks/send_emails echo tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py echo tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py @@ -88,6 +89,7 @@ legacy_paths() { proxy-db-auth-checks) echo tests/unit/proxy/auth/test_auth_checks.py echo tests/unit/proxy/auth/test_user_api_key_auth.py + echo tests/unit/proxy/test_credential_slot_registry.py echo tests/unit/proxy/test_deprecated_key_grace_period.py ;; proxy-db-budgets) echo tests/unit/proxy/auth/test_default_end_user_budget_simple.py @@ -105,6 +107,7 @@ legacy_paths() { echo tests/unit/proxy/test_update_spend.py echo tests/unit/skills/test_skills_db.py ;; proxy-db-endpoints-and-responses) + echo tests/unit/proxy/engine echo tests/unit/proxy/auth/test_models_fallback_endpoint.py echo tests/unit/proxy/common_utils/test_check_batch_cost.py echo tests/unit/proxy/common_utils/test_check_responses_cost.py @@ -146,7 +149,10 @@ legacy_paths() { echo tests/unit/proxy/test_proxy_server.py ;; proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;; proxy-extras) echo tests/unit/litellm_proxy_extras ;; - proxy-infra) echo tests/unit/gateway ;; + proxy-infra) + echo tests/unit/gateway + echo tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py + echo tests/unit/proxy/roi_calculator ;; responses-caching-types) find tests/unit/responses -name 'test_*.py' -not -path 'tests/unit/responses/mcp/*' echo tests/unit/types ;; diff --git a/.githooks/commit-msg b/.githooks/commit-msg index b64e38a2286..b602861c672 100755 --- a/.githooks/commit-msg +++ b/.githooks/commit-msg @@ -42,7 +42,7 @@ case "$subject" in ;; esac -ALLOWED_TYPES="feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert" +ALLOWED_TYPES="feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert|security" # Description must not start with an uppercase letter — kept in sync with the # subjectPattern in .github/workflows/conventional-commits.yml so the local # hook is the strictly tighter of the two gates. (Without this guard, a commit @@ -61,7 +61,7 @@ cat >&2 <()!: (description must start with a lowercase letter) - Allowed types: feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert + Allowed types: feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert, security Examples: feat(router): add weighted round-robin strategy fix(bedrock): decouple STS region from aws_region_name diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 70a50d7f06e..582cf0f5217 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,10 +1,2 @@ -/ui/ @yuneng-berri @ryan-crabbe-berri -/litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri -/ui/Dockerfile -/ui/nginx.conf -/ui/litellm-dashboard/src/lib/http/schema.d.ts -/ui/litellm-dashboard/tsconfig.tsbuildinfo /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri -/litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri -/.github/CODEOWNERS @yuneng-berri diff --git a/.github/assets/roi-calculator/00-original-setup.png b/.github/assets/roi-calculator/00-original-setup.png new file mode 100644 index 00000000000..95bdeb56907 Binary files /dev/null and b/.github/assets/roi-calculator/00-original-setup.png differ diff --git a/.github/assets/roi-calculator/01-connect-github.png b/.github/assets/roi-calculator/01-connect-github.png new file mode 100644 index 00000000000..4214298785a Binary files /dev/null and b/.github/assets/roi-calculator/01-connect-github.png differ diff --git a/.github/assets/roi-calculator/02-repositories.png b/.github/assets/roi-calculator/02-repositories.png new file mode 100644 index 00000000000..81e69c20c2b Binary files /dev/null and b/.github/assets/roi-calculator/02-repositories.png differ diff --git a/.github/assets/roi-calculator/03-estimator-schedule.png b/.github/assets/roi-calculator/03-estimator-schedule.png new file mode 100644 index 00000000000..2934bc969d8 Binary files /dev/null and b/.github/assets/roi-calculator/03-estimator-schedule.png differ diff --git a/.github/assets/roi-calculator/04-backfill-progress.png b/.github/assets/roi-calculator/04-backfill-progress.png new file mode 100644 index 00000000000..19026b8042f Binary files /dev/null and b/.github/assets/roi-calculator/04-backfill-progress.png differ diff --git a/.github/assets/roi-calculator/06-overview.png b/.github/assets/roi-calculator/06-overview.png new file mode 100644 index 00000000000..abf2f7a0aaa Binary files /dev/null and b/.github/assets/roi-calculator/06-overview.png differ diff --git a/.github/assets/roi-calculator/07-people-unmatched.png b/.github/assets/roi-calculator/07-people-unmatched.png new file mode 100644 index 00000000000..a605d980f20 Binary files /dev/null and b/.github/assets/roi-calculator/07-people-unmatched.png differ diff --git a/.github/assets/roi-calculator/08-match-email.png b/.github/assets/roi-calculator/08-match-email.png new file mode 100644 index 00000000000..9f578fd783c Binary files /dev/null and b/.github/assets/roi-calculator/08-match-email.png differ diff --git a/.github/assets/roi-calculator/09-people-matched.png b/.github/assets/roi-calculator/09-people-matched.png new file mode 100644 index 00000000000..6d72179ae67 Binary files /dev/null and b/.github/assets/roi-calculator/09-people-matched.png differ diff --git a/.github/assets/roi-calculator/10-pr-reasoning.png b/.github/assets/roi-calculator/10-pr-reasoning.png new file mode 100644 index 00000000000..423c6bdc3e3 Binary files /dev/null and b/.github/assets/roi-calculator/10-pr-reasoning.png differ diff --git a/.github/assets/roi-calculator/11-settings.png b/.github/assets/roi-calculator/11-settings.png new file mode 100644 index 00000000000..1ef5c446408 Binary files /dev/null and b/.github/assets/roi-calculator/11-settings.png differ diff --git a/.github/assets/roi-calculator/12-restart-setup.png b/.github/assets/roi-calculator/12-restart-setup.png new file mode 100644 index 00000000000..7a2f410a5e2 Binary files /dev/null and b/.github/assets/roi-calculator/12-restart-setup.png differ diff --git a/.github/assets/roi-calculator/13-advanced-settings.png b/.github/assets/roi-calculator/13-advanced-settings.png new file mode 100644 index 00000000000..61549454c88 Binary files /dev/null and b/.github/assets/roi-calculator/13-advanced-settings.png differ diff --git a/.github/assets/roi-calculator/14-overview-pulls.png b/.github/assets/roi-calculator/14-overview-pulls.png new file mode 100644 index 00000000000..0f07752c4c3 Binary files /dev/null and b/.github/assets/roi-calculator/14-overview-pulls.png differ diff --git a/.github/assets/roi-calculator/15-sample-preview.png b/.github/assets/roi-calculator/15-sample-preview.png new file mode 100644 index 00000000000..6128d0a5dff Binary files /dev/null and b/.github/assets/roi-calculator/15-sample-preview.png differ diff --git a/.github/assets/roi-calculator/16-calculator-sidebar.png b/.github/assets/roi-calculator/16-calculator-sidebar.png new file mode 100644 index 00000000000..8ed3042f36c Binary files /dev/null and b/.github/assets/roi-calculator/16-calculator-sidebar.png differ diff --git a/.github/assets/roi-calculator/19-matching-calculator-icons.png b/.github/assets/roi-calculator/19-matching-calculator-icons.png new file mode 100644 index 00000000000..af12106e315 Binary files /dev/null and b/.github/assets/roi-calculator/19-matching-calculator-icons.png differ diff --git a/.github/assets/roi-calculator/20-partial-repository-report.png b/.github/assets/roi-calculator/20-partial-repository-report.png new file mode 100644 index 00000000000..eac03deddae Binary files /dev/null and b/.github/assets/roi-calculator/20-partial-repository-report.png differ diff --git a/.github/assets/roi-calculator/21-empty-repository-preserved-report.png b/.github/assets/roi-calculator/21-empty-repository-preserved-report.png new file mode 100644 index 00000000000..4c6add87f95 Binary files /dev/null and b/.github/assets/roi-calculator/21-empty-repository-preserved-report.png differ diff --git a/.github/assets/roi-calculator/22-partial-calculation-explanation.png b/.github/assets/roi-calculator/22-partial-calculation-explanation.png new file mode 100644 index 00000000000..5415956b3fa Binary files /dev/null and b/.github/assets/roi-calculator/22-partial-calculation-explanation.png differ diff --git a/.github/assets/roi-calculator/23-estimator-outage-preserved-report.png b/.github/assets/roi-calculator/23-estimator-outage-preserved-report.png new file mode 100644 index 00000000000..346cc2acab7 Binary files /dev/null and b/.github/assets/roi-calculator/23-estimator-outage-preserved-report.png differ diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index 69d9f427212..445a8519436 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -111,3 +111,10 @@ dockerfiles: An example image under cookbook/ that is documentation rather than a shipped artifact paths: - cookbook/litellm-ollama-docker-image/Dockerfile + - reason: >- + The Rust gateway image compiles the whole workspace in release mode, which is too slow for + a per-pull-request job while the gateway binary is still being assembled; the Rust lint, + clippy, and compile jobs already cover the code it packages. Revisit when the gateway is + published + paths: + - litellm-rust/crates/gateway/Dockerfile diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index 6b7fcd57bbc..465918f5a81 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -214,7 +214,7 @@ def main( native_module: Final = load_native_module(native_path) native_module_loads: Final = native_module is not None panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") - native_size_limit: Final = 40_000_000 + native_size_limit: Final = 45_000_000 native_size_within_limit: Final = native_member.file_size <= native_size_limit validations: Final = ( (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), diff --git a/.github/workflows/conventional-commits.yml b/.github/workflows/conventional-commits.yml index eb9eb69f8b6..4ae59ed581f 100644 --- a/.github/workflows/conventional-commits.yml +++ b/.github/workflows/conventional-commits.yml @@ -41,6 +41,7 @@ jobs: ci chore revert + security requireScope: false subjectPattern: ^(?![A-Z]).+$ subjectPatternError: | diff --git a/.github/workflows/create-rc-branch.yml b/.github/workflows/create-rc-branch.yml index 53760ad553e..5269460cc93 100644 --- a/.github/workflows/create-rc-branch.yml +++ b/.github/workflows/create-rc-branch.yml @@ -15,6 +15,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: write + outputs: + version: ${{ steps.version.outputs.version }} steps: - name: Require main env: @@ -64,3 +66,14 @@ jobs: sha: context.sha, }); core.info(`Created branch ${branchName} at ${context.sha}`); + + linear-release: + name: Move the Linear release to rc + needs: create-rc-branch + permissions: + contents: read + uses: ./.github/workflows/linear-release.yml + with: + rc_version: ${{ needs.create-rc-branch.outputs.version }} + secrets: + LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }} diff --git a/.github/workflows/lens-worker.yml b/.github/workflows/lens-worker.yml new file mode 100644 index 00000000000..0798425fd76 --- /dev/null +++ b/.github/workflows/lens-worker.yml @@ -0,0 +1,60 @@ +name: Lens Worker Image + +on: + pull_request: + branches: [main, litellm_oss_branch, "litellm_**"] + paths: + - deploy/lens/** + - litellm/proxy/engine/** + - .github/workflows/lens-worker.yml + push: + branches: [main, litellm_agent_engine] + paths: + - deploy/lens/** + - litellm/proxy/engine/** + - .github/workflows/lens-worker.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + lens-worker-image: + permissions: + contents: read + packages: write + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + - name: Build Lens worker + run: docker build -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} . + - name: Verify standalone imports with a read-only filesystem + run: | + docker run --rm --network none --read-only --cap-drop ALL \ + --security-opt no-new-privileges --entrypoint python \ + lens-worker:${{ github.sha }} -c ' + import os + import engine.worker + from engine.trace_store import trace_store + assert os.getuid() == 65532 + with trace_store() as store: + assert store.count() == 0 + ' + - name: Publish versioned Lens worker + if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' + env: + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REGISTRY_USER: ${{ github.actor }} + IMAGE: ghcr.io/berriai/litellm-lens-worker:sha-${{ github.sha }} + run: | + printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin + docker tag lens-worker:${{ github.sha }} "$IMAGE" + docker push "$IMAGE" + printf 'Lens worker image: `%s`\n' "$IMAGE" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/linear-release.yml b/.github/workflows/linear-release.yml new file mode 100644 index 00000000000..a1457fc0778 --- /dev/null +++ b/.github/workflows/linear-release.yml @@ -0,0 +1,131 @@ +name: Linear Release + +on: + push: + branches: + - main + - "rc/**" + release: + types: [published] + workflow_call: + inputs: + rc_version: + description: "X.Y.0 release whose rc branch was just cut" + required: true + type: string + secrets: + LINEAR_API_KEY: + required: true + +permissions: {} + +jobs: + linear-release: + name: Linear Release + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Plan + id: plan + env: + EVENT: ${{ github.event_name }} + REF_NAME: ${{ github.ref_name }} + BEFORE: ${{ github.event.before }} + CREATED: ${{ github.event.created }} + RC_VERSION: ${{ inputs.rc_version }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + PRERELEASE: ${{ github.event.release.prerelease }} + run: | + set -euo pipefail + sync_base="${BEFORE}" + if [ "${CREATED}" = "true" ]; then + sync_base="" + fi + if [ -n "${RC_VERSION}" ]; then + echo "version=${RC_VERSION}" >> "$GITHUB_OUTPUT" + echo "stage=rc" >> "$GITHUB_OUTPUT" + elif [ "${EVENT}" = "release" ]; then + if [ "${PRERELEASE}" = "true" ] || ! echo "${RELEASE_TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.0$'; then + echo "::notice::${RELEASE_TAG} is not an X.Y.0 stable release; nothing to complete" + exit 0 + fi + echo "version=${RELEASE_TAG#v}" >> "$GITHUB_OUTPUT" + echo "complete=true" >> "$GITHUB_OUTPUT" + elif [ "${REF_NAME}" = "main" ]; then + version="$(python3 .github/scripts/read_rc_version.py | cut -d= -f2)" + status=0 + git ls-remote --exit-code --heads origin "rc/${version}" > /dev/null || status=$? + case "${status}" in + 0) + IFS=. read -r major minor _ <<< "${version}" + version="${major}.$((minor + 1)).0" + ;; + 2) ;; + *) + echo "::error::could not check whether rc/${version} exists (git ls-remote exit ${status})" + exit 1 + ;; + esac + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "sync_base=${sync_base}" >> "$GITHUB_OUTPUT" + echo "main=true" >> "$GITHUB_OUTPUT" + else + echo "version=${REF_NAME#rc/}" >> "$GITHUB_OUTPUT" + echo "sync_base=${sync_base}" >> "$GITHUB_OUTPUT" + echo "stage=rc" >> "$GITHUB_OUTPUT" + fi + + - name: Sync commits into the release + if: steps.plan.outputs.sync_base != '' + uses: linear/linear-release-action@d4af10092984f9bc6d5efa075b242bdf01333463 # v0.18.0 + with: + access_key: ${{ secrets.LINEAR_API_KEY }} + command: sync + name: LiteLLM ${{ steps.plan.outputs.version }} + version: ${{ steps.plan.outputs.version }} + base_ref: ${{ steps.plan.outputs.sync_base }} + cli_version: v0.18.0 + + - name: Keep the main stage unless the rc branch was cut during this run + id: main_stage + if: steps.plan.outputs.main == 'true' + env: + VERSION: ${{ steps.plan.outputs.version }} + run: | + set -euo pipefail + status=0 + git ls-remote --exit-code --heads origin "rc/${VERSION}" > /dev/null || status=$? + case "${status}" in + 0) echo "::notice::rc/${VERSION} was cut during this run; leaving the release in its rc stage" ;; + 2) echo "stage=main" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::could not check whether rc/${VERSION} exists (git ls-remote exit ${status})" + exit 1 + ;; + esac + + - name: Move the release to its stage + if: steps.plan.outputs.stage != '' || steps.main_stage.outputs.stage != '' + uses: linear/linear-release-action@d4af10092984f9bc6d5efa075b242bdf01333463 # v0.18.0 + with: + access_key: ${{ secrets.LINEAR_API_KEY }} + command: update + stage: ${{ steps.plan.outputs.stage || steps.main_stage.outputs.stage }} + version: ${{ steps.plan.outputs.version }} + cli_version: v0.18.0 + + - name: Complete the release + if: steps.plan.outputs.complete == 'true' + uses: linear/linear-release-action@d4af10092984f9bc6d5efa075b242bdf01333463 # v0.18.0 + with: + access_key: ${{ secrets.LINEAR_API_KEY }} + command: complete + version: ${{ steps.plan.outputs.version }} + cli_version: v0.18.0 diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index 23955e33dec..b4c01865583 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -80,6 +80,11 @@ jobs: - name: test_e2e_changed_gate run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py + - name: test_e2e_metadata + env: + PYTHONPATH: tests/e2e + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py + - name: Check merge smoke harness run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py diff --git a/.github/workflows/test-postgres.yml b/.github/workflows/test-postgres.yml index a1e6bf54135..1ffb7f67f16 100644 --- a/.github/workflows/test-postgres.yml +++ b/.github/workflows/test-postgres.yml @@ -24,6 +24,7 @@ jobs: timeout-minutes: ${{ matrix.job-timeout-minutes }} permissions: contents: read + id-token: write services: postgres: @@ -134,9 +135,19 @@ jobs: env: TEST_PATH: ${{ matrix.test-path }} WORKERS: ${{ matrix.workers }} + PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=litellm/proxy/engine --cov-report=xml:coverage-lens-postgres.xml' || '' }} run: | if [ "${WORKERS}" = "0" ]; then uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 else uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}" fi + + - name: Upload Lens database coverage + if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'proxy-behavior' && !cancelled() + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + use_oidc: true + files: coverage-lens-postgres.xml + flags: lens-postgres + fail_ci_if_error: true diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f55e186e3b2..d3604a81e8f 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -79,7 +79,9 @@ jobs: - shard: integrations artifact-name: integrations - test-path: "" + test-path: >- + tests/test_litellm/integrations + tests/test_litellm/tracing unit-flag: integrations workers: 2 reruns: 3 diff --git a/Makefile b/Makefile index 79c18f6fe82..cad3242fbce 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ .PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \ test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \ test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ - test-rust-extension \ + test-rust-extension rust-sqlx-prepare \ info lint lint-inner lint-dev lint-checks format \ lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \ lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \ @@ -56,6 +56,7 @@ help: @echo " make test-integration - Run integration tests" @echo " make test-unit-helm - Run helm unit tests" @echo " make test-rust-extension - Build the Rust extension and run its public Python tests" + @echo " make rust-sqlx-prepare - Refresh litellm-rust/crates/db/.sqlx against a migrated Postgres container" @echo "" @echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide" @echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine." @@ -306,6 +307,9 @@ test-rust-extension: LITELLM_RUST=1 LITELLM_LOCAL_MODEL_COST_MAP=True \ "$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust +rust-sqlx-prepare: + cd litellm-rust && cargo run -p litellm-db-testing --bin sqlx-prepare + test: install-test-deps $(UV_RUN) pytest tests/ diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py index 232561dd154..51a4d8f716c 100644 --- a/backend/routes/allowlist.py +++ b/backend/routes/allowlist.py @@ -81,6 +81,8 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = ( # Spend / analytics "/spend/", "/analytics/", + "/engine/", + "/v1/traces", "/global/", "/user_agent", "/usage/", @@ -144,6 +146,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset( { "/", "/routes", + "/engine", "/openapi.json", "/docs", "/docs/oauth2-redirect", diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 26e4e06a796..92dc89eb0b8 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -99,7 +99,7 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 44358 + "limit": 44802 }, "reportUnknownLambdaType": { "limit": 109 diff --git a/cookbook/litellm_proxy_server/mcp/README.md b/cookbook/litellm_proxy_server/mcp/README.md deleted file mode 100644 index aeee0719019..00000000000 --- a/cookbook/litellm_proxy_server/mcp/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# Publish MCP servers in the AI Hub - -Set `litellm_settings.public_mcp_servers` to the concrete IDs of the servers you want listed in the public AI Hub. Pin `server_id` in each configuration entry so the publication list stays stable across deployments - -```yaml -mcp_servers: - documentation: - server_id: documentation-mcp - url: https://mcp.example.com/mcp - transport: http - available_on_public_internet: true - -litellm_settings: - public_mcp_hub_strict_whitelist: true - public_mcp_servers: - - documentation-mcp -``` - -Use `documentation-mcp`, the `server_id`, in the publication list. The configuration key `documentation`, display names, and aliases are not publication IDs. Database-created servers use the ID returned by `/v1/mcp/server` - -The dashboard's **AI Hub > MCP Hub > Manage MCP Hub Visibility** dialog edits this same list. Its YAML example includes the selected server IDs. With database-backed configuration (`store_model_in_db: true`), a value declared in YAML is owned by that file: edit the file and reload, or remove that key from YAML to let the dashboard manage it in the database. File-backed deployments can save the list directly to their configuration file - -To remove all explicit entries, save an empty selection in the dialog or configure: - -```yaml -litellm_settings: - public_mcp_hub_strict_whitelist: true - public_mcp_servers: [] -``` - -## Hub listing and network access - -The **Hub listing** column in AI Hub identifies servers that appear in `/public/mcp_hub`. The dashboard derives this status from the current registry and publication settings. Setting `mcp_info.is_public` on a server does not publish it; that response field is derived metadata. `mcp_info.is_public_explicit` identifies registered servers included in the explicit publication list - -Gateway cards and server details show **All Networks** when `available_on_public_internet` is enabled or the server is explicitly published in `public_mcp_servers`. They show **Internal Only** when both are false. The per-server flag defaults to `true`; explicit publication overrides a disabled flag for compatibility. Older proxies that omit the metadata needed to determine access show **Unknown**. These labels describe allowed client IPs; authentication and tool permissions still apply - -The default `public_mcp_hub_strict_whitelist: true` lists only registered servers in `public_mcp_servers`. Legacy mode (`false`) additionally lists registered servers with `available_on_public_internet: true`. In legacy mode, clearing the explicit publication list leaves these automatically listed servers visible. Enable strict mode when the publication list should fully determine hub visibility diff --git a/cookbook/misc/config.yaml b/cookbook/misc/config.yaml index 27a6332a882..a485bf825fc 100644 --- a/cookbook/misc/config.yaml +++ b/cookbook/misc/config.yaml @@ -24,7 +24,7 @@ model_list: - model_name: sagemaker-completion-model litellm_params: model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4 - input_cost_per_second: 0.000420 + cost_per_second: 0.000420 - model_name: text-embedding-ada-002 litellm_params: model: azure/azure-embedding-model diff --git a/cookbook/misc/test_responses_api.py b/cookbook/misc/test_responses_api.py index 0011db4664d..68da5fb6cd0 100644 --- a/cookbook/misc/test_responses_api.py +++ b/cookbook/misc/test_responses_api.py @@ -12,7 +12,7 @@ def encode_image(image_path): # Path to your image -image_path = "litellm/proxy/logo.jpg" +image_path = "litellm/proxy/logo.png" # Getting the Base64 string base64_image = encode_image(image_path) @@ -27,7 +27,7 @@ response = client.responses.create( {"type": "input_text", "text": "what color is the image"}, { "type": "input_image", - "image_url": f"data:image/jpeg;base64,{base64_image}", + "image_url": f"data:image/png;base64,{base64_image}", }, ], } diff --git a/deploy/lens/Dockerfile b/deploy/lens/Dockerfile new file mode 100644 index 00000000000..dc6f61a4d94 --- /dev/null +++ b/deploy/lens/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.12-slim +WORKDIR /app +RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7 +COPY litellm/proxy/engine/__init__.py litellm/proxy/engine/models.py litellm/proxy/engine/trace_store.py litellm/proxy/engine/analysis.py litellm/proxy/engine/worker.py /app/engine/ +VOLUME /tmp +USER 65532:65532 +CMD ["python", "-m", "engine.worker"] diff --git a/deploy/lens/Dockerfile.dockerignore b/deploy/lens/Dockerfile.dockerignore new file mode 100644 index 00000000000..8478be71be7 --- /dev/null +++ b/deploy/lens/Dockerfile.dockerignore @@ -0,0 +1,8 @@ +** +!litellm/ +!litellm/proxy/ +!litellm/proxy/engine/ +!litellm/proxy/engine/__init__.py +!litellm/proxy/engine/models.py +!litellm/proxy/engine/analysis.py +!litellm/proxy/engine/worker.py diff --git a/deploy/lens/README.md b/deploy/lens/README.md new file mode 100644 index 00000000000..22754f89287 --- /dev/null +++ b/deploy/lens/README.md @@ -0,0 +1,105 @@ +# Lens worker + +Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`) + +## Start a worker + +Upgrade your existing LiteLLM proxy to a release that includes Lens with PostgreSQL, agent tracing (`general_settings.tracing: {store: clickhouse}`), and ClickHouse configured through `CLICKHOUSE_URL` and a separate SELECT-only `CLICKHOUSE_READER_URL`. Enable the ClickHouse callback and request/response logging to analyze LLM requests. Lens can only inspect content you actually retain + +In Lens, click **Connect worker**, then **Generate setup command**. The LiteLLM address is filled in for you; change it only if the server running Docker needs a different network address. Copy the command and run it on your server. The dialog changes to **Worker connected** when the container checks in + +The command already contains the compatible worker image and one worker token. No separate API key, source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis + +The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. Worker image releases are independent of proxy releases: update the pinned image when changing their API contract. CI also publishes immutable commit tags for reproducible builds + +For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL` and `LENS_WORKER_TOKEN` in an environment file. Its default image is already selected: + +```bash +docker compose --env-file /path/to/lens.env -f compose.yaml up -d +``` + +Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build` + +The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its configured router; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles one scan at a time and can serve multiple lenses. For more throughput, start another worker with a separate credential + +V1 setup, manual runs, feedback, and worker credentials are restricted to proxy administrators. Proxy-admin viewers can inspect results. Regular user and team keys cannot access the Lens API. Worker credentials can serve the administrator’s lenses. Revoke it in the connection dialog when retiring a worker. Redeploy the worker alongside proxy upgrades so their API versions match + +## Configure a lens + +Choose agent runs, individual LLM requests, or both. The matching-activity preview updates as you choose an application (the recorded OpenTelemetry service.name) or, for request activity, a LiteLLM model group and add metadata conditions. It shows run names, timestamps, and trace IDs; open a run to inspect its original steps before starting analysis. Suggestions come from up to 100 recent executions and may not include every recorded attribute. You can enter other exact keys and values. Leave service and filters blank for all activity your account can access. Filters are exact key/value matches, combined with AND. Trace filters match span or resource attributes on the same span. Request filters match logged metadata, including caller metadata stored under `requester_metadata`; `tag=value` matches request tags. `swarm=research` works only if your instrumentation records that attribute + +Describe how the agent should behave and optionally add specific checks. Select the lookback window, team and metadata, then choose the percentage to review and an optional maximum. **100% with no maximum selects every matching run**. The preview pages through all matching activity and lets you select particular runs. Percentage sampling uses a stable hash order, rounds up, and applies the optional maximum after the percentage + +Choose your analysis model, parallelism and monthly budget. Parallelism controls simultaneous model calls, not the number of runs selected. New lenses run once by default. Turn on monitoring to repeat the same setup at a custom interval. **Run now** uses the same saved settings immediately, including the same lookback window and sampling. Every scan recalculates the window, so overlapping windows can review the same activity again. Duplicate a lens when you want a separate investigation without changing an existing monitor + +Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every 10 seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. Configuration edits apply to the next scan. A running scan retains its settings and selected execution IDs across retries + +## Read the results + +Needs attention shows issues, highest priority first. Patterns contains useful trends and successful behavior that may not need a fix. Each finding starts with a short explanation and a next step when useful. Expand the limitations for uncertainty and counterexamples. Evidence is grouped by run and collapsed until you need it; each quote opens the original step + +Use the batch selector or Scans tab to reopen previous results. Each batch keeps its own findings, settings, selected runs, coverage and cost. Older batches created before snapshot support remain available through accumulated findings. The Runs tab lists the selected batch's sample and can filter per-run observations, including runs without an observed issue and runs with insufficient evidence. These observations precede the final evidence investigation. Linked-run counts on findings include cited counterexamples, so they are not failure counts + +Choose **This is expected** and explain why to teach later scans about acceptable behavior. Feedback is kept with the lens and included in subsequent reviews. It does not alter historical evidence or exempt different problems + +## What a scan does + +The proxy selects executions received or updated within the configured lookback window, with a two-minute settling period. Older rows without receipt timestamps use execution end time. Overlapping scans do not increment a finding's occurrence count for the same execution ID + +A trace is spans sharing a trace ID within one team, not an automatically reconstructed conversation session. Requests are individual LLM calls. When both sources are enabled, requests correlated to a recorded span by response ID are excluded to reduce double counting + +The worker reviews the selected executions in parallel. It pages through their recorded spans and gives the first reviewer a catalog, task and outcome excerpts. The reviewer can read more original content to resolve uncertainties. Large catalogs and groups of observations are processed in bounded context windows, with every page available. Grouping retains supporting run IDs in code, so a pattern occurring thousands of times does not require a model to repeat thousands of IDs. Candidate investigators can page through supporting observations, other runs and original evidence + +There is no fixed total run, span, candidate or investigation-turn cutoff. Repeated or empty evidence requests stop a stalled investigation. Context windows, the configured budget, available model capacity and recorded evidence still bound practical work. The dashboard reports completed work and gaps. The investigator has no shell, browsing, code-editing or production-action tools + +Each model response must match a bounded JSON schema. A malformed response gets one repair attempt through the same budget controls; repeated invalid output fails the scan. Both the worker and proxy validate quoted evidence. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed + +Coverage distinguishes eligible, sampled, reviewed, partial, and unassessable executions. Findings describe observations in the sample, not population-wide success rates or proven causes. A root span does not prove that a trace contains every expected span. Long, missing, redacted, or expired content limits the conclusions + +## Operations and limits + +PostgreSQL stores configurations, findings and all scan history, returned in pages of 50 jobs. Workers claim jobs with optimistic concurrency and a five-minute lease, renewed every 30 seconds. A disconnected job can be reclaimed up to three times. Cancellation stops subsequent work; a model call already in flight may finish and incur cost + +Before every model call, Lens reserves a conservative amount against the monthly lens budget. Successful calls reconcile to reported cost where pricing is available. Interrupted calls retain their reservation because the provider may have charged. A scan stops when the next reservation would exceed the limit, so it can stop with some budget remaining. Lens budgets are separate from virtual-key budgets; analysis calls use the proxy router directly + +V1 requires ClickHouse for both sources. It does not reconstruct sessions from unrelated trace IDs, guarantee exhaustive reviews, cache all per-execution observations across scans, or automatically fix agent code. Trace contents can change as late spans arrive, even though a job's selected IDs are fixed. Findings should be reviewed by a person before acting on them + + +## API access + +The UI and API use the same scan lifecycle. Authenticate with a proxy administrator credential for writes, or a proxy-admin viewer credential for reads. Worker credentials are only for worker operations + +```bash +curl "$LITELLM_URL/engine" -H "Authorization: Bearer $LITELLM_API_KEY" \ + -H 'Content-Type: application/json' -d '{ + "name": "Research quality", "model": "your-model-alias", + "context": "Answer the requested question using cited, retrieved evidence.", + "source": "traces", "lookback_hours": 24, + "sample_percent": 100, "sample_size": null, "concurrency": 8, + "enabled": true, "interval_minutes": 1440, "monthly_budget": 50 + }' + +curl "$LITELLM_URL/engine/$LENS_ID/runs" -X POST \ + -H "Authorization: Bearer $LITELLM_API_KEY" -H 'Content-Type: application/json' -d '{}' + +curl "$LITELLM_URL/engine/$LENS_ID/runs?offset=0" -H "Authorization: Bearer $LITELLM_API_KEY" +curl "$LITELLM_URL/engine/$LENS_ID/runs/$BATCH_ID" -H "Authorization: Bearer $LITELLM_API_KEY" +``` + +Creation queues the first batch. Posting to `/engine/{id}/runs` queues another, or returns the existing active batch. The run response contains its ID under `jobs[0].id`. Poll the batch URL for status, findings and assessments. List responses omit large result payloads; request a batch to retrieve them. Supply an optional complete `settings` object on the runs POST for a one-off override; the saved lens stays unchanged. Selection accepts `team_id`, exact `filters`, and opaque `execution_ids` returned by `/engine/preview/sample`. Preview accepts `offset` and `as_of` to keep the time window fixed while paging. Feedback uses `PATCH /engine/{id}/findings/{finding_id}` with `status` and `reason` + +## Quality evaluation + +Run the checked-in cases against a configured real model. Expected labels are used only for scoring, never passed to the model. Dev and held-out cases include missing outcomes, failed tools, recovery, handoffs, unsupported claims, repeated work, long evidence and prompt injection. The background option adds clean arithmetic traces to test rare-issue discovery at scale; those repeated synthetic cases do not establish accuracy on every production workload + +```bash +python -m tests.proxy_behavior.lens.evaluate --api-base "$LITELLM_URL" \ + --model your-model-alias --split all --background 1000 --concurrency 16 \ + --output /tmp/lens-quality.json +``` + +Set `LITELLM_API_KEY` privately. This makes paid model calls. Inspect missed and unexpected per-run labels, final findings and coverage; do not equate a passing dataset with guaranteed detection on arbitrary traces + +The worker uses temporary disk space for trace content while reviewing it, and removes those files after each review. Its Docker image supplies a writable temporary volume while keeping the application filesystem read-only + +To check that accepted behavior stays accepted without hiding new problems, run the evaluator with `--dataset tests/proxy_behavior/lens/feedback_cases.json`. Reports include elapsed time, model call count, reported cost when the proxy provides it, missed checks, unexpected checks, and inconclusive candidates diff --git a/deploy/lens/compose.build.yaml b/deploy/lens/compose.build.yaml new file mode 100644 index 00000000000..e4237d8de23 --- /dev/null +++ b/deploy/lens/compose.build.yaml @@ -0,0 +1,6 @@ +services: + lens-worker: + build: + context: ../.. + dockerfile: deploy/lens/Dockerfile + image: litellm-lens-worker:local diff --git a/deploy/lens/compose.yaml b/deploy/lens/compose.yaml new file mode 100644 index 00000000000..773ff00113a --- /dev/null +++ b/deploy/lens/compose.yaml @@ -0,0 +1,10 @@ +services: + lens-worker: + image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:40fdb82113dd4474cb6e833cf28552487d87c8baf61693a1c3fc2863b7968c6a} + environment: + LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container} + LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI} + restart: unless-stopped + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] diff --git a/deploy/lens/screenshots/after.png b/deploy/lens/screenshots/after.png new file mode 100644 index 00000000000..983625e2f42 Binary files /dev/null and b/deploy/lens/screenshots/after.png differ diff --git a/deploy/lens/screenshots/before.png b/deploy/lens/screenshots/before.png new file mode 100644 index 00000000000..5022cd2bb18 Binary files /dev/null and b/deploy/lens/screenshots/before.png differ diff --git a/deploy/lens/screenshots/finding.png b/deploy/lens/screenshots/finding.png new file mode 100644 index 00000000000..dc8250f976e Binary files /dev/null and b/deploy/lens/screenshots/finding.png differ diff --git a/deploy/lens/screenshots/progress.png b/deploy/lens/screenshots/progress.png new file mode 100644 index 00000000000..f69ff1c45b3 Binary files /dev/null and b/deploy/lens/screenshots/progress.png differ diff --git a/deploy/lens/screenshots/setup.png b/deploy/lens/screenshots/setup.png new file mode 100644 index 00000000000..731fa012dbe Binary files /dev/null and b/deploy/lens/screenshots/setup.png differ diff --git a/deploy/lens/screenshots/trace.png b/deploy/lens/screenshots/trace.png new file mode 100644 index 00000000000..ef0178376d5 Binary files /dev/null and b/deploy/lens/screenshots/trace.png differ diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index d4c07d56d90..eca12855afa 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -103,7 +103,7 @@ ENV LITELLM_NON_ROOT=true RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \ cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \ - cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \ + cp /app/litellm/proxy/logo.png /var/lib/litellm/assets/logo.png && \ touch /var/lib/litellm/ui/.litellm_ui_ready RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ diff --git a/docker/docker-compose.tracing.yml b/docker/docker-compose.tracing.yml new file mode 100644 index 00000000000..b39fc8f4561 --- /dev/null +++ b/docker/docker-compose.tracing.yml @@ -0,0 +1,62 @@ +name: litellm-tracing + +services: + litellm: + build: + context: .. + target: runtime + command: ["--config", "/app/tracing-config.yaml", "--port", "4000"] + environment: + LITELLM_MASTER_KEY: local-tracing-master-key + LITELLM_SALT_KEY: sk-local-tracing-salt-key + DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm + STORE_MODEL_IN_DB: "True" + CLICKHOUSE_URL: http://default:local-tracing@clickhouse:8123 + CLICKHOUSE_READER_URL: http://default:local-tracing@clickhouse:8123 + CLICKHOUSE_DATABASE: litellm + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + volumes: + - ./tracing-config.yaml:/app/tracing-config.yaml:ro + ports: + - "127.0.0.1:4002:4000" + depends_on: + db: + condition: service_healthy + clickhouse: + condition: service_healthy + + db: + image: postgres:16 + environment: + POSTGRES_DB: litellm + POSTGRES_USER: litellm + POSTGRES_PASSWORD: litellm + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "127.0.0.1:15432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"] + interval: 5s + timeout: 5s + retries: 10 + + clickhouse: + image: clickhouse/clickhouse-server:26.9.6.6 + environment: + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: local-tracing + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + volumes: + - clickhouse_data:/var/lib/clickhouse + ports: + - "127.0.0.1:18123:8123" + healthcheck: + test: ["CMD", "clickhouse-client", "--user", "default", "--password", "local-tracing", "--query", "SELECT 1"] + interval: 5s + timeout: 5s + retries: 20 + +volumes: + postgres_data: + clickhouse_data: diff --git a/docker/tracing-config.yaml b/docker/tracing-config.yaml new file mode 100644 index 00000000000..03637cfa9fb --- /dev/null +++ b/docker/tracing-config.yaml @@ -0,0 +1,10 @@ +model_list: + - model_name: gpt-6.1-sol + litellm_params: + model: openai/gpt-6.1-sol + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + tracing: + store: clickhouse diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index 2114dfd9849..d134c39c91b 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -22,10 +22,8 @@ from litellm._uuid import uuid from litellm.proxy._types import * from litellm.proxy.auth.auth_checks import delete_cached_project_object from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.management_endpoints.common_utils import ( - _is_user_team_admin, # pyright: ignore[reportPrivateUsage] # shared owner of team-admin membership - _set_object_metadata_field, -) +from litellm.proxy.management.teams.access import is_team_admin +from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects from litellm.proxy.management_helpers.utils import ( management_endpoint_wrapper, @@ -117,7 +115,7 @@ async def _check_user_permission_for_project( return False team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump()) - return _is_user_team_admin(user_api_key_dict, team) or user_api_key_dict.user_id in (team.admins or []) + return is_team_admin(user_api_key_dict, team) or user_api_key_dict.user_id in (team.admins or []) async def _validate_team_exists( diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index e5e54a3df2c..74cedb9d84d 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.71" +version = "0.1.72" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.71" +version = "0.1.72" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index c4a3d3f7473..6e91f5486d0 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -73,6 +73,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/v1/containers", "/containers", "/v1/evals", + "/v1/traces", "/v1/memory", "/queue/chat/", # Google data plane (v1beta is the Google AI Studio version) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921190000_agent_identity/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921190000_agent_identity/migration.sql new file mode 100644 index 00000000000..06cf03b26b5 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921190000_agent_identity/migration.sql @@ -0,0 +1,97 @@ +-- AlterTable +ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN IF NOT EXISTS "enabled" BOOLEAN NOT NULL DEFAULT true, +ADD COLUMN IF NOT EXISTS "execution_mode" TEXT NOT NULL DEFAULT 'autonomous', +ADD COLUMN IF NOT EXISTS "identity_managed" BOOLEAN NOT NULL DEFAULT false; + +-- AlterTable +ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN IF NOT EXISTS "billing_agent_id" TEXT; + +-- CreateTable +CREATE TABLE IF NOT EXISTS "LiteLLM_AgentIdentity" ( + "agent_id" TEXT NOT NULL, + "active" BOOLEAN NOT NULL DEFAULT true, + "provider" TEXT NOT NULL, + "issuer" TEXT NOT NULL, + "tenant_id" TEXT NOT NULL, + "client_id" TEXT NOT NULL, + "service_principal_id" TEXT, + "required_roles" TEXT[] DEFAULT ARRAY[]::TEXT[], + "required_scopes" TEXT[] DEFAULT ARRAY['user_impersonation']::TEXT[], + "revision" TEXT NOT NULL, + "last_authenticated_at" TIMESTAMP(3), + + CONSTRAINT "LiteLLM_AgentIdentity_pkey" PRIMARY KEY ("agent_id") +); + +-- CreateTable +CREATE TABLE IF NOT EXISTS "LiteLLM_RetiredAgentIdentity" ( + "binding_id" TEXT NOT NULL, + "agent_id" TEXT, + "provider" TEXT NOT NULL, + "issuer" TEXT NOT NULL, + "tenant_id" TEXT NOT NULL, + "client_id" TEXT NOT NULL, + + CONSTRAINT "LiteLLM_RetiredAgentIdentity_pkey" PRIMARY KEY ("binding_id") +); + +-- CreateTable +CREATE TABLE IF NOT EXISTS "LiteLLM_RetiredAgent" ( + "original_agent_id" TEXT NOT NULL, + "retired_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_RetiredAgent_pkey" PRIMARY KEY ("original_agent_id") +); + +-- CreateTable +CREATE TABLE IF NOT EXISTS "LiteLLM_VerifiedSubject" ( + "subject_id" TEXT NOT NULL, + "issuer" TEXT NOT NULL, + "tenant_id" TEXT NOT NULL, + "oid" TEXT NOT NULL, + "kind" TEXT NOT NULL DEFAULT 'human', + "user_id" TEXT, + "verified_via" TEXT NOT NULL DEFAULT 'sso_interactive', + "verified_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_VerifiedSubject_pkey" PRIMARY KEY ("subject_id") +); + +-- CreateIndex +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_AgentIdentity_provider_tenant_id_client_id_key" ON "LiteLLM_AgentIdentity"("provider", "tenant_id", "client_id"); + +-- CreateIndex +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_AgentIdentity_issuer_service_principal_id_key" ON "LiteLLM_AgentIdentity"("issuer", "service_principal_id"); + +-- CreateIndex +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_RetiredAgentIdentity_provider_tenant_id_client_id_key" ON "LiteLLM_RetiredAgentIdentity"("provider", "tenant_id", "client_id"); + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "LiteLLM_VerifiedSubject_user_id_idx" ON "LiteLLM_VerifiedSubject"("user_id"); + +-- CreateIndex +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_VerifiedSubject_issuer_tenant_id_oid_key" ON "LiteLLM_VerifiedSubject"("issuer", "tenant_id", "oid"); + +-- AddForeignKey +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_AgentIdentity_agent_id_fkey') THEN + ALTER TABLE "LiteLLM_AgentIdentity" ADD CONSTRAINT "LiteLLM_AgentIdentity_agent_id_fkey" FOREIGN KEY ("agent_id") REFERENCES "LiteLLM_AgentsTable"("agent_id") ON DELETE CASCADE ON UPDATE CASCADE; + END IF; +END $$; + +-- AddForeignKey +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_RetiredAgentIdentity_agent_id_fkey') THEN + ALTER TABLE "LiteLLM_RetiredAgentIdentity" ADD CONSTRAINT "LiteLLM_RetiredAgentIdentity_agent_id_fkey" FOREIGN KEY ("agent_id") REFERENCES "LiteLLM_AgentsTable"("agent_id") ON DELETE SET NULL ON UPDATE CASCADE; + END IF; +END $$; + +-- AddForeignKey +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_VerifiedSubject_user_id_fkey') THEN + ALTER TABLE "LiteLLM_VerifiedSubject" ADD CONSTRAINT "LiteLLM_VerifiedSubject_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "LiteLLM_UserTable"("user_id") ON DELETE CASCADE ON UPDATE CASCADE; + END IF; +END $$; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260925000000_add_mcp_pinned_tools/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260925000000_add_mcp_pinned_tools/migration.sql new file mode 100644 index 00000000000..61d037f4771 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260925000000_add_mcp_pinned_tools/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "pinned_tools" JSONB DEFAULT '{}'; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260928000000_add_daily_model_usage/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260928000000_add_daily_model_usage/migration.sql new file mode 100644 index 00000000000..1395296ea61 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260928000000_add_daily_model_usage/migration.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyModelUsage" ( + "date" TEXT NOT NULL, + "model_group" TEXT NOT NULL, + "model" TEXT NOT NULL, + "custom_llm_provider" TEXT NOT NULL, + "task_type" TEXT NOT NULL, + "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "prompt_tokens" BIGINT NOT NULL DEFAULT 0, + "completion_tokens" BIGINT NOT NULL DEFAULT 0, + "request_count" BIGINT NOT NULL DEFAULT 0, + "successful_requests" BIGINT NOT NULL DEFAULT 0, + "failed_requests" BIGINT NOT NULL DEFAULT 0, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + CONSTRAINT "LiteLLM_DailyModelUsage_pkey" PRIMARY KEY ("date", "model_group", "model", "custom_llm_provider", "task_type") +); + +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyModelUsage_date_idx" ON "LiteLLM_DailyModelUsage"("date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyModelUsage_model_group_idx" ON "LiteLLM_DailyModelUsage"("model_group"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260930000000_agent_engine/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260930000000_agent_engine/migration.sql new file mode 100644 index 00000000000..2d41b2ef12d --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260930000000_agent_engine/migration.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS "LiteLLM_Engine" ( + "id" TEXT NOT NULL PRIMARY KEY, + "version" INTEGER NOT NULL DEFAULT 0, + "data" JSONB NOT NULL +); +CREATE TABLE IF NOT EXISTS "LiteLLM_EngineWorker" ( + "id" TEXT NOT NULL PRIMARY KEY, + "token_hash" TEXT NOT NULL UNIQUE, + "data" JSONB NOT NULL +); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261001000000_lens_run_history/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261001000000_lens_run_history/migration.sql new file mode 100644 index 00000000000..8b242d15d17 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261001000000_lens_run_history/migration.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS "LiteLLM_EngineRun" ( + "id" TEXT NOT NULL PRIMARY KEY, + "engine_id" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL, + "data" JSONB NOT NULL +); +CREATE INDEX IF NOT EXISTS "LiteLLM_EngineRun_engine_id_created_at_idx" ON "LiteLLM_EngineRun"("engine_id", "created_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 69c63d9ecd6..75dc7ddde9d 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -78,6 +78,11 @@ model LiteLLM_AgentsTable { object_permission_id String? object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id]) spend Float @default(0.0) + identity_managed Boolean @default(false) + enabled Boolean @default(true) + execution_mode String @default("autonomous") + identity LiteLLM_AgentIdentity? + retired_identities LiteLLM_RetiredAgentIdentity[] tpm_limit Int? rpm_limit Int? session_tpm_limit Int? @@ -88,6 +93,56 @@ model LiteLLM_AgentsTable { updated_by String } +model LiteLLM_AgentIdentity { + agent_id String @id + active Boolean @default(true) + agent LiteLLM_AgentsTable @relation(fields: [agent_id], references: [agent_id], onDelete: Cascade) + provider String + issuer String + tenant_id String + client_id String + service_principal_id String? + required_roles String[] @default([]) + required_scopes String[] @default(["user_impersonation"]) + revision String @default(uuid()) + last_authenticated_at DateTime? + @@unique([provider, tenant_id, client_id]) + @@unique([issuer, service_principal_id]) +} + +model LiteLLM_RetiredAgentIdentity { + binding_id String @id @default(uuid()) + agent_id String? + agent LiteLLM_AgentsTable? @relation(fields: [agent_id], references: [agent_id], onDelete: SetNull) + provider String + issuer String + tenant_id String + client_id String + @@unique([provider, tenant_id, client_id]) +} + +model LiteLLM_RetiredAgent { + original_agent_id String @id + retired_at DateTime @default(now()) +} + +model LiteLLM_VerifiedSubject { + subject_id String @id @default(uuid()) + issuer String + tenant_id String + oid String + kind String @default("human") + user_id String? + user LiteLLM_UserTable? @relation(fields: [user_id], references: [user_id], onDelete: Cascade) + verified_via String @default("sso_interactive") + verified_at DateTime @default(now()) + @@unique([issuer, tenant_id, oid]) + @@index([user_id]) +} + + + + model LiteLLM_OrganizationTable { organization_id String @id @default(uuid()) organization_alias String @@ -241,6 +296,7 @@ model LiteLLM_DeletedTeamTable { // Track spend, rate limit, budget Users model LiteLLM_UserTable { + verified_subjects LiteLLM_VerifiedSubject[] user_id String @id user_alias String? team_id String? @@ -322,6 +378,7 @@ model LiteLLM_MCPServerTable { allowed_tools String[] @default([]) tool_name_to_display_name Json? @default("{}") tool_name_to_description Json? @default("{}") + pinned_tools Json? @default("{}") extra_headers String[] @default([]) static_headers Json? @default("{}") // Admin-configured environment variables interpolated into static_headers @@ -674,6 +731,7 @@ model LiteLLM_SpendLogs { session_id String? status String? mcp_namespaced_tool_name String? + billing_agent_id String? agent_id String? proxy_server_request Json? @default("{}") litellm_call_id String? @@ -1259,6 +1317,26 @@ model LiteLLM_DailyToolSpend { @@id([date, tool_name]) } +model LiteLLM_DailyModelUsage { + date String + model_group String + model String + custom_llm_provider String + task_type String + spend Float @default(0.0) + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + request_count BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@id([date, model_group, model, custom_llm_provider, task_type]) + @@index([date]) + @@index([model_group]) +} + // Gateway request counts recorded at the ASGI edge by // BillableRequestMetricsMiddleware. This is the source of truth for SGR // (successful gateway requests): it counts what the proxy actually answered, @@ -1816,3 +1894,24 @@ model LiteLLM_WorkflowMessage { @@unique([run_id, sequence_number]) @@index([run_id]) } + +model LiteLLM_Engine { + id String @id + version Int @default(0) + data Json +} + +model LiteLLM_EngineRun { + id String @id + engine_id String + created_at DateTime + data Json + + @@index([engine_id, created_at]) +} + +model LiteLLM_EngineWorker { + id String @id + token_hash String @unique + data Json +} diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 2835715ef30..e92af4b0861 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.102" +version = "0.4.103" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.102" +version = "0.4.103" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm-rust/.agents/skills/rust-tracing/SKILL.md b/litellm-rust/.agents/skills/rust-tracing/SKILL.md new file mode 100644 index 00000000000..c1c7f43c456 --- /dev/null +++ b/litellm-rust/.agents/skills/rust-tracing/SKILL.md @@ -0,0 +1,22 @@ +--- +name: rust-tracing +description: Add or change Rust diagnostic tracing in litellm-rust, including route spans, subscriber layers, and Python logger delivery +--- + +# Rust tracing + +Use upstream `tracing` throughout Rust, including `#[tracing::instrument]`, events, and span propagation. Centralize collection and delivery infrastructure in `crates/tracing`. Direct upstream imports still reach our configured subscriber; re-exporting macros does not control delivery. Do not introduce Rust `log` or `pyo3-log` for this path + +`litellm-tracing` owns shared subscriber layers, span field collection, and diagnostic processing. Keep adapters composable as `tracing_subscriber::Layer`s, with `Logger` providing host setup. Runtime-specific delivery belongs in the host bridge. The Python bridge delivers directly to the existing Python SDK logger, preserving its handlers, filtering, redaction, and request correlation. Keep Python dependencies out of `crates/tracing` + +Hosts configure subscribers. Keep Python execution scoped to its captured dispatch rather than installing a process-wide subscriber. Propagate both span context and dispatch across spawned work and returned streams + +In core, instrument execution shared by native calls and hosted machines. Use consistent route, model, provider, streaming, and outcome fields. Put status recording at shared provider boundaries instead of scattering basic logging through handlers. Keep upstream HTTP status separate from route success + +Use `skip_all` and explicitly selected fields. Basic tracing excludes bodies, credentials, headers, and raw error strings. Avoid automatic `ret` or `err` capture of sensitive values. Keep payload diagnostics separate and subject to existing redaction + +A returned stream retains its route span until exhaustion, error, or drop, with exactly one terminal outcome. Builder construction does not start a trace. Never hold a span entry guard across an await. Diagnostic tracing remains separate from lifecycle callbacks and `CustomLogger` dispatch + +Use `litellm_tracing::sink_layer` to compose a sink with other subscriber layers. It inherits span fields into events and emits span-close summaries with elapsed time. Test observable records, concurrent isolation, dynamic filtering, sensitive-field exclusion, and stream cancellation when changing this behavior + +Consult the [tracing API](https://docs.rs/tracing/latest/tracing/) and [subscriber layers](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/layer/index.html) for implementation details diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index bc6a2552e4c..fe0aac56f0c 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -1,5 +1,7 @@ # Rust workspace rules +For diagnostic tracing changes, follow [.agents/skills/rust-tracing/SKILL.md](.agents/skills/rust-tracing/SKILL.md) + ## Test placement - Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;` @@ -16,7 +18,9 @@ Use [`#[rstest]`](https://docs.rs/rstest/latest/rstest/attr.rstest.html) for new ## Error definitions - A crate's errors live in `src/error.rs`, defined with `thiserror`, and re-exported from `lib.rs` +- Put message templates in the variant's `#[error(...)]` declaration. Callers pass only the small typed arguments needed to fill them, never `Error::Variant(format!(...))` or a preformatted message. Keep the smallest set of neutral variants that callers need to distinguish; different wording or providers do not justify new variants - Default to one top-level `Error` enum per crate, with one variant per failure mode and a `#[error(...)]` message on each. A failure mode is something a caller handles differently (phase, status code, retry, a message Python parity pins exactly); failures no caller tells apart share one variant and differ only in its message +- Keep shared error enums minimal and provider-neutral. Provider names, credential types, configuration fields, and setup guidance belong in caller-supplied data, not dedicated variants or hardcoded shared messages. Reuse a variant for the same failure mode across providers, such as `MissingApiBase { provider: "Azure", guidance: "..." }`. An exact parity message does not justify a provider-specific variant when caller-supplied context can preserve it - Wrap a lower-level error as a variant with `#[from]` or `#[source]` instead of flattening it to a string - Exception: split into separate types when different functions fail in disjoint ways, especially when different callers see them. A shared enum would force every caller to match variants its function can never return - Name a split type after what went wrong (a unit struct is fine for a single failure mode), not after the function that returns it diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index d67623feffd..0ad05d99e76 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -146,6 +146,22 @@ dependencies = [ "serde_json", ] +[[package]] +name = "astral-tokio-tar" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b18457efd137254e016bbde5e1d88df61c4e1a5ae2223746e56123bac6af2463" +dependencies = [ + "futures-core", + "libc", + "portable-atomic", + "rustc-hash", + "rustix", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "async-compression" version = "0.4.46" @@ -202,6 +218,15 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -354,7 +379,7 @@ dependencies = [ "bytes", "fastrand", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.4.2", "http-body 1.1.0", @@ -433,7 +458,7 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.4.2", "percent-encoding", @@ -706,6 +731,7 @@ checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", "bytes", + "form_urlencoded", "futures-util", "http 1.4.2", "http-body 1.1.0", @@ -722,6 +748,7 @@ dependencies = [ "serde_core", "serde_json", "serde_path_to_error", + "serde_urlencoded", "sync_wrapper", "tokio", "tower", @@ -747,6 +774,25 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-login" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "964ea6eb764a227baa8c3368e45c94d23b6863cc7b880c6c9e341c143c5a5ff7" +dependencies = [ + "axum", + "form_urlencoded", + "serde", + "subtle", + "thiserror 2.0.19", + "tower-cookies", + "tower-layer", + "tower-service", + "tower-sessions", + "tracing", + "urlencoding", +] + [[package]] name = "azure_core" version = "1.1.0" @@ -830,6 +876,12 @@ dependencies = [ "time", ] +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + [[package]] name = "base64" version = "0.13.1" @@ -858,6 +910,12 @@ dependencies = [ "vsimd", ] +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bit-set" version = "0.8.0" @@ -893,6 +951,9 @@ name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] [[package]] name = "block-buffer" @@ -912,6 +973,80 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "bollard" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" +dependencies = [ + "async-stream", + "base64 0.22.1", + "bitflags 2.13.1", + "bollard-buildkit-proto", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http 1.4.2", + "http-body-util", + "hyper 1.10.1", + "hyper-named-pipe", + "hyper-rustls 0.27.9", + "hyper-util", + "hyperlocal", + "log", + "num", + "pin-project-lite", + "rand 0.9.5", + "rustls 0.23.42", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.19", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-buildkit-proto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a885520bf6249ab931a764ffdb87b0ceef48e6e7d807cfdb21b751e086e1ad" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tonic-prost", + "ureq", +] + +[[package]] +name = "bollard-stubs" +version = "1.52.1-rc.29.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" +dependencies = [ + "base64 0.22.1", + "bollard-buildkit-proto", + "bytes", + "prost", + "serde", + "serde_json", + "serde_repr", + "time", +] + [[package]] name = "borrow-or-share" version = "0.2.4" @@ -1139,12 +1274,41 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" +[[package]] +name = "const-hex" +version = "1.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e59eef12462b0f9b0a3620219be5d639afd79fe39dff0a42c3997061f9298b4" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "proptest", + "serde_core", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-oid" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1179,6 +1343,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc-fast" version = "1.10.0" @@ -1276,6 +1455,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.22" @@ -1288,6 +1476,18 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -1316,6 +1516,33 @@ dependencies = [ "cmov", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "daachorse" version = "3.0.3" @@ -1462,6 +1689,17 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "der-parser" version = "10.0.0" @@ -1534,7 +1772,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", + "const-oid 0.9.6", "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -1544,7 +1784,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", - "const-oid", + "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", ] @@ -1560,6 +1800,23 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "docker_credential" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" +dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dunce" version = "1.0.5" @@ -1572,11 +1829,73 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +dependencies = [ + "serde", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "hkdf 0.12.4", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] [[package]] name = "email_address" @@ -1596,6 +1915,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "envy" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f47e0157f2cb54f5ae1bd371b30a2ae4311e1c028f575cd4e81de7353215965" +dependencies = [ + "serde", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1618,6 +1946,16 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + [[package]] name = "event-listener" version = "5.4.2" @@ -1678,6 +2016,33 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "ferroid" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" +dependencies = [ + "portable-atomic", + "rand 0.10.2", + "web-time", +] + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "filetime" version = "0.2.29" @@ -1716,6 +2081,17 @@ dependencies = [ "serde", ] +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.9", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1795,6 +2171,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.33" @@ -1882,6 +2269,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -1942,7 +2330,7 @@ dependencies = [ "bytes", "google-cloud-gax", "hex", - "hmac", + "hmac 0.13.0", "http 1.4.2", "jiff", "reqwest 0.13.5", @@ -1996,9 +2384,9 @@ dependencies = [ "http-body-util", "hyper 1.10.1", "lazy_static", - "opentelemetry", + "opentelemetry 0.32.0", "opentelemetry-semantic-conventions", - "opentelemetry_sdk", + "opentelemetry_sdk 0.32.1", "percent-encoding", "pin-project", "prost", @@ -2149,6 +2537,36 @@ dependencies = [ "url", ] +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "futures-sink", + "futures-timer", + "futures-util", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "smallvec", + "spinning_top", + "web-time", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.3.27" @@ -2210,6 +2628,8 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", ] @@ -2226,11 +2646,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.12.2" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ - "hashbrown 0.17.1", + "hashbrown 0.16.1", ] [[package]] @@ -2251,6 +2671,33 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac 0.13.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" @@ -2260,6 +2707,15 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "0.2.12" @@ -2315,6 +2771,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + [[package]] name = "httparse" version = "1.10.1" @@ -2382,6 +2844,20 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" +dependencies = [ + "hex", + "hyper 1.10.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -2450,6 +2926,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.10.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -2802,11 +3293,36 @@ dependencies = [ "zmij", ] +[[package]] +name = "jsonwebtoken" +version = "11.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75fe14a82d81e5f5af639997db37d8b96045938a7ac6ab18cdbe1c7467e05e1" +dependencies = [ + "base64 0.22.1", + "ed25519-dalek", + "getrandom 0.2.17", + "hmac 0.12.1", + "js-sys", + "p256", + "p384", + "rand 0.8.7", + "rsa", + "serde", + "serde_json", + "sha2 0.10.9", + "signature", + "zeroize", +] + [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin 0.9.9", +] [[package]] name = "libc" @@ -2815,10 +3331,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] -name = "libsqlite3-sys" -version = "0.38.2" +name = "libm" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libsqlite3-sys" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ "cc", "pkg-config", @@ -2891,6 +3413,7 @@ dependencies = [ "http 1.4.2", "litellm-auth-types", "moka", + "rstest", "serde_json", "sha2 0.10.9", "tokio", @@ -2900,6 +3423,7 @@ dependencies = [ name = "litellm-auth-types" version = "0.1.0" dependencies = [ + "rstest", "serde", "subtle", "thiserror 2.0.19", @@ -3040,8 +3564,10 @@ name = "litellm-cache-response" version = "0.1.0" dependencies = [ "litellm-cache", + "litellm-cache-gcs", "litellm-cache-memory", "litellm-cache-redis", + "litellm-http", "py_literal", "redis", "redis-test", @@ -3050,6 +3576,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tokio", + "wiremock", ] [[package]] @@ -3133,21 +3660,24 @@ dependencies = [ "litellm-auth", "litellm-auth-aws", "litellm-auth-gcp", + "litellm-cache", + "litellm-cache-memory", + "litellm-cache-response", "litellm-core-utils", + "litellm-framing", "litellm-host", + "litellm-host-native", "litellm-http", "litellm-llms", + "litellm-llms-types", "litellm-secrets", "litellm-tracing", - "litellm-types", "mime_guess", "moka", "rand 0.8.7", "reqwest 0.12.28", "rstest", "rstest_reuse", - "rustls 0.23.42", - "rustls-native-certs", "serde", "serde_json", "sha2 0.10.9", @@ -3157,6 +3687,8 @@ dependencies = [ "time", "tokio", "tokio-tungstenite", + "tokio-util", + "tracing", "url", "veil", "wiremock", @@ -3167,13 +3699,12 @@ name = "litellm-core-utils" version = "0.1.0" dependencies = [ "fancy-regex 0.19.2", + "litellm-llms-types", "litellm-tracing", - "litellm-types", "rstest", "serde", "serde_json", "serde_path_to_error", - "serde_with", "strum", "thiserror 2.0.19", "url", @@ -3194,6 +3725,27 @@ version = "0.1.0" dependencies = [ "criterion", "proptest", + "rstest", +] + +[[package]] +name = "litellm-db" +version = "0.1.0" +dependencies = [ + "serde", + "sqlx", +] + +[[package]] +name = "litellm-db-testing" +version = "0.1.0" +dependencies = [ + "rstest", + "sqlx", + "tempfile", + "testcontainers-modules", + "thiserror 2.0.19", + "tokio", ] [[package]] @@ -3216,20 +3768,30 @@ name = "litellm-gateway" version = "0.1.0" dependencies = [ "axum", + "base64 0.22.1", + "envy", "futures-util", "http-body-util", + "litellm-auth-types", "litellm-config", "litellm-core", "litellm-gateway-auth", "litellm-gateway-inference", + "litellm-gateway-mcp", + "litellm-gateway-ui", "litellm-http", "litellm-llms", "litellm-secrets", "litellm-tracing", "rstest", + "serde", "serde_json", + "tempfile", + "thiserror 2.0.19", "tokio", + "tokio-util", "tower", + "tower-sessions-moka-store", "tracing", "uuid", ] @@ -3239,16 +3801,20 @@ name = "litellm-gateway-auth" version = "0.1.0" dependencies = [ "axum", + "axum-login", "futures-util", "litellm-auth-types", "litellm-config", "litellm-secrets", "rstest", + "serde", "sha2 0.10.9", "subtle", "thiserror 2.0.19", "tokio", "tower", + "tower-sessions", + "veil", ] [[package]] @@ -3260,13 +3826,19 @@ dependencies = [ "bytes", "futures-util", "litellm-auth", + "litellm-cache-memory", + "litellm-cache-response", "litellm-core", + "litellm-gateway-auth", + "litellm-host", + "litellm-host-http", "litellm-http", "litellm-llms", + "litellm-llms-types", "litellm-router", "litellm-secrets", - "litellm-types", "rstest", + "serde", "serde_json", "thiserror 2.0.19", "tokio", @@ -3274,10 +3846,75 @@ dependencies = [ "wiremock", ] +[[package]] +name = "litellm-gateway-management" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "litellm-auth-types", + "litellm-gateway-auth", + "rand 0.8.7", + "rstest", + "thiserror 2.0.19", + "tokio", +] + +[[package]] +name = "litellm-gateway-mcp" +version = "0.1.0" +dependencies = [ + "axum", + "base64 0.22.1", + "futures-util", + "http 1.4.2", + "litellm-auth-types", + "litellm-config", + "litellm-secrets", + "moka", + "rmcp", + "rstest", + "serde", + "serde_json", + "sha2 0.10.9", + "sse-stream", + "thiserror 2.0.19", + "tokio", + "tokio-util", + "tower", + "url", + "uuid", +] + +[[package]] +name = "litellm-gateway-ui" +version = "0.1.0" +dependencies = [ + "axum", + "axum-login", + "base64 0.22.1", + "governor", + "jsonwebtoken", + "litellm-auth-types", + "litellm-gateway-auth", + "rand 0.8.7", + "rstest", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.19", + "time", + "tokio", + "tower", + "tower-cookies", + "tower-http", + "tower-sessions", +] + [[package]] name = "litellm-host" version = "0.1.0" dependencies = [ + "futures-util", "litellm-auth", "litellm-coroutine", "rstest", @@ -3285,6 +3922,33 @@ dependencies = [ "tokio", ] +[[package]] +name = "litellm-host-http" +version = "0.1.0" +dependencies = [ + "axum", + "bytes", + "futures-util", + "http 1.4.2", + "litellm-host", + "litellm-host-native", + "rstest", + "serde_json", + "thiserror 2.0.19", + "tokio", +] + +[[package]] +name = "litellm-host-native" +version = "0.1.0" +dependencies = [ + "futures-util", + "litellm-host", + "rstest", + "serde_json", + "tokio", +] + [[package]] name = "litellm-host-python" version = "0.1.0" @@ -3305,18 +3969,24 @@ dependencies = [ name = "litellm-http" version = "0.1.0" dependencies = [ + "futures-util", "http 1.4.2", "hyper-util", "litellm-core-utils", "rcgen", "reqwest 0.12.28", + "reqwest 0.13.5", + "rmcp", "rstest", "rustls 0.23.42", + "rustls-native-certs", "serde", "serde_json", "tempfile", "thiserror 2.0.19", "tokio", + "tokio-tungstenite", + "tracing", "veil", "webpki-roots", ] @@ -3339,9 +4009,9 @@ dependencies = [ "litellm-framing", "litellm-host", "litellm-http", + "litellm-llms-types", "litellm-python-compat", "litellm-secrets", - "litellm-types", "reqwest 0.12.28", "rstest", "serde", @@ -3355,13 +4025,26 @@ dependencies = [ "url", ] +[[package]] +name = "litellm-llms-types" +version = "0.1.0" +dependencies = [ + "macro_rules_attribute", + "rstest", + "schemars 1.2.2", + "serde", + "serde_json", + "serde_with", + "strum", +] + [[package]] name = "litellm-model-catalog" version = "0.1.0" dependencies = [ "indexmap 2.14.0", "jsonschema", - "litellm-types", + "litellm-llms-types", "rstest", "schemars 1.2.2", "serde", @@ -3399,12 +4082,13 @@ dependencies = [ "litellm-host-python", "litellm-http", "litellm-llms", + "litellm-llms-types", "litellm-secrets", "litellm-secrets-aws", "litellm-secrets-types", "litellm-token-counter", + "litellm-traces", "litellm-tracing", - "litellm-types", "pyo3", "pyo3-async-runtimes", "qdrant-client", @@ -3419,6 +4103,7 @@ dependencies = [ "thiserror 2.0.19", "tokio", "tokio-tungstenite", + "tracing", "url", "veil", "wiremock", @@ -3661,6 +4346,7 @@ dependencies = [ name = "litellm-token-counter-huggingface" version = "0.1.0" dependencies = [ + "rstest", "serde_json", "thiserror 2.0.19", "tokenizers", @@ -3672,11 +4358,32 @@ version = "0.1.0" dependencies = [ "base64 0.22.1", "once_cell", + "rstest", "rustc-hash", "thiserror 2.0.19", "tiktoken-rs", ] +[[package]] +name = "litellm-traces" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "flate2", + "litellm-http", + "opentelemetry-proto", + "prost", + "rstest", + "serde", + "serde_json", + "sha2 0.10.9", + "testcontainers-modules", + "thiserror 2.0.19", + "time", + "tokio", + "url", +] + [[package]] name = "litellm-tracing" version = "0.1.0" @@ -3691,17 +4398,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "litellm-types" -version = "0.1.0" -dependencies = [ - "rstest", - "schemars 1.2.2", - "serde", - "serde_json", - "strum", -] - [[package]] name = "litemap" version = "0.8.2" @@ -3715,6 +4411,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ "scopeguard", + "serde", ] [[package]] @@ -3884,6 +4581,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -3894,6 +4603,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + [[package]] name = "num" version = "0.4.3" @@ -3928,6 +4643,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.7", + "smallvec", + "zeroize", +] + [[package]] name = "num-cmp" version = "0.1.0" @@ -3986,6 +4717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -4061,6 +4793,33 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 2.0.19", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" +dependencies = [ + "base64 0.22.1", + "const-hex", + "opentelemetry 0.33.0", + "opentelemetry_sdk 0.33.0", + "prost", + "serde", +] + [[package]] name = "opentelemetry-semantic-conventions" version = "0.32.1" @@ -4076,7 +4835,23 @@ dependencies = [ "futures-channel", "futures-executor", "futures-util", - "opentelemetry", + "opentelemetry 0.32.0", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror 2.0.19", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry 0.33.0", "percent-encoding", "portable-atomic", "rand 0.9.5", @@ -4089,6 +4864,30 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + [[package]] name = "page_size" version = "0.6.0" @@ -4128,6 +4927,31 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn 2.0.119", +] + [[package]] name = "paste" version = "1.0.15" @@ -4150,6 +4974,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -4230,6 +5063,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.33" @@ -4303,6 +5157,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -4321,6 +5184,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "process-wrap" +version = "10.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f21b97672d2dc848e7b25701ab4535618b92f4861c13cc3f7f7bed52ad3c8da" +dependencies = [ + "futures", + "indexmap 2.14.0", + "nix", + "tokio", + "tracing", + "windows", +] + [[package]] name = "proptest" version = "1.11.0" @@ -4903,8 +5780,10 @@ checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ "base64 0.23.1", "bytes", + "encoding_rs", "futures-core", "futures-util", + "h2 0.4.15", "http 1.4.2", "http-body 1.1.0", "http-body-util", @@ -4913,6 +5792,7 @@ dependencies = [ "hyper-util", "js-sys", "log", + "mime", "percent-encoding", "pin-project-lite", "quinn", @@ -4936,6 +5816,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac 0.12.1", + "subtle", +] + [[package]] name = "ring" version = "0.17.14" @@ -4950,6 +5840,60 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmcp" +version = "3.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b6317cd8c13e3ec9033cf2aa5aa92cd743f0f4f8a93cddc42ea0ae6ce3b8898" +dependencies = [ + "async-trait", + "base64 0.23.1", + "bytes", + "chrono", + "futures", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "indexmap 2.14.0", + "pastey", + "pin-project-lite", + "process-wrap", + "rand 0.10.2", + "reqwest 0.13.5", + "schemars 1.2.2", + "serde", + "serde_json", + "sse-stream", + "thiserror 2.0.19", + "tokio", + "tokio-stream", + "tokio-util", + "tower-service", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rsqlite-vfs" version = "0.1.1" @@ -5002,9 +5946,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.40.2" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" dependencies = [ "bitflags 2.13.1", "fallible-iterator", @@ -5254,6 +6198,7 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" dependencies = [ + "chrono", "dyn-clone", "ref-cast", "schemars_derive", @@ -5289,6 +6234,20 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -5397,6 +6356,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -5537,6 +6507,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -5570,6 +6550,9 @@ name = "smallvec" version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -5596,6 +6579,9 @@ name = "spin" version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] [[package]] name = "spin" @@ -5603,6 +6589,25 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "spm_precompiled" version = "0.1.4" @@ -5627,6 +6632,196 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "sqlx" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378620ccc25c62c89d8be1c819e76a88d59bdcc3304733330788948e619bfd71" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" +dependencies = [ + "base64 0.22.1", + "bytes", + "cfg-if", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.16.1", + "hashlink", + "indexmap 2.14.0", + "log", + "memchr", + "percent-encoding", + "rustls 0.23.42", + "rustls-native-certs", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "thiserror 2.0.19", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2b84f2bc39a5705ef27ec785a11c934a41bbd4a24941e257927cddc26b60bf" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.119", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb8d96de5fdc85a5c4ec813432b523ec637e80ba98f046555f75f7908ddac7c3" +dependencies = [ + "cfg-if", + "dotenvy", + "either", + "heck", + "hex", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.119", + "thiserror 2.0.19", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27" +dependencies = [ + "bitflags 2.13.1", + "byteorder", + "bytes", + "chrono", + "crc", + "digest 0.11.3", + "dotenvy", + "either", + "futures-core", + "futures-util", + "generic-array", + "log", + "percent-encoding", + "serde", + "sha1 0.11.0", + "sha2 0.11.0", + "sqlx-core", + "thiserror 2.0.19", + "tracing", +] + +[[package]] +name = "sqlx-postgres" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.13.1", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf 0.13.0", + "hmac 0.13.0", + "itoa", + "log", + "md-5", + "memchr", + "rand 0.10.2", + "serde", + "serde_json", + "sha2 0.11.0", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.19", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488e99c397a62007e4229aec669a179816339afc6d2620ca6fa420dbee2e982c" +dependencies = [ + "atoi", + "chrono", + "flume", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "thiserror 2.0.19", + "tracing", + "url", +] + +[[package]] +name = "sse-stream" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c25ac7aff0abd1dbc474536e40416e1102c7dd9bfba0b9861c6d357f835dcfb4" +dependencies = [ + "bytes", + "futures-util", + "http-body 1.1.0", + "http-body-util", + "pin-project-lite", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -5639,12 +6834,46 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.119", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "strum" version = "0.28.0" @@ -5773,6 +7002,47 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "testcontainers" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfd5785b5483672915ed5fe3cddf9f546802779fc1eceff0a6fb7321fac81c1e" +dependencies = [ + "astral-tokio-tar", + "async-trait", + "bollard", + "bytes", + "docker_credential", + "either", + "etcetera", + "ferroid", + "futures", + "http 1.4.2", + "itertools 0.14.0", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "reqwest 0.13.5", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.19", + "tokio", + "tokio-stream", + "tokio-util", + "url", +] + +[[package]] +name = "testcontainers-modules" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5985fde5befe4ffa77a052e035e16c2da86e8bae301baa9f9904ad3c494d357" +dependencies = [ + "testcontainers", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -6145,6 +7415,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-cookies" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "151b5a3e3c45df17466454bb74e9ecedecc955269bdedbf4d150dfa393b55a36" +dependencies = [ + "axum-core", + "cookie", + "futures-util", + "http 1.4.2", + "parking_lot", + "pin-project-lite", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-http" version = "0.6.11" @@ -6159,6 +7445,11 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", "pin-project-lite", "tokio", "tokio-util", @@ -6180,6 +7471,69 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" +[[package]] +name = "tower-sessions" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a05911f23e8fae446005fe9b7b97e66d95b6db589dc1c4d59f6a2d4d4927d3" +dependencies = [ + "async-trait", + "http 1.4.2", + "time", + "tokio", + "tower-cookies", + "tower-layer", + "tower-service", + "tower-sessions-core", + "tower-sessions-memory-store", + "tracing", +] + +[[package]] +name = "tower-sessions-core" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce8cce604865576b7751b7a6bc3058f754569a60d689328bb74c52b1d87e355b" +dependencies = [ + "async-trait", + "axum-core", + "base64 0.22.1", + "futures", + "http 1.4.2", + "parking_lot", + "rand 0.8.7", + "serde", + "serde_json", + "thiserror 2.0.19", + "time", + "tokio", + "tracing", +] + +[[package]] +name = "tower-sessions-memory-store" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb05909f2e1420135a831dd5df9f5596d69196d0a64c3499ca474c4bd3d33242" +dependencies = [ + "async-trait", + "time", + "tokio", + "tower-sessions-core", +] + +[[package]] +name = "tower-sessions-moka-store" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a5e622001aa59953f422ade78a0fa0d1f4d2566c9bf697bffe6aa89f1438f08" +dependencies = [ + "async-trait", + "moka", + "time", + "tower-sessions-core", +] + [[package]] name = "tracing" version = "0.1.44" @@ -6230,7 +7584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", - "opentelemetry", + "opentelemetry 0.32.0", "tracing", "tracing-core", "tracing-subscriber", @@ -6350,6 +7704,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -6362,6 +7722,15 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-normalization-alignments" version = "0.1.12" @@ -6371,6 +7740,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -6401,6 +7776,33 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" +dependencies = [ + "base64 0.23.1", + "log", + "percent-encoding", + "rustls 0.23.42", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" +dependencies = [ + "base64 0.23.1", + "http 1.4.2", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -6411,6 +7813,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -6425,6 +7828,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -6678,6 +8087,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" + [[package]] name = "winapi" version = "0.3.9" @@ -6709,6 +8124,27 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -6722,6 +8158,17 @@ dependencies = [ "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -6750,6 +8197,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -6802,6 +8259,15 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -7019,6 +8485,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "zerotrie" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index ed703396c22..257a47268e4 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -12,12 +12,18 @@ repository = "https://github.com/BerriAI/litellm" litellm-config = { path = "crates/config" } litellm-router = { path = "crates/router" } litellm-tracing = { path = "crates/tracing" } +litellm-traces = { path = "crates/traces" } litellm-core = { path = "crates/core" } +litellm-gateway-mcp = { path = "crates/gateway-mcp" } litellm-gateway = { path = "crates/gateway" } litellm-gateway-inference = { path = "crates/gateway-inference" } litellm-gateway-auth = { path = "crates/gateway-auth" } +litellm-gateway-management = { path = "crates/gateway-management" } +litellm-gateway-ui = { path = "crates/gateway-ui" } litellm-coroutine = { path = "crates/coroutine" } litellm-host = { path = "crates/host" } +litellm-host-http = { path = "crates/host-http" } +litellm-host-native = { path = "crates/host-native" } litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" } litellm-framing = { path = "crates/framer" } litellm-auth = { path = "crates/auth" } @@ -34,8 +40,10 @@ litellm-secrets-azure = { path = "crates/secrets-azure" } litellm-secrets-cyberark = { path = "crates/secrets-cyberark" } litellm-http = { path = "crates/http" } litellm-llms = { path = "crates/llms" } -litellm-types = { path = "crates/types" } +litellm-llms-types = { path = "crates/llms-types" } litellm-core-utils = { path = "crates/core-utils" } +litellm-db = { path = "crates/db" } +litellm-db-testing = { path = "crates/db-testing" } litellm-cache = { path = "crates/cache" } litellm-cache-azure-blob = { path = "crates/cache-azure-blob" } litellm-cache-memory = { path = "crates/cache-memory" } @@ -56,6 +64,8 @@ litellm-python-compat = { path = "crates/python-compat" } tracing = "0.1" axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "multipart"] } +axum-login = "0.18.0" +tower-sessions = { version = "0.14.0", features = ["memory-store"] } bytes = "1" http = "1" google-cloud-auth = { version = "1.16.0", default-features = false } @@ -65,6 +75,7 @@ proptest = "1.7.0" pyo3 = "0.29.2" pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } rand = "0.8" +macro_rules_attribute = "0.2.3" schemars = "1" reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "http2", "stream"] } qdrant-client = { version = "1.19.0", default-features = false } @@ -80,6 +91,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["float_roundtrip"] } serde_with = { version = "=3.16.1", default-features = false, features = ["std", "macros"] } sha2 = "0.10" +sqlx = { version = "0.9.0", default-features = false, features = ["json", "macros", "postgres", "runtime-tokio", "chrono", "tls-rustls-ring-native-roots"] } subtle = "2" thiserror = "2.0" tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] } diff --git a/litellm-rust/clippy.toml b/litellm-rust/clippy.toml index 0e2ff770d27..18663cfdffb 100644 --- a/litellm-rust/clippy.toml +++ b/litellm-rust/clippy.toml @@ -1,4 +1,4 @@ -# The Tokio runtime is reached only through `host-python/src/execution.rs`, whose fork gate +# The Tokio runtime is reached only through `host-python/src/runtime.rs`, whose fork gate # must see every entry. Going around it makes a fork-after-use hang instead of raising. disallowed-methods = [ { path = "pyo3_async_runtimes::tokio::get_runtime", reason = "use litellm_host_python::run_sync / run_sync_value" }, @@ -12,6 +12,13 @@ disallowed-methods = [ { path = "reqwest::ClientBuilder::danger_accept_invalid_certs", reason = "set HttpClientConfig::verify instead" }, { path = "reqwest::ClientBuilder::identity", reason = "set HttpClientConfig::client_certificate instead" }, { path = "reqwest::ClientBuilder::use_preconfigured_tls", reason = "HttpClientConfig owns the TLS configuration" }, + { path = "sqlx::query", reason = "use sqlx::query! or query_file! so the SQL is checked against the migrated schema" }, + { path = "sqlx::query_as", reason = "use sqlx::query_as! or query_file_as! so the SQL is checked against the migrated schema" }, + { path = "sqlx::query_scalar", reason = "use sqlx::query_scalar! so the SQL is checked against the migrated schema" }, + { path = "sqlx::query_with", reason = "use sqlx::query! or query_file! so the SQL is checked against the migrated schema" }, + { path = "sqlx::query_as_with", reason = "use sqlx::query_as! or query_file_as! so the SQL is checked against the migrated schema" }, + { path = "sqlx::query_scalar_with", reason = "use sqlx::query_scalar! so the SQL is checked against the migrated schema" }, + { path = "sqlx::raw_sql", reason = "raw_sql is unchecked; use the checked query macros" }, ] # Every outbound client comes from litellm_http::HttpClientPool so it honors the host's TLS, diff --git a/litellm-rust/crates/auth-azure/src/native.rs b/litellm-rust/crates/auth-azure/src/native.rs index d635e559641..64752162384 100644 --- a/litellm-rust/crates/auth-azure/src/native.rs +++ b/litellm-rust/crates/auth-azure/src/native.rs @@ -133,7 +133,7 @@ impl NativeAzureTokenAcquirer { let token = credential .get_token(&[scope.as_str()], None) .await - .map_err(|error| Error::AzureTokenAcquisition(error.to_string()))?; + .map_err(|error| Error::CredentialAcquisition(error.to_string().into()))?; let expires_on = u64::try_from(token.expires_on.unix_timestamp()) .ok() .map(|seconds| UNIX_EPOCH + Duration::from_secs(seconds)); @@ -250,7 +250,12 @@ fn validate_authority(request: &NativeAzureRequest) -> Result<(), Error> { let Some(authority) = authority else { return Ok(()); }; - let url = url::Url::parse(authority.value()).map_err(|_| Error::InvalidAzureAuthority)?; + let url = url::Url::parse(authority.value()).map_err(|_| { + Error::InvalidConfiguration( + "Azure authority must be an HTTPS origin without credentials, query, or fragment" + .into(), + ) + })?; if url.scheme() != "https" || url.host_str().is_none() || !url.username().is_empty() @@ -259,7 +264,10 @@ fn validate_authority(request: &NativeAzureRequest) -> Result<(), Error> { || url.fragment().is_some() || !matches!(url.path(), "" | "/") { - return Err(Error::InvalidAzureAuthority); + return Err(Error::InvalidConfiguration( + "Azure authority must be an HTTPS origin without credentials, query, or fragment" + .into(), + )); } Ok(()) } @@ -368,7 +376,9 @@ fn trusted_source(sources: &[InputSource]) -> InputSource { } fn mixed_sources() -> Result { - Err(Error::MixedAzureCredentialSources) + Err(Error::InvalidConfiguration( + "request-controlled Azure auth inputs cannot be combined with host credentials".into(), + )) } fn build_credential( @@ -433,7 +443,12 @@ fn build_credential( NativeAzureRequest::DeveloperTools { .. } => DeveloperToolsCredential::new(None) .map(|credential| credential as Arc), } - .map_err(|error| Error::AzureCredentialInitialization(error.to_string())) + .map_err(|error| { + Error::InvalidConfiguration(litellm_auth_types::ErrorDetail::failed( + "Azure credential initialization", + error, + )) + }) } fn client_options( @@ -638,7 +653,7 @@ mod tests { assert_eq!(transport.requests.lock().unwrap().len(), 6); } - #[test] + #[rstest::rstest] fn request_authority_requires_request_owned_client_secret_identity() { let error = ValidatedAzureRequest::new(sourced_client_secret( InputSource::Deployment, @@ -647,10 +662,13 @@ mod tests { )) .unwrap_err(); - assert!(matches!( + assert_eq!( error, - litellm_auth_types::Error::MixedAzureCredentialSources - )); + litellm_auth_types::Error::InvalidConfiguration( + "request-controlled Azure auth inputs cannot be combined with host credentials" + .into() + ) + ); } #[test] @@ -665,24 +683,24 @@ mod tests { assert_eq!(request.credential_source(), InputSource::Request); } - #[test] - fn authority_is_restricted_to_an_https_origin() { - for authority in [ - "http://login.example", - "https://user@login.example", - "https://login.example/tenant", - "https://login.example?target=other", - ] { - let error = ValidatedAzureRequest::new(sourced_client_secret( - InputSource::Deployment, - InputSource::Deployment, - authority, - )) - .unwrap_err(); - assert!(matches!( - error, - litellm_auth_types::Error::InvalidAzureAuthority - )); - } + #[rstest::rstest] + #[case::http("http://login.example")] + #[case::userinfo("https://user@login.example")] + #[case::path("https://login.example/tenant")] + #[case::query("https://login.example?target=other")] + fn authority_is_restricted_to_an_https_origin(#[case] authority: &str) { + let error = ValidatedAzureRequest::new(sourced_client_secret( + InputSource::Deployment, + InputSource::Deployment, + authority, + )) + .unwrap_err(); + assert_eq!( + error, + litellm_auth_types::Error::InvalidConfiguration( + "Azure authority must be an HTTPS origin without credentials, query, or fragment" + .into() + ) + ); } } diff --git a/litellm-rust/crates/auth-azure/src/resolve.rs b/litellm-rust/crates/auth-azure/src/resolve.rs index 9a7afe645db..2142e22db50 100644 --- a/litellm-rust/crates/auth-azure/src/resolve.rs +++ b/litellm-rust/crates/auth-azure/src/resolve.rs @@ -91,7 +91,9 @@ impl AzureAuthService { AzureCredentialPlan::Caller(caller) => { let credential = caller.acquire().await?; if credential.secret().expose().is_empty() { - return Err(Error::EmptyAzureToken); + return Err(Error::EmptyCallerCredential( + "Azure AD token provider returned an empty token", + )); } Ok(Some(Sourced::new(credential, InputSource::Deployment))) } @@ -104,7 +106,11 @@ impl AzureAuthService { } => { let assertion = resolve_reference(inputs, env_lookup, reference.value()) .await? - .ok_or(Error::UnresolvedOidcReference)?; + .ok_or_else(|| { + Error::CredentialAcquisition( + "Azure OIDC reference did not resolve to a value".into(), + ) + })?; let request = ValidatedAzureRequest::new(NativeAzureRequest::ClientAssertion { tenant_id, client_id, @@ -167,7 +173,7 @@ pub(crate) fn select_auth_plan( .map(|selector| Sourced::new(selector, value.source())) }) .transpose() - .map_err(|_| Error::InvalidAzureSelector)?; + .map_err(|_| Error::InvalidConfiguration("invalid Azure credential selector".into()))?; let federated_token_file = configured_string( &inputs.federated_token_file, AZURE_FEDERATED_TOKEN_FILE_ENV, @@ -257,7 +263,9 @@ fn select_native_plan( let selection_source = selected.source(); match selected.into_value() { - AzureCredentialType::ClientSecretCredential => Err(Error::MissingClientSecretFields), + AzureCredentialType::ClientSecretCredential => Err(Error::InvalidConfiguration( + "ClientSecretCredential requires tenant_id, client_id, and client_secret".into(), + )), AzureCredentialType::WorkloadIdentityCredential => { Ok(AzureCredentialPlan::Native(ValidatedAzureRequest::new( workload_request(tenant_id, client_id, federated_token_file, scope, authority)?, @@ -341,9 +349,17 @@ fn workload_request( authority: Option>, ) -> Result { Ok(NativeAzureRequest::WorkloadIdentity { - tenant_id: tenant_id.ok_or(Error::MissingWorkloadTenant)?, - client_id: client_id.ok_or(Error::MissingWorkloadClient)?, - token_file_path: token_file_path.ok_or(Error::MissingWorkloadTokenFile)?, + tenant_id: tenant_id.ok_or_else(|| { + Error::InvalidConfiguration("WorkloadIdentityCredential requires tenant_id".into()) + })?, + client_id: client_id.ok_or_else(|| { + Error::InvalidConfiguration("WorkloadIdentityCredential requires client_id".into()) + })?, + token_file_path: token_file_path.ok_or_else(|| { + Error::InvalidConfiguration( + "WorkloadIdentityCredential requires azure_federated_token_file".into(), + ) + })?, scope, authority, }) @@ -394,10 +410,11 @@ async fn resolve_reference( .map_or(CredentialLookup::Missing, CredentialLookup::Found), CredentialRef::None => return Ok(None), CredentialRef::File(_) | CredentialRef::Request(_) | CredentialRef::Host(_) => { - let resolver = inputs - .credential_resolver - .as_ref() - .ok_or(Error::MissingHostResolver)?; + let resolver = inputs.credential_resolver.as_ref().ok_or_else(|| { + Error::InvalidConfiguration( + "credential reference requires a host credential resolver".into(), + ) + })?; resolver.resolve(reference).await? } }; @@ -415,7 +432,9 @@ fn oidc_reference( }; let value = token.value().expose(); if token.source() == InputSource::Request && value.starts_with("oidc/") { - return Err(Error::RequestAzureCredentialReference); + return Err(Error::InvalidConfiguration( + "request-controlled Azure credential references are not allowed".into(), + )); } if let Some(name) = value.strip_prefix("oidc/env/") { return non_empty_reference(name, "OIDC environment reference") @@ -437,14 +456,20 @@ fn oidc_reference( ))); } if value.starts_with("oidc/") { - return Err(Error::UnsupportedOidcReference); + return Err(Error::InvalidConfiguration( + "unsupported OIDC reference".into(), + )); } Ok(None) } fn non_empty_reference(value: &str, kind: &str) -> Result { if value.is_empty() { - return Err(Error::EmptyReference(kind.to_string())); + return Err(Error::InvalidConfiguration( + litellm_auth_types::ErrorDetail::Empty { + subject: kind.into(), + }, + )); } Ok(value.to_string()) } @@ -493,7 +518,7 @@ mod tests { expires_on: None, }) } else { - Err(Error::AzureTokenAcquisition(format!("{kind} failed"))) + Err(Error::CredentialAcquisition(kind.into())) } }) } @@ -602,7 +627,7 @@ mod tests { assert!(error.to_string().contains("unsupported OIDC reference")); } - #[test] + #[rstest::rstest] fn request_oidc_reference_is_rejected_before_lookup() { let params = json!({ "azure_ad_token": "oidc/env/ASSERTION", @@ -624,7 +649,12 @@ mod tests { }) .unwrap_err(); - assert!(matches!(error, Error::RequestAzureCredentialReference)); + assert_eq!( + error, + Error::InvalidConfiguration( + "request-controlled Azure credential references are not allowed".into() + ) + ); } #[tokio::test] @@ -723,6 +753,7 @@ mod tests { assert_eq!(credential.value().secret().expose(), "caller-token"); } + #[rstest::rstest] #[tokio::test] async fn empty_caller_token_is_rejected() { let error = AzureAuthService::default() @@ -730,6 +761,9 @@ mod tests { .await .unwrap_err(); - assert!(matches!(error, Error::EmptyAzureToken)); + assert_eq!( + error, + Error::EmptyCallerCredential("Azure AD token provider returned an empty token") + ); } } diff --git a/litellm-rust/crates/auth-azure/src/types.rs b/litellm-rust/crates/auth-azure/src/types.rs index a3a898f000f..a042937a047 100644 --- a/litellm-rust/crates/auth-azure/src/types.rs +++ b/litellm-rust/crates/auth-azure/src/types.rs @@ -117,7 +117,12 @@ fn string_config( None => Ok(ConfigValue::Absent), Some(Value::Null) => Ok(ConfigValue::ExplicitNone(source)), Some(Value::String(value)) => Ok(ConfigValue::Value(Sourced::new(value.clone(), source))), - Some(_) => Err(Error::InvalidFieldType(name.to_string())), + Some(_) => Err(Error::InvalidConfiguration( + litellm_auth_types::ErrorDetail::InvalidType { + field: name.into(), + expected: "a string or null", + }, + )), } } diff --git a/litellm-rust/crates/auth-gcp/Cargo.toml b/litellm-rust/crates/auth-gcp/Cargo.toml index 0c6258a193c..8a3598234e1 100644 --- a/litellm-rust/crates/auth-gcp/Cargo.toml +++ b/litellm-rust/crates/auth-gcp/Cargo.toml @@ -19,3 +19,6 @@ tokio.workspace = true gcp_auth = "0.12.7" google-cloud-auth = { workspace = true, optional = true } http = { workspace = true, optional = true } + +[dev-dependencies] +rstest.workspace = true diff --git a/litellm-rust/crates/auth-gcp/src/lib.rs b/litellm-rust/crates/auth-gcp/src/lib.rs index 4374dff95aa..97bc2c482c3 100644 --- a/litellm-rust/crates/auth-gcp/src/lib.rs +++ b/litellm-rust/crates/auth-gcp/src/lib.rs @@ -299,7 +299,7 @@ fn validate_request_credentials(configured: &str) -> Result<&str, Error> { .map(str::to_string) }); if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) { - return Err(Error::RequestVertexTokenEndpoint); + return Err(Error::InvalidConfiguration("request-controlled Vertex credentials must use the canonical Google OAuth token endpoint".into())); } Ok(configured) } @@ -376,10 +376,20 @@ fn optional_credentials( .map(SecretValue::new) .map(|value| Sourced::new(value, source)) .map(Some) - .map_err(|error| Error::InvalidFieldType(format!("{}: {error}", names[0]))); + .map_err(|error| { + Error::InvalidConfiguration(litellm_auth_types::ErrorDetail::failed( + "credential serialization", + error, + )) + }); } Some(_) => { - return Err(Error::InvalidFieldType(names[0].to_string())); + return Err(Error::InvalidConfiguration( + litellm_auth_types::ErrorDetail::InvalidType { + field: names[0].into(), + expected: "a string or null", + }, + )); } } } @@ -397,7 +407,12 @@ fn optional_string(params: &Map, names: &[&str]) -> Result