diff --git a/.github/workflows/guard-main-branch.yml b/.github/workflows/guard-main-branch.yml index 21aad18d298..aa4968f0c1e 100644 --- a/.github/workflows/guard-main-branch.yml +++ b/.github/workflows/guard-main-branch.yml @@ -31,12 +31,12 @@ jobs: echo "PR head repo: $HEAD_REPO" echo "PR head branch: $HEAD_REF" if [ "$HEAD_REPO" != "$BASE_REPO" ]; then - echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the 'litellm_oss_staging' branch instead." + echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the current daily OSS branch (named litellm_oss_daily_YYYY_MM_DD; a fresh one is cut each weekday, so target the most recent) instead." exit 1 fi if [ "$HEAD_REF" = "litellm_internal_staging" ] || [[ "$HEAD_REF" == litellm_hotfix_?* ]]; then echo "Allowed source branch." exit 0 fi - echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_oss_staging' instead." + echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against the current daily OSS branch (named litellm_oss_daily_YYYY_MM_DD; a fresh one is cut each weekday, so target the most recent) instead." exit 1 diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index 67bec6715f0..525e2c5b949 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -36,83 +36,3 @@ jobs: - name: Build run: npm run build - - frontend-lint: - runs-on: ubuntu-latest - timeout-minutes: 8 - defaults: - run: - working-directory: ui/litellm-dashboard - - steps: - - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Collect changed files - id: changed - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - : > "$RUNNER_TEMP/prettier_files.txt" - : > "$RUNNER_TEMP/eslint_files.txt" - while IFS= read -r f; do - [ -f "$f" ] || continue - case "$f" in - *.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs) - printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" - printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;; - *.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html) - printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;; - esac - done < <(git diff --name-only --diff-filter=ACMR --relative "$BASE_SHA"...HEAD -- .) - if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then - echo "has_files=true" >> "$GITHUB_OUTPUT" - else - echo "has_files=false" >> "$GITHUB_OUTPUT" - echo "No lintable UI files changed in this PR; nothing to check." - fi - - - name: Setup Node.js - if: steps.changed.outputs.has_files == 'true' - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 - with: - node-version: "20" - cache: "npm" - cache-dependency-path: ui/litellm-dashboard/package-lock.json - - - name: Install dependencies - if: steps.changed.outputs.has_files == 'true' - run: npm ci - - - name: Lint changed files (prettier + eslint) - if: steps.changed.outputs.has_files == 'true' - run: | - prettier_files=() - eslint_files=() - while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt" - while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt" - status=0 - if [ ${#prettier_files[@]} -gt 0 ]; then - echo "::group::Prettier (${#prettier_files[@]} files)" - npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; } - echo "::endgroup::" - fi - if [ ${#eslint_files[@]} -gt 0 ]; then - echo "::group::ESLint (${#eslint_files[@]} files)" - npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1 - echo "::endgroup::" - fi - exit $status - - - name: Check lint budgets - if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }} - run: | - npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true - node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json --check eslint-metrics.json - - - name: Check for dead code (knip) - if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }} - run: npm run knip diff --git a/.github/workflows/test-litellm-ui-lint.yml b/.github/workflows/test-litellm-ui-lint.yml new file mode 100644 index 00000000000..226cbf6a879 --- /dev/null +++ b/.github/workflows/test-litellm-ui-lint.yml @@ -0,0 +1,92 @@ +name: UI Lint +permissions: + contents: read + +on: + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_staging + - "litellm_**" + +jobs: + frontend-lint: + runs-on: ubuntu-latest + timeout-minutes: 8 + defaults: + run: + working-directory: ui/litellm-dashboard + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Collect changed files + id: changed + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + : > "$RUNNER_TEMP/prettier_files.txt" + : > "$RUNNER_TEMP/eslint_files.txt" + while IFS= read -r f; do + [ -f "$f" ] || continue + case "$f" in + *.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs) + printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" + printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;; + *.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html) + printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;; + esac + done < <(git diff --name-only --diff-filter=ACMR --relative "$BASE_SHA"...HEAD -- .) + if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then + echo "has_files=true" >> "$GITHUB_OUTPUT" + else + echo "has_files=false" >> "$GITHUB_OUTPUT" + echo "No lintable UI files changed in this PR; nothing to check." + fi + + - name: Setup Node.js + if: steps.changed.outputs.has_files == 'true' + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: ui/litellm-dashboard/package-lock.json + + - name: Install dependencies + if: steps.changed.outputs.has_files == 'true' + run: npm ci + + - name: Lint changed files (prettier + eslint) + if: steps.changed.outputs.has_files == 'true' + run: | + prettier_files=() + eslint_files=() + while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt" + while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt" + status=0 + if [ ${#prettier_files[@]} -gt 0 ]; then + echo "::group::Prettier (${#prettier_files[@]} files)" + npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; } + echo "::endgroup::" + fi + if [ ${#eslint_files[@]} -gt 0 ]; then + echo "::group::ESLint (${#eslint_files[@]} files)" + npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1 + echo "::endgroup::" + fi + exit $status + + - name: Check lint budgets + if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }} + run: | + npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true + node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json --check eslint-metrics.json + + - name: Check for dead code (knip) + if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }} + run: npm run knip diff --git a/CLAUDE.md b/CLAUDE.md index 5affa7748d7..78da2c65d96 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ Same thing for bug fixes. The tests should make it so that this specific bug can End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md` -When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose +When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose for internal contributors; external / OSS contributions target the current daily OSS branch instead, named `litellm_oss_daily_YYYY_MM_DD` (a fresh one is cut each weekday, so use the most recent) When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1080579d0fa..0202965ec4b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -322,7 +322,7 @@ npm run build ## Submitting Your PR 1. **Push your branch**: `git push origin your-feature-branch` -2. **Create a PR**: Go to GitHub and create a pull request +2. **Create a PR**: Go to GitHub and open a pull request against the current daily OSS branch, named `litellm_oss_daily_YYYY_MM_DD`. A fresh one is cut each weekday, so pick the most recent from the [branch list](https://github.com/BerriAI/litellm/branches/all?query=litellm_oss_daily). Do not target `main`. 3. **Fill out the PR template**: Provide clear description of changes 4. **Wait for review**: Maintainers will review and provide feedback 5. **Address feedback**: Make requested changes and push updates diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py index 3e4ac61b637..c6274249cce 100644 --- a/backend/routes/allowlist.py +++ b/backend/routes/allowlist.py @@ -47,6 +47,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = ( "/fallback", "/fallbacks", "/cache_settings", + "/coordination_redis/", "/cost_tracking", "/cost/", "/credentials", diff --git a/helm/litellm-helm/templates/_helpers.tpl b/helm/litellm-helm/templates/_helpers.tpl index 25b02dd5f37..469d52c03a7 100644 --- a/helm/litellm-helm/templates/_helpers.tpl +++ b/helm/litellm-helm/templates/_helpers.tpl @@ -76,10 +76,13 @@ so fall back to "default" (or an explicit override) to avoid a cyclic dependency {{- end }} {{/* -Get redis service name +Get redis service name. +The bundled Redis subchart only serves sentinel in "replication" architecture +(it rejects standalone + sentinel outright), and in that mode the sentinel +Service is named "-redis", not "-redis-master". */}} {{- define "litellm.redis.serviceName" -}} -{{- if and (eq .Values.redis.architecture "standalone") .Values.redis.sentinel.enabled -}} +{{- if .Values.redis.sentinel.enabled -}} {{- printf "%s-%s" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}} {{- else -}} {{- printf "%s-%s-master" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}} diff --git a/helm/litellm-helm/templates/configmap-litellm.yaml b/helm/litellm-helm/templates/configmap-litellm.yaml index acbe4e3a4b5..03e4f620206 100644 --- a/helm/litellm-helm/templates/configmap-litellm.yaml +++ b/helm/litellm-helm/templates/configmap-litellm.yaml @@ -1,9 +1,22 @@ {{- if .Values.proxyConfigMap.create }} +{{- $config := deepCopy .Values.proxy_config }} +{{- if and .Values.redis.enabled (dig "coordination" "enabled" true .Values.redis) }} +{{- $generalSettings := (get $config "general_settings") | default dict }} +{{- if not (hasKey $generalSettings "coordination_redis") }} +{{- $coordinationRedis := dict "host" "os.environ/REDIS_HOST" "port" "os.environ/REDIS_PORT" "password" "os.environ/REDIS_PASSWORD" }} +{{- if .Values.redis.sentinel.enabled }} +{{- $sentinelNode := list (include "litellm.redis.serviceName" .) (include "litellm.redis.port" . | int) }} +{{- $coordinationRedis = dict "sentinel_nodes" (list $sentinelNode) "service_name" (default "mymaster" .Values.redis.sentinel.masterSet) "password" "os.environ/REDIS_PASSWORD" }} +{{- end }} +{{- $_ := set $generalSettings "coordination_redis" $coordinationRedis }} +{{- $_ := set $config "general_settings" $generalSettings }} +{{- end }} +{{- end }} apiVersion: v1 kind: ConfigMap metadata: name: {{ include "litellm.fullname" . }}-config data: config.yaml: | -{{ .Values.proxy_config | toYaml | indent 6 }} +{{ $config | toYaml | indent 6 }} {{- end }} diff --git a/helm/litellm-helm/tests/coordination_redis_tests.yaml b/helm/litellm-helm/tests/coordination_redis_tests.yaml new file mode 100644 index 00000000000..0b58b1e6bc8 --- /dev/null +++ b/helm/litellm-helm/tests/coordination_redis_tests.yaml @@ -0,0 +1,143 @@ +suite: test coordination redis +templates: + - configmap-litellm.yaml + - deployment.yaml +tests: + - it: should not render coordination_redis when redis is disabled + template: configmap-litellm.yaml + set: + redis.enabled: false + asserts: + - notMatchRegex: + path: data["config.yaml"] + pattern: coordination_redis + + - it: should not emit redis env vars when redis is disabled + template: deployment.yaml + set: + redis.enabled: false + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: RELEASE-NAME-redis-master + any: true + + - it: should render coordination_redis pointing at the bundled redis when enabled + template: configmap-litellm.yaml + set: + redis.enabled: true + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "coordination_redis:\n host: os.environ/REDIS_HOST\n password: os.environ/REDIS_PASSWORD\n port: os.environ/REDIS_PORT\n" + - matchRegex: + path: data["config.yaml"] + pattern: "master_key: os.environ/PROXY_MASTER_KEY" + + - it: should emit redis env vars backing the coordination_redis os.environ refs + template: deployment.yaml + set: + redis.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: RELEASE-NAME-redis-master + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PORT + value: "6379" + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: RELEASE-NAME-redis + key: redis-password + + - it: should not render coordination_redis when coordination is opted out + template: configmap-litellm.yaml + set: + redis.enabled: true + redis.coordination.enabled: false + asserts: + - notMatchRegex: + path: data["config.yaml"] + pattern: coordination_redis + + - it: should keep emitting redis env vars when coordination is opted out + template: deployment.yaml + set: + redis.enabled: true + redis.coordination.enabled: false + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: RELEASE-NAME-redis-master + + - it: should not clobber a user supplied coordination_redis block + template: configmap-litellm.yaml + set: + redis.enabled: true + proxy_config.general_settings.coordination_redis: + url: os.environ/COORDINATION_REDIS_URL + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "coordination_redis:\n url: os.environ/COORDINATION_REDIS_URL\n" + - notMatchRegex: + path: data["config.yaml"] + pattern: "host: os.environ/REDIS_HOST" + + - it: should render sentinel_nodes and service_name in sentinel mode + template: configmap-litellm.yaml + set: + redis.enabled: true + redis.architecture: replication + redis.sentinel.enabled: true + asserts: + # The sentinel Service the redis subchart renders is "-redis", and a + # plain client cannot speak the sentinel protocol, so host/port must not appear + - matchRegex: + path: data["config.yaml"] + pattern: "coordination_redis:\n password: os.environ/REDIS_PASSWORD\n sentinel_nodes:\n - - RELEASE-NAME-redis\n - 26379\n service_name: mymaster\n" + - notMatchRegex: + path: data["config.yaml"] + pattern: "host: os.environ/REDIS_HOST" + + - it: should carry a custom sentinel masterSet into service_name + template: configmap-litellm.yaml + set: + redis.enabled: true + redis.architecture: replication + redis.sentinel.enabled: true + redis.sentinel.masterSet: litellm-master + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: "service_name: litellm-master" + + - it: should point REDIS_HOST at the sentinel service in sentinel mode + template: deployment.yaml + set: + redis.enabled: true + redis.architecture: replication + redis.sentinel.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: RELEASE-NAME-redis + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PORT + value: "26379" diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 6e30a6af444..d3821a547e5 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -331,12 +331,28 @@ postgresql: # secretKeys: # userPasswordKey: password -# requires cache: true in config file -# either enable this or pass a secret for REDIS_HOST, REDIS_PORT, REDIS_PASSWORD or REDIS_URL -# with cache: true to use existing redis instance +# Redis is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend +# tracking, and the pod lock manager. Enabling this deploys the bundled Redis +# subchart, wires REDIS_HOST / REDIS_PORT / REDIS_PASSWORD into the proxy, and +# renders a `general_settings.coordination_redis` block into the proxy config. +# +# To point at an existing Redis instead, leave `enabled: false` and pass a +# secret for REDIS_HOST, REDIS_PORT, REDIS_PASSWORD or REDIS_URL; the proxy +# falls back to those env vars for coordination. Set `cache: true` in the proxy +# config only if you also want LLM response caching, which is independent of +# coordination +# +# When `redis.sentinel.enabled` is set, the coordination block is rendered with +# `sentinel_nodes` and `service_name` (from `redis.sentinel.masterSet`) instead +# of host/port, because a plain Redis client cannot talk to the sentinel port redis: enabled: false architecture: standalone + coordination: + # Set to false to keep the bundled Redis for response caching only and leave + # `general_settings.coordination_redis` out of the rendered config. A + # `coordination_redis` block you define yourself in `proxy_config` always wins + enabled: true # Prisma migration job settings migrationJob: diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 4319907883e..7c281aa158b 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -213,6 +213,10 @@ harmless no-op for the Job and authoritative for the app pods. */}} - name: DISABLE_SCHEMA_UPDATE value: "true" +{{/* These feed the proxy's coordination Redis (cross-pod rate limits, spend + tracking, pod lock manager) via its REDIS_* env fallback. An explicit + `general_settings.coordination_redis` block in proxy_config takes + precedence over anything emitted here. */}} {{- if $root.Values.redis.host }} - name: REDIS_HOST value: {{ $root.Values.redis.host | quote }} @@ -226,10 +230,11 @@ harmless no-op for the Job and authoritative for the app pods. key: {{ $root.Values.redis.passwordSecret.passwordKey | default "password" }} {{- end }} {{- if $root.Values.redis.cluster }} -{{/* The proxy's Cache() reads REDIS_CLUSTER_NODES as JSON and constructs a - RedisClusterCache when it's set (litellm/caching/caching.py:169-192). - We seed with the single configured endpoint — the cluster client - discovers the remaining nodes from CLUSTER SLOTS at startup. */}} +{{/* The proxy falls back to REDIS_CLUSTER_NODES (JSON) to build a cluster-mode + coordination client when `general_settings.coordination_redis` is absent + and no plain-Redis response cache is configured. We seed with the single + configured endpoint; the cluster client discovers the remaining nodes from + CLUSTER SLOTS at startup. */}} - name: REDIS_CLUSTER_NODES value: {{ printf "[{\"host\":%q,\"port\":%v}]" $root.Values.redis.host (int $root.Values.redis.port) | quote }} {{- end }} diff --git a/helm/litellm/tests/redis_env_tests.yaml b/helm/litellm/tests/redis_env_tests.yaml new file mode 100644 index 00000000000..684d7071b35 --- /dev/null +++ b/helm/litellm/tests/redis_env_tests.yaml @@ -0,0 +1,109 @@ +suite: test redis coordination env vars +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml + - backend/deployment.yaml +values: + - ./values/required.yaml +tests: + - it: gateway omits redis env vars when no host is configured + template: gateway/deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: redis.example.com + any: true + - notContains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_CLUSTER_NODES + any: true + + - it: gateway emits host, port and password when redis is configured + template: gateway/deployment.yaml + set: + redis.host: redis.example.com + redis.port: 6380 + redis.passwordSecret.name: redis-secret + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: redis.example.com + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PORT + value: "6380" + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: password + + - it: backend emits the same redis env vars so both pods coordinate on one redis + template: backend/deployment.yaml + set: + redis.host: redis.example.com + redis.passwordSecret.name: redis-secret + redis.passwordSecret.passwordKey: redis-password + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: redis.example.com + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + + - it: gateway omits REDIS_PASSWORD for an auth-less redis + template: gateway/deployment.yaml + set: + redis.host: redis.example.com + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_PASSWORD + any: true + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_HOST + value: redis.example.com + + - it: gateway seeds REDIS_CLUSTER_NODES from host and port in cluster mode + template: gateway/deployment.yaml + set: + redis.host: redis.example.com + redis.port: 6380 + redis.cluster: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_CLUSTER_NODES + value: '[{"host":"redis.example.com","port":6380}]' + + - it: gateway omits REDIS_CLUSTER_NODES when cluster mode is off + template: gateway/deployment.yaml + set: + redis.host: redis.example.com + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: REDIS_CLUSTER_NODES + any: true diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 6aa5dd39cd0..a8f2d39663e 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -100,7 +100,18 @@ database: usernameKey: username passwordKey: password -# Optional Redis (caching, rate limiting). Leave host empty to disable. +# Optional Redis. Leave host empty to disable. +# +# This is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend +# tracking, and the pod lock manager. The chart emits REDIS_HOST / REDIS_PORT / +# REDIS_PASSWORD, which the proxy picks up through its coordination Redis env +# fallback. Response caching is separate and off unless you enable it in +# `proxy_config.litellm_settings.cache`. +# +# For full control, define `general_settings.coordination_redis` in +# `proxy_config` (host/port/password/username/url/ssl/startup_nodes/ +# sentinel_nodes/sentinel_password/service_name, each accepting os.environ/VAR +# refs). An explicit block overrides these env vars. # # Set `cluster: true` for Redis Cluster mode (e.g. AWS ElastiCache Cluster, # self-hosted Redis Cluster). The chart emits REDIS_CLUSTER_NODES from diff --git a/litellm/_redis.py b/litellm/_redis.py index bb3a0974241..053514c0dbe 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -325,8 +325,19 @@ def _get_redis_client_logic(**env_overrides): value = get_secret(v) # type: ignore env_overrides[k] = value + environment_kwargs = _redis_kwargs_from_environment() + + # An explicitly configured connection target outranks REDIS_URL from the + # environment. Without this, the url branch below strips the caller's + # host/port/password and silently connects to whatever REDIS_URL names. + caller_named_a_target = any( + env_overrides.get(key) is not None for key in ("host", "startup_nodes", "sentinel_nodes") + ) + if caller_named_a_target and env_overrides.get("url") is None: + environment_kwargs.pop("url", None) + redis_kwargs = { - **_redis_kwargs_from_environment(), + **environment_kwargs, **env_overrides, } diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 17011bb8db7..3038bdb90b2 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -223,6 +223,15 @@ lives in [`plumbing/`](./plumbing): readers/exporters receive them alongside the server metrics, and one is built and registered as the global only when none is set (mirroring how V2 owns trace export). +- [`events.py`](./plumbing/events.py) — GenAI client events. Gated on + `enable_events` (`LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS`), a failed LLM call + records the semconv `gen_ai.client.operation.exception` log event at severity + WARN, carrying `exception.type` / `exception.message` / `exception.stacktrace` + and correlated to the failed span through the trace and span ids. The + `LoggerProvider` is resolved like the meter provider, except that an explicit + `NoOpLoggerProvider` global is an operator opt-out that builds no recorder at + all. The deprecated `error.*` span attributes and the `exception` span event + are still stamped by the emitter for backwards compatibility. ### Adapter diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 46aa166a8bb..f97f8b8394c 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -18,6 +18,7 @@ from litellm.integrations.otel.model.payloads import ( ServiceSpanData, SpanError, ) +from litellm.integrations.otel.plumbing.events import GenAIEventRecorder from litellm.integrations.otel.plumbing.providers import to_otel_span_kind from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError from litellm.integrations.otel.model.spans import ( @@ -77,9 +78,11 @@ class SpanEmitter: tracer: Tracer, config: OpenTelemetryV2Config, mappers: Sequence[AttributeMapper] | None = None, + event_recorder: GenAIEventRecorder | None = None, ) -> None: self._tracer = tracer self._config = config + self._event_recorder = event_recorder # The mapper chain is the sole source of span attributes. When not # passed in, resolve it from the config so there's one source of truth. self._mappers: list[AttributeMapper] = ( @@ -223,6 +226,14 @@ class SpanEmitter: ExceptionEvent.NAME, {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message}, ) + if self._event_recorder is not None and role is SpanRole.LLM_CALL: + self._event_recorder.record_operation_exception( + span_context=span.get_span_context(), + error_type=error_type, + message=message, + stack_trace=error.stack_trace, + timestamp_ns=end_time_ns, + ) # On success leave the status UNSET (the semconv default) rather than # forcing OK — that matches the FastAPI server span and avoids implying a # span-level health signal litellm doesn't actually evaluate. Only a diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index e258b239d93..be72fabd387 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -6,6 +6,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Callable, Iterator, Mapping, Sequence, cast from opentelemetry.context import Context, attach, get_current +from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk.trace import TracerProvider from opentelemetry.trace import Span, Tracer, get_current_span, use_span @@ -40,14 +41,17 @@ from litellm.integrations.otel.model.payloads import ( is_mcp_list_tools, is_mcp_tool_call, ) +from litellm.integrations.otel.plumbing.events import GenAIEventRecorder from litellm.integrations.otel.plumbing.metrics import ( GenAIMetricRecorder, create_genai_metrics, ) from litellm.integrations.otel.plumbing.providers import ( build_tracer_provider, + get_event_logger, get_meter, get_tracer, + resolve_logger_provider, resolve_meter_provider, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache @@ -104,7 +108,7 @@ class OpenTelemetryV2(CustomLogger): config: OpenTelemetryV2Config | None = None, callback_name: str | None = None, tracer_provider: TracerProvider | None = None, - logger_provider: Any | None = None, # reserved for OTel logs + logger_provider: LoggerProvider | None = None, meter_provider: Any | None = None, **kwargs: Any, ) -> None: @@ -117,7 +121,12 @@ class OpenTelemetryV2(CustomLogger): self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME) self._metrics_recorder = self._init_metrics(meter_provider) self._metric_filter_error_logged = False - self._emitter = SpanEmitter(self.tracer, self.config, mappers=resolve_mappers(self.config.mapper_names)) + self._emitter = SpanEmitter( + self.tracer, + self.config, + mappers=resolve_mappers(self.config.mapper_names), + event_recorder=self._init_events(logger_provider), + ) self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME) self._open_llm_calls: "OrderedDict[str, _LLMCallSpan]" = OrderedDict() self._init_otel_logger_on_litellm_proxy() @@ -136,6 +145,22 @@ class OpenTelemetryV2(CustomLogger): meter = get_meter(provider, LITELLM_TRACER_NAME) return GenAIMetricRecorder(create_genai_metrics(meter), self.callback_name) + def _init_events(self, logger_provider: LoggerProvider | None) -> "GenAIEventRecorder | None": + """Create the GenAI event recorder when events are enabled, else ``None``. + + ``logger_provider`` is an explicit override (tests inject one); otherwise the + provider is resolved from the OTel global so an operator-configured logs + pipeline receives the events, building and registering one only when no + global provider is set. A ``None`` resolution means the operator opted out + of the logs signal, so no recorder is built. + """ + if not self.config.enable_events: + return None + provider = resolve_logger_provider(self.config, logger_provider) + if provider is None: + return None + return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME)) + # ====================================================================== # # Proxy global registration # ====================================================================== # diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index aab80c7e5c4..44b2f7e0488 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -177,6 +177,19 @@ class ExceptionEvent: NAME: Final = "exception" TYPE: Final = "exception.type" MESSAGE: Final = "exception.message" + STACKTRACE: Final = "exception.stacktrace" + + +class GenAIEvent: + """GenAI semconv event names, from the GenAI registry's *events* section. + + ``gen_ai.client.operation.exception`` is defined as a log-based event + (severity WARN) carrying the ``exception.*`` trio, correlated to the failed + span via the trace/span ids — the semconv-compliant home for GenAI failure + details, unlike the deprecated ``error.message`` span attribute. + """ + + OPERATION_EXCEPTION: Final = "gen_ai.client.operation.exception" class Server: diff --git a/litellm/integrations/otel/plumbing/events.py b/litellm/integrations/otel/plumbing/events.py new file mode 100644 index 00000000000..f674526d04f --- /dev/null +++ b/litellm/integrations/otel/plumbing/events.py @@ -0,0 +1,52 @@ +"""GenAI client events: the ``gen_ai.client.operation.exception`` log event. + +The GenAI semantic conventions define exception recording for client +operations as a log-based event (severity WARN) carrying the ``exception.*`` +attribute trio, correlated to the failed span through the trace/span ids — +not as a span attribute or span event. This module owns building and +emitting that event; the exporter pipeline it rides is built in +:mod:`litellm.integrations.otel.plumbing.providers`. +""" + +from dataclasses import dataclass + +from opentelemetry._events import Event, EventLogger +from opentelemetry._logs.severity import SeverityNumber +from opentelemetry.trace import SpanContext + +from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + +@dataclass(frozen=True, slots=True) +class GenAIEventRecorder: + event_logger: EventLogger + + def record_operation_exception( + self, + span_context: SpanContext, + error_type: str, + message: str, + stack_trace: str | None, + timestamp_ns: int | None, + ) -> None: + # ``exception.type`` and ``exception.message`` are the semconv-required + # pair and always ride the event; only the recommended stacktrace is + # conditional on the payload carrying one. + stacktrace = ((ExceptionEvent.STACKTRACE, stack_trace),) if stack_trace else () + self.event_logger.emit( + Event( + name=GenAIEvent.OPERATION_EXCEPTION, + timestamp=timestamp_ns, + trace_id=span_context.trace_id, + span_id=span_context.span_id, + trace_flags=span_context.trace_flags, + severity_number=SeverityNumber.WARN, + attributes=dict( + ( + (ExceptionEvent.TYPE, error_type), + (ExceptionEvent.MESSAGE, message), + *stacktrace, + ) + ), + ) + ) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index ac971c6daa8..ced65aa1ec3 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -2,9 +2,20 @@ from typing import TYPE_CHECKING, Any, Callable, Iterable -from opentelemetry import baggage, metrics +from opentelemetry import _logs, baggage, metrics +from opentelemetry._events import EventLogger +from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider from opentelemetry.context import Context from opentelemetry.metrics import MeterProvider, NoOpMeterProvider +from opentelemetry.sdk._events import EventLoggerProvider +from opentelemetry.sdk._logs import LoggerProvider as SDKLoggerProvider +from opentelemetry.sdk._logs.export import ( + BatchLogRecordProcessor, + ConsoleLogExporter, + InMemoryLogExporter, + LogExporter, + SimpleLogRecordProcessor, +) from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider @@ -224,6 +235,112 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader": return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) +def _otlp_logs_endpoint(endpoint: str | None) -> str | None: + """Point an OTLP/HTTP base endpoint at the ``/v1/logs`` signal path. + + The OTLP/HTTP exporter only appends ``/v1/logs`` when it reads + ``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; an explicitly passed endpoint is used + verbatim, so a base URL would POST to the root. Mirror ``_otlp_traces_endpoint`` + for the logs signal (rewriting a sibling signal path when present). + """ + if not endpoint: + return endpoint + endpoint = endpoint.rstrip("/") + if endpoint.endswith("/v1/logs"): + return endpoint + for other_signal in ("/v1/traces", "/v1/metrics"): + if endpoint.endswith(other_signal): + return endpoint[: -len(other_signal)] + "/v1/logs" + return endpoint + "/v1/logs" + + +def build_log_exporter(config: OpenTelemetryV2Config) -> LogExporter: + """Build a log exporter mirroring the exporter selection of the other signals. + + ``console`` (and any unrecognized kind) exports to the console; ``otlp_http`` + and ``otlp_grpc`` export over OTLP with the configured endpoint/headers; + ``in_memory`` buffers for tests. Like GenAI metrics, events ride the + single-destination shorthand fields, not the multi-exporter ``exporters`` list. + """ + kind = (config.exporter or "console").lower() + if kind in ("in_memory", "inmemory", "memory"): + return InMemoryLogExporter() + if kind in ("otlp_http", "http", "http/protobuf", "http/json"): + from opentelemetry.exporter.otlp.proto.http._log_exporter import ( + OTLPLogExporter as HTTPLogExporter, + ) + + return HTTPLogExporter( + endpoint=_otlp_logs_endpoint(config.endpoint), + headers=parse_headers(config.headers), + ) + if kind in ("otlp_grpc", "grpc"): + try: + from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter as GRPCLogExporter, + ) + except ImportError as exc: + raise ImportError( + "OpenTelemetry OTLP gRPC log exporter is not available. Install " + "`opentelemetry-exporter-otlp` and `grpcio` (or `litellm[grpc]`)." + ) from exc + + return GRPCLogExporter(endpoint=config.endpoint, headers=parse_headers(config.headers)) + return ConsoleLogExporter() + + +def build_logger_provider( + config: OpenTelemetryV2Config, + log_exporter: LogExporter | None = None, +) -> SDKLoggerProvider: + """Build the :class:`LoggerProvider` GenAI events export through. + + ``log_exporter`` is an explicit override (tests inject an + ``InMemoryLogExporter``); otherwise the exporter is selected from the config's + exporter kind via :func:`build_log_exporter`. Console and in-memory exporters + get a Simple processor (synchronous export, which tests rely on), everything + else a Batch processor — the same split as span processing. + """ + exporter = log_exporter if log_exporter is not None else build_log_exporter(config) + provider = SDKLoggerProvider(resource=build_resource(config)) + use_simple = isinstance(exporter, (ConsoleLogExporter, InMemoryLogExporter)) + provider.add_log_record_processor( + SimpleLogRecordProcessor(exporter) if use_simple else BatchLogRecordProcessor(exporter) + ) + return provider + + +def resolve_logger_provider( + config: OpenTelemetryV2Config, + logger_provider: SDKLoggerProvider | None = None, +) -> SDKLoggerProvider | None: + """Resolve the :class:`LoggerProvider` GenAI events record through, or ``None`` + when the operator has opted out of the logs signal. + + Same resolution order as :func:`resolve_meter_provider`: an injected provider + wins (DI/tests); an operator-configured SDK global is reused so events ride + their pipeline; an explicit ``NoOpLoggerProvider`` global is an opt-out and + yields ``None``, so no event is ever built. Only the default placeholder + global makes V2 build a provider from the config and publish it as the global. + """ + if logger_provider is not None: + return logger_provider + + existing: LoggerProvider = _logs.get_logger_provider() + if isinstance(existing, SDKLoggerProvider): + return existing + if isinstance(existing, NoOpLoggerProvider): + return None + + provider = build_logger_provider(config) + _logs.set_logger_provider(provider) + return provider + + +def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> EventLogger: + return EventLoggerProvider(logger_provider=provider).get_event_logger(name, litellm_version) + + def build_meter_provider( config: OpenTelemetryV2Config, metric_reader: "MetricReader | None" = None, diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 9dc202c4717..fdab3d5b9d4 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -95,6 +95,9 @@ class ExceptionCheckers: if "current length is" in _error_str_lowercase and "while limit is" in _error_str_lowercase: return True + if "maximum input length is" in _error_str_lowercase and "tokens" in _error_str_lowercase: + return True + return False @staticmethod diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index daee3369a3c..155fba008d4 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -94,25 +94,30 @@ class AmazonAnthropicClaudeMessagesConfig( return [value] def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict) -> None: - """Bedrock Invoke rejects ``role: "system"`` entries inside ``messages`` on - some Claude aliases; Anthropic Messages carries that content in the - top-level ``system`` field. Move any such entries into ``system`` before - the Invoke request is built.""" + """Bedrock Invoke rejects a conversation that opens with ``role: "system"`` + entries inside ``messages`` ("messages.0: use the top-level 'system' + parameter for the initial system prompt"); Anthropic Messages carries that + content in the top-level ``system`` field, so hoist the leading run of + system entries there. Mid-conversation system entries (e.g. Claude Code's + ``mid-conversation-system-2026-04-07`` reminders) are accepted by Invoke in + place and MUST stay in place: hoisting one mutates the ``system`` prefix + and invalidates the prompt cache for the entire message history. + Billing-header system blocks are stripped from the top-level ``system`` + field regardless of whether anything was hoisted.""" messages = anthropic_messages_request.get("messages") if not isinstance(messages, list): return - system_role_messages = [m for m in messages if isinstance(m, dict) and m.get("role") == "system"] - if not system_role_messages: - return - - anthropic_messages_request["messages"] = [ - m for m in messages if not (isinstance(m, dict) and m.get("role") == "system") - ] + leading_count = next( + (i for i, m in enumerate(messages) if not (isinstance(m, dict) and m.get("role") == "system")), + len(messages), + ) + if leading_count: + anthropic_messages_request["messages"] = messages[leading_count:] system_content = [ block for source in ( anthropic_messages_request.get("system"), - *(m.get("content") for m in system_role_messages), + *(m.get("content") for m in messages[:leading_count]), ) for block in self._as_system_content_blocks(source) ] diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index f746a42080a..9689a7cb047 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -499,23 +499,86 @@ def _raise_unless_oauth2_discovery_server( mcp_server_name: Optional[str], description: str, ) -> None: - """404 a NAMED discovery request unless it resolves to an oauth2 server. + """404 a NAMED discovery request unless it resolves to an oauth2 or DCR-bridge server. A named server that is unknown (or hidden from the caller) and one that exists but is non-oauth2 both return the same 404, so the well-known discovery paths cannot be used to enumerate non-OAuth server names. Root discovery (no name) is unaffected, and pass-through servers are resolved by the caller before this runs. + DCR-bridge servers are admitted because they serve the gateway's own authorization + server metadata (the register, authorize, and token relays). """ if mcp_server_name is None: return if mcp_server is not None and mcp_server.auth_type == MCPAuth.oauth2: return + if mcp_server is not None and mcp_server.is_dcr_bridge: + return raise HTTPException( status_code=404, detail=f"MCP server '{mcp_server_name}' is {description}", ) +def _dcr_bridge_relays_client_registration(mcp_server: MCPServer) -> bool: + """True when a DCR-bridge server relays client registration to the upstream authorization + server instead of short-circuiting to an admin-configured OAuth client. In the relay arm the + upstream holds each client's own registration, so the authorize and token relays pass the + client's ``client_id`` and ``redirect_uri`` through verbatim and the authorization code + returns directly to the client's redirect URI without transiting the gateway. Gateway-side + redirect trust and the ``/callback`` state relay therefore only apply to the short-circuit + arm, where the upstream only knows the gateway's own callback.""" + return mcp_server.is_dcr_bridge and bool(mcp_server.registration_url) and not mcp_server.client_id + + +def _require_s256_pkce( + code_challenge: Optional[str], + code_challenge_method: Optional[str], +) -> Tuple[str, str]: + """DCR-bridge servers serve unauthenticated public OAuth clients, so the PKCE downgrade + paths (no challenge, or a non-S256 method; RFC 7636 defaults a missing method to ``plain``) + are rejected at the gateway instead of relying on upstream enforcement. Returns the + validated pair so callers get non-optional values.""" + if code_challenge and code_challenge_method == "S256": + return code_challenge, code_challenge_method + raise HTTPException( + status_code=400, + detail=( + "This server requires PKCE: send code_challenge with " + "code_challenge_method=S256 on the authorization request" + ), + ) + + +def _redirect_to_upstream_authorize( + *, + mcp_server: MCPServer, + client_id: str, + redirect_uri: str, + state: str, + code_challenge: str, + code_challenge_method: str, + response_type: Optional[str], + scope: Optional[str], +) -> RedirectResponse: + """The bridge relay arm's authorize redirect: every client-supplied parameter passes through + to the upstream authorize endpoint verbatim, no relay state cookie is set, and the upstream + enforces its own registered redirect binding for the client.""" + scope_value = scope or (" ".join(mcp_server.scopes) if mcp_server.scopes else None) + passthrough_params = { + "client_id": client_id, + "redirect_uri": redirect_uri, + "state": state, + "response_type": response_type or "code", + "code_challenge": code_challenge, + "code_challenge_method": code_challenge_method, + **({"scope": scope_value} if scope_value else {}), + } + parsed_auth_url = urlparse(mcp_server.authorization_url or "") + merged_params = {**dict(parse_qsl(parsed_auth_url.query)), **passthrough_params} + return RedirectResponse(urlunparse(parsed_auth_url._replace(query=urlencode(merged_params)))) + + async def authorize_with_server( request: Request, mcp_server: MCPServer, @@ -531,6 +594,24 @@ async def authorize_with_server( if mcp_server.authorization_url is None: raise HTTPException(status_code=400, detail="MCP server authorization url is not set") + if mcp_server.is_dcr_bridge: + # Enforce S256 PKCE on both bridge arms. The relay arm forwards the validated, + # now-non-optional pair to the upstream authorize; the short-circuit arm keeps + # calling this for its enforcement side effect, then falls through to the gateway + # /callback flow below, which reads the original code_challenge names. + bridge_challenge, bridge_method = _require_s256_pkce(code_challenge, code_challenge_method) + if _dcr_bridge_relays_client_registration(mcp_server): + return _redirect_to_upstream_authorize( + mcp_server=mcp_server, + client_id=client_id, + redirect_uri=redirect_uri, + state=state, + code_challenge=bridge_challenge, + code_challenge_method=bridge_method, + response_type=response_type, + scope=scope, + ) + # Trusted redirect_uri: same-origin, loopback, or ops-allowlisted. # The URI is encrypted into the OAuth state and decoded on # /callback to redirect the user back; a non-trusted URI would be @@ -626,11 +707,21 @@ async def exchange_token_with_server( status_code=400, detail="code is required for authorization_code grant", ) + bridge_token_relay = _dcr_bridge_relays_client_registration(mcp_server) + if bridge_token_relay and not redirect_uri: + raise HTTPException( + status_code=400, + detail=( + "redirect_uri is required for the authorization_code grant on this server; " + "send the same redirect_uri used on the authorization request" + ), + ) proxy_base_url = get_request_base_url(request) + resolved_redirect_uri = redirect_uri if bridge_token_relay else f"{proxy_base_url}/callback" token_data = { "grant_type": "authorization_code", "code": code, - "redirect_uri": f"{proxy_base_url}/callback", + "redirect_uri": resolved_redirect_uri, **client_auth.body, } if code_verifier: @@ -648,7 +739,17 @@ async def exchange_token_with_server( detail="MCP upstream token endpoint returned no response", ) - response.raise_for_status() + try: + response.raise_for_status() + except httpx.HTTPStatusError as exc: + if "invalid_target" in exc.response.text: + verbose_logger.warning( + "MCP server %s: the upstream authorization server rejected the token request with " + "invalid_target; it may require RFC 8707 resource indicators, which the gateway " + "does not send yet (tracked as LIT-4339)", + mcp_server.server_id, + ) + raise token_response = response.json() access_token = token_response["access_token"] @@ -888,6 +989,21 @@ async def _persist_dcr_client_registration( return "failed" +_MAX_UPSTREAM_ERROR_CHARS = 500 + + +def _safe_upstream_error_detail(response: httpx.Response) -> str: + """Bounded plaintext summary of an upstream registration failure for the client. + + RFC 7591 error bodies are small JSON objects (``error`` / ``error_description``); relaying the + text lets the client read the real reason instead of a bare 500, and the length bound keeps a + hostile or oversized upstream body from bloating the gateway response.""" + body = response.text + if not body: + return response.reason_phrase or "upstream registration failed" + return body[:_MAX_UPSTREAM_ERROR_CHARS] + + async def register_client_with_server( request: Request, mcp_server: MCPServer, @@ -897,6 +1013,7 @@ async def register_client_with_server( token_endpoint_auth_method: Optional[str], fallback_client_id: Optional[str] = None, persist_credentials: bool = False, + client_redirect_uris: Optional[list] = None, ): _raise_if_not_oauth2(mcp_server) request_base_url = get_request_base_url(request) @@ -918,12 +1035,19 @@ async def register_client_with_server( if mcp_server.registration_url is None: return dummy_return + bridge_relay = _dcr_bridge_relays_client_registration(mcp_server) + if bridge_relay and not client_redirect_uris: + raise HTTPException( + status_code=400, + detail="redirect_uris is required to register a client with this server", + ) + register_data = { "client_name": client_name, - "redirect_uris": [f"{request_base_url}/callback"], - "grant_types": grant_types or [], - "response_types": response_types or [], - "token_endpoint_auth_method": token_endpoint_auth_method or "", + "redirect_uris": client_redirect_uris if bridge_relay else [f"{request_base_url}/callback"], + "grant_types": grant_types or (["authorization_code", "refresh_token"] if bridge_relay else []), + "response_types": response_types or (["code"] if bridge_relay else []), + "token_endpoint_auth_method": token_endpoint_auth_method or ("none" if bridge_relay else ""), } headers = { "Content-Type": "application/json", @@ -941,11 +1065,13 @@ async def register_client_with_server( status_code=502, detail="MCP upstream registration endpoint returned no response", ) + if bridge_relay and response.status_code >= 400: + raise HTTPException(status_code=response.status_code, detail=_safe_upstream_error_detail(response)) response.raise_for_status() token_response = response.json() - if persist_credentials: + if persist_credentials and not bridge_relay: persistence_result = await _persist_dcr_client_registration(mcp_server, token_response) if persistence_result == "reused": return dummy_return @@ -1369,6 +1495,13 @@ async def _build_oauth_protected_resource_response( else: resource_url = f"{request_base_url}/mcp" + if mcp_server is not None and mcp_server_name and mcp_server.is_dcr_bridge: + return { + "authorization_servers": [f"{request_base_url}/{mcp_server_name}"], + "resource": resource_url, + "scopes_supported": (mcp_server.scopes if mcp_server.scopes else []), + } + # Pass-through branch: proxy the upstream's own metadata so discovery # directs the client at the real IdP (Okta, Keycloak, …) instead of us. if mcp_server is not None and ( @@ -1698,6 +1831,7 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non response_types=data.get("response_types", []), token_endpoint_auth_method=data.get("token_endpoint_auth_method", ""), fallback_client_id=resolved.server_name or resolved.name, + client_redirect_uris=data.get("redirect_uris"), ) return dummy_return @@ -1712,4 +1846,5 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non response_types=data.get("response_types", []), token_endpoint_auth_method=data.get("token_endpoint_auth_method", ""), fallback_client_id=mcp_server_name, + client_redirect_uris=data.get("redirect_uris"), ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index a3da237ca09..1d681b43b9e 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -2630,10 +2630,16 @@ class MCPServerManager: return prefixed_or_original_tools - except MCPUpstreamAuthError: + except MCPUpstreamAuthError as upstream_auth_error: # Pass-through 401 must surface to single-server routes so the # client triggers the upstream OAuth flow. The multi-server # aggregator catches this explicitly to keep absorbing. + if server.is_dcr_bridge and upstream_auth_error.www_authenticate is not None: + raise MCPUpstreamAuthError( + status_code=upstream_auth_error.status_code, + www_authenticate=None, + server_name=upstream_auth_error.server_name, + ) from upstream_auth_error raise except HTTPException as e: # A v2 resolver auth challenge (token_exchange's RFC 9728 401, authorization_code's @@ -2643,9 +2649,10 @@ class MCPServerManager: # Non-auth HTTP errors stay absorbed so one misconfigured server can't blank the listing. if e.status_code in (401, 403): headers = e.headers or {} + challenge_header = headers.get("WWW-Authenticate") or headers.get("www-authenticate") raise MCPUpstreamAuthError( status_code=e.status_code, - www_authenticate=headers.get("WWW-Authenticate") or headers.get("www-authenticate"), + www_authenticate=None if server.is_dcr_bridge else challenge_header, server_name=server.name, ) from e verbose_logger.warning(f"Failed to get tools from server {server.name}: {str(e)}") diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py new file mode 100644 index 00000000000..298bc8d98cc --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py @@ -0,0 +1,358 @@ +"""Client-held sealed envelope for the oauth_delegate DCR bridge. + +A DCR-bridge client holds ONE bearer that must carry BOTH a litellm identity and the +upstream OAuth grant, with zero server-side storage. The gateway token endpoint mints a +litellm-signed envelope (:func:`mint_envelope`); the MCP edge validates it, recovers the +identity claims and the inner upstream grant (:func:`open_envelope`), and forwards the +inner access token upstream. This module is pure and unwired: it imports nothing from +endpoint or edge code, reads no proxy globals, and takes all key material and the clock +as explicit parameters. + +Wire shape: ``llm_env_`` + an HS256 JWT (same signing approach as the BYOK session +bearer in ``byok_oauth_endpoints.py``). Registered claims are ``iss``/``iat``/``exp``; +custom claims are ``user_id``, ``server_id``, and ``grant``, where ``grant`` is the +upstream token grant serialized to JSON, encrypted with the repo's symmetric +encryption helpers (``encrypt_value``/``decrypt_value`` from +``encrypt_decrypt_utils`` — the same family ``encrypt_value_helper`` applies to +persisted DCR credentials), and base64url-encoded, so the inner token never appears +in plaintext anywhere in the envelope. + +Failures are values: :func:`open_envelope` returns one of the frozen +``EnvelopeOpenError`` variants (discriminated on ``tag``) for invalid, expired, +tampered, or undecryptable input, and :func:`mint_envelope` returns +``EnvelopeTooLarge`` for oversized grants. Error values carry tags and sizes only, +never token material. + +The pydantic input models reject programmer errors at construction (e.g. a +non-positive ``expires_in`` or an empty required field). :func:`open_envelope` is +additionally total over hostile, attacker-controlled input: it never raises, only +returns an ``EnvelopeOpenError``. :func:`mint_envelope` operates on a +gateway-supplied grant (an upstream IdP's UTF-8 JSON token response), so it does not +defend against non-UTF-8 field content that cannot survive JSON parsing; its only +value-typed failure is ``EnvelopeTooLarge``. +""" + +from __future__ import annotations + +import base64 +from datetime import datetime, timedelta +from typing import Literal, TypeAlias + +import jwt +from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError + +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value, encrypt_value + +ENVELOPE_PREFIX = "llm_env_" +"""Marker prefix on every serialized envelope so the edge can cheaply tell an envelope +from a raw upstream token before doing any cryptography.""" + +ENVELOPE_ISSUER = "litellm-mcp-bridge" +"""``iss`` claim stamped into every envelope and required back on open.""" + +MAX_ENVELOPE_TTL_SECONDS = 3600 +"""Hard ceiling on envelope lifetime. ``exp`` is ``min(upstream expires_in, this cap)`` +(the cap alone when the upstream omits ``expires_in``), matching the 1h lifetime of the +BYOK session bearer this module's signing approach is borrowed from: a client-held +credential should never outlive a bounded window even when the upstream token does.""" + +MAX_ENVELOPE_BYTES = 12288 +"""Size cap on the final serialized envelope (prefix + JWT, in bytes). Upstream JWTs +commonly run 2-4KB; base64 plus encryption overhead roughly doubles that inside the +envelope, and common proxy/server header limits sit around 16KB total. 12288 leaves +comfortable headroom for a large upstream token while keeping the envelope safely +transmittable as a single Authorization header. Oversized grants are rejected with a +typed error, never truncated.""" + +_ENVELOPE_JWT_ALGORITHM = "HS256" + + +class EnvelopeIdentity(BaseModel): + """The litellm identity the envelope binds the inner grant to.""" + + model_config = ConfigDict(frozen=True) + user_id: str = Field(min_length=1) + server_id: str = Field(min_length=1) + + +class UpstreamTokenGrant(BaseModel): + """The upstream OAuth token response fields sealed inside the envelope. + + ``expires_in`` must be positive when present; a non-positive value is a programmer + error rejected at construction. Token fields are ``SecretStr`` so reprs never leak + them. + """ + + model_config = ConfigDict(frozen=True) + access_token: SecretStr = Field(min_length=1) + token_type: str = Field(min_length=1) + refresh_token: SecretStr | None = None + scope: str | None = None + expires_in: int | None = Field(default=None, gt=0) + + +class EnvelopeKeys(BaseModel): + """Injected key material: the HS256 signing key and the symmetric encryption key. + + ``signing_key`` must be at least 32 bytes: HS256's HMAC-SHA256 has a 256-bit + security level, RFC 7518 requires a key of at least that size, and a shorter key + makes PyJWT emit ``InsecureKeyLengthWarning``. + """ + + model_config = ConfigDict(frozen=True) + signing_key: SecretStr = Field(min_length=32) + encryption_key: SecretStr = Field(min_length=1) + + +class SealedEnvelope(BaseModel): + """A minted envelope: the client-held bearer value and when it expires.""" + + model_config = ConfigDict(frozen=True) + token: SecretStr + expires_at: datetime + + +class OpenedEnvelope(BaseModel): + """A validated envelope: the identity it was minted for and the recovered grant.""" + + model_config = ConfigDict(frozen=True) + identity: EnvelopeIdentity + grant: UpstreamTokenGrant + + +class EnvelopeTooLarge(BaseModel): + """The serialized envelope exceeded ``MAX_ENVELOPE_BYTES``; carries sizes only.""" + + model_config = ConfigDict(frozen=True) + tag: Literal["envelope_too_large"] = "envelope_too_large" + size_bytes: int + max_bytes: int + + +EnvelopeMintError: TypeAlias = EnvelopeTooLarge + + +class NotAnEnvelope(BaseModel): + """The candidate does not carry the envelope prefix.""" + + model_config = ConfigDict(frozen=True) + tag: Literal["not_an_envelope"] = "not_an_envelope" + + +class BadSignature(BaseModel): + """The JWT signature does not verify under the provided signing key.""" + + model_config = ConfigDict(frozen=True) + tag: Literal["bad_signature"] = "bad_signature" + + +class Expired(BaseModel): + """The envelope's ``exp`` is not in the future relative to the provided ``now``.""" + + model_config = ConfigDict(frozen=True) + tag: Literal["expired"] = "expired" + + +class MalformedPayload(BaseModel): + """The token is not a well-formed envelope: undecodable JWT, wrong issuer, missing + or mistyped claims, or a decrypted grant that fails validation.""" + + model_config = ConfigDict(frozen=True) + tag: Literal["malformed_payload"] = "malformed_payload" + + +class DecryptFailed(BaseModel): + """The signed ``grant`` blob could not be decrypted under the provided key.""" + + model_config = ConfigDict(frozen=True) + tag: Literal["decrypt_failed"] = "decrypt_failed" + + +EnvelopeOpenError: TypeAlias = NotAnEnvelope | BadSignature | Expired | MalformedPayload | DecryptFailed + + +class _EnvelopeClaims(BaseModel): + """Decoded-claims boundary that pins the exact shape :func:`mint_envelope` emits. + + ``user_id``/``server_id`` mirror the ``min_length`` constraints of + :class:`EnvelopeIdentity` so any claim set that validates here also constructs an + identity, keeping :func:`open_envelope` raise-free: a correctly signed JWT with an + empty identity claim fails here and maps to ``MalformedPayload``. + + ``strict`` rejects coerced types (``exp: "123"``, ``exp: 123.0``) rather than opening + on them, and ``extra="forbid"`` rejects any claim the gateway never mints (a hostile + ``nbf``/``aud``/... rides along on a re-signed token). Since PyJWT's own ``iat``/ + ``nbf``/``exp`` validators are disabled at decode (they raise on hostile claim types + and, for ``iat``/``nbf``, compare against the wall clock rather than the injected + ``now``), this model is the sole, total type gate for every registered claim. + """ + + model_config = ConfigDict(frozen=True, strict=True, extra="forbid") + iss: str + iat: int + exp: int + user_id: str = Field(min_length=1) + server_id: str = Field(min_length=1) + grant: str = Field(min_length=1) + + +class _GrantWire(BaseModel): + model_config = ConfigDict(frozen=True) + access_token: str + token_type: str + refresh_token: str | None = None + scope: str | None = None + expires_in: int | None = None + + +def is_envelope(candidate: str) -> bool: + """Cheap prefix check so the edge can route envelopes vs raw tokens without crypto.""" + return candidate.startswith(ENVELOPE_PREFIX) + + +def mint_envelope( + identity: EnvelopeIdentity, + grant: UpstreamTokenGrant, + keys: EnvelopeKeys, + now: datetime, +) -> SealedEnvelope | EnvelopeMintError: + """Seal ``grant`` for ``identity`` into a client-held envelope. + + ``exp`` is ``min(grant.expires_in, MAX_ENVELOPE_TTL_SECONDS)`` seconds from ``now`` + (the cap alone when ``expires_in`` is absent). Returns ``EnvelopeTooLarge`` when the + serialized envelope exceeds ``MAX_ENVELOPE_BYTES``. + """ + expires_at = now + timedelta(seconds=_envelope_ttl_seconds(grant.expires_in)) + claims = _EnvelopeClaims( + iss=ENVELOPE_ISSUER, + iat=int(now.timestamp()), + exp=int(expires_at.timestamp()), + user_id=identity.user_id, + server_id=identity.server_id, + grant=_encrypt_grant_blob(_grant_plaintext(grant), keys.encryption_key), + ) + token = ENVELOPE_PREFIX + jwt.encode( + claims.model_dump(), + keys.signing_key.get_secret_value(), + algorithm=_ENVELOPE_JWT_ALGORITHM, + ) + size_bytes = len(token.encode("utf-8")) + if size_bytes > MAX_ENVELOPE_BYTES: + return EnvelopeTooLarge(size_bytes=size_bytes, max_bytes=MAX_ENVELOPE_BYTES) + return SealedEnvelope(token=SecretStr(token), expires_at=expires_at) + + +def open_envelope( + candidate: str, + keys: EnvelopeKeys, + now: datetime, +) -> OpenedEnvelope | EnvelopeOpenError: + """Validate ``candidate`` and recover the identity and inner grant. + + Never raises for bad input: every invalid, expired, tampered, or undecryptable + candidate maps to a distinct ``EnvelopeOpenError`` variant. The recovered + ``grant.expires_in`` is the value the upstream reported at mint time and is not + re-derived, so it is stale by up to the envelope's lifetime; callers that need a + live remaining lifetime should use ``now`` against the upstream, not this field. + """ + if not is_envelope(candidate): + return NotAnEnvelope() + # UTF-8 byte length is never below character length, so a character count already over the + # cap rejects an oversize candidate in O(1) without encoding it; the exact byte check then + # runs only on candidates already bounded to <= MAX_ENVELOPE_BYTES characters. + if len(candidate) > MAX_ENVELOPE_BYTES: + return MalformedPayload() + if len(candidate.encode("utf-8", "surrogatepass")) > MAX_ENVELOPE_BYTES: + return MalformedPayload() + claims = _decode_claims(candidate.removeprefix(ENVELOPE_PREFIX), keys.signing_key) + if not isinstance(claims, _EnvelopeClaims): + return claims + if now.timestamp() >= claims.exp: + return Expired() + grant = _decrypt_grant(claims.grant, keys.encryption_key) + if not isinstance(grant, UpstreamTokenGrant): + return grant + return OpenedEnvelope( + identity=EnvelopeIdentity(user_id=claims.user_id, server_id=claims.server_id), + grant=grant, + ) + + +def _envelope_ttl_seconds(upstream_expires_in: int | None) -> int: + if upstream_expires_in is None: + return MAX_ENVELOPE_TTL_SECONDS + return min(upstream_expires_in, MAX_ENVELOPE_TTL_SECONDS) + + +def _grant_plaintext(grant: UpstreamTokenGrant) -> str: + wire = _GrantWire( + access_token=grant.access_token.get_secret_value(), + token_type=grant.token_type, + refresh_token=None if grant.refresh_token is None else grant.refresh_token.get_secret_value(), + scope=grant.scope, + expires_in=grant.expires_in, + ) + return wire.model_dump_json(exclude_none=True) + + +def _decode_claims( + compact: str, + signing_key: SecretStr, +) -> _EnvelopeClaims | BadSignature | MalformedPayload: + """Verify the HS256 signature and shape of an attacker-controlled compact JWT. + + ``compact`` is fully hostile and bounded to ``MAX_ENVELOPE_BYTES`` by the caller. + PyJWT's ``iat``/``nbf``/``exp`` validators are disabled: they raise on hostile claim + types and, for ``iat``/``nbf``, compare against the wall clock rather than the + injected ``now`` (``exp`` is checked by the caller against ``now``). Apart from a + signature mismatch (``BadSignature``), every decode failure is ``MalformedPayload``: + a non-UTF-8 candidate surfaces as ``UnicodeEncodeError`` (a ``ValueError``), a + non-string registered claim such as ``iss`` as a ``TypeError`` from PyJWT's claim + validators, and a wrong issuer or structurally invalid token as an + ``InvalidTokenError``. ``_EnvelopeClaims`` is the total type gate for the payload. + """ + try: + payload = jwt.decode( + compact, + signing_key.get_secret_value(), + algorithms=[_ENVELOPE_JWT_ALGORITHM], + issuer=ENVELOPE_ISSUER, + options={ + "verify_exp": False, + "verify_iat": False, + "verify_nbf": False, + "require": ["iss", "iat", "exp"], + }, + ) + except jwt.InvalidSignatureError: + return BadSignature() + except (jwt.InvalidTokenError, ValueError, TypeError): + return MalformedPayload() + try: + return _EnvelopeClaims.model_validate(payload) + except ValidationError: + return MalformedPayload() + + +def _encrypt_grant_blob(plaintext: str, encryption_key: SecretStr) -> str: + ciphertext = bytes(encrypt_value(value=plaintext, signing_key=encryption_key.get_secret_value())) + return base64.urlsafe_b64encode(ciphertext).decode("ascii") + + +def _decrypt_grant( + blob: str, + encryption_key: SecretStr, +) -> UpstreamTokenGrant | DecryptFailed | MalformedPayload: + from nacl.exceptions import CryptoError + + try: + plaintext = decrypt_value( + value=base64.urlsafe_b64decode(blob), + signing_key=encryption_key.get_secret_value(), + ) + except (CryptoError, ValueError): + return DecryptFailed() + try: + return UpstreamTokenGrant.model_validate_json(plaintext) + except ValidationError: + return MalformedPayload() diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py index f24d5715e83..e12c6cdbd56 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py @@ -7,6 +7,8 @@ Filters MCP tools semantically for /chat/completions and /responses endpoints. from typing import TYPE_CHECKING, Any, Dict, List, Optional from litellm._logging import verbose_logger +from litellm.exceptions import ContextWindowExceededError +from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR if TYPE_CHECKING: @@ -15,6 +17,36 @@ if TYPE_CHECKING: from litellm.router import Router +class SemanticToolFilterContextWindowError(Exception): + """Raised when the embedding model exceeds its context window, so semantic filtering cannot run.""" + + def __init__(self, embedding_model: str, stage: str, original_error: str): + self.embedding_model = embedding_model + self.stage = stage + self.original_error = original_error + super().__init__( + f"MCP semantic tool filtering could not run: embedding model '{embedding_model}' " + f"exceeded its context window while embedding {stage}. " + f"The request was blocked instead of silently passing all tools through. " + f"Switch to an embedding model with a larger context window, or disable " + f"semantic tool filtering." + ) + + +def _is_context_window_error(error: Optional[BaseException], max_depth: int = 5) -> bool: + """Detect a context-window overflow anywhere in an exception's cause chain.""" + current = error + for _ in range(max_depth): + if current is None: + return False + if isinstance(current, ContextWindowExceededError): + return True + if ExceptionCheckers.is_error_str_context_window_exceeded(str(current)): + return True + current = current.__cause__ or current.__context__ + return False + + class SemanticMCPToolFilter: """Filters MCP tools using semantic similarity to reduce context window size.""" @@ -42,6 +74,7 @@ class SemanticMCPToolFilter: self.embedding_model = embedding_model self.router_instance = litellm_router_instance self.tool_router: Optional["SemanticRouter"] = None + self.context_window_error: Optional[str] = None self._tool_map: Dict[str, Any] = {} # MCPTool objects or OpenAI function dicts async def build_router_from_mcp_registry(self) -> None: @@ -111,6 +144,7 @@ class SemanticMCPToolFilter: return try: + self.context_window_error = None # Convert tools to routes routes = [] self._tool_map = {} @@ -143,6 +177,9 @@ class SemanticMCPToolFilter: except Exception as e: verbose_logger.error(f"Failed to build semantic router: {e}") self.tool_router = None + if _is_context_window_error(e): + self.context_window_error = str(e) + return raise async def filter_tools( @@ -169,6 +206,13 @@ class SemanticMCPToolFilter: if not available_tools: return available_tools + if self.context_window_error is not None: + raise SemanticToolFilterContextWindowError( + embedding_model=self.embedding_model, + stage="the MCP tool descriptions during semantic router build", + original_error=self.context_window_error, + ) + if not query or not query.strip(): return available_tools @@ -189,6 +233,16 @@ class SemanticMCPToolFilter: return self._get_tools_by_names(matched_tool_names, available_tools) except Exception as e: + if _is_context_window_error(e): + verbose_logger.error( + f"Semantic tool filter embedding exceeded its context window: {e}", + exc_info=True, + ) + raise SemanticToolFilterContextWindowError( + embedding_model=self.embedding_model, + stage="the user query", + original_error=str(e), + ) from e verbose_logger.error(f"Semantic tool filter failed: {e}", exc_info=True) return available_tools diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 238cb51bb64..5090aa7d7d5 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -3700,6 +3700,17 @@ if MCP_AVAILABLE: and not _scope_has_authorization_header(scope) and not _client_has_per_server_auth_header(server, mcp_server_auth_headers) ): + if server.is_dcr_bridge: + raise HTTPException( + status_code=401, + detail="Unauthorized", + headers={ + "www-authenticate": _get_passthrough_www_authenticate( + scope=scope, + server_name=server_name, + ) + }, + ) upstream_status, upstream_www_authenticate = await _probe_upstream_auth(server.url or "", "") if upstream_status == 401 and upstream_www_authenticate: raise HTTPException( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 43ddf302692..f441a1d3f84 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2151,6 +2151,41 @@ class PluginConfig(LiteLLMPydanticObjectBase): ) +class CoordinationRedisNode(LiteLLMPydanticObjectBase): + """A single startup node of a cluster-mode Redis used for proxy coordination.""" + + host: str = Field(description="hostname of the cluster node") + port: int = Field(description="port of the cluster node") + + +class CoordinationRedisParams(LiteLLMPydanticObjectBase): + """ + Connection params for the proxy's coordination Redis (cross-pod tpm/rpm rate + limits, spend tracking, pod lock manager, shared health checks), configured + independently of the response-cache backend in `litellm_settings.cache_params`. + """ + + model_config = ConfigDict(extra="allow", protected_namespaces=()) + + host: Optional[str] = Field(None, description="Redis hostname") + port: Optional[int] = Field(None, description="Redis port") + password: Optional[str] = Field(None, description="Redis password") + username: Optional[str] = Field(None, description="Redis username") + url: Optional[str] = Field(None, description="full Redis connection url, e.g. redis://:pass@host:6379") + ssl: Optional[bool] = Field(None, description="connect over TLS") + startup_nodes: Optional[List[CoordinationRedisNode]] = Field( + None, description="cluster-mode startup nodes; when set a cluster client is used" + ) + sentinel_nodes: Optional[List[List[Union[str, int]]]] = Field( + None, description="sentinel [host, port] pairs; when set a sentinel-managed client is used" + ) + sentinel_password: Optional[str] = Field(None, description="password for the sentinel nodes") + service_name: Optional[str] = Field(None, description="sentinel service name") + + def has_connection_target(self) -> bool: + return any(value is not None for value in (self.host, self.url, self.startup_nodes, self.sentinel_nodes)) + + class ConfigGeneralSettings(LiteLLMPydanticObjectBase): """ Documents all the fields supported by `general_settings` in config.yaml @@ -2166,6 +2201,15 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): use_google_kms: Optional[bool] = Field(None, description="decrypt keys with google kms") use_azure_key_vault: Optional[bool] = Field(None, description="load keys from azure key vault") master_key: Optional[str] = Field(None, description="require a key for all calls to proxy") + coordination_redis: Optional[CoordinationRedisParams] = Field( + None, + description=( + "standalone Redis for cross-pod coordination (tpm/rpm rate limits, " + "spend tracking, pod lock manager, shared health checks), configured " + "independently of the response-cache backend; takes precedence over " + "borrowing the `cache_params` Redis and over the REDIS_* env fallback" + ), + ) allow_cli_sso_verification_uri_complete: bool | None = Field( None, description="opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine", diff --git a/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml b/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml index b353924f000..eb091cc72c5 100644 --- a/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml +++ b/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml @@ -5,8 +5,9 @@ model_list: api_key: my-fake-key api_base: os.environ/FAKE_OPENAI_API_BASE -litellm_settings: - cache: True - cache_params: - type: redis +general_settings: + coordination_redis: + host: os.environ/REDIS_HOST + port: os.environ/REDIS_PORT + password: os.environ/REDIS_PASSWORD diff --git a/litellm/proxy/example_config_yaml/spend_tracking_config.yaml b/litellm/proxy/example_config_yaml/spend_tracking_config.yaml index 60adadbd8d4..d66fd5fa601 100644 --- a/litellm/proxy/example_config_yaml/spend_tracking_config.yaml +++ b/litellm/proxy/example_config_yaml/spend_tracking_config.yaml @@ -7,9 +7,6 @@ model_list: general_settings: use_redis_transaction_buffer: true - -litellm_settings: - cache: True - cache_params: - type: redis - supported_call_types: [] \ No newline at end of file + coordination_redis: + host: os.environ/REDIS_HOST + port: os.environ/REDIS_PORT \ No newline at end of file diff --git a/litellm/proxy/hooks/mcp_semantic_filter/hook.py b/litellm/proxy/hooks/mcp_semantic_filter/hook.py index 7379096bf9b..bad6ef44ccd 100644 --- a/litellm/proxy/hooks/mcp_semantic_filter/hook.py +++ b/litellm/proxy/hooks/mcp_semantic_filter/hook.py @@ -7,6 +7,8 @@ Reduces context window size and improves tool selection accuracy. from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from fastapi import HTTPException + from litellm._logging import verbose_proxy_logger from litellm.constants import ( DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL, @@ -14,6 +16,9 @@ from litellm.constants import ( DEFAULT_MCP_SEMANTIC_FILTER_TOP_K, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + SemanticToolFilterContextWindowError, +) if TYPE_CHECKING: from litellm.caching.caching import DualCache @@ -294,6 +299,8 @@ class SemanticToolFilterHook(CustomLogger): ) return data + except SemanticToolFilterContextWindowError as e: + raise HTTPException(status_code=400, detail={"error": str(e)}) from e except Exception as e: verbose_proxy_logger.error(f"Failed to expand MCP references: {e}", exc_info=True) return None @@ -366,6 +373,8 @@ class SemanticToolFilterHook(CustomLogger): return data + except SemanticToolFilterContextWindowError as e: + raise HTTPException(status_code=400, detail={"error": str(e)}) from e except Exception as e: verbose_proxy_logger.warning(f"Semantic tool filter hook failed: {e}. Proceeding with all tools.") return None diff --git a/litellm/proxy/management_endpoints/coordination_redis_endpoints.py b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py new file mode 100644 index 00000000000..7ab4e3019c3 --- /dev/null +++ b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py @@ -0,0 +1,431 @@ +""" +COORDINATION REDIS SETTINGS MANAGEMENT + +Endpoints for managing `general_settings.coordination_redis` - the standalone +Redis the proxy uses for cross-pod coordination (tpm/rpm rate limits, spend +tracking, pod lock manager, shared health checks), configured independently of +the response-cache backend. + +GET /coordination_redis/settings - Get the coordination Redis settings, field metadata, and which source is active +POST /coordination_redis/settings - Save coordination Redis settings to the database +POST /coordination_redis/settings/test - Test a coordination Redis connection with the provided credentials +""" + +import asyncio +import json +from collections.abc import Mapping +from contextlib import suppress +from datetime import datetime, timezone +from typing import Optional + +from fastapi import APIRouter, Depends, Header, HTTPException +from pydantic import BaseModel, Field, TypeAdapter, ValidationError + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid +from litellm.caching.caching import RedisCache +from litellm.caching.redis_cluster_cache import RedisClusterCache +from litellm.proxy._types import ( + AUDIT_ACTIONS, + CoordinationRedisParams, + LiteLLM_AuditLogs, + LitellmTableNames, + LitellmUserRoles, + UserAPIKeyAuth, +) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.utils import invalidate_config_param +from litellm.repositories.config_repository import ConfigRepository +from litellm.secret_managers.main import get_secret_str +from litellm.types.management_endpoints import ( + COORDINATION_REDIS_SETTINGS_FIELDS, + CoordinationRedisSettingsField, + CoordinationRedisSource, +) + +router = APIRouter() + +_GENERAL_SETTINGS_PARAM_NAME = "general_settings" +_COORDINATION_REDIS_KEY = "coordination_redis" + +# Fields that carry credentials. Redacted on read so a plaintext Redis / +# Sentinel password never leaves the server, and scrubbed out of connection-test +# error strings. `url` is here because a Redis url can embed a password inline +# (e.g. redis://:secret@host:6379/1). +_SENSITIVE_FIELDS: frozenset[str] = frozenset({"password", "sentinel_password", "url"}) + +_REDACTED_VALUE = "***REDACTED***" + +_ENV_REF_PREFIX = "os.environ/" + +_PING_TIMEOUT_SECONDS = 5.0 + +_SETTINGS_ADAPTER: TypeAdapter[dict[str, object]] = TypeAdapter(dict[str, object]) + + +def _enforce_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail={"error": "Only proxy admins can manage coordination Redis settings"}, + ) + + +def _resolve_env_ref(value: object) -> object: + """Resolve an `os.environ/VAR` reference to its value, passing anything else through.""" + if isinstance(value, str) and value.startswith(_ENV_REF_PREFIX): + return get_secret_str(value) + return value + + +def _resolve_env_refs(settings: Mapping[str, object]) -> dict[str, object]: + return {key: _resolve_env_ref(value) for key, value in settings.items()} + + +def _redact_credentials(settings: Mapping[str, object]) -> dict[str, object]: + """Replace credential-bearing values with a fixed marker, keeping the rest intact.""" + return { + key: (_REDACTED_VALUE if key in _SENSITIVE_FIELDS and value is not None else value) + for key, value in settings.items() + } + + +def _redact_all_values(settings: Optional[Mapping[str, object]]) -> dict[str, object]: + """Replace every value with a fixed marker, preserving the key set. + + The audit row shows *which* fields changed without the audit table becoming + a credential-harvest sink. + """ + if not settings: + return {} + return {key: _REDACTED_VALUE for key in settings} + + +def _credential_values(settings: Mapping[str, object]) -> tuple[str, ...]: + return tuple( + str(value) for key, value in settings.items() if key in _SENSITIVE_FIELDS and isinstance(value, (str, int)) + ) + + +def _scrub_credentials(message: str, settings: Mapping[str, object]) -> str: + """Strip any credential value the caller supplied out of an error string. + + Redis client errors routinely echo the connection url (password inline) or + the auth error back to the caller. + """ + scrubbed = message + for secret in _credential_values(settings): + if secret: + scrubbed = scrubbed.replace(secret, _REDACTED_VALUE) + return scrubbed + + +def _merge_over_saved( + incoming: Mapping[str, object], + saved: Mapping[str, object], +) -> dict[str, object]: + """Restore the real credential behind every value the caller echoed back redacted. + + GET returns credentials as ``***REDACTED***``; an admin who edits the + non-secret fields and re-submits would otherwise test (and save) the marker + as the password. + """ + return { + key: (saved[key] if value == _REDACTED_VALUE and key in saved else value) for key, value in incoming.items() + } + + +def _validated_params(settings: Mapping[str, object]) -> CoordinationRedisParams: + """Validate settings the way startup does: resolve env refs, then require a connection target.""" + try: + params = CoordinationRedisParams(**_resolve_env_refs(settings)) + except ValidationError as e: + invalid_fields = sorted({str(error["loc"][0]) for error in e.errors() if error["loc"]}) + raise HTTPException( + status_code=400, + detail={"error": f"Invalid coordination_redis settings for fields: {invalid_fields}"}, + ) + if not params.has_connection_target(): + raise HTTPException( + status_code=400, + detail={ + "error": ( + "coordination_redis needs a connection target: " + "set one of host, url, startup_nodes, or sentinel_nodes" + ) + }, + ) + return params + + +async def _read_general_settings() -> dict[str, object]: + """Read the persisted `general_settings` config row (empty when unset or no DB).""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return {} + config_param = await ConfigRepository(prisma_client).get_param(_GENERAL_SETTINGS_PARAM_NAME) + if config_param is None or config_param.param_value is None: + return {} + return _SETTINGS_ADAPTER.validate_python(config_param.param_value) + + +async def get_persisted_coordination_redis_settings() -> Optional[dict[str, object]]: + """The coordination_redis block saved to the database, if any. + + Read at startup so settings saved from the admin UI take effect on the next + boot, and used here so a read reports what the proxy would boot with. + """ + persisted = (await _read_general_settings()).get(_COORDINATION_REDIS_KEY) + if isinstance(persisted, dict): + return _SETTINGS_ADAPTER.validate_python(persisted) + return None + + +async def _current_coordination_redis_settings() -> Optional[dict[str, object]]: + """The coordination_redis block the proxy would boot with. + + The persisted row wins over the yaml-loaded config state because startup + applies the DB `general_settings` row over the file config. + """ + from litellm.proxy.proxy_server import proxy_config + + persisted = await get_persisted_coordination_redis_settings() + if persisted is not None: + return persisted + + config_state = _SETTINGS_ADAPTER.validate_python(proxy_config.get_config_state()) + general_settings = config_state.get(_GENERAL_SETTINGS_PARAM_NAME) + if not isinstance(general_settings, dict): + return None + from_file = general_settings.get(_COORDINATION_REDIS_KEY) + if isinstance(from_file, dict): + return _SETTINGS_ADAPTER.validate_python(from_file) + return None + + +def _coordination_redis_source(settings: Optional[Mapping[str, object]]) -> Optional[CoordinationRedisSource]: + """Which source the proxy's coordination Redis comes from, in startup precedence order. + + Mirrors `ProxyConfig._init_coordination_redis` -> `ProxyConfig._init_cache`: + an explicit block wins, else a plain-Redis response-cache backend is + borrowed, else the REDIS_* environment fallback applies. + """ + from litellm.proxy.proxy_server import _environment_has_redis_connection_target + + if settings: + return "coordination_redis" + cache_backend = litellm.cache.cache if litellm.cache is not None else None + if isinstance(cache_backend, (RedisCache, RedisClusterCache)): + return "cache_backend" + if _environment_has_redis_connection_target(): + return "environment" + return None + + +def _log_audit_task_exception(task: "asyncio.Task[None]") -> None: + """Surface a fire-and-forget audit-log task failure as a warning.""" + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + verbose_proxy_logger.warning("Failed to write coordination-redis-settings audit log: %s", exc) + + +async def _emit_coordination_redis_audit_log( + *, + action: AUDIT_ACTIONS, + before_settings: Optional[Mapping[str, object]], + after_settings: Optional[Mapping[str, object]], + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: Optional[str], +) -> None: + """Emit an audit-log row for a /coordination_redis/settings mutation.""" + if litellm.store_audit_logs is not True: + return + + from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update + from litellm.proxy.proxy_server import litellm_proxy_admin_name + + task = asyncio.create_task( + create_audit_log_for_update( + request_data=LiteLLM_AuditLogs( + id=str(uuid.uuid4()), + updated_at=datetime.now(timezone.utc), + changed_by=litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name, + changed_by_api_key=user_api_key_dict.api_key, + table_name=LitellmTableNames.CONFIG_TABLE_NAME, + object_id=_COORDINATION_REDIS_KEY, + action=action, + updated_values=json.dumps({"settings": _redact_all_values(after_settings)}, default=str), + before_value=json.dumps({"settings": _redact_all_values(before_settings)}, default=str), + ) + ) + ) + task.add_done_callback(_log_audit_task_exception) + + +class CoordinationRedisSettingsResponse(BaseModel): + values: dict[str, object] = Field(description="Current coordination Redis settings, with credentials redacted") + fields: list[CoordinationRedisSettingsField] = Field( + description="List of all configurable coordination Redis settings with metadata" + ) + source: Optional[CoordinationRedisSource] = Field( + description="Where the proxy's coordination Redis comes from; null when it has none" + ) + + +class CoordinationRedisSettingsRequest(BaseModel): + settings: dict[str, object] = Field(description="Coordination Redis connection params") + + +class CoordinationRedisTestResponse(BaseModel): + status: str = Field(description="Connection status: 'healthy' or 'unhealthy'") + error: Optional[str] = Field(default=None, description="Error message if the connection failed") + + +@router.get( + "/coordination_redis/settings", + tags=["Coordination Redis Settings"], + dependencies=[Depends(user_api_key_auth)], + response_model=CoordinationRedisSettingsResponse, +) +async def get_coordination_redis_settings( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> CoordinationRedisSettingsResponse: + """ + Get the coordination Redis configuration and available settings. + + Returns: + - values: current coordination Redis settings, with password/sentinel_password/url redacted + - fields: all configurable settings with their metadata (type, description, default, section) + - source: "coordination_redis" | "cache_backend" | "environment" | null + """ + _enforce_proxy_admin(user_api_key_dict) + + settings = await _current_coordination_redis_settings() + source = _coordination_redis_source(settings) + + values = _redact_credentials(settings or {}) + fields = [field.model_copy(deep=True) for field in COORDINATION_REDIS_SETTINGS_FIELDS] + for field in fields: + if field.field_name in values: + field.field_value = values[field.field_name] + + return CoordinationRedisSettingsResponse(values=values, fields=fields, source=source) + + +@router.post( + "/coordination_redis/settings", + tags=["Coordination Redis Settings"], + dependencies=[Depends(user_api_key_auth)], +) +async def update_coordination_redis_settings( + request: CoordinationRedisSettingsRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + litellm_changed_by: Optional[str] = Header( + None, + description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + ), +) -> dict[str, object]: + """ + Save coordination Redis settings under `general_settings.coordination_redis`. + + Parameters: + - settings: dict - Redis connection params (host, port, username, password, url, ssl, startup_nodes, sentinel_nodes, sentinel_password, service_name). Values may be `os.environ/VAR` references, which are stored as written and resolved at startup + + The settings are written to the `general_settings` row of LiteLLM_Config, + which startup merges over the yaml config; the proxy picks them up on its + next restart. + """ + from litellm.proxy.proxy_server import prisma_client, store_model_in_db + + _enforce_proxy_admin(user_api_key_dict) + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": "Database not connected. Please connect a database."}, + ) + + if store_model_in_db is not True: + raise HTTPException( + status_code=500, + detail={"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."}, + ) + + saved_settings = await _current_coordination_redis_settings() + settings = _merge_over_saved(request.settings, saved_settings or {}) + _validated_params(settings) + + general_settings = await _read_general_settings() + before_settings = general_settings.get(_COORDINATION_REDIS_KEY) + action: AUDIT_ACTIONS = "updated" if isinstance(before_settings, dict) else "created" + + await ConfigRepository(prisma_client).set_param( + param_name=_GENERAL_SETTINGS_PARAM_NAME, + param_value={**general_settings, _COORDINATION_REDIS_KEY: settings}, + ) + await invalidate_config_param(_GENERAL_SETTINGS_PARAM_NAME) + + # coordination_redis carries Redis credentials and decides where cross-pod + # rate-limit and spend state lives; an admin repointing it is a + # data-routing pivot, so make the change traceable. + await _emit_coordination_redis_audit_log( + action=action, + before_settings=before_settings if isinstance(before_settings, dict) else None, + after_settings=settings, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + + return { + "message": "Coordination Redis settings updated successfully. Restart the proxy to apply them.", + "status": "success", + "settings": _redact_credentials(settings), + } + + +@router.post( + "/coordination_redis/settings/test", + tags=["Coordination Redis Settings"], + dependencies=[Depends(user_api_key_auth)], + response_model=CoordinationRedisTestResponse, +) +async def check_coordination_redis_connection( + request: CoordinationRedisSettingsRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> CoordinationRedisTestResponse: + """ + Test a coordination Redis connection with the provided credentials. + + Parameters: + - settings: dict - Redis connection params to test. Credential fields sent back as `***REDACTED***` fall back to the saved value + + Builds a throwaway client (never touching global state) and pings it. + """ + from litellm.proxy.proxy_server import _build_redis_usage_cache + + _enforce_proxy_admin(user_api_key_dict) + + saved_settings = await _current_coordination_redis_settings() + settings = _merge_over_saved(request.settings, saved_settings or {}) + params = _validated_params(settings) + + redis_cache: Optional[RedisCache] = None + try: + redis_cache = _build_redis_usage_cache(params.model_dump(exclude_none=True)) + await asyncio.wait_for(redis_cache.ping(), timeout=_PING_TIMEOUT_SECONDS) + return CoordinationRedisTestResponse(status="healthy") + except asyncio.TimeoutError: + return CoordinationRedisTestResponse( + status="unhealthy", + error=f"Connection timed out after {_PING_TIMEOUT_SECONDS}s", + ) + except Exception as e: # noqa: BLE001 # any client/connection failure is a health verdict, not a 500 + return CoordinationRedisTestResponse(status="unhealthy", error=_scrub_credentials(str(e), settings)) + finally: + if redis_cache is not None: + with suppress(Exception): + await redis_cache.disconnect() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ceb04a8d5d0..b07ee9e02cb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -75,6 +75,7 @@ from litellm.proxy._types import ( ConfigGeneralSettings, ConfigList, ConfigYAML, + CoordinationRedisParams, EnterpriseLicenseData, FieldDetail, InvitationClaim, @@ -212,6 +213,7 @@ from contextlib import asynccontextmanager from functools import lru_cache import litellm +import litellm._redis from litellm import Router from litellm._logging import verbose_proxy_logger, verbose_router_logger from litellm.caching.caching import DualCache, RedisCache @@ -364,6 +366,10 @@ from litellm.proxy.secure_share.secure_share_endpoints import ( from litellm.proxy.management_endpoints.callback_management_endpoints import ( router as callback_management_endpoints_router, ) +from litellm.proxy.management_endpoints.coordination_redis_endpoints import ( + get_persisted_coordination_redis_settings, + router as coordination_redis_settings_router, +) from litellm.proxy.management_endpoints.common_utils import ( _user_has_admin_privileges, _user_has_admin_view, @@ -928,6 +934,16 @@ async def proxy_startup_event(app: FastAPI): asyncio.create_task(_run_pw_migration()) + ## A coordination_redis block saved from the admin UI lives in the database, + ## which is only reachable once the prisma client exists. Apply it here, before + ## the coordination Redis is published to its consumers below. + db_coordination_redis_cache = await ProxyStartupEvent._init_coordination_redis_from_db( + litellm_settings=proxy_config.get_config_state().get("litellm_settings") or {}, + llm_router=llm_router, + ) + if db_coordination_redis_cache is not None: + _set_redis_usage_cache(db_coordination_redis_cache) + ## use_redis_transaction_buffer: fall back to a standalone Redis (REDIS_* env) ## when the proxy cache backend is not Redis ## transaction_buffer_redis_cache = redis_usage_cache @@ -3552,6 +3568,101 @@ def _apply_ssrf_general_settings(settings: Mapping[str, object]) -> None: ) +def _set_redis_usage_cache(coordination_redis_cache: RedisCache | None) -> None: + """Publish the resolved coordination Redis to the consumers that read it directly.""" + global redis_usage_cache + redis_usage_cache = coordination_redis_cache + + +def _resolve_coordination_redis_env_refs(raw_params: Mapping[str, object]) -> dict[str, object]: + """Resolve `os.environ/VAR` references in a coordination_redis block.""" + return { + key: (get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value) + for key, value in raw_params.items() + } + + +def _build_redis_usage_cache(redis_params: Mapping[str, object]) -> RedisCache: + """ + Builds the proxy's coordination Redis client from resolved connection + params. Cluster-mode targets (explicit `startup_nodes` or the + REDIS_CLUSTER_NODES env var) get a `RedisClusterCache`, so consumers that + branch on cluster mode (e.g. the v3 rate limiter) take the cluster path; + everything else (host/url/sentinel) gets a plain `RedisCache`. + """ + startup_nodes = redis_params.get("startup_nodes") + if startup_nodes is None: + env_cluster_nodes = get_secret_str("REDIS_CLUSTER_NODES") + if env_cluster_nodes is not None: + startup_nodes = json.loads(env_cluster_nodes) + non_node_params = {key: value for key, value in redis_params.items() if key != "startup_nodes"} + if startup_nodes: + return RedisClusterCache(startup_nodes=startup_nodes, **non_node_params) + return RedisCache(**non_node_params) + + +def _environment_has_redis_connection_target() -> bool: + """ + Whether the REDIS_* environment variables name a Redis to connect to (host, + url, cluster nodes, or sentinel nodes). Read-only: callers that only need to + know whether the env fallback would apply use this instead of building a + client. + """ + redis_env_kwargs = litellm._redis._redis_kwargs_from_environment() + return ( + "host" in redis_env_kwargs + or "url" in redis_env_kwargs + or get_secret_str("REDIS_CLUSTER_NODES") is not None + or get_secret_str("REDIS_SENTINEL_NODES") is not None + ) + + +def _build_redis_usage_cache_from_environment() -> RedisCache | None: + """ + Builds a standalone coordination Redis from REDIS_* environment variables. + + Lets the proxy's coordination Redis (cross-pod tpm/rpm rate limits, spend + tracking, pod lock manager) run when the response-cache backend is not a + plain Redis KV cache (e.g. a semantic cache, disk, or s3). + + Returns None when the environment carries no connection target (host, url, + cluster nodes, or sentinel nodes). + """ + if not _environment_has_redis_connection_target(): + return None + return _build_redis_usage_cache(litellm._redis._redis_kwargs_from_environment()) + + +def _attach_redis_usage_cache(redis_cache: RedisCache, enable_redis_auth_cache: bool) -> None: + """ + Wires an established coordination Redis into the proxy-level caches that + consume it directly: the spend counter cache, the cluster-wide config + cache, and (only when opted in) the virtual-key auth cache. + """ + spend_counter_cache.attach_redis_cache( + redis_cache, + default_redis_ttl=litellm.default_redis_ttl, + ) + if enable_redis_auth_cache is True: + user_api_key_cache.attach_redis_cache( + redis_cache, + default_redis_ttl=litellm.default_redis_ttl, + ) + verbose_proxy_logger.info( + "enable_redis_auth_cache=True: attached Redis to " + "user_api_key_cache — virtual-key lookups are now " + "shared across all proxy workers." + ) + else: + verbose_proxy_logger.info( + "enable_redis_auth_cache is not set: user_api_key_cache " + "remains in-memory only (per-worker). Set " + "litellm_settings.enable_redis_auth_cache: true to share " + "the auth cache across workers and reduce DB load." + ) + litellm_config_cache.redis_cache = redis_cache + + class ProxyConfig: """ Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic. @@ -3750,12 +3861,52 @@ class ProxyConfig: team_config = self._get_team_config(team_id=team_id, all_teams_config=all_teams_config) return team_config + def _init_coordination_redis(self, config: dict) -> RedisCache | None: + """ + Builds the coordination Redis from `general_settings.coordination_redis` + when present, attaching it to the proxy-level caches. Runs before cache + init, so an explicit block takes precedence over borrowing the + response-cache Redis and over the REDIS_* env fallback. Returns the + built client (None when the block is absent) for the caller to publish. + """ + settings = config.get("general_settings") or {} + litellm_settings = config.get("litellm_settings") or {} + raw_params = settings.get("coordination_redis") + if raw_params is None: + return None + if not isinstance(raw_params, dict): + raise ValueError("general_settings.coordination_redis must be a mapping of Redis connection params") + + coordination_params = CoordinationRedisParams(**_resolve_coordination_redis_env_refs(raw_params)) + if not coordination_params.has_connection_target(): + raise ValueError( + "general_settings.coordination_redis needs a connection target: " + "set one of host, url, startup_nodes, or sentinel_nodes" + ) + + coordination_redis_cache = _build_redis_usage_cache(coordination_params.model_dump(exclude_none=True)) + _attach_redis_usage_cache( + coordination_redis_cache, + enable_redis_auth_cache=litellm_settings.get("enable_redis_auth_cache", False) is True, + ) + verbose_proxy_logger.info( + "coordination_redis: using a standalone Redis from general_settings " + "for usage tracking, rate limiting, and cross-pod coordination." + ) + return coordination_redis_cache + def _init_cache( self, cache_params: dict, enable_redis_auth_cache: bool = False, - ): - global redis_usage_cache, llm_router, general_settings + ) -> RedisCache | None: + """ + Initializes the response cache and resolves the coordination Redis. + + Returns the coordination Redis for the caller to publish: an explicit + coordination_redis block already set wins, else a plain-Redis response + cache backend is borrowed, else the REDIS_* environment fallback applies. + """ from litellm import Cache if "default_in_memory_ttl" in cache_params: @@ -3766,37 +3917,29 @@ class ProxyConfig: litellm.cache = Cache(**cache_params) - if litellm.cache is not None and isinstance(litellm.cache.cache, (RedisCache, RedisClusterCache)): - ## INIT PROXY REDIS USAGE CLIENT ## - redis_usage_cache = litellm.cache.cache - spend_counter_cache.attach_redis_cache( - redis_usage_cache, - default_redis_ttl=litellm.default_redis_ttl, - ) - # Note: PKCE verifier storage uses redis_usage_cache directly (not - # user_api_key_cache) to avoid routing all API-key lookups through Redis. - if enable_redis_auth_cache is True: - user_api_key_cache.attach_redis_cache( - redis_usage_cache, - default_redis_ttl=litellm.default_redis_ttl, - ) - verbose_proxy_logger.info( - "enable_redis_auth_cache=True: attached Redis to " - "user_api_key_cache — virtual-key lookups are now " - "shared across all proxy workers." - ) + resolved_usage_cache = redis_usage_cache + cache_backend = litellm.cache.cache if litellm.cache is not None else None + if resolved_usage_cache is None: + if isinstance(cache_backend, (RedisCache, RedisClusterCache)): + ## INIT PROXY REDIS USAGE CLIENT ## + resolved_usage_cache = cache_backend else: - verbose_proxy_logger.info( - "enable_redis_auth_cache is not set: user_api_key_cache " - "remains in-memory only (per-worker). Set " - "litellm_settings.enable_redis_auth_cache: true to share " - "the auth cache across workers and reduce DB load." - ) - litellm_config_cache.redis_cache = redis_usage_cache + resolved_usage_cache = _build_redis_usage_cache_from_environment() + if resolved_usage_cache is not None: + verbose_proxy_logger.info( + "Cache backend %s is not a Redis KV cache; built a standalone " + "Redis from REDIS_* environment variables for usage tracking, " + "rate limiting, and cross-pod coordination.", + type(cache_backend).__name__, + ) + + if resolved_usage_cache is not None: # Note: PKCE verifier storage uses redis_usage_cache directly (not # user_api_key_cache) to avoid routing all API-key lookups through Redis. + _attach_redis_usage_cache(resolved_usage_cache, enable_redis_auth_cache) elif litellm_config_cache.redis_cache is None: verbose_proxy_logger.info("litellm_config_cache: no Redis configured; cluster-wide cache sharing disabled.") + return resolved_usage_cache def switch_on_llm_response_caching(self): """ @@ -4041,6 +4184,11 @@ class ProxyConfig: self._load_environment_variables(config=config) + ## Coordination Redis (before cache init, so the explicit block wins) + coordination_redis_cache = self._init_coordination_redis(config=config) + if coordination_redis_cache is not None: + _set_redis_usage_cache(coordination_redis_cache) + ## Callback settings callback_settings = config.get("callback_settings", {}) if callback_settings: @@ -4121,9 +4269,11 @@ class ProxyConfig: cache_params[key] = get_secret(value) ## to pass a complete url, or set ssl=True, etc. just set it as `os.environ[REDIS_URL] = `, _redis.py checks for REDIS specific environment variables - self._init_cache( - cache_params=cache_params, - enable_redis_auth_cache=litellm_settings.get("enable_redis_auth_cache", False) is True, + _set_redis_usage_cache( + self._init_cache( + cache_params=cache_params, + enable_redis_auth_cache=litellm_settings.get("enable_redis_auth_cache", False) is True, + ) ) if litellm.cache is not None: verbose_proxy_logger.debug(f"{blue_color_code}Set Cache on LiteLLM Proxy{reset_color_code}") @@ -7268,6 +7418,46 @@ class ProxyStartupEvent: "Redis for the transaction buffer." ) + @staticmethod + async def _init_coordination_redis_from_db( + litellm_settings: Mapping[str, object], + llm_router: Optional[Router], + ) -> RedisCache | None: + """ + Applies a coordination_redis block saved to the database, which the admin + UI writes and the config file therefore never carries. + + Returns None when nothing is persisted or the persisted block names no + connection target, leaving the file/env resolution untouched. + """ + try: + persisted = await get_persisted_coordination_redis_settings() + except Exception as e: # noqa: BLE001 # a config-row read failure must not block proxy startup + verbose_proxy_logger.warning("Could not read coordination_redis from the database: %s", e) + return None + if persisted is None: + return None + + coordination_params = CoordinationRedisParams(**_resolve_coordination_redis_env_refs(persisted)) + if not coordination_params.has_connection_target(): + verbose_proxy_logger.warning( + "coordination_redis saved in the database names no connection target; ignoring it." + ) + return None + + coordination_redis_cache = _build_redis_usage_cache(coordination_params.model_dump(exclude_none=True)) + _attach_redis_usage_cache( + coordination_redis_cache, + enable_redis_auth_cache=litellm_settings.get("enable_redis_auth_cache", False) is True, + ) + if llm_router is not None and llm_router.cache.redis_cache is None: + llm_router._update_redis_cache(cache=coordination_redis_cache) + verbose_proxy_logger.info( + "coordination_redis: using the standalone Redis saved in the database " + "for usage tracking, rate limiting, and cross-pod coordination." + ) + return coordination_redis_cache + @staticmethod def _get_transaction_buffer_redis_cache( general_settings: dict, @@ -7280,7 +7470,6 @@ class ProxyStartupEvent: Returns None when the buffer is disabled, or when no Redis host or url is set in the environment. """ - from litellm._redis import _redis_kwargs_from_environment from litellm.secret_managers.main import str_to_bool _use_redis_transaction_buffer: bool | str | None = general_settings.get("use_redis_transaction_buffer", False) @@ -7290,11 +7479,7 @@ class ProxyStartupEvent: if not _use_redis_transaction_buffer: return None - redis_env_kwargs = _redis_kwargs_from_environment() - if "host" not in redis_env_kwargs and "url" not in redis_env_kwargs: - return None - - return RedisCache(**redis_env_kwargs) + return _build_redis_usage_cache_from_environment() @classmethod async def _initialize_semantic_tool_filter( @@ -15815,6 +16000,7 @@ app.include_router(router_settings_router) app.include_router(fallback_management_router) app.include_router(cache_settings_router) app.include_router(secure_share_router) +app.include_router(coordination_redis_settings_router) app.include_router(user_agent_analytics_router) app.include_router(enterprise_router) app.include_router(ui_discovery_endpoints_router) diff --git a/litellm/types/management_endpoints/__init__.py b/litellm/types/management_endpoints/__init__.py index 5c5bcb2e754..3b501443edd 100644 --- a/litellm/types/management_endpoints/__init__.py +++ b/litellm/types/management_endpoints/__init__.py @@ -7,6 +7,12 @@ from .cache_settings_endpoints import ( REDIS_TYPE_DESCRIPTIONS, CacheSettingsField, ) +from .coordination_redis_endpoints import ( + COORDINATION_REDIS_SETTINGS_FIELDS, + CoordinationRedisSection, + CoordinationRedisSettingsField, + CoordinationRedisSource, +) from .router_settings_endpoints import ( ROUTER_SETTINGS_FIELDS, ROUTING_STRATEGY_DESCRIPTIONS, @@ -20,4 +26,8 @@ __all__ = [ "CACHE_SETTINGS_FIELDS", "REDIS_TYPE_DESCRIPTIONS", "CacheSettingsField", + "COORDINATION_REDIS_SETTINGS_FIELDS", + "CoordinationRedisSection", + "CoordinationRedisSettingsField", + "CoordinationRedisSource", ] diff --git a/litellm/types/management_endpoints/coordination_redis_endpoints.py b/litellm/types/management_endpoints/coordination_redis_endpoints.py new file mode 100644 index 00000000000..b6889d83323 --- /dev/null +++ b/litellm/types/management_endpoints/coordination_redis_endpoints.py @@ -0,0 +1,105 @@ +""" +Types and field definitions for coordination Redis settings management endpoints +""" + +from typing import Literal, Optional + +from pydantic import BaseModel + +CoordinationRedisSection = Literal["connection", "cluster", "sentinel"] + +CoordinationRedisSource = Literal["coordination_redis", "cache_backend", "environment"] + + +class CoordinationRedisSettingsField(BaseModel): + field_name: str + field_type: str + field_value: Optional[object] = None + field_description: str + field_default: Optional[object] = None + ui_field_name: str + section: CoordinationRedisSection + + +COORDINATION_REDIS_SETTINGS_FIELDS: list[CoordinationRedisSettingsField] = [ + CoordinationRedisSettingsField( + field_name="host", + field_type="String", + field_description="Redis server hostname or IP address", + ui_field_name="Host", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="port", + field_type="Integer", + field_description="Redis server port number", + field_default=6379, + ui_field_name="Port", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="username", + field_type="String", + field_description="Redis server username (if required)", + ui_field_name="Username", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="password", + field_type="String", + field_description="Redis server password", + ui_field_name="Password", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="url", + field_type="String", + field_description=( + "Full Redis connection URL (e.g. redis://:password@host:6379/1). " + "Set this instead of the discrete host/port/username/password fields." + ), + ui_field_name="Redis URL", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="ssl", + field_type="Boolean", + field_description="Connect to Redis over TLS", + field_default=False, + ui_field_name="SSL", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="startup_nodes", + field_type="List", + field_description=( + "Cluster-mode startup nodes (e.g. [{'host': '127.0.0.1', 'port': 7001}]). " + "When set, a Redis Cluster client is used." + ), + ui_field_name="Cluster Startup Nodes", + section="cluster", + ), + CoordinationRedisSettingsField( + field_name="sentinel_nodes", + field_type="List", + field_description=( + "Sentinel [host, port] pairs (e.g. [['localhost', 26379]]). When set, a Sentinel-managed client is used." + ), + ui_field_name="Sentinel Nodes", + section="sentinel", + ), + CoordinationRedisSettingsField( + field_name="sentinel_password", + field_type="String", + field_description="Password for the Redis Sentinel nodes", + ui_field_name="Sentinel Password", + section="sentinel", + ), + CoordinationRedisSettingsField( + field_name="service_name", + field_type="String", + field_description="Master service name for Redis Sentinel", + ui_field_name="Service Name", + section="sentinel", + ), +] diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index 6feffe036bd..1a51dd0d0a7 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -189,7 +189,7 @@ litellm_settings: langfuse_host: https://us.cloud.langfuse.com # cache: true # [OPTIONAL] use for caching responses # enable_caching_on_provider_specific_optional_params: True # Include provider-specific params in cache keys - # cache_params: # And for shared health check + # cache_params: # type: redis # host: localhost # port: 6379 @@ -228,8 +228,11 @@ general_settings: proxy_batch_write_at: 1 database_connection_pool_limit: 10 # background_health_checks: true - # use_shared_health_check: true + # use_shared_health_check: true # needs a coordination Redis (below) # health_check_interval: 30 + # coordination_redis: # standalone Redis for cross-pod coordination: rate limits, spend tracking, pod locks, shared health checks + # host: localhost + # port: 6379 # cancel_on_disconnect: true # cancel the in-flight upstream LLM request (non-streaming) when the client disconnects, freeing backend capacity (e.g. a vLLM GPU slot) # database_url: "postgresql://:@:/" # [OPTIONAL] use for token-based auth to proxy diff --git a/terraform/litellm/README.md b/terraform/litellm/README.md index 8f09cb53407..d4b40741052 100644 --- a/terraform/litellm/README.md +++ b/terraform/litellm/README.md @@ -178,6 +178,7 @@ only where the underlying cloud forces it. | Force destroy of object store | `s3_force_destroy` | `gcs_force_destroy` | | Database deletion protection | `skip_final_snapshot` | `cloudsql_deletion_protection` | | `proxy_config` (typed YAML map) | `proxy_config` | `proxy_config` | +| Coordination Redis | `REDIS_*` from ElastiCache (automatic) | `REDIS_*` from Memorystore (automatic) | | Extra plain env per component | `gateway_extra_env`, `backend_extra_env` | `gateway_extra_env`, `backend_extra_env` | | Extra secret-backed env | `gateway_extra_secrets`, `backend_extra_secrets` (ARNs) | `gateway_extra_secrets`, `backend_extra_secrets` (resource IDs) | | Uvicorn `--workers` on gateway | `gateway_num_workers` | `gateway_num_workers` | @@ -189,6 +190,19 @@ Each module stamps its own stack-identity tag (`litellm:stack` on AWS, merges `var.tags` / `var.labels` on top. Provider `default_tags` on AWS merge on top of all of these. +Coordination Redis needs no input on either cloud. Each module provisions the +managed Redis (ElastiCache on AWS, Memorystore on GCP) and exports `REDIS_HOST`, +`REDIS_PORT` and `REDIS_SSL` (plus `REDIS_SSL_CA_CERTS` on GCP) into the gateway +and backend env. The proxy falls back to those variables to build its +coordination Redis, which backs cross-pod tpm/rpm rate limits, spend tracking +and the pod lock manager. This is independent of LLM response caching, which +stays off unless you enable `litellm_settings.cache` in `proxy_config`. + +To coordinate through a Redis the module does not manage, set +`general_settings.coordination_redis` in `var.proxy_config`. An explicit block +overrides the `REDIS_*` env fallback; see the commented example in each +stack's `examples/default/terraform.tfvars.example` + OTel is opt-in on both clouds: leave `otel_endpoint` empty and nothing OTel-related is added to the container env; set it and both gateway and backend get `LITELLM_OTEL_V2=true` plus the full `OTEL_*` block, with diff --git a/terraform/litellm/aws/examples/default/terraform.tfvars.example b/terraform/litellm/aws/examples/default/terraform.tfvars.example index 4fdfb47e678..061ca2a9b82 100644 --- a/terraform/litellm/aws/examples/default/terraform.tfvars.example +++ b/terraform/litellm/aws/examples/default/terraform.tfvars.example @@ -56,6 +56,19 @@ env = "stage" # general_settings = { # master_key = "os.environ/LITELLM_MASTER_KEY" # database_url = "os.environ/DATABASE_URL" +# +# # Optional. The module already exports REDIS_HOST / REDIS_PORT / REDIS_SSL +# # from the ElastiCache group it provisions, and the proxy falls back to +# # those for cross-pod rate limits, spend tracking and the pod lock manager. +# # Set this block only to coordinate through a Redis the module does not +# # manage; it overrides the REDIS_* env fallback. Cluster mode takes +# # `startup_nodes` and sentinel takes `sentinel_nodes` + `service_name` +# # coordination_redis = { +# # host = "os.environ/COORDINATION_REDIS_HOST" +# # port = "os.environ/COORDINATION_REDIS_PORT" +# # password = "os.environ/COORDINATION_REDIS_PASSWORD" +# # ssl = true +# # } # } # } diff --git a/terraform/litellm/gcp/examples/default/terraform.tfvars.example b/terraform/litellm/gcp/examples/default/terraform.tfvars.example index 6358ec96e6d..4416cf0ee5d 100644 --- a/terraform/litellm/gcp/examples/default/terraform.tfvars.example +++ b/terraform/litellm/gcp/examples/default/terraform.tfvars.example @@ -51,6 +51,20 @@ env = "stage" # general_settings = { # master_key = "os.environ/LITELLM_MASTER_KEY" # database_url = "os.environ/DATABASE_URL" +# +# # Optional. The module already exports REDIS_HOST / REDIS_PORT / REDIS_SSL +# # (plus REDIS_SSL_CA_CERTS) from the Memorystore instance it provisions, and +# # the proxy falls back to those for cross-pod rate limits, spend tracking +# # and the pod lock manager. Set this block only to coordinate through a +# # Redis the module does not manage; it overrides the REDIS_* env fallback. +# # Cluster mode takes `startup_nodes` and sentinel takes `sentinel_nodes` +# # plus `service_name` +# # coordination_redis = { +# # host = "os.environ/COORDINATION_REDIS_HOST" +# # port = "os.environ/COORDINATION_REDIS_PORT" +# # password = "os.environ/COORDINATION_REDIS_PASSWORD" +# # ssl = true +# # } # } # } diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index eb795a64b79..5191414edeb 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -485,6 +485,74 @@ def test_build_span_exporter_variants(): OpenTelemetryV2Config(exporter="otlp_http", endpoint="http://h:4318") ) assert "OTLPSpanExporter" in type(http_exporter).__name__ + + +def test_otlp_logs_endpoint_normalization(): + norm = providers._otlp_logs_endpoint + # A base endpoint gets the signal path appended (the common OTLP env shape). + assert norm("http://collector:4318") == "http://collector:4318/v1/logs" + assert norm("http://collector:4318/") == "http://collector:4318/v1/logs" + # An already-correct path is left intact. + assert norm("http://collector:4318/v1/logs") == "http://collector:4318/v1/logs" + # A sibling signal's path is rewritten to logs, so one OTEL_ENDPOINT works + # for every signal rather than POSTing events at the traces path. + assert norm("http://collector:4318/v1/traces") == "http://collector:4318/v1/logs" + assert norm("http://collector:4318/v1/metrics") == "http://collector:4318/v1/logs" + assert norm(None) is None + + +def test_build_log_exporter_variants(): + from opentelemetry.sdk._logs.export import ConsoleLogExporter, InMemoryLogExporter + + assert isinstance( + providers.build_log_exporter(OpenTelemetryV2Config(exporter="console")), + ConsoleLogExporter, + ) + assert isinstance( + providers.build_log_exporter(OpenTelemetryV2Config(exporter="in_memory")), + InMemoryLogExporter, + ) + # An unrecognized kind falls back to console rather than dropping events. + assert isinstance( + providers.build_log_exporter(OpenTelemetryV2Config(exporter="unknown")), + ConsoleLogExporter, + ) + http_exporter = providers.build_log_exporter( + OpenTelemetryV2Config(exporter="otlp_http", endpoint="http://h:4318") + ) + assert "OTLPLogExporter" in type(http_exporter).__name__ + + +def test_build_logger_provider_picks_processor_by_exporter_kind(): + """Console and in-memory exporters export synchronously (tests depend on it); + every other destination gets the batch processor.""" + from opentelemetry.sdk._logs.export import ( + BatchLogRecordProcessor, + ConsoleLogExporter, + InMemoryLogExporter, + SimpleLogRecordProcessor, + ) + + cfg = OpenTelemetryV2Config(exporter="in_memory") + + def processor_of(provider): + return provider._multi_log_record_processor._log_record_processors[0] + + assert isinstance( + processor_of(providers.build_logger_provider(cfg, log_exporter=InMemoryLogExporter())), + SimpleLogRecordProcessor, + ) + assert isinstance( + processor_of(providers.build_logger_provider(cfg, log_exporter=ConsoleLogExporter())), + SimpleLogRecordProcessor, + ) + http_exporter = providers.build_log_exporter( + OpenTelemetryV2Config(exporter="otlp_http", endpoint="http://h:4318") + ) + assert isinstance( + processor_of(providers.build_logger_provider(cfg, log_exporter=http_exporter)), + BatchLogRecordProcessor, + ) grpc_exporter = providers.build_span_exporter( OpenTelemetryV2Config(exporter="otlp_grpc", endpoint="http://h:4317") ) @@ -719,6 +787,177 @@ def test_success_span_records_no_exception_event(): assert all(e.name != ExceptionEvent.NAME for e in span.events) +def _engine_with_event_recorder(): + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + from litellm.integrations.otel.emitter import SpanEmitter + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + provider, span_exporter = providers.in_memory_provider(cfg) + log_exporter = InMemoryLogExporter() + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider)) + engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg, event_recorder=recorder) + return engine, span_exporter, log_exporter + + +def _llm_call_data(error): + return LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model=None, + response_id=None, + request_params=LLMRequestParams(), + usage=LLMUsage(), + finish_reasons=(), + error=error, + response_cost=None, + server=None, + identity=RequestIdentity(call_id=None), + ) + + +def test_operation_exception_log_event_emitted_on_failed_llm_call(): + """A failed LLM call records the GenAI semconv ``gen_ai.client.operation.exception`` + event on the logs signal: severity WARN, the full ``exception.*`` trio (including + the stacktrace, which span-side only exists under a vendor key), correlated to + the failed span via trace/span ids. The span-side error surface stays intact.""" + from opentelemetry._logs.severity import SeverityNumber + + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + engine, span_exporter, log_exporter = _engine_with_event_recorder() + engine.emit( + SpanRole.LLM_CALL, + _llm_call_data( + SpanError( + error_type="RateLimitError", + message="rate limited", + code="429", + stack_trace="Traceback (most recent call last) ...", + llm_provider="openai", + ) + ), + ) + (span,) = span_exporter.get_finished_spans() + (log,) = log_exporter.get_finished_logs() + record = log.log_record + + assert record.attributes["event.name"] == GenAIEvent.OPERATION_EXCEPTION + assert record.severity_number == SeverityNumber.WARN + assert record.attributes[ExceptionEvent.TYPE] == "RateLimitError" + assert record.attributes[ExceptionEvent.MESSAGE] == "rate limited" + assert record.attributes[ExceptionEvent.STACKTRACE] == "Traceback (most recent call last) ..." + assert record.trace_id == span.context.trace_id + assert record.span_id == span.context.span_id + + assert [e.name for e in span.events] == [ExceptionEvent.NAME] + assert span.attributes["error.type"] == "RateLimitError" + + +def test_operation_exception_log_event_omits_absent_stacktrace(): + from litellm.integrations.otel.model.semconv import ExceptionEvent + + engine, _, log_exporter = _engine_with_event_recorder() + engine.emit(SpanRole.LLM_CALL, _llm_call_data(SpanError(error_type="APIError", message="boom"))) + (log,) = log_exporter.get_finished_logs() + + assert ExceptionEvent.STACKTRACE not in log.log_record.attributes + assert log.log_record.attributes[ExceptionEvent.MESSAGE] == "boom" + + +def test_operation_exception_log_event_always_carries_required_pair(): + """``exception.type`` and ``exception.message`` are the semconv-required pair: + they ride the event even when the recorder is handed empty strings, so an + event is never emitted with no required field. Only the stacktrace is + conditional.""" + from opentelemetry.sdk._logs.export import InMemoryLogExporter + from opentelemetry.trace import INVALID_SPAN_CONTEXT + + from litellm.integrations.otel.model.semconv import ExceptionEvent + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + log_exporter = InMemoryLogExporter() + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider)) + + recorder.record_operation_exception( + span_context=INVALID_SPAN_CONTEXT, + error_type="", + message="", + stack_trace="", + timestamp_ns=None, + ) + (log,) = log_exporter.get_finished_logs() + attributes = log.log_record.attributes + assert attributes[ExceptionEvent.TYPE] == "" + assert attributes[ExceptionEvent.MESSAGE] == "" + assert ExceptionEvent.STACKTRACE not in attributes + + +def test_operation_exception_log_event_not_emitted_on_success(): + engine, span_exporter, log_exporter = _engine_with_event_recorder() + engine.emit(SpanRole.LLM_CALL, _llm_call_data(None)) + + assert len(span_exporter.get_finished_spans()) == 1 + assert log_exporter.get_finished_logs() == () + + +def test_operation_exception_log_event_only_for_llm_call_role(): + """The event is scoped to GenAI client operations; a failed guardrail span + keeps its span-side error surface but records no GenAI exception event.""" + engine, span_exporter, log_exporter = _engine_with_event_recorder() + engine.emit( + SpanRole.GUARDRAIL, + GuardrailSpanData("presidio", status="failure", error=SpanError(error_type="X", message="denied")), + ) + (span,) = span_exporter.get_finished_spans() + + assert span.attributes["error.type"] == "X" + assert log_exporter.get_finished_logs() == () + + +def test_resolve_logger_provider_honors_explicit_noop_optout(monkeypatch): + """A ``NoOpLoggerProvider`` global is an explicit operator opt-out from the logs + signal: resolve to ``None`` so no recorder (and so no event) is ever built, + rather than emitting into a provider that drops everything.""" + from opentelemetry import _logs + from opentelemetry._logs import NoOpLoggerProvider + + from litellm.integrations.otel.logger import OpenTelemetryV2 + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + tracer_provider, _ = providers.in_memory_provider(cfg) + monkeypatch.setattr(_logs, "get_logger_provider", lambda: NoOpLoggerProvider()) + + assert providers.resolve_logger_provider(cfg) is None + logger = OpenTelemetryV2(config=cfg, tracer_provider=tracer_provider) + assert logger._emitter._event_recorder is None + + +def test_resolve_logger_provider_reuses_operator_sdk_global(monkeypatch): + """Events ride an operator-configured logs pipeline rather than a second one + built by litellm, so they land wherever the operator's other logs land.""" + from opentelemetry import _logs + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + operator_provider = providers.build_logger_provider(cfg, log_exporter=InMemoryLogExporter()) + monkeypatch.setattr(_logs, "get_logger_provider", lambda: operator_provider) + + assert providers.resolve_logger_provider(cfg) is operator_provider + + +def test_operation_exception_event_keys_are_pinned(): + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + assert GenAIEvent.OPERATION_EXCEPTION == "gen_ai.client.operation.exception" + assert ExceptionEvent.STACKTRACE == "exception.stacktrace" + + # --- service taxonomy: which calls become spans, and of what kind ----------- # diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 697b9293eea..b5e077e3561 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -217,6 +217,61 @@ def test_async_log_failure_event_marks_error_status(): assert span.attributes["error.type"] == "RateLimitError" +def _logger_with_events(enable_events): + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=enable_events) + span_exporter = InMemorySpanExporter() + tracer_provider = providers.build_tracer_provider(cfg, exporter=span_exporter) + log_exporter = InMemoryLogExporter() + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + logger = OpenTelemetryV2(config=cfg, tracer_provider=tracer_provider, logger_provider=logger_provider) + return logger, span_exporter, log_exporter + + +def test_enable_events_records_operation_exception_through_failure_callback(): + """With ``enable_events`` on, a real failure callback records the GenAI + ``gen_ai.client.operation.exception`` log event, carrying the traceback from + the standard logging payload and correlated to the LLM-call span.""" + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + logger, span_exporter, log_exporter = _logger_with_events(enable_events=True) + payload = _payload( + status="failure", + error_information={ + "error_class": "RateLimitError", + "error_message": "429 rate limited", + "traceback": "Traceback (most recent call last) ...", + }, + ) + _emit_llm(logger, _kwargs(payload=payload), fail=True) + + (span,) = span_exporter.get_finished_spans() + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.attributes["event.name"] == GenAIEvent.OPERATION_EXCEPTION + assert record.attributes[ExceptionEvent.TYPE] == "RateLimitError" + assert record.attributes[ExceptionEvent.MESSAGE] == "429 rate limited" + assert record.attributes[ExceptionEvent.STACKTRACE] == "Traceback (most recent call last) ..." + assert record.trace_id == span.context.trace_id + assert record.span_id == span.context.span_id + + +def test_events_off_by_default_records_no_log_event_on_failure(): + """``enable_events`` defaults to off: even with a logs pipeline injected, a + failure records only the span-side error surface, no log event.""" + logger, span_exporter, log_exporter = _logger_with_events(enable_events=False) + payload = _payload( + status="failure", + error_information={"error_class": "RateLimitError", "error_message": "429"}, + ) + _emit_llm(logger, _kwargs(payload=payload), fail=True) + + assert len(span_exporter.get_finished_spans()) == 1 + assert log_exporter.get_finished_logs() == () + assert OpenTelemetryV2Config(exporter="in_memory").enable_events is False + + def test_sync_log_event_is_noop(): """V2 closes the span async-only; the sync callback runs out-of-context, so it no-ops (the span stays open on the carrier until the async callback).""" diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index f441270be7c..1fcee1b1c42 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -78,6 +78,18 @@ context_window_test_cases = [ "CerebrasException - Please reduce the length of the messages or completion. Current length is 50000 while limit is 40000", True, ), + ( + "Invalid 'input[0]': maximum input length is 8192 tokens.", + True, + ), + ( + "OpenAIException - Error code: 400 - {'error': {'message': \"Invalid 'input[0]': maximum input length is 8192 tokens.\", 'type': 'invalid_request_error'}}", + True, + ), + ( + "Invalid 'metadata': maximum input length is 512 characters.", + False, + ), # Negative cases (should return False) ("A generic API error occurred.", False), ("Invalid API Key provided.", False), diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 532c6ff3598..ffe45cd5e79 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -1974,6 +1974,79 @@ def test_bedrock_invoke_transform_merges_list_content_system_role_into_system(): ] +def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(): + """Regression test for the Bedrock prompt-cache collapse: hoisting a + mid-conversation ``role: "system"`` message (e.g. Claude Code's + ``mid-conversation-system-2026-04-07`` reminders) into the top-level + ``system`` field mutates the cache prefix and invalidates the cached message + history, so such entries must be forwarded in place. Invoke only rejects a + system entry at ``messages.0``. Billing-header blocks must still be stripped + from the top-level ``system`` field even when nothing is hoisted.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "user", "content": "read the file"}, + {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-4-8", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={ + "max_tokens": 256, + "stream": False, + "system": [ + {"type": "text", "text": "x-anthropic-billing-header: cc_version=2.1.205;"}, + {"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}}, + ], + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == messages + assert result["system"] == [ + {"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}} + ] + + +def test_bedrock_invoke_transform_hoists_only_leading_system_run(): + """Only the leading run of ``role: "system"`` messages is hoisted into the + top-level ``system`` field; a later system entry keeps its position in + ``messages`` so the serialized prefix stays stable across turns.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "system", "content": "You are terse."}, + {"role": "system", "content": "Cite sources."}, + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-4-8", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [ + {"type": "text", "text": "You are terse."}, + {"type": "text", "text": "Cite sources."}, + ] + + def test_as_system_content_blocks_handles_each_shape(): """``_as_system_content_blocks`` normalizes every system shape: ``None`` -> empty, a string -> a single text block, a list -> a shallow copy, and any other value diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_envelope.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_envelope.py new file mode 100644 index 00000000000..71de206aa1e --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_envelope.py @@ -0,0 +1,487 @@ +"""Spec tests for the sealed-envelope module (oauth_delegate DCR bridge). + +The envelope is the single client-held bearer carrying both a litellm identity and the +encrypted upstream grant, with zero server-side storage. These tests pin the security +contract: an envelope opens only under the exact keys that minted it, tampering with any +signed byte is detected, expiry is enforced against the injected clock (capped by the +module TTL ceiling), oversized envelopes are rejected rather than truncated, and no +error value, model repr, or raised exception ever contains the inner access token. +""" + +import base64 +import hashlib +import hmac +import json +from datetime import datetime, timedelta, timezone + +import jwt +import pytest +from cryptography.hazmat.primitives.asymmetric import rsa +from pydantic import SecretStr, ValidationError + +from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( + ENVELOPE_ISSUER, + ENVELOPE_PREFIX, + MAX_ENVELOPE_BYTES, + MAX_ENVELOPE_TTL_SECONDS, + BadSignature, + DecryptFailed, + EnvelopeIdentity, + EnvelopeKeys, + EnvelopeTooLarge, + Expired, + MalformedPayload, + NotAnEnvelope, + OpenedEnvelope, + SealedEnvelope, + UpstreamTokenGrant, + is_envelope, + mint_envelope, + open_envelope, +) +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value, encrypt_value + +_NOW = datetime(2026, 7, 9, 12, 0, 0, tzinfo=timezone.utc) +_SIGNING_KEY = "unit-test-signing-key-0123456789abcdef0123456789abcdef" +_ENCRYPTION_KEY = "unit-test-encryption-key-fedcba9876543210fedcba9876543210" +_OTHER_SIGNING_KEY = "other-signing-key-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +_OTHER_ENCRYPTION_KEY = "other-encryption-key-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +_KEYS = EnvelopeKeys(signing_key=SecretStr(_SIGNING_KEY), encryption_key=SecretStr(_ENCRYPTION_KEY)) +_WRONG_SIGNING = EnvelopeKeys(signing_key=SecretStr(_OTHER_SIGNING_KEY), encryption_key=SecretStr(_ENCRYPTION_KEY)) +_WRONG_ENCRYPTION = EnvelopeKeys(signing_key=SecretStr(_SIGNING_KEY), encryption_key=SecretStr(_OTHER_ENCRYPTION_KEY)) +_ACCESS_TOKEN = "upstream-access-token-do-not-leak-8f14e45fceea" +_REFRESH_TOKEN = "upstream-refresh-token-do-not-leak-1d0aa4b7" +_IDENTITY = EnvelopeIdentity(user_id="user-123", server_id="srv-456") + + +def _full_grant() -> UpstreamTokenGrant: + return UpstreamTokenGrant( + access_token=SecretStr(_ACCESS_TOKEN), + token_type="Bearer", + refresh_token=SecretStr(_REFRESH_TOKEN), + scope="read:tools write:tools", + expires_in=600, + ) + + +def _minimal_grant() -> UpstreamTokenGrant: + return UpstreamTokenGrant(access_token=SecretStr(_ACCESS_TOKEN), token_type="Bearer") + + +def _sealed_token(grant: UpstreamTokenGrant, keys: EnvelopeKeys = _KEYS) -> str: + sealed = mint_envelope(_IDENTITY, grant, keys, _NOW) + assert isinstance(sealed, SealedEnvelope) + return sealed.token.get_secret_value() + + +def _unverified_claims(sealed_token: str) -> dict[str, object]: + return jwt.decode(sealed_token.removeprefix(ENVELOPE_PREFIX), options={"verify_signature": False}) + + +def _forge(claims: dict[str, object], signing_key: str = _SIGNING_KEY) -> str: + return ENVELOPE_PREFIX + jwt.encode(claims, signing_key, algorithm="HS256") + + +def _b64url(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") + + +def _hand_crafted_hs256(payload: dict[str, object], signing_key: str = _SIGNING_KEY) -> str: + """Assemble an HS256 envelope from raw bytes, bypassing PyJWT's encode-side claim + guards (it refuses to build a token with a non-string ``iss``). This is the real + attacker path: a client crafts the compact JWT directly, so any registered claim can + carry a hostile JSON type.""" + header = _b64url(json.dumps({"alg": "HS256", "typ": "JWT"}).encode("utf-8")) + body = _b64url(json.dumps(payload).encode("utf-8")) + signing_input = f"{header}.{body}".encode("ascii") + signature = _b64url(hmac.new(signing_key.encode("utf-8"), signing_input, hashlib.sha256).digest()) + return ENVELOPE_PREFIX + f"{header}.{body}.{signature}" + + +def _tampered(sealed_token: str, segment: int, index: int) -> str: + parts = sealed_token.removeprefix(ENVELOPE_PREFIX).split(".") + original = parts[segment][index] + replacement = "A" if original in "QRST" else "Q" + mutated = parts[segment][:index] + replacement + parts[segment][index + 1 :] + rebuilt = ".".join(parts[:segment] + [mutated] + parts[segment + 1 :]) + return ENVELOPE_PREFIX + rebuilt + + +def test_round_trip_recovers_identity_and_grant_exactly(): + grant = _full_grant() + token = _sealed_token(grant) + assert is_envelope(token) + opened = open_envelope(token, _KEYS, _NOW) + assert isinstance(opened, OpenedEnvelope) + assert opened.identity == _IDENTITY + assert opened.grant == grant + assert opened.grant.access_token.get_secret_value() == _ACCESS_TOKEN + assert opened.grant.refresh_token is not None + assert opened.grant.refresh_token.get_secret_value() == _REFRESH_TOKEN + + +def test_minimal_grant_round_trips_without_none_leakage_into_claims(): + token = _sealed_token(_minimal_grant()) + claims = _unverified_claims(token) + blob = claims["grant"] + assert isinstance(blob, str) + plaintext = decrypt_value(value=base64.urlsafe_b64decode(blob), signing_key=_ENCRYPTION_KEY) + assert set(json.loads(plaintext)) == {"access_token", "token_type"} + opened = open_envelope(token, _KEYS, _NOW) + assert isinstance(opened, OpenedEnvelope) + assert opened.grant.refresh_token is None + assert opened.grant.scope is None + assert opened.grant.expires_in is None + + +def test_claim_layout_and_no_plaintext_token_in_envelope(): + token = _sealed_token(_full_grant()) + claims = _unverified_claims(token) + assert set(claims) == {"iss", "iat", "exp", "user_id", "server_id", "grant"} + assert claims["iss"] == ENVELOPE_ISSUER + assert claims["iat"] == int(_NOW.timestamp()) + assert claims["exp"] == int(_NOW.timestamp()) + 600 + assert claims["user_id"] == "user-123" + assert claims["server_id"] == "srv-456" + assert _ACCESS_TOKEN not in token + assert _ACCESS_TOKEN not in json.dumps(claims) + assert _REFRESH_TOKEN not in json.dumps(claims) + + +@pytest.mark.parametrize( + "expires_in, expected_ttl", + [ + (600, 600), + (MAX_ENVELOPE_TTL_SECONDS + 82800, MAX_ENVELOPE_TTL_SECONDS), + (None, MAX_ENVELOPE_TTL_SECONDS), + ], +) +def test_exp_is_min_of_upstream_expires_in_and_cap(expires_in, expected_ttl): + grant = UpstreamTokenGrant(access_token=SecretStr(_ACCESS_TOKEN), token_type="Bearer", expires_in=expires_in) + sealed = mint_envelope(_IDENTITY, grant, _KEYS, _NOW) + assert isinstance(sealed, SealedEnvelope) + assert sealed.expires_at == _NOW + timedelta(seconds=expected_ttl) + + +def test_expiry_honored_against_injected_clock(): + token = _sealed_token(_full_grant()) + assert isinstance(open_envelope(token, _KEYS, _NOW + timedelta(seconds=599)), OpenedEnvelope) + assert isinstance(open_envelope(token, _KEYS, _NOW + timedelta(seconds=600)), Expired) + assert isinstance(open_envelope(token, _KEYS, _NOW + timedelta(seconds=601)), Expired) + + +def test_ttl_cap_enforced_on_open_even_when_upstream_token_lives_longer(): + grant = UpstreamTokenGrant(access_token=SecretStr(_ACCESS_TOKEN), token_type="Bearer", expires_in=86400) + token = _sealed_token(grant) + just_before_cap = _NOW + timedelta(seconds=MAX_ENVELOPE_TTL_SECONDS - 1) + at_cap = _NOW + timedelta(seconds=MAX_ENVELOPE_TTL_SECONDS) + assert isinstance(open_envelope(token, _KEYS, just_before_cap), OpenedEnvelope) + assert isinstance(open_envelope(token, _KEYS, at_cap), Expired) + + +def test_tampering_any_payload_or_signature_byte_is_bad_signature(): + token = _sealed_token(_full_grant()) + parts = token.removeprefix(ENVELOPE_PREFIX).split(".") + for segment in (1, 2): + for index in range(len(parts[segment])): + result = open_envelope(_tampered(token, segment, index), _KEYS, _NOW) + assert isinstance(result, BadSignature), f"segment {segment} index {index}: {result!r}" + + +def test_tampering_header_bytes_never_opens(): + token = _sealed_token(_full_grant()) + parts = token.removeprefix(ENVELOPE_PREFIX).split(".") + for index in range(len(parts[0])): + result = open_envelope(_tampered(token, 0, index), _KEYS, _NOW) + assert isinstance(result, (BadSignature, MalformedPayload)), f"header index {index}: {result!r}" + + +def test_alg_none_is_rejected(): + claims = _unverified_claims(_sealed_token(_full_grant())) + unsigned = ENVELOPE_PREFIX + jwt.encode(claims, None, algorithm="none") + assert isinstance(open_envelope(unsigned, _KEYS, _NOW), MalformedPayload) + + +def test_wrong_signing_key_is_bad_signature(): + token = _sealed_token(_full_grant()) + assert isinstance(open_envelope(token, _WRONG_SIGNING, _NOW), BadSignature) + + +def test_wrong_encryption_key_is_decrypt_failed(): + token = _sealed_token(_full_grant()) + assert isinstance(open_envelope(token, _WRONG_ENCRYPTION, _NOW), DecryptFailed) + + +def test_ciphertext_swapped_from_another_envelope_is_decrypt_failed(): + claims_a = _unverified_claims(_sealed_token(_full_grant(), keys=_KEYS)) + claims_b = _unverified_claims(_sealed_token(_minimal_grant(), keys=_WRONG_ENCRYPTION)) + swapped = _forge({**claims_a, "grant": claims_b["grant"]}) + assert isinstance(open_envelope(swapped, _KEYS, _NOW), DecryptFailed) + + +def test_wrong_issuer_is_malformed_payload(): + claims = _unverified_claims(_sealed_token(_full_grant())) + assert isinstance(open_envelope(_forge({**claims, "iss": "evil-issuer"}), _KEYS, _NOW), MalformedPayload) + + +def test_missing_identity_claim_is_malformed_payload(): + claims = _unverified_claims(_sealed_token(_full_grant())) + forged = _forge({key: value for key, value in claims.items() if key != "user_id"}) + assert isinstance(open_envelope(forged, _KEYS, _NOW), MalformedPayload) + + +@pytest.mark.parametrize("identity_claim", ["user_id", "server_id"]) +def test_signed_empty_identity_claim_is_malformed_payload_not_a_raise(identity_claim): + claims = _unverified_claims(_sealed_token(_full_grant())) + forged = _forge({**claims, identity_claim: ""}) + assert isinstance(open_envelope(forged, _KEYS, _NOW), MalformedPayload) + intact = open_envelope(_forge(claims), _KEYS, _NOW) + assert isinstance(intact, OpenedEnvelope) + assert intact.identity == _IDENTITY + + +def test_lone_surrogate_candidate_is_malformed_payload_not_a_raise(): + surrogate_candidate = ENVELOPE_PREFIX + "\ud800abc.def.ghi" + result = open_envelope(surrogate_candidate, _KEYS, _NOW) + assert isinstance(result, MalformedPayload) + + +@pytest.mark.parametrize( + "override", + [ + {"iat": [1]}, + {"iat": {}}, + {"iat": float("inf")}, + {"nbf": None}, + {"nbf": [1]}, + ], +) +def test_hostile_iat_nbf_types_are_malformed_payload_not_a_raise(override): + claims = _unverified_claims(_sealed_token(_full_grant())) + forged = _forge({**claims, **override}) + result = open_envelope(forged, _KEYS, _NOW) + assert isinstance(result, MalformedPayload) + + +@pytest.mark.parametrize("hostile_iss", [["litellm-mcp-bridge"], 5, {"iss": "x"}]) +def test_non_string_issuer_claim_is_malformed_payload_not_a_raise(hostile_iss): + claims = _unverified_claims(_sealed_token(_full_grant())) + forged = _hand_crafted_hs256({**claims, "iss": hostile_iss}) + result = open_envelope(forged, _KEYS, _NOW) + assert isinstance(result, MalformedPayload) + + +@pytest.mark.parametrize("hostile_exp", ["600", 600.5, [600]]) +def test_non_int_exp_claim_is_malformed_payload_not_a_raise(hostile_exp): + claims = _unverified_claims(_sealed_token(_full_grant())) + forged = _hand_crafted_hs256({**claims, "exp": hostile_exp}) + result = open_envelope(forged, _KEYS, _NOW) + assert isinstance(result, MalformedPayload) + + +def test_unexpected_extra_claim_is_malformed_payload(): + claims = _unverified_claims(_sealed_token(_full_grant())) + forged = _forge({**claims, "role": "admin"}) + assert isinstance(open_envelope(forged, _KEYS, _NOW), MalformedPayload) + + +def test_future_iat_opens_against_injected_now_not_wall_clock(): + future = _NOW + timedelta(seconds=100_000) + sealed = mint_envelope(_IDENTITY, _full_grant(), _KEYS, future) + assert isinstance(sealed, SealedEnvelope) + opened = open_envelope(sealed.token.get_secret_value(), _KEYS, future) + assert isinstance(opened, OpenedEnvelope) + assert opened.identity == _IDENTITY + assert opened.grant == _full_grant() + + +def test_rs256_signed_token_is_rejected_against_the_hs256_pin(): + claims = _unverified_claims(_sealed_token(_full_grant())) + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + rs256_token = ENVELOPE_PREFIX + jwt.encode(claims, private_key, algorithm="RS256") + result = open_envelope(rs256_token, _KEYS, _NOW) + assert isinstance(result, MalformedPayload) + + +@pytest.mark.parametrize("short_key", ["", "too-short", "x" * 31]) +def test_signing_key_below_hs256_minimum_is_rejected_at_construction(short_key): + with pytest.raises(ValidationError): + EnvelopeKeys(signing_key=SecretStr(short_key), encryption_key=SecretStr(_ENCRYPTION_KEY)) + + +def test_signing_key_at_hs256_minimum_is_accepted(): + keys = EnvelopeKeys(signing_key=SecretStr("y" * 32), encryption_key=SecretStr(_ENCRYPTION_KEY)) + assert keys.signing_key.get_secret_value() == "y" * 32 + + +def test_correctly_signed_garbage_grant_blob_is_decrypt_failed(): + claims = _unverified_claims(_sealed_token(_full_grant())) + forged = _forge({**claims, "grant": "not-a-ciphertext"}) + assert isinstance(open_envelope(forged, _KEYS, _NOW), DecryptFailed) + + +def test_decryptable_blob_that_is_not_a_grant_is_malformed_payload(): + claims = _unverified_claims(_sealed_token(_full_grant())) + wrong_shape = base64.urlsafe_b64encode( + bytes(encrypt_value(value=json.dumps({"nope": 1}), signing_key=_ENCRYPTION_KEY)) + ).decode("ascii") + forged = _forge({**claims, "grant": wrong_shape}) + assert isinstance(open_envelope(forged, _KEYS, _NOW), MalformedPayload) + + +def _mint_with_token_len(n: int) -> SealedEnvelope | EnvelopeTooLarge: + grant = UpstreamTokenGrant(access_token=SecretStr("a" * n), token_type="Bearer") + return mint_envelope(_IDENTITY, grant, _KEYS, _NOW) + + +def _largest_token_len_that_mints(lo: int, hi: int) -> int: + if hi - lo <= 1: + return lo + mid = (lo + hi) // 2 + if isinstance(_mint_with_token_len(mid), SealedEnvelope): + return _largest_token_len_that_mints(mid, hi) + return _largest_token_len_that_mints(lo, mid) + + +def test_oversized_grant_is_a_typed_mint_error_never_truncated(): + result = _mint_with_token_len(30000) + assert isinstance(result, EnvelopeTooLarge) + assert result.tag == "envelope_too_large" + assert result.size_bytes > MAX_ENVELOPE_BYTES + assert result.max_bytes == MAX_ENVELOPE_BYTES + + +def test_size_cap_boundary_just_under_succeeds_and_just_over_fails(): + assert isinstance(_mint_with_token_len(1), SealedEnvelope) + assert isinstance(_mint_with_token_len(30000), EnvelopeTooLarge) + largest = _largest_token_len_that_mints(1, 30000) + assert largest > 6000 + sealed = _mint_with_token_len(largest) + assert isinstance(sealed, SealedEnvelope) + assert len(sealed.token.get_secret_value().encode("utf-8")) <= MAX_ENVELOPE_BYTES + overflowing = _mint_with_token_len(largest + 1) + assert isinstance(overflowing, EnvelopeTooLarge) + assert overflowing.size_bytes > MAX_ENVELOPE_BYTES + opened = open_envelope(sealed.token.get_secret_value(), _KEYS, _NOW) + assert isinstance(opened, OpenedEnvelope) + + +def test_open_size_guard_measures_bytes_not_characters(): + """The open-side size guard must reject on UTF-8 byte length, matching mint's cap, so a + hostile multi-byte candidate whose character count is under the cap but whose byte count is + over it is rejected up front rather than reaching the expensive HMAC/decrypt path. Patching + _decode_claims to fail loudly proves the guard short-circuits before decode.""" + from unittest.mock import patch + + from litellm.proxy._experimental.mcp_server.outbound_credentials import envelope + + multibyte_body = "é" * 7000 # 7000 chars, 14000 UTF-8 bytes + candidate = ENVELOPE_PREFIX + multibyte_body + assert len(candidate) <= MAX_ENVELOPE_BYTES + assert len(candidate.encode("utf-8")) > MAX_ENVELOPE_BYTES + + with patch.object(envelope, "_decode_claims", side_effect=AssertionError("decode reached")) as decode: + result = open_envelope(candidate, _KEYS, _NOW) + + assert isinstance(result, MalformedPayload) + decode.assert_not_called() + + +def test_open_size_guard_rejects_oversize_character_count_before_decode(): + """A candidate whose character count already exceeds the cap is rejected up front, before the + decode path, so an arbitrarily long hostile string is not run through HMAC/decrypt. The cheap + character precheck makes this O(1) since UTF-8 byte length is never below character length.""" + from unittest.mock import patch + + from litellm.proxy._experimental.mcp_server.outbound_credentials import envelope + + candidate = ENVELOPE_PREFIX + ("a" * (MAX_ENVELOPE_BYTES + 1)) + assert len(candidate) > MAX_ENVELOPE_BYTES + + with patch.object(envelope, "_decode_claims", side_effect=AssertionError("decode reached")) as decode: + result = open_envelope(candidate, _KEYS, _NOW) + + assert isinstance(result, MalformedPayload) + decode.assert_not_called() + + +def test_is_envelope_detects_only_prefixed_values(): + assert is_envelope(_sealed_token(_full_grant())) + raw_jwt = jwt.encode({"sub": "user-123"}, _SIGNING_KEY, algorithm="HS256") + assert not is_envelope(raw_jwt) + assert not is_envelope("some-random-opaque-token") + assert not is_envelope("") + + +def test_open_on_non_envelope_input_is_not_an_envelope(): + raw_jwt = jwt.encode({"sub": "user-123"}, _SIGNING_KEY, algorithm="HS256") + assert isinstance(open_envelope(raw_jwt, _KEYS, _NOW), NotAnEnvelope) + assert isinstance(open_envelope("", _KEYS, _NOW), NotAnEnvelope) + assert isinstance(open_envelope(_ACCESS_TOKEN, _KEYS, _NOW), NotAnEnvelope) + + +def test_open_on_prefixed_garbage_is_malformed_payload(): + assert isinstance(open_envelope(ENVELOPE_PREFIX + "garbage", _KEYS, _NOW), MalformedPayload) + assert isinstance(open_envelope(ENVELOPE_PREFIX + _ACCESS_TOKEN, _KEYS, _NOW), MalformedPayload) + + +def test_no_result_value_ever_reveals_the_access_token(): + grant = _full_grant() + sealed = mint_envelope(_IDENTITY, grant, _KEYS, _NOW) + assert isinstance(sealed, SealedEnvelope) + token = sealed.token.get_secret_value() + oversized_grant = UpstreamTokenGrant(access_token=SecretStr(_ACCESS_TOKEN + "x" * 30000), token_type="Bearer") + values = ( + sealed, + open_envelope(token, _KEYS, _NOW), + mint_envelope(_IDENTITY, oversized_grant, _KEYS, _NOW), + open_envelope(_ACCESS_TOKEN, _KEYS, _NOW), + open_envelope(ENVELOPE_PREFIX + _ACCESS_TOKEN, _KEYS, _NOW), + open_envelope(token, _WRONG_SIGNING, _NOW), + open_envelope(token, _WRONG_ENCRYPTION, _NOW), + open_envelope(token, _KEYS, _NOW + timedelta(seconds=601)), + grant, + ) + for value in values: + assert _ACCESS_TOKEN not in repr(value) + assert _ACCESS_TOKEN not in str(value) + assert _REFRESH_TOKEN not in repr(value) + assert _REFRESH_TOKEN not in str(value) + + +def test_non_positive_expires_in_is_rejected_at_construction_without_leaking(): + for bad_expires_in in (0, -5): + with pytest.raises(ValidationError) as excinfo: + UpstreamTokenGrant( + access_token=SecretStr(_ACCESS_TOKEN), + token_type="Bearer", + expires_in=bad_expires_in, + ) + assert _ACCESS_TOKEN not in str(excinfo.value) + assert _ACCESS_TOKEN not in repr(excinfo.value) + + +def test_empty_identity_and_key_fields_are_rejected_at_construction(): + with pytest.raises(ValidationError): + EnvelopeIdentity(user_id="", server_id="srv-456") + with pytest.raises(ValidationError): + EnvelopeIdentity(user_id="user-123", server_id="") + with pytest.raises(ValidationError): + EnvelopeKeys(signing_key=SecretStr(""), encryption_key=SecretStr(_ENCRYPTION_KEY)) + with pytest.raises(ValidationError): + EnvelopeKeys(signing_key=SecretStr(_SIGNING_KEY), encryption_key=SecretStr("")) + with pytest.raises(ValidationError): + UpstreamTokenGrant(access_token=SecretStr(""), token_type="Bearer") + + +def test_public_models_are_frozen(): + sealed = mint_envelope(_IDENTITY, _full_grant(), _KEYS, _NOW) + assert isinstance(sealed, SealedEnvelope) + opened = open_envelope(sealed.token.get_secret_value(), _KEYS, _NOW) + assert isinstance(opened, OpenedEnvelope) + with pytest.raises(ValidationError): + sealed.token = SecretStr("overwritten") + with pytest.raises(ValidationError): + opened.grant = _minimal_grant() + with pytest.raises(ValidationError): + _IDENTITY.user_id = "someone-else" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index c8e871b3bc1..11a6fea0cea 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -3577,6 +3577,465 @@ async def test_token_exchange_passes_through_upstream_expires_in(): assert body["expires_in"] == 43200 +_BRIDGE_CLIENT_REDIRECT = "https://claude.ai/api/mcp/auth_callback" + + +def _bridge_server(**overrides): + from litellm.proxy._types import MCPTransport + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + fields = { + "server_id": "bridge_srv", + "name": "bridge_srv", + "server_name": "bridge_srv", + "alias": "bridge_srv", + "transport": MCPTransport.http, + "auth_type": MCPAuth.true_passthrough, + "dcr_bridge": True, + "authorization_url": "https://provider.com/oauth/authorize", + "token_url": "https://provider.com/oauth/token", + "registration_url": "https://provider.com/oauth/register", + **overrides, + } + return MCPServer(**fields) + + +def _bridge_mock_request(): + from fastapi import Request + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://litellm.example.com/" + mock_request.headers = {} + return mock_request + + +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_type_value", ["true_passthrough", "oauth_delegate"]) +async def test_authorize_bridge_relay_passes_client_params_verbatim(auth_type_value): + """The bridge relay arm (registration relayed upstream, no admin-configured client) passes the + client's client_id, redirect_uri, state, and PKCE through verbatim: the code returns straight + to the client's own redirect URI, so the gateway sets no state cookie, injects no /callback, + and applies no gateway-side redirect trust (the upstream enforces its registered binding).""" + from urllib.parse import parse_qs, urlparse + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + authorize_with_server, + ) + from litellm.types.mcp import MCPAuth + + response = await authorize_with_server( + request=_bridge_mock_request(), + mcp_server=_bridge_server(auth_type=MCPAuth(auth_type_value)), + client_id="dcr-client-123", + redirect_uri=_BRIDGE_CLIENT_REDIRECT, + state="client-state", + code_challenge="chal", + code_challenge_method="S256", + ) + + assert response.status_code == 307 + location = response.headers["location"] + assert location.startswith("https://provider.com/oauth/authorize") + query = parse_qs(urlparse(location).query) + assert query["client_id"] == ["dcr-client-123"] + assert query["redirect_uri"] == [_BRIDGE_CLIENT_REDIRECT] + assert query["state"] == ["client-state"] + assert query["code_challenge"] == ["chal"] + assert query["code_challenge_method"] == ["S256"] + assert "litellm.example.com" not in location + assert "set-cookie" not in {key.lower() for key in response.headers.keys()} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "code_challenge,code_challenge_method", + [(None, None), ("chal", None), ("chal", "plain"), (None, "S256")], +) +async def test_authorize_bridge_requires_s256_pkce(code_challenge, code_challenge_method): + """Bridge servers serve unauthenticated public clients, so the PKCE downgrade paths (missing + challenge, or a method that is not S256; RFC 7636 defaults a missing method to plain) are + rejected at the gateway on both bridge arms.""" + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + authorize_with_server, + ) + + with pytest.raises(HTTPException) as exc: + await authorize_with_server( + request=_bridge_mock_request(), + mcp_server=_bridge_server(), + client_id="dcr-client-123", + redirect_uri=_BRIDGE_CLIENT_REDIRECT, + state="s", + code_challenge=code_challenge, + code_challenge_method=code_challenge_method, + ) + + assert exc.value.status_code == 400 + assert "S256" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_authorize_bridge_short_circuit_keeps_callback_and_redirect_trust(): + """The bridge short-circuit arm (admin-configured OAuth client, upstream only knows the + gateway callback) keeps the /callback state relay and the gateway redirect trust: a public + client redirect target is rejected unless ops allowlist it, and a trusted target still routes + through the gateway callback with the state cookie.""" + from urllib.parse import parse_qs, urlparse + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + authorize_with_server, + ) + + short_circuit_server = _bridge_server(client_id="admin-client", registration_url=None) + + with pytest.raises(HTTPException) as exc: + await authorize_with_server( + request=_bridge_mock_request(), + mcp_server=short_circuit_server, + client_id="ignored", + redirect_uri=_BRIDGE_CLIENT_REDIRECT, + state="s", + code_challenge="chal", + code_challenge_method="S256", + ) + assert exc.value.status_code in (400, 403) + + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper", + return_value="mocked_encrypted_state", + ): + response = await authorize_with_server( + request=_bridge_mock_request(), + mcp_server=short_circuit_server, + client_id="ignored", + redirect_uri="http://127.0.0.1:60108/callback", + state="s", + code_challenge="chal", + code_challenge_method="S256", + ) + + query = parse_qs(urlparse(response.headers["location"]).query) + assert query["redirect_uri"] == ["https://litellm.example.com/callback"] + assert query["client_id"] == ["admin-client"] + + +@pytest.mark.asyncio +async def test_authorize_non_bridge_client_forwarded_keeps_pre_bridge_contract(): + """A client-forwarded server without dcr_bridge keeps the pre-bridge behavior: no PKCE + requirement and the gateway /callback relay (this is the browser-only Authorize path).""" + from urllib.parse import parse_qs, urlparse + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + authorize_with_server, + ) + + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper", + return_value="mocked_encrypted_state", + ): + response = await authorize_with_server( + request=_bridge_mock_request(), + mcp_server=_bridge_server(dcr_bridge=None), + client_id="cid", + redirect_uri="http://127.0.0.1:60108/callback", + state="s", + ) + + assert response.status_code == 307 + query = parse_qs(urlparse(response.headers["location"]).query) + assert query["redirect_uri"] == ["https://litellm.example.com/callback"] + + +async def _bridge_token_post_data(server, redirect_uri): + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + exchange_token_with_server, + ) + + fake_http_response = MagicMock() + fake_http_response.json.return_value = {"access_token": "tok", "token_type": "Bearer"} + fake_http_response.raise_for_status = MagicMock() + fake_http_client = MagicMock() + fake_http_client.post = AsyncMock(return_value=fake_http_response) + + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=fake_http_client, + ): + await exchange_token_with_server( + request=_bridge_mock_request(), + mcp_server=server, + grant_type="authorization_code", + code="auth-code", + redirect_uri=redirect_uri, + client_id="dcr-client-123", + client_secret=None, + code_verifier="verifier", + ) + return fake_http_client.post.call_args.kwargs["data"] + + +@pytest.mark.asyncio +async def test_token_bridge_relay_posts_client_redirect_uri(): + """The bridge relay arm's token exchange sends the client's own redirect_uri upstream (it must + match the authorize leg) with the caller's public client_id and PKCE verifier.""" + data = await _bridge_token_post_data(_bridge_server(), redirect_uri=_BRIDGE_CLIENT_REDIRECT) + + assert data["redirect_uri"] == _BRIDGE_CLIENT_REDIRECT + assert data["client_id"] == "dcr-client-123" + assert data["code_verifier"] == "verifier" + assert "client_secret" not in data + + +@pytest.mark.asyncio +async def test_token_bridge_relay_requires_redirect_uri(): + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + exchange_token_with_server, + ) + + with pytest.raises(HTTPException) as exc: + await exchange_token_with_server( + request=_bridge_mock_request(), + mcp_server=_bridge_server(), + grant_type="authorization_code", + code="auth-code", + redirect_uri=None, + client_id="dcr-client-123", + client_secret=None, + code_verifier="verifier", + ) + + assert exc.value.status_code == 400 + assert "redirect_uri" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_token_non_bridge_keeps_gateway_callback(): + """Without dcr_bridge the token exchange keeps posting the gateway callback as redirect_uri, + pinning the pre-bridge contract for the browser-only Authorize path.""" + data = await _bridge_token_post_data(_bridge_server(dcr_bridge=None), redirect_uri=_BRIDGE_CLIENT_REDIRECT) + + assert data["redirect_uri"] == "https://litellm.example.com/callback" + + +def _named_as_metadata_response(server): + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _build_oauth_authorization_server_response, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + global_mcp_server_manager.registry.clear() + global_mcp_server_manager.registry[server.server_id] = server + try: + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", + return_value=None, + ): + return _build_oauth_authorization_server_response( + request=_bridge_mock_request(), + mcp_server_name=server.server_name, + ) + finally: + global_mcp_server_manager.registry.clear() + + +def test_oauth_authorization_server_metadata_served_for_bridge_server(): + """Bridge servers get the gateway's AS metadata (the register, authorize, and token relays), + which is what makes the DCR front door discoverable to standard MCP clients.""" + result = _named_as_metadata_response(_bridge_server()) + + assert result["authorization_endpoint"] == "https://litellm.example.com/bridge_srv/authorize" + assert result["token_endpoint"] == "https://litellm.example.com/bridge_srv/token" + assert result["registration_endpoint"] == "https://litellm.example.com/bridge_srv/register" + + +def test_oauth_authorization_server_404_for_non_bridge_client_forwarded_server(): + """Without dcr_bridge a client-forwarded server keeps 404ing AS-metadata discovery: verbatim + upstream discovery is the contract and the gateway must not advertise itself as its AS.""" + with pytest.raises(HTTPException) as exc: + _named_as_metadata_response(_bridge_server(dcr_bridge=None)) + + assert exc.value.status_code == 404 + + +async def _bridge_register_response(server, request_payload, persist_credentials=False): + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + register_client_with_server, + ) + + mock_response = MagicMock() + mock_response.status_code = 201 + mock_response.json.return_value = { + "client_id": "upstream-issued-client", + "redirect_uris": request_payload.get("redirect_uris", []), + "token_endpoint_auth_method": "none", + } + mock_response.raise_for_status = MagicMock() + mock_async_client = MagicMock() + mock_async_client.post = AsyncMock(return_value=mock_response) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=mock_async_client, + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._persist_dcr_client_registration", + new_callable=AsyncMock, + ) as mock_persist, + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", + new_callable=AsyncMock, + return_value=False, + ), + ): + response = await register_client_with_server( + request=_bridge_mock_request(), + mcp_server=server, + client_name=request_payload.get("client_name", ""), + grant_types=request_payload.get("grant_types"), + response_types=request_payload.get("response_types"), + token_endpoint_auth_method=request_payload.get("token_endpoint_auth_method"), + persist_credentials=persist_credentials, + client_redirect_uris=request_payload.get("redirect_uris"), + ) + return response, mock_async_client, mock_persist + + +@pytest.mark.asyncio +async def test_register_bridge_relay_forwards_client_redirect_uris(): + """The bridge relay arm registers the client's own redirect_uris upstream with public-client + defaults and relays the upstream response verbatim, so the upstream AS enforces the redirect + binding for that client and the auth code never transits the gateway.""" + import json + + response, mock_async_client, _ = await _bridge_register_response( + _bridge_server(), + {"client_name": "Claude", "redirect_uris": [_BRIDGE_CLIENT_REDIRECT]}, + ) + + posted = mock_async_client.post.call_args.kwargs["json"] + assert posted["redirect_uris"] == [_BRIDGE_CLIENT_REDIRECT] + assert posted["grant_types"] == ["authorization_code", "refresh_token"] + assert posted["response_types"] == ["code"] + assert posted["token_endpoint_auth_method"] == "none" + + payload = json.loads(response.body.decode("utf-8")) + assert payload["client_id"] == "upstream-issued-client" + + +@pytest.mark.asyncio +async def test_register_bridge_relay_requires_redirect_uris(): + with pytest.raises(HTTPException) as exc: + await _bridge_register_response(_bridge_server(), {"client_name": "Claude"}) + + assert exc.value.status_code == 400 + assert "redirect_uris" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_register_bridge_relay_surfaces_upstream_error_not_500(): + """A bridge relay registration the upstream rejects must surface the upstream status and its + RFC 7591 error body to the client, not a bare 500 that hides the real reason.""" + import httpx + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + register_client_with_server, + ) + + error_response = MagicMock() + error_response.status_code = 400 + error_response.text = '{"error":"invalid_redirect_uri","error_description":"redirect_uri not allowed"}' + error_response.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError("bad", request=MagicMock(), response=error_response) + ) + mock_async_client = MagicMock() + mock_async_client.post = AsyncMock(return_value=error_response) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=mock_async_client, + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", + new_callable=AsyncMock, + return_value=False, + ), + ): + with pytest.raises(HTTPException) as exc: + await register_client_with_server( + request=_bridge_mock_request(), + mcp_server=_bridge_server(), + client_name="Claude", + grant_types=None, + response_types=None, + token_endpoint_auth_method=None, + client_redirect_uris=[_BRIDGE_CLIENT_REDIRECT], + ) + + assert exc.value.status_code == 400 + assert "invalid_redirect_uri" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_register_non_bridge_upstream_error_still_raises_500(): + """Non-bridge DCR keeps its pre-change behavior: raise_for_status propagates so the flag-off + contract is byte-identical; only the bridge relay arm relays the upstream status.""" + import httpx + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + register_client_with_server, + ) + + error_response = MagicMock() + error_response.status_code = 400 + error_response.text = '{"error":"invalid_client_metadata"}' + error_response.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError("bad", request=MagicMock(), response=error_response) + ) + mock_async_client = MagicMock() + mock_async_client.post = AsyncMock(return_value=error_response) + + oauth2_server = _bridge_server(auth_type=MCPAuth.oauth2, dcr_bridge=None) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=mock_async_client, + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._reuse_persisted_dcr_client_if_available", + new_callable=AsyncMock, + return_value=False, + ), + ): + with pytest.raises(httpx.HTTPStatusError): + await register_client_with_server( + request=_bridge_mock_request(), + mcp_server=oauth2_server, + client_name="Claude", + grant_types=None, + response_types=None, + token_endpoint_auth_method=None, + ) + + +@pytest.mark.asyncio +async def test_register_bridge_relay_never_persists(): + """Relayed registrations belong to individual clients; persisting one as the server's own DCR + client would hand every future caller the first client's identity.""" + _, _, mock_persist = await _bridge_register_response( + _bridge_server(), + {"client_name": "Claude", "redirect_uris": [_BRIDGE_CLIENT_REDIRECT]}, + persist_credentials=True, + ) + + mock_persist.assert_not_called() + + async def _exchange_persistence_attempted_for_auth_type(auth_type) -> bool: """Run exchange_token_with_server for a server of ``auth_type`` and report whether it attempted to persist the exchanged token server-side. The client-forwarded token modes must not persist: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py index 6de35ebc524..ec285f8eba0 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py @@ -571,3 +571,53 @@ async def test_oauth_protected_resource_true_passthrough_returns_upstream_metada assert result["resource"] == "https://upstream.example.com/mcp" finally: global_mcp_server_manager.registry.clear() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_type", [MCPAuth.true_passthrough, MCPAuth.oauth_delegate]) +@pytest.mark.parametrize("use_standard_pattern", [True, False]) +async def test_oauth_protected_resource_dcr_bridge_returns_gateway_facade(auth_type, use_standard_pattern): + """With dcr_bridge on, discovery flips from the upstream-verbatim contract to the gateway + facade: resource is the gateway URL the client dialed and authorization_servers names the + gateway's per-server AS, so DCR-only clients (which enforce the RFC 9728 resource match) + can register and sign in through the gateway. No upstream metadata fetch happens.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + global_mcp_server_manager.registry.clear() + bridge_server = MCPServer( + server_id="bridge-1", + name="sample_docs", + server_name="sample_docs", + alias="sample_docs", + url="https://upstream.example.com/mcp", + transport=MCPTransport.http, + auth_type=auth_type, + dcr_bridge=True, + scopes=["read"], + registration_url="https://okta.example.com/register", + ) + global_mcp_server_manager.registry[bridge_server.server_id] = bridge_server + + try: + with patch.object(discoverable_endpoints, "get_async_httpx_client") as mock_client_factory: + result = await _build_oauth_protected_resource_response( + request=_make_request(), + mcp_server_name="sample_docs", + use_standard_pattern=use_standard_pattern, + ) + finally: + global_mcp_server_manager.registry.clear() + + expected_resource = ( + "https://gateway.example.com/mcp/sample_docs" + if use_standard_pattern + else "https://gateway.example.com/sample_docs/mcp" + ) + assert result == { + "authorization_servers": ["https://gateway.example.com/sample_docs"], + "resource": expected_resource, + "scopes_supported": ["read"], + } + mock_client_factory.assert_not_called() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index f81e460a8be..7c55bd4560f 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -6678,6 +6678,67 @@ class TestMCPToolsListAuthSurfacing: assert await manager._get_tools_from_server(server) == [] + @pytest.mark.asyncio + async def test_get_tools_from_server_suppresses_upstream_challenge_for_dcr_bridge(self): + """A dcr_bridge server must never relay the upstream's own WWW-Authenticate: it points + clients at the upstream protected-resource metadata, which fails the RFC 9728 resource + match against the gateway URL they dialed. Stripping it makes the single-server route + fabricate the gateway well-known challenge, whose content is the bridge facade.""" + from litellm.proxy._experimental.mcp_server.exceptions import ( + MCPUpstreamAuthError, + ) + from litellm.types.mcp import MCPAuth + + manager = MCPServerManager() + bridge_server = MCPServer( + server_id="bridge-srv", + name="bridge-srv", + transport=MCPTransport.http, + auth_type=MCPAuth.true_passthrough, + dcr_bridge=True, + ) + upstream_challenge = 'Bearer resource_metadata="https://upstream.example/.well-known/oauth-protected-resource"' + client = MagicMock() + client.list_tools = AsyncMock(side_effect=_upstream_status_error(401, upstream_challenge)) + manager._create_mcp_client = AsyncMock(return_value=client) + + with pytest.raises(MCPUpstreamAuthError) as exc_info: + await manager._get_tools_from_server(bridge_server) + + assert exc_info.value.status_code == 401 + assert exc_info.value.www_authenticate is None + assert exc_info.value.server_name == "bridge-srv" + + @pytest.mark.asyncio + async def test_get_tools_from_server_suppresses_resolver_challenge_for_dcr_bridge(self): + """The client-build-time HTTPException conversion path applies the same suppression.""" + from litellm.proxy._experimental.mcp_server.exceptions import ( + MCPUpstreamAuthError, + ) + from litellm.types.mcp import MCPAuth + + manager = MCPServerManager() + bridge_server = MCPServer( + server_id="bridge-srv", + name="bridge-srv", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + dcr_bridge=True, + ) + manager._create_mcp_client = AsyncMock( + side_effect=HTTPException( + status_code=401, + detail="Unauthorized", + headers={"WWW-Authenticate": 'Bearer resource_metadata="https://upstream.example/prm"'}, + ) + ) + + with pytest.raises(MCPUpstreamAuthError) as exc_info: + await manager._get_tools_from_server(bridge_server) + + assert exc_info.value.status_code == 401 + assert exc_info.value.www_authenticate is None + @pytest.mark.asyncio async def test_aggregate_list_tools_absorbs_unauthenticated_server(self): from litellm.proxy._experimental.mcp_server.exceptions import ( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py index da183e8d02a..be95b3f3f73 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py @@ -1400,6 +1400,76 @@ async def test_handle_streamable_http_mcp_true_passthrough_without_token_surface probe_client.post.assert_awaited_once() +@pytest.mark.asyncio +async def test_handle_streamable_http_mcp_true_passthrough_dcr_bridge_challenges_with_gateway_metadata(): + """With dcr_bridge on, the missing-token challenge names the GATEWAY's well-known instead of + relaying the upstream's: the gateway is the authorization server for bridge clients, and the + upstream's own challenge would point them at metadata that fails the RFC 9728 resource match. + The upstream probe is skipped entirely; the gateway can answer authoritatively.""" + from fastapi import HTTPException + + try: + from litellm.proxy._experimental.mcp_server.server import ( + handle_streamable_http_mcp, + session_manager_stateful, + ) + except ImportError: + pytest.skip("MCP server not available") + + probe_client = MagicMock() + probe_client.post = AsyncMock() + + scope = _passthrough_mode_scope("tp_bridge_server") + receive = AsyncMock( + return_value={ + "type": "http.request", + "body": b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}', + "more_body": False, + } + ) + send = AsyncMock() + user_auth = MagicMock() + user_auth.user_id = None + bridge_server = _build_passthrough_mode_server("tp_bridge_server", MCPAuth.true_passthrough).model_copy( + update={"dcr_bridge": True} + ) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + new_callable=AsyncMock, + return_value=(user_auth, None, ["tp_bridge_server"], None, None, None), + ), + patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch( + "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + True, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", + return_value=probe_client, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + return_value=bridge_server, + ), + patch.object( + session_manager_stateful, + "handle_request", + new_callable=AsyncMock, + ) as mock_handle_request, + ): + with pytest.raises(HTTPException) as exc_info: + await handle_streamable_http_mcp(scope, receive, send) + + assert mock_handle_request.await_count == 0 + assert exc_info.value.status_code == 401 + challenge = exc_info.value.headers["www-authenticate"] + assert "/.well-known/oauth-protected-resource/tp_bridge_server/mcp" in challenge + assert "upstream.example.com" not in challenge + probe_client.post.assert_not_awaited() + + @pytest.mark.asyncio async def test_handle_streamable_http_mcp_true_passthrough_with_token_skips_probe_and_challenge(): """When the true_passthrough caller already carries an Authorization the diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py index 82c0aa3ccda..9bc0a525326 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py @@ -1356,3 +1356,311 @@ def test_truncate_csv_at_tool_name_boundary_edges(): assert _truncate_csv_at_tool_name_boundary(tool_names_csv="ab,cd,ef", max_length=5) == "ab,cd" assert _truncate_csv_at_tool_name_boundary(tool_names_csv="ab,cd,ef", max_length=4) == "ab" assert _truncate_csv_at_tool_name_boundary(tool_names_csv="single_name_longer_than_cap", max_length=10) == "" + + +def _make_context_window_raising_router(state): + """ + Mock litellm Router whose embedding call raises ContextWindowExceededError + once state["raise_context_error"] is flipped to True. + """ + import litellm + from litellm.types.utils import Embedding, EmbeddingResponse + + def mock_embedding_sync(*args, **kwargs): + if state["raise_context_error"]: + raise litellm.ContextWindowExceededError( + message="Invalid 'input[0]': maximum input length is 8192 tokens.", + model="text-embedding-3-small", + llm_provider="openai", + ) + return EmbeddingResponse( + data=[Embedding(embedding=[0.1] * 1536, index=0, object="embedding")], + model="text-embedding-3-small", + object="list", + usage={"prompt_tokens": 10, "total_tokens": 10}, + ) + + async def mock_embedding_async(*args, **kwargs): + return mock_embedding_sync(*args, **kwargs) + + mock_router = Mock() + mock_router.embedding = mock_embedding_sync + mock_router.aembedding = mock_embedding_async + return mock_router + + +def _make_context_window_filter(state, top_k: int = 3): + from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + SemanticMCPToolFilter, + ) + + return SemanticMCPToolFilter( + embedding_model="text-embedding-3-small", + litellm_router_instance=_make_context_window_raising_router(state), + top_k=top_k, + similarity_threshold=0.3, + enabled=True, + ) + + +@pytest.mark.asyncio +async def test_semantic_filter_fails_closed_on_query_time_context_window_error(): + """ + Regression test (LIT-4284): a context-window overflow while embedding the + user query must fail closed with a typed error instead of silently + returning all tools (previously reported as N->N "success"). + """ + from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + SemanticToolFilterContextWindowError, + ) + + state = {"raise_context_error": False} + filter_instance = _make_context_window_filter(state) + + tools = [ + MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"}) + for i in range(5) + ] + filter_instance._build_router(tools) + assert filter_instance.tool_router is not None + + state["raise_context_error"] = True + with pytest.raises(SemanticToolFilterContextWindowError) as exc_info: + await filter_instance.filter_tools(query="send an email", available_tools=tools) + + message = str(exc_info.value) + assert "context window" in message + assert "text-embedding-3-small" in message + print("✅ Query-time context window overflow fails closed") + + +@pytest.mark.asyncio +async def test_semantic_filter_records_build_time_context_window_error(): + """ + Regression test (LIT-4284): a context-window overflow while embedding the + tool descriptions at router-build time must be recorded (not raised out of + the build, which previously left the hook unregistered and filtering + silently disabled) and must fail subsequent filtering closed. + """ + from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + SemanticToolFilterContextWindowError, + ) + + state = {"raise_context_error": True} + filter_instance = _make_context_window_filter(state) + + tools = [ + MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"}) + for i in range(5) + ] + filter_instance._build_router(tools) + + assert filter_instance.tool_router is None + assert filter_instance.context_window_error is not None + + with pytest.raises(SemanticToolFilterContextWindowError) as exc_info: + await filter_instance.filter_tools(query="send an email", available_tools=tools) + + message = str(exc_info.value) + assert "context window" in message + assert "tool descriptions" in message + print("✅ Build-time context window overflow is recorded and fails closed") + + +@pytest.mark.asyncio +async def test_semantic_filter_hook_fails_closed_on_context_window_error(): + """ + Regression test (LIT-4284): the pre-call hook must reject the request with + an actionable HTTP 400 when the embedding model overflows its context + window, instead of forwarding all tools and emitting an N->N success + header. + """ + from fastapi import HTTPException + + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + + state = {"raise_context_error": False} + filter_instance = _make_context_window_filter(state) + + tools = [ + MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"}) + for i in range(5) + ] + filter_instance._build_router(tools) + hook = SemanticToolFilterHook(filter_instance) + + state["raise_context_error"] = True + data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Send an email"}], + "tools": tools, + "metadata": {}, + } + + with pytest.raises(HTTPException) as exc_info: + await hook.async_pre_call_hook( + user_api_key_dict=Mock(), + cache=Mock(), + data=data, + call_type="completion", + ) + + assert exc_info.value.status_code == 400 + error_message = exc_info.value.detail["error"] + assert "context window" in error_message + assert "text-embedding-3-small" in error_message + assert "larger context window" in error_message + assert "maximum input length" not in error_message + print("✅ Hook fails closed with actionable 400 on context window overflow") + + +@pytest.mark.asyncio +async def test_semantic_filter_hook_fails_closed_on_expanded_tools_context_window_error(): + """ + Regression test (LIT-4284): the litellm_proxy MCP expansion path (driven + by the dashboard test panel via /v1/responses) must also fail closed with + an actionable HTTP 400 instead of being swallowed by the expansion + catch-all. + """ + from fastapi import HTTPException + + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + + state = {"raise_context_error": False} + filter_instance = _make_context_window_filter(state) + + registry_tools = [ + MCPTool(name=f"srv-tool_{i}", description=f"Registry tool {i}", inputSchema={"type": "object"}) + for i in range(5) + ] + filter_instance._build_router(registry_tools) + + expanded_tools = [ + { + "type": "function", + "name": f"srv-tool_{i}", + "description": f"Registry tool {i}", + "parameters": {"type": "object", "properties": {}}, + } + for i in range(5) + ] + + hook = SemanticToolFilterHook(filter_instance) + hook._expand_mcp_tools = AsyncMock( # type: ignore[method-assign] + return_value=expanded_tools + ) + + state["raise_context_error"] = True + data = { + "model": "gpt-4", + "input": [{"role": "user", "content": "Send an email", "type": "message"}], + "tools": [ + { + "type": "mcp", + "server_url": "litellm_proxy", + "require_approval": "never", + } + ], + "metadata": {}, + } + + with pytest.raises(HTTPException) as exc_info: + await hook.async_pre_call_hook( + user_api_key_dict=Mock(), + cache=Mock(), + data=data, + call_type="aresponses", + ) + + assert exc_info.value.status_code == 400 + error_message = exc_info.value.detail["error"] + assert "context window" in error_message + assert "larger context window" in error_message + assert "maximum input length" not in error_message + print("✅ Expansion path fails closed with actionable 400 on context window overflow") + + +@pytest.mark.asyncio +async def test_semantic_filter_hook_ignores_build_error_for_native_only_tools(): + """ + A recorded build-time context-window error must only block requests that + rely on MCP tool filtering; requests carrying only native tools pass + through untouched. + """ + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + + state = {"raise_context_error": True} + filter_instance = _make_context_window_filter(state) + + mcp_tools = [ + MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"}) + for i in range(3) + ] + filter_instance._build_router(mcp_tools) + assert filter_instance.context_window_error is not None + + hook = SemanticToolFilterHook(filter_instance) + + native_tools = [ + { + "type": "function", + "function": {"name": "local_fn", "description": "A local function", "parameters": {}}, + } + ] + data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Send an email"}], + "tools": native_tools, + "metadata": {}, + } + + result = await hook.async_pre_call_hook( + user_api_key_dict=Mock(), + cache=Mock(), + data=data, + call_type="completion", + ) + + assert result is not None + assert result["tools"] == native_tools + print("✅ Native-only requests pass through despite recorded build error") + + +def test_is_context_window_error_detection_variants(): + """ + _is_context_window_error must detect the overflow in every shape it + reaches filter_tools in: the raw typed exception, the encoder's + explicitly chained ValueError wrapper, an implicitly chained wrapper, + and a bare error whose message carries a known overflow phrase; a + generic error must not match. + """ + import litellm + from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + _is_context_window_error, + ) + + cwe = litellm.ContextWindowExceededError( + message="Invalid 'input[0]': maximum input length is 8192 tokens.", + model="text-embedding-3-small", + llm_provider="openai", + ) + assert _is_context_window_error(cwe) + + try: + raise ValueError("Internal_litellm_router API call failed") from cwe + except ValueError as explicitly_chained: + assert _is_context_window_error(explicitly_chained) + + try: + try: + raise litellm.ContextWindowExceededError( + message="overflow", model="m", llm_provider="openai" + ) + except litellm.ContextWindowExceededError: + raise ValueError("wrapper without explicit chaining") + except ValueError as implicitly_chained: + assert _is_context_window_error(implicitly_chained) + + assert _is_context_window_error(ValueError("Invalid 'input[0]': maximum input length is 8192 tokens.")) + assert not _is_context_window_error(ValueError("A generic API error occurred.")) + assert not _is_context_window_error(None) diff --git a/tests/test_litellm/proxy/management_endpoints/test_coordination_redis_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_coordination_redis_endpoints.py new file mode 100644 index 00000000000..4e6bfc4c063 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_coordination_redis_endpoints.py @@ -0,0 +1,577 @@ +""" +Unit tests for coordination Redis settings management endpoints +""" + +import asyncio +import json +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import HTTPException + +sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path + +import litellm +from litellm.caching.caching import RedisCache +from litellm.caching.redis_cluster_cache import RedisClusterCache +from litellm.proxy._types import LitellmTableNames, LitellmUserRoles +from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.proxy.management_endpoints.coordination_redis_endpoints import ( + _REDACTED_VALUE, + CoordinationRedisSettingsRequest, + get_coordination_redis_settings, + check_coordination_redis_connection, + update_coordination_redis_settings, +) +from litellm.types.management_endpoints.coordination_redis_endpoints import ( + COORDINATION_REDIS_SETTINGS_FIELDS, +) + +_SAVED_SETTINGS = { + "host": "coord-redis.example.com", + "port": 6379, + "password": "super-secret-redis-pw", + "url": "redis://:super-secret-redis-pw@coord-redis.example.com:6379", + "sentinel_password": "super-secret-sentinel-pw", +} + + +def _admin_auth() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="hashed", + user_id="admin-user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + +def _prisma_with_general_settings(general_settings: dict | None) -> MagicMock: + """A prisma client whose LiteLLM_Config `general_settings` row holds ``general_settings``.""" + row = None + if general_settings is not None: + row = MagicMock() + row.param_value = json.dumps(general_settings) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=row) + mock_prisma.db.litellm_config.upsert = AsyncMock() + return mock_prisma + + +def _proxy_config(file_general_settings: dict | None = None) -> MagicMock: + proxy_config = MagicMock() + proxy_config.get_config_state = MagicMock( + return_value={"general_settings": file_general_settings or {}}, + ) + return proxy_config + + +# ── GET /coordination_redis/settings ────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_get_redacts_every_credential_field(): + """password, sentinel_password and the (password-bearing) url never leave the + server in plaintext; non-credential fields come back untouched.""" + with ( + patch( + "litellm.proxy.proxy_server.prisma_client", + _prisma_with_general_settings({"coordination_redis": _SAVED_SETTINGS}), + ), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + serialized = json.dumps(response.model_dump()) + assert "super-secret-redis-pw" not in serialized + assert "super-secret-sentinel-pw" not in serialized + + assert response.values["password"] == _REDACTED_VALUE + assert response.values["sentinel_password"] == _REDACTED_VALUE + assert response.values["url"] == _REDACTED_VALUE + assert response.values["host"] == "coord-redis.example.com" + assert response.values["port"] == 6379 + + # field metadata is hydrated with the same redacted values + by_name = {field.field_name: field for field in response.fields} + assert by_name["password"].field_value == _REDACTED_VALUE + assert by_name["host"].field_value == "coord-redis.example.com" + + +@pytest.mark.asyncio +async def test_get_source_is_coordination_redis_when_block_present(monkeypatch): + """An explicit block wins even when a Redis cache backend and REDIS_* env both exist.""" + monkeypatch.setattr(litellm, "cache", MagicMock(cache=MagicMock(spec=RedisCache))) + monkeypatch.setenv("REDIS_HOST", "env-redis") + + with ( + patch( + "litellm.proxy.proxy_server.prisma_client", + _prisma_with_general_settings({"coordination_redis": _SAVED_SETTINGS}), + ), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source == "coordination_redis" + + +@pytest.mark.asyncio +async def test_get_source_reads_block_from_yaml_config_when_db_row_absent(monkeypatch): + """A block set in config.yaml (not the DB) still reports source=coordination_redis.""" + monkeypatch.setattr(litellm, "cache", None) + monkeypatch.delenv("REDIS_HOST", raising=False) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings(None)), + patch( + "litellm.proxy.proxy_server.proxy_config", + _proxy_config({"coordination_redis": {"host": "yaml-redis"}}), + ), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source == "coordination_redis" + assert response.values["host"] == "yaml-redis" + + +@pytest.mark.parametrize("cache_backend_cls", [RedisCache, RedisClusterCache]) +@pytest.mark.asyncio +async def test_get_source_is_cache_backend_when_no_block(monkeypatch, cache_backend_cls): + """With no explicit block, a plain-Redis response-cache backend is borrowed — + which beats the REDIS_* env fallback.""" + monkeypatch.setattr(litellm, "cache", MagicMock(cache=MagicMock(spec=cache_backend_cls))) + monkeypatch.setenv("REDIS_HOST", "env-redis") + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source == "cache_backend" + assert response.values == {} + + +@pytest.mark.asyncio +async def test_get_source_is_environment_when_no_block_and_non_redis_cache(monkeypatch): + """A non-Redis cache backend falls through to the REDIS_* env fallback.""" + monkeypatch.setattr(litellm, "cache", MagicMock(cache=MagicMock())) + monkeypatch.setenv("REDIS_HOST", "env-redis") + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source == "environment" + + +@pytest.mark.asyncio +async def test_get_source_is_none_when_nothing_configured(monkeypatch): + monkeypatch.setattr(litellm, "cache", None) + for env_var in ("REDIS_HOST", "REDIS_URL", "REDIS_CLUSTER_NODES", "REDIS_SENTINEL_NODES"): + monkeypatch.delenv(env_var, raising=False) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source is None + + +@pytest.mark.asyncio +async def test_get_source_does_not_build_a_client(monkeypatch): + """The env-fallback probe is read-only: no Redis client is constructed on GET.""" + monkeypatch.setattr(litellm, "cache", None) + monkeypatch.setenv("REDIS_HOST", "env-redis") + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server._build_redis_usage_cache") as mock_build, + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source == "environment" + mock_build.assert_not_called() + + +@pytest.mark.asyncio +async def test_get_rejects_non_admin(): + with pytest.raises(HTTPException) as exc_info: + await get_coordination_redis_settings( + user_api_key_dict=UserAPIKeyAuth(api_key="hashed", user_role=LitellmUserRoles.INTERNAL_USER) + ) + assert exc_info.value.status_code == 403 + + +def test_fields_cover_every_coordination_redis_param(): + """The declarative field list drives the Admin UI form; it must stay in sync + with the model the backend validates against.""" + from litellm.proxy._types import CoordinationRedisParams + + assert {field.field_name for field in COORDINATION_REDIS_SETTINGS_FIELDS} == set( + CoordinationRedisParams.model_fields.keys() + ) + + by_name = {field.field_name: field for field in COORDINATION_REDIS_SETTINGS_FIELDS} + assert by_name["startup_nodes"].section == "cluster" + assert by_name["sentinel_nodes"].section == "sentinel" + assert by_name["host"].section == "connection" + + +# ── POST /coordination_redis/settings ───────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_update_rejects_settings_without_a_connection_target(monkeypatch): + """A block with no host/url/startup_nodes/sentinel_nodes would blow up at + startup; reject it at write time and persist nothing.""" + monkeypatch.setattr(litellm, "store_audit_logs", False) + mock_prisma = _prisma_with_general_settings({}) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + with pytest.raises(HTTPException) as exc_info: + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest(settings={"ssl": True, "service_name": "mymaster"}), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + assert exc_info.value.status_code == 400 + mock_prisma.db.litellm_config.upsert.assert_not_called() + + +@pytest.mark.asyncio +async def test_update_persists_into_the_general_settings_config_row(monkeypatch): + """Settings land under `general_settings.coordination_redis` in LiteLLM_Config + (the row startup merges over the yaml config), and sibling general_settings + keys survive the write.""" + monkeypatch.setattr(litellm, "store_audit_logs", False) + mock_prisma = _prisma_with_general_settings({"master_key": "sk-1234"}) + invalidated: list[str] = [] + + async def _capture_invalidate(param_name: str) -> None: + invalidated.append(param_name) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints.invalidate_config_param", + new=_capture_invalidate, + ), + ): + response = await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest( + settings={"host": "coord-redis.example.com", "port": 6379, "password": "pw"} + ), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + upsert_kwargs = mock_prisma.db.litellm_config.upsert.call_args.kwargs + assert upsert_kwargs["where"] == {"param_name": "general_settings"} + persisted = json.loads(upsert_kwargs["data"]["update"]["param_value"]) + assert persisted["coordination_redis"] == { + "host": "coord-redis.example.com", + "port": 6379, + "password": "pw", + } + assert persisted["master_key"] == "sk-1234" + assert invalidated == ["general_settings"] + + # the response echoes the saved settings back redacted + assert response["settings"]["password"] == _REDACTED_VALUE + assert response["settings"]["host"] == "coord-redis.example.com" + + +@pytest.mark.asyncio +async def test_update_persists_os_environ_refs_verbatim(monkeypatch): + """`os.environ/VAR` refs are resolved only to validate; the ref itself is what + gets stored, so the credential never lands in the DB.""" + monkeypatch.setattr(litellm, "store_audit_logs", False) + monkeypatch.setenv("MY_REDIS_HOST", "resolved-host") + mock_prisma = _prisma_with_general_settings({}) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints.invalidate_config_param", + new=AsyncMock(), + ), + ): + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest(settings={"host": "os.environ/MY_REDIS_HOST"}), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + persisted = json.loads(mock_prisma.db.litellm_config.upsert.call_args.kwargs["data"]["update"]["param_value"]) + assert persisted["coordination_redis"] == {"host": "os.environ/MY_REDIS_HOST"} + + +@pytest.mark.asyncio +async def test_update_keeps_saved_credential_when_client_echoes_the_redaction_marker(monkeypatch): + """The UI reads settings back redacted; re-submitting them must not persist + `***REDACTED***` as the password.""" + monkeypatch.setattr(litellm, "store_audit_logs", False) + mock_prisma = _prisma_with_general_settings({"coordination_redis": _SAVED_SETTINGS}) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints.invalidate_config_param", + new=AsyncMock(), + ), + ): + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest( + settings={"host": "new-host", "port": 6380, "password": _REDACTED_VALUE} + ), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + persisted = json.loads(mock_prisma.db.litellm_config.upsert.call_args.kwargs["data"]["update"]["param_value"]) + assert persisted["coordination_redis"]["password"] == "super-secret-redis-pw" + assert persisted["coordination_redis"]["host"] == "new-host" + + +@pytest.mark.asyncio +async def test_update_emits_audit_log_with_values_redacted(monkeypatch): + monkeypatch.setattr(litellm, "store_audit_logs", True) + mock_prisma = _prisma_with_general_settings({}) + audit_calls = [] + + async def capture(request_data): + audit_calls.append(request_data) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints.invalidate_config_param", + new=AsyncMock(), + ), + patch("litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", new=capture), + ): + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest( + settings={"host": "coord-redis.example.com", "password": "super-secret-redis-pw"} + ), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + for _ in range(3): + await asyncio.sleep(0) + + assert len(audit_calls) == 1 + log = audit_calls[0] + assert log.table_name == LitellmTableNames.CONFIG_TABLE_NAME + assert log.object_id == "coordination_redis" + assert log.action == "created" # no prior block → create + + after = json.loads(log.updated_values) + assert set(after["settings"].keys()) == {"host", "password"} + assert "super-secret-redis-pw" not in log.updated_values + assert "coord-redis.example.com" not in log.updated_values + + +@pytest.mark.asyncio +async def test_update_audit_action_is_updated_when_a_block_already_exists(monkeypatch): + monkeypatch.setattr(litellm, "store_audit_logs", True) + mock_prisma = _prisma_with_general_settings({"coordination_redis": {"host": "old-host"}}) + audit_calls = [] + + async def capture(request_data): + audit_calls.append(request_data) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints.invalidate_config_param", + new=AsyncMock(), + ), + patch("litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", new=capture), + ): + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest(settings={"host": "new-host"}), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + for _ in range(3): + await asyncio.sleep(0) + + assert audit_calls[0].action == "updated" + assert json.loads(audit_calls[0].before_value)["settings"] == {"host": _REDACTED_VALUE} + + +@pytest.mark.asyncio +async def test_update_rejects_non_admin(): + with pytest.raises(HTTPException) as exc_info: + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest(settings={"host": "coord-redis.example.com"}), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed", user_role=LitellmUserRoles.INTERNAL_USER), + litellm_changed_by=None, + ) + assert exc_info.value.status_code == 403 + + +# ── POST /coordination_redis/settings/test ──────────────────────────────────── + + +@pytest.mark.asyncio +async def test_connection_test_returns_healthy_on_successful_ping(): + mock_client = MagicMock() + mock_client.ping = AsyncMock(return_value=True) + mock_client.disconnect = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server._build_redis_usage_cache", return_value=mock_client) as mock_build, + ): + response = await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest( + settings={"host": "coord-redis.example.com", "port": 6379, "password": "pw"} + ), + user_api_key_dict=_admin_auth(), + ) + + assert response.status == "healthy" + assert response.error is None + assert mock_build.call_args.args[0] == {"host": "coord-redis.example.com", "port": 6379, "password": "pw"} + mock_client.ping.assert_awaited_once() + mock_client.disconnect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_connection_test_reports_unhealthy_without_leaking_the_password(): + """Redis client errors echo the connection url back; the password must be + scrubbed out of the error the admin sees.""" + mock_client = MagicMock() + mock_client.ping = AsyncMock( + side_effect=ConnectionError("Error connecting to redis://:super-secret-redis-pw@coord-redis.example.com:6379") + ) + mock_client.disconnect = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server._build_redis_usage_cache", return_value=mock_client), + ): + response = await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest( + settings={ + "host": "coord-redis.example.com", + "url": "redis://:super-secret-redis-pw@coord-redis.example.com:6379", + "password": "super-secret-redis-pw", + } + ), + user_api_key_dict=_admin_auth(), + ) + + assert response.status == "unhealthy" + assert response.error is not None + assert "super-secret-redis-pw" not in response.error + assert _REDACTED_VALUE in response.error + mock_client.disconnect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_connection_test_uses_the_saved_password_for_a_redacted_field(): + """An admin re-testing settings read back from GET sends `***REDACTED***`; + the saved credential is what actually gets dialed.""" + mock_client = MagicMock() + mock_client.ping = AsyncMock(return_value=True) + mock_client.disconnect = AsyncMock() + + with ( + patch( + "litellm.proxy.proxy_server.prisma_client", + _prisma_with_general_settings({"coordination_redis": _SAVED_SETTINGS}), + ), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server._build_redis_usage_cache", return_value=mock_client) as mock_build, + ): + response = await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest( + settings={"host": "coord-redis.example.com", "password": _REDACTED_VALUE} + ), + user_api_key_dict=_admin_auth(), + ) + + assert response.status == "healthy" + assert mock_build.call_args.args[0]["password"] == "super-secret-redis-pw" + + +@pytest.mark.asyncio +async def test_connection_test_times_out_instead_of_hanging(): + async def _never_returns(): + await asyncio.sleep(60) + + mock_client = MagicMock() + mock_client.ping = MagicMock(side_effect=lambda: _never_returns()) + mock_client.disconnect = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server._build_redis_usage_cache", return_value=mock_client), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints._PING_TIMEOUT_SECONDS", + 0.01, + ), + ): + response = await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest(settings={"host": "unreachable"}), + user_api_key_dict=_admin_auth(), + ) + + assert response.status == "unhealthy" + assert "timed out" in (response.error or "") + + +@pytest.mark.asyncio +async def test_connection_test_rejects_settings_without_a_connection_target(): + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + with pytest.raises(HTTPException) as exc_info: + await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest(settings={"ssl": True}), + user_api_key_dict=_admin_auth(), + ) + + assert exc_info.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_connection_test_rejects_non_admin(): + with pytest.raises(HTTPException) as exc_info: + await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest(settings={"host": "coord-redis.example.com"}), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed", user_role=LitellmUserRoles.INTERNAL_USER), + ) + assert exc_info.value.status_code == 403 diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index d06a1c16ab9..603d5cc15b7 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -5,6 +5,7 @@ import os import socket import subprocess import sys +import types from datetime import datetime, timedelta, timezone from pathlib import Path from unittest import mock @@ -23,6 +24,10 @@ sys.path.insert( ) # Adds the parent directory to the system-path import litellm +import litellm.proxy.proxy_server as proxy_server_module +from litellm.caching.caching import RedisCache +from litellm.caching.redis_cluster_cache import RedisClusterCache +from litellm.caching.dual_cache import DualCache from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.proxy_server import app, initialize @@ -9358,3 +9363,326 @@ def test_update_config_redacts_all_environment_variable_values( assert "db.internal" not in data["updated_values"] finally: restore() + + +class _EnvBuiltRedisCache(RedisCache): + """RedisCache stand-in that records its constructor kwargs and never + opens a network connection, so tests can assert which connection params + the proxy used to build its coordination Redis.""" + + def __init__(self, **kwargs): + self.init_kwargs = kwargs + + +def _run_init_cache_with_backend(cache_backend, redis_env_kwargs): + """Run ProxyConfig._init_cache with a stubbed response-cache backend and a + controlled REDIS_* environment, returning (redis_usage_cache, + spend_counter redis, config-cache redis) as observed after the call.""" + mock_litellm_cache = MagicMock() + mock_litellm_cache.cache = cache_backend + fresh_spend_cache = DualCache() + fresh_config_cache = types.SimpleNamespace(redis_cache=None) + + with ( + patch.object(proxy_server_module, "redis_usage_cache", None), + patch.object(proxy_server_module, "spend_counter_cache", fresh_spend_cache), + patch.object(proxy_server_module, "user_api_key_cache", DualCache()), + patch.object(proxy_server_module, "llm_router", None), + patch.object(proxy_server_module, "litellm_config_cache", fresh_config_cache), + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch( + "litellm._redis._redis_kwargs_from_environment", + return_value=redis_env_kwargs, + ), + patch("litellm.Cache", return_value=mock_litellm_cache), + ): + litellm.cache = None + resolved = proxy_server_module.ProxyConfig()._init_cache(cache_params={"type": "qdrant-semantic"}) + return ( + resolved, + fresh_spend_cache.redis_cache, + fresh_config_cache.redis_cache, + ) + + +def test_init_cache_non_redis_backend_builds_usage_redis_from_environment(): + """A semantic (non-Redis-KV) response cache must not disable the proxy's + coordination Redis: when REDIS_* env vars provide a connection, + _init_cache builds a standalone usage cache so cross-pod rate limits, + spend tracking, and the pod lock manager stay Redis-backed.""" + usage_cache, spend_redis, config_redis = _run_init_cache_with_backend( + cache_backend=object(), + redis_env_kwargs={"host": "coordination-redis", "port": "6379"}, + ) + + assert isinstance(usage_cache, _EnvBuiltRedisCache) + assert usage_cache.init_kwargs["host"] == "coordination-redis" + assert spend_redis is usage_cache + assert config_redis is usage_cache + + +def test_init_cache_non_redis_backend_without_redis_env_stays_in_memory(): + """Without any REDIS_* connection info, a non-Redis response cache must + leave the coordination Redis unset instead of building a broken client.""" + usage_cache, spend_redis, config_redis = _run_init_cache_with_backend( + cache_backend=object(), + redis_env_kwargs={}, + ) + + assert usage_cache is None + assert spend_redis is None + assert config_redis is None + + +def test_init_cache_redis_backend_reuses_cache_backend_over_environment(): + """When the response cache itself is a plain Redis KV cache, it must be + reused as the coordination Redis; the REDIS_* environment fallback must + not construct a second client.""" + redis_backend = _EnvBuiltRedisCache(host="cache-params-host") + usage_cache, spend_redis, _ = _run_init_cache_with_backend( + cache_backend=redis_backend, + redis_env_kwargs={"host": "env-host"}, + ) + + assert usage_cache is redis_backend + assert usage_cache.init_kwargs["host"] == "cache-params-host" + assert spend_redis is redis_backend + + +class _EnvBuiltClusterCache(RedisClusterCache): + """RedisClusterCache stand-in that records constructor kwargs and never + opens a network connection.""" + + def __init__(self, **kwargs): + self.init_kwargs = kwargs + + +def _run_init_coordination_redis(config, env=None): + """Run ProxyConfig._init_coordination_redis against a stubbed module state, + returning (redis_usage_cache, spend_counter redis, config-cache redis).""" + fresh_spend_cache = DualCache() + fresh_config_cache = types.SimpleNamespace(redis_cache=None) + + with ( + patch.object(proxy_server_module, "redis_usage_cache", None), + patch.object(proxy_server_module, "spend_counter_cache", fresh_spend_cache), + patch.object(proxy_server_module, "user_api_key_cache", DualCache()), + patch.object(proxy_server_module, "litellm_config_cache", fresh_config_cache), + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + mock.patch.dict(os.environ, env or {}, clear=False), + ): + built = proxy_server_module.ProxyConfig()._init_coordination_redis(config=config) + return ( + built, + fresh_spend_cache.redis_cache, + fresh_config_cache.redis_cache, + ) + + +def test_init_coordination_redis_explicit_block_builds_standalone_client(): + """general_settings.coordination_redis must build the coordination Redis + even when no response cache is configured at all, and attach it to the + spend counter and config caches.""" + usage_cache, spend_redis, config_redis = _run_init_coordination_redis( + config={"general_settings": {"coordination_redis": {"host": "coord-host", "port": 6380}}}, + ) + + assert isinstance(usage_cache, _EnvBuiltRedisCache) + assert usage_cache.init_kwargs["host"] == "coord-host" + assert usage_cache.init_kwargs["port"] == 6380 + assert spend_redis is usage_cache + assert config_redis is usage_cache + + +def test_init_coordination_redis_resolves_os_environ_references(): + """os.environ/ values inside the coordination_redis block must be resolved + the same way cache_params values are.""" + usage_cache, _, _ = _run_init_coordination_redis( + config={"general_settings": {"coordination_redis": {"host": "os.environ/COORD_REDIS_HOST"}}}, + env={"COORD_REDIS_HOST": "resolved-host"}, + ) + + assert usage_cache.init_kwargs["host"] == "resolved-host" + + +def test_init_coordination_redis_startup_nodes_builds_cluster_client(): + """A coordination_redis block with startup_nodes must construct a cluster + client, so cluster-aware consumers (v3 rate limiter) take the cluster path.""" + usage_cache, _, _ = _run_init_coordination_redis( + config={ + "general_settings": { + "coordination_redis": {"startup_nodes": [{"host": "node-1", "port": 7000}]} + } + }, + ) + + assert isinstance(usage_cache, _EnvBuiltClusterCache) + assert usage_cache.init_kwargs["startup_nodes"] == [{"host": "node-1", "port": 7000}] + + +def test_init_coordination_redis_without_connection_target_raises(): + """A coordination_redis block with no host, url, startup_nodes, or + sentinel_nodes is a config error and must fail startup loudly instead of + silently running without coordination.""" + with pytest.raises(ValueError, match="connection target"): + _run_init_coordination_redis( + config={"general_settings": {"coordination_redis": {"ssl": True}}}, + ) + + +def test_init_coordination_redis_non_mapping_block_raises(): + """A scalar coordination_redis value is a config error.""" + with pytest.raises(ValueError, match="mapping"): + _run_init_coordination_redis( + config={"general_settings": {"coordination_redis": "redis://host:6379"}}, + ) + + +def test_init_coordination_redis_absent_leaves_usage_cache_unset(): + """Without the block, nothing changes: the coordination Redis stays unset + for the downstream borrow / env fallback logic to decide.""" + usage_cache, spend_redis, _ = _run_init_coordination_redis( + config={"general_settings": {}}, + ) + + assert usage_cache is None + assert spend_redis is None + + +def test_explicit_coordination_redis_takes_precedence_over_cache_backend(): + """When both an explicit coordination_redis block and a plain-Redis + response cache are configured, the explicit block must win; the cache + backend must not overwrite it.""" + fresh_spend_cache = DualCache() + fresh_config_cache = types.SimpleNamespace(redis_cache=None) + cache_backend = _EnvBuiltRedisCache(host="cache-backend-host") + mock_litellm_cache = MagicMock() + mock_litellm_cache.cache = cache_backend + + with ( + patch.object(proxy_server_module, "redis_usage_cache", None), + patch.object(proxy_server_module, "spend_counter_cache", fresh_spend_cache), + patch.object(proxy_server_module, "user_api_key_cache", DualCache()), + patch.object(proxy_server_module, "llm_router", None), + patch.object(proxy_server_module, "litellm_config_cache", fresh_config_cache), + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + patch("litellm.Cache", return_value=mock_litellm_cache), + ): + litellm.cache = None + proxy_config = proxy_server_module.ProxyConfig() + built = proxy_config._init_coordination_redis( + config={"general_settings": {"coordination_redis": {"host": "explicit-coord-host"}}} + ) + assert built is not None + proxy_server_module.redis_usage_cache = built + usage_cache = proxy_config._init_cache(cache_params={"type": "redis"}) + + assert isinstance(usage_cache, _EnvBuiltRedisCache) + assert usage_cache is not cache_backend + assert usage_cache.init_kwargs["host"] == "explicit-coord-host" + assert fresh_spend_cache.redis_cache is usage_cache + + +def test_env_fallback_builds_cluster_client_from_cluster_nodes_env(): + """A deployment whose only Redis env is REDIS_CLUSTER_NODES must still get + a coordination Redis from the env fallback, and it must be a cluster + client so cluster-aware consumers take the cluster path.""" + nodes = '[{"host": "cnode-1", "port": 7000}]' + with ( + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + patch("litellm._redis._redis_kwargs_from_environment", return_value={}), + mock.patch.dict(os.environ, {"REDIS_CLUSTER_NODES": nodes}, clear=False), + ): + result = proxy_server_module._build_redis_usage_cache_from_environment() + + assert isinstance(result, _EnvBuiltClusterCache) + assert result.init_kwargs["startup_nodes"] == [{"host": "cnode-1", "port": 7000}] + + +def test_env_fallback_builds_client_from_sentinel_nodes_env(): + """A sentinel-only environment (REDIS_SENTINEL_NODES, no host or url) must + also produce a coordination Redis from the env fallback.""" + with ( + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + patch("litellm._redis._redis_kwargs_from_environment", return_value={}), + mock.patch.dict(os.environ, {"REDIS_SENTINEL_NODES": '[["s1", 26379]]'}, clear=False), + ): + result = proxy_server_module._build_redis_usage_cache_from_environment() + + assert isinstance(result, _EnvBuiltRedisCache) + + +@pytest.mark.asyncio +async def test_startup_applies_coordination_redis_saved_in_database(): + """A coordination_redis block saved from the admin UI lives only in the + database, so startup must read it and build the coordination Redis from it. + Without this the save endpoint's "restart to apply" promise is false and the + proxy silently coordinates in per-pod memory.""" + fresh_spend_cache = DualCache() + fresh_config_cache = types.SimpleNamespace(redis_cache=None) + + with ( + patch.object(proxy_server_module, "spend_counter_cache", fresh_spend_cache), + patch.object(proxy_server_module, "user_api_key_cache", DualCache()), + patch.object(proxy_server_module, "litellm_config_cache", fresh_config_cache), + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + patch.object( + proxy_server_module, + "get_persisted_coordination_redis_settings", + AsyncMock(return_value={"host": "db-host", "port": 6381}), + ), + ): + result = await proxy_server_module.ProxyStartupEvent._init_coordination_redis_from_db( + litellm_settings={}, + llm_router=None, + ) + + assert isinstance(result, _EnvBuiltRedisCache) + assert result.init_kwargs["host"] == "db-host" + assert fresh_spend_cache.redis_cache is result + assert fresh_config_cache.redis_cache is result + + +@pytest.mark.asyncio +async def test_startup_ignores_database_coordination_redis_without_connection_target(): + """A persisted block with no host/url/cluster/sentinel must be ignored rather + than crashing startup or building a client that cannot connect.""" + with ( + patch.object(proxy_server_module, "spend_counter_cache", DualCache()), + patch.object(proxy_server_module, "litellm_config_cache", types.SimpleNamespace(redis_cache=None)), + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object( + proxy_server_module, + "get_persisted_coordination_redis_settings", + AsyncMock(return_value={"ssl": True}), + ), + ): + result = await proxy_server_module.ProxyStartupEvent._init_coordination_redis_from_db( + litellm_settings={}, + llm_router=None, + ) + + assert result is None + + +@pytest.mark.asyncio +async def test_startup_survives_database_read_failure_for_coordination_redis(): + """A config-row read failure must not block proxy startup.""" + with ( + patch.object( + proxy_server_module, + "get_persisted_coordination_redis_settings", + AsyncMock(side_effect=RuntimeError("db unreachable")), + ), + ): + result = await proxy_server_module.ProxyStartupEvent._init_coordination_redis_from_db( + litellm_settings={}, + llm_router=None, + ) + + assert result is None diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 0081e1c819f..305b402821b 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -629,3 +629,46 @@ def test_sync_client_url_used_when_no_cluster(mock_from_url, monkeypatch): get_redis_client() mock_from_url.assert_called_once() + + +@patch("litellm._redis.redis.Redis.from_url") +def test_explicit_host_outranks_environment_redis_url(mock_from_url, monkeypatch): + """ + An explicitly configured host must win over REDIS_URL in the environment. + + Otherwise the url branch strips the caller's host/port and the client + silently connects to whatever REDIS_URL names, so an explicit config block + (or a connection test typed into the admin UI) targets the wrong server. + """ + monkeypatch.setenv("REDIS_URL", "redis://env-host:6379") + monkeypatch.delenv("REDIS_CLUSTER_NODES", raising=False) + + client = get_redis_client(host="explicit-host", port=6380) + + mock_from_url.assert_not_called() + assert client.connection_pool.connection_kwargs["host"] == "explicit-host" + assert client.connection_pool.connection_kwargs["port"] == 6380 + + +@patch("litellm._redis.redis.Redis.from_url") +def test_explicit_url_still_wins_over_environment_host(mock_from_url, monkeypatch): + """An explicit url argument keeps taking the from_url path.""" + monkeypatch.setenv("REDIS_HOST", "env-host") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.delenv("REDIS_CLUSTER_NODES", raising=False) + + get_redis_client(url="redis://explicit-host:6380") + + mock_from_url.assert_called_once() + assert mock_from_url.call_args.kwargs["url"] == "redis://explicit-host:6380" + + +@patch("litellm._redis.redis.Redis.from_url") +def test_environment_redis_url_used_when_caller_names_no_target(mock_from_url, monkeypatch): + """With no caller-supplied connection target, REDIS_URL still drives the client.""" + monkeypatch.setenv("REDIS_URL", "redis://env-host:6379") + monkeypatch.delenv("REDIS_CLUSTER_NODES", raising=False) + + get_redis_client() + + mock_from_url.assert_called_once() diff --git a/ui/litellm-dashboard/e2e_tests/tests/auth/logout.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/auth/logout.spec.ts index d8644babfe3..1e2f2269dbe 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/auth/logout.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/auth/logout.spec.ts @@ -6,7 +6,8 @@ test.describe("Logout", () => { test("Clicking Logout clears the session and forces re-login on a protected page", async ({ page }) => { await page.goto("/ui"); - await expect(page.getByText("Virtual Keys")).toBeVisible({ timeout: 10_000 }); + // Scope to the sidebar; the top-bar breadcrumb also shows "Virtual Keys". + await expect(page.getByRole("complementary").getByText("Virtual Keys")).toBeVisible({ timeout: 10_000 }); // Open the navbar User dropdown. The trigger button exposes an aria-label // of "Account menu — — signed in as ", and the antd Dropdown diff --git a/ui/litellm-dashboard/e2e_tests/tests/auth/proxyLogoutUrl.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/auth/proxyLogoutUrl.spec.ts index 6358fcf438e..9faf6741333 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/auth/proxyLogoutUrl.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/auth/proxyLogoutUrl.spec.ts @@ -42,7 +42,8 @@ test.describe("PROXY_LOGOUT_URL redirect", () => { timeout: 30_000, }); await page.goto("/ui"); - await expect(page.getByText("Virtual Keys")).toBeVisible({ timeout: 15_000 }); + // Scope to the sidebar; the top-bar breadcrumb also shows "Virtual Keys". + await expect(page.getByRole("complementary").getByText("Virtual Keys")).toBeVisible({ timeout: 15_000 }); await settingsLoaded; // Pre-condition: we start authenticated. The admin storage state carries a diff --git a/ui/litellm-dashboard/e2e_tests/tests/internal-user/internalUserNoTeam.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/internal-user/internalUserNoTeam.spec.ts index 548639d6877..92e46d6b27c 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/internal-user/internalUserNoTeam.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/internal-user/internalUserNoTeam.spec.ts @@ -16,7 +16,8 @@ test.describe("Internal User with no team memberships", () => { await page.getByPlaceholder("Enter your username").fill("noteam@test.local"); await page.getByPlaceholder("Enter your password").fill("test"); await page.getByRole("button", { name: "Login", exact: true }).click(); - await expect(page.getByText("Virtual Keys")).toBeVisible({ timeout: 15_000 }); + // Scope to the sidebar; the top-bar breadcrumb also shows "Virtual Keys". + await expect(page.getByRole("complementary").getByText("Virtual Keys")).toBeVisible({ timeout: 15_000 }); await dismissFeedbackPopup(page); // Open the Create Key modal. diff --git a/ui/litellm-dashboard/e2e_tests/tests/login/internalUserIdentity.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/login/internalUserIdentity.spec.ts index 6008049a2aa..569908c5f75 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/login/internalUserIdentity.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/login/internalUserIdentity.spec.ts @@ -14,7 +14,8 @@ test.describe("Navbar identity scoping", () => { test("Internal user navbar dropdown shows their own role and user id, not the admin's", async ({ page }) => { await page.goto("/ui"); - await expect(page.getByText("Virtual Keys")).toBeVisible({ timeout: 10_000 }); + // Scope to the sidebar; the top-bar breadcrumb also shows "Virtual Keys". + await expect(page.getByRole("complementary").getByText("Virtual Keys")).toBeVisible({ timeout: 10_000 }); // The account menu button carries the user's role and email/id in its // aria-label (see UserDropdown.tsx). Match by partial role. diff --git a/ui/litellm-dashboard/e2e_tests/tests/login/login.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/login/login.spec.ts index d1b64f37156..11febf0ed48 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/login/login.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/login/login.spec.ts @@ -9,7 +9,8 @@ test("user can log in", async ({ page }) => { const loginButton = page.getByRole("button", { name: "Login", exact: true }); await expect(loginButton).toBeEnabled(); await loginButton.click(); - await expect(page.getByText("Virtual Keys")).toBeVisible(); + // Scope to the sidebar; the top-bar breadcrumb also shows "Virtual Keys". + await expect(page.getByRole("complementary").getByText("Virtual Keys")).toBeVisible(); // Match the navbar account button by its stable aria-label (UserDropdown.tsx // emits "Account menu — — signed in as "). Earlier this used diff --git a/ui/litellm-dashboard/e2e_tests/tests/migration/migratedPages.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/migration/migratedPages.spec.ts index 0a3be326e42..3ad4b217d08 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/migration/migratedPages.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/migration/migratedPages.spec.ts @@ -17,7 +17,11 @@ const ROOT = process.env.SERVER_ROOT_PATH ?? ""; const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const pathRe = (segment: string) => new RegExp(`${esc(ROOT)}/ui/${esc(segment)}/?($|\\?)`); -const virtualKeysLink = (page: Page) => page.getByRole("link", { name: "Virtual Keys", exact: true }); +// Scope nav lookups to the sidebar (a `complementary` landmark). The top bar +// now renders a breadcrumb whose current-page item is also a "Virtual Keys" +// link, so an unscoped locator would match two elements. +const sidebar = (page: Page) => page.getByRole("complementary"); +const virtualKeysLink = (page: Page) => sidebar(page).getByRole("link", { name: "Virtual Keys", exact: true }); /** The dashboard shell is present (sidebar rendered); page didn't 404 / crash. */ async function expectRendered(page: Page) { @@ -26,16 +30,21 @@ async function expectRendered(page: Page) { /** * Click a migrated page's sidebar link. Migrated items render as ; - * nested ones live under collapsible submenus, so expand submenus until the link is clickable. + * nested ones live under collapsible groups whose children only render while the + * group is open, so expand collapsed groups until the link is clickable. */ async function clickSidebar(page: Page, segment: string) { - const link = page.locator(`a[href$="/ui/${segment}"]`).first(); + const link = sidebar(page).locator(`a[href$="/ui/${segment}"]`).first(); for (let i = 0; i < 8 && !(await link.isVisible().catch(() => false)); i++) { - const collapsedSubmenu = page - .locator(".ant-menu-submenu:not(.ant-menu-submenu-open) > .ant-menu-submenu-title") + // A collapsed group is a menu item with a group-toggle button but no + // rendered submenu yet; clicking the toggle expands it. + const collapsedGroup = sidebar(page) + .locator( + '[data-slot="sidebar-menu-item"]:has(> [data-slot="sidebar-menu-button"]):not(:has(> [data-slot="sidebar-menu-sub"])) > [data-slot="sidebar-menu-button"]', + ) .first(); - if (!(await collapsedSubmenu.isVisible().catch(() => false))) break; - await collapsedSubmenu.click(); + if (!(await collapsedGroup.isVisible().catch(() => false))) break; + await collapsedGroup.click(); await page.waitForTimeout(250); } await link.click(); diff --git a/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts index 7ac2e7df39d..7e42d07ae7c 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts @@ -42,7 +42,9 @@ for (const { role, storage } of roles) { throw new Error(`No page mapping found for menu label: ${buttonLabel}`); } - const tab = page.getByRole("menuitem", { name: buttonLabel }); + // Sidebar items are links inside the `complementary` landmark; scoping + // there avoids the top-bar breadcrumb, which also links the page name. + const tab = page.getByRole("complementary").getByRole("link", { name: buttonLabel }); await expect(tab).toBeVisible(); await tab.click(); diff --git a/ui/litellm-dashboard/e2e_tests/tests/settings/adminSettings.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/settings/adminSettings.spec.ts index f61532b05a5..c4a14a891d4 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/settings/adminSettings.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/settings/adminSettings.spec.ts @@ -6,8 +6,11 @@ test.describe("Add Model", () => { test("admin settings test", async ({ page }) => { await page.goto("/ui"); - await page.getByRole("menuitem", { name: /Settings/ }).click(); - await page.getByRole("menuitem", { name: /Admin Settings/ }).click(); + // "Settings" is a collapsible group (button) in the sidebar; expand it, then + // click the "Admin Settings" child link. Scope to the complementary landmark. + const sidebar = page.getByRole("complementary"); + await sidebar.getByRole("button", { name: /Settings/ }).click(); + await sidebar.getByRole("link", { name: /Admin Settings/ }).click(); await page.getByRole("tab", { name: "UI Settings" }).click(); await expect(page.getByText("Configuration for UI-specific")).toBeVisible(); }); diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index 8e1f4de5872..dac15a21620 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,7 +1,7 @@ { "@typescript-eslint/no-explicit-any": 1977, "complexity": 129, - "local/no-large-inline-object-arg": 512, + "local/no-large-inline-object-arg": 509, "local/no-long-condition-chain": 233, "max-depth": 59, "no-console": 16 diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 544565ee207..2448348bbce 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1261,7 +1261,7 @@ "count": 1 } }, - "src/app/(dashboard)/agents/_components/index.tsx": { + "src/app/(dashboard)/agents/_components/AgentsPanel.tsx": { "no-restricted-imports": { "count": 1 }, @@ -1363,7 +1363,7 @@ "count": 2 } }, - "src/components/claude_code_plugins.tsx": { + "src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx": { "no-restricted-imports": { "count": 1 }, @@ -1379,12 +1379,12 @@ "count": 1 } }, - "src/components/claude_code_plugins/add_plugin_form.tsx": { + "src/app/(dashboard)/skills/_components/add_plugin_form.tsx": { "no-restricted-imports": { "count": 1 } }, - "src/components/claude_code_plugins/plugin_table.tsx": { + "src/app/(dashboard)/skills/_components/plugin_table.tsx": { "no-nested-ternary": { "count": 1 }, @@ -1534,7 +1534,7 @@ "count": 1 } }, - "src/app/(dashboard)/guardrails/_components/index.tsx": { + "src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx": { "react-hooks/set-state-in-effect": { "count": 1 } @@ -2324,7 +2324,7 @@ }, "src/components/team/TeamVirtualKeysTable.tsx": { "no-nested-ternary": { - "count": 2 + "count": 1 }, "no-restricted-imports": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/index.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.test.tsx similarity index 99% rename from ui/litellm-dashboard/src/app/(dashboard)/agents/_components/index.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.test.tsx index 6228253222a..48674f21883 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/index.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.test.tsx @@ -1,7 +1,7 @@ import React from "react"; import { render, screen, waitFor, act, fireEvent, within } from "@testing-library/react"; import { describe, it, expect, vi, beforeEach } from "vitest"; -import AgentsPanel from "./index"; +import AgentsPanel from "./AgentsPanel"; import * as networking from "@/components/networking"; vi.mock("@/components/networking", () => ({ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/index.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/agents/_components/index.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentsPanel.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/page.tsx index 98085ab20a9..885c9a36f30 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/page.tsx @@ -1,6 +1,6 @@ "use client"; -import AgentsPanel from "./_components"; +import AgentsPanel from "./_components/AgentsPanel"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx index 944983da0e1..c53e6318d6b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx @@ -25,6 +25,7 @@ import { adminGlobalCacheActivity, cachingHealthCheckCall } from "@/components/n // Import the new component import { CacheHealthTab } from "./cache_health"; import CacheSettings from "./cache_settings"; +import CoordinationRedisSettings from "./coordination_redis_settings"; const formatDateWithoutTZ = (date: Date | undefined) => { if (!date) return undefined; @@ -264,6 +265,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole Cache Analytics Cache Health Cache Settings + Coordination Redis
@@ -383,6 +385,9 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole + + + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFieldSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFieldSection.tsx new file mode 100644 index 00000000000..af807926ed5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFieldSection.tsx @@ -0,0 +1,46 @@ +import React from "react"; +import CoordinationRedisFormField from "./CoordinationRedisFormField"; +import { fieldsForSection } from "./coordinationRedisUtils"; +import { CoordinationRedisType, CoordinationSection } from "./coordinationRedisFields"; + +interface CoordinationRedisFieldSectionProps { + title: string; + section: CoordinationSection; + redisType: CoordinationRedisType; + configuredSecrets: ReadonlySet; + gridCols?: string; + headingLevel?: "h4" | "h5"; +} + +const CoordinationRedisFieldSection: React.FC = ({ + title, + section, + redisType, + configuredSecrets, + gridCols = "grid-cols-1 gap-6 sm:grid-cols-2", + headingLevel = "h4", +}) => { + const fields = fieldsForSection(section, redisType); + if (fields.length === 0) { + return null; + } + + const Heading = headingLevel; + + return ( +
+ {title} +
+ {fields.map((field) => ( + + ))} +
+
+ ); +}; + +export default CoordinationRedisFieldSection; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFormField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFormField.tsx new file mode 100644 index 00000000000..50c2a39567a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFormField.tsx @@ -0,0 +1,39 @@ +import { Form, Input, Switch } from "antd"; +import React from "react"; +import { CoordinationField } from "./coordinationRedisFields"; + +export const SECRET_ALREADY_SET_PLACEHOLDER = "Already set. Enter a new value to replace it."; + +interface CoordinationRedisFormFieldProps { + field: CoordinationField; + isSecretConfigured: boolean; +} + +const renderControl = (field: CoordinationField, placeholder: string): React.ReactNode => { + switch (field.type) { + case "boolean": + return ; + case "password": + return ; + case "integer": + return ; + case "list": + return ; + default: + return ; + } +}; + +const CoordinationRedisFormField: React.FC = ({ field, isSecretConfigured }) => ( + + {renderControl(field, isSecretConfigured ? SECRET_ALREADY_SET_PLACEHOLDER : field.helpText)} + +); + +export default CoordinationRedisFormField; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisTypeSelector.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisTypeSelector.tsx new file mode 100644 index 00000000000..daab8505890 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisTypeSelector.tsx @@ -0,0 +1,33 @@ +import React from "react"; +import { Select } from "antd"; +import { + COORDINATION_REDIS_TYPES, + COORDINATION_REDIS_TYPE_DESCRIPTIONS, + COORDINATION_REDIS_TYPE_LABELS, + CoordinationRedisType, +} from "./coordinationRedisFields"; + +interface CoordinationRedisTypeSelectorProps { + redisType: CoordinationRedisType; + onTypeChange: (type: CoordinationRedisType) => void; +} + +const OPTIONS = COORDINATION_REDIS_TYPES.map((type) => ({ value: type, label: COORDINATION_REDIS_TYPE_LABELS[type] })); + +const CoordinationRedisTypeSelector: React.FC = ({ redisType, onTypeChange }) => ( +
+ + +
+ ), + }, +]; + +const expansionColumns: ColumnDef[] = [ + { + id: "expander", + header: "", + cell: ({ row }) => ( + + ), + }, + { + accessorKey: "name", + header: "Name", + cell: ({ row }) => {row.original.name}, + }, +]; + +const CHARLIE_ALICE_BOB: Person[] = [person("c", "Charlie"), person("a", "Alice"), person("b", "Bob")]; + +describe("DataTable sorting", () => { + it("client mode reorders rows when the sort header is clicked", async () => { + const user = userEvent.setup(); + render(); + + expect(names()).toEqual(["Charlie", "Alice", "Bob"]); + await user.click(screen.getByTestId("sort-header-name")); + expect(names()).toEqual(["Alice", "Bob", "Charlie"]); + }); + + it("server mode fires the callback but never reorders locally", async () => { + const user = userEvent.setup(); + const onSortingChange = vi.fn(); + render( + , + ); + + // sorting state says ascending, but server mode must render data as given + expect(names()).toEqual(["Charlie", "Alice", "Bob"]); + await user.click(screen.getByTestId("sort-header-name")); + expect(onSortingChange).toHaveBeenCalledTimes(1); + expect(names()).toEqual(["Charlie", "Alice", "Bob"]); + }); + + it("dropdown-tristate variant sorts ascending, descending, then resets", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByTestId("sort-trigger-name")); + await user.click(await screen.findByText("Ascending")); + expect(names()).toEqual(["Alice", "Bob", "Charlie"]); + + await user.click(screen.getByTestId("sort-trigger-name")); + await user.click(await screen.findByText("Descending")); + expect(names()).toEqual(["Charlie", "Bob", "Alice"]); + + await user.click(screen.getByTestId("sort-trigger-name")); + await user.click(await screen.findByText("Reset")); + expect(names()).toEqual(["Charlie", "Alice", "Bob"]); + }); +}); + +describe("DataTable pagination", () => { + const fivePeople: Person[] = Array.from({ length: 5 }, (_, i) => person(String(i), `P${i}`)); + + it("client mode slices rows and advances pages", async () => { + const user = userEvent.setup(); + render(); + + expect(names()).toEqual(["P0", "P1"]); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-2 of 5"); + + await user.click(screen.getByTestId("pagination-next")); + expect(names()).toEqual(["P2", "P3"]); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 3-4 of 5"); + }); + + it("server mode shows X-Y of Z from rowCount and does NOT slice the given rows", async () => { + const user = userEvent.setup(); + const onPaginationChange = vi.fn(); + const pageSlice: Person[] = [person("10", "P10"), person("11", "P11"), person("12", "P12")]; + render( + , + ); + + expect(names()).toEqual(["P10", "P11", "P12"]); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 11-20 of 25"); + + await user.click(screen.getByTestId("pagination-next")); + expect(onPaginationChange).toHaveBeenCalledTimes(1); + }); +}); + +describe("DataTable column visibility", () => { + it("hides a column when toggled off in the view-options menu", async () => { + const user = userEvent.setup(); + render( + } + />, + ); + + expect(screen.getByText("Email")).toBeInTheDocument(); + await user.click(screen.getByTestId("view-options-trigger")); + await user.click(await screen.findByTestId("view-option-email")); + await waitFor(() => expect(screen.queryByText("Email")).not.toBeInTheDocument()); + + await user.click(screen.getByTestId("view-option-email")); + await waitFor(() => expect(screen.getByText("Email")).toBeInTheDocument()); + }); + + it("omits columns that opt out of hiding from the menu", async () => { + const user = userEvent.setup(); + const columns: ColumnDef[] = [ + { + accessorKey: "name", + header: "Name", + enableHiding: false, + cell: ({ row }) => {row.original.name}, + }, + { + accessorKey: "email", + header: "Email", + cell: ({ row }) => {row.original.email}, + }, + ]; + render( + } + />, + ); + + await user.click(screen.getByTestId("view-options-trigger")); + expect(await screen.findByTestId("view-option-email")).toBeInTheDocument(); + expect(screen.queryByTestId("view-option-name")).toBeNull(); + }); +}); + +describe("DataTable pinned columns", () => { + it("applies sticky positioning to a pinned column only", () => { + const { container } = render(); + + const pinnedHead = container.querySelector('th[data-header-id="name"]'); + const normalHead = container.querySelector('th[data-header-id="email"]'); + + expect(pinnedHead?.style.position).toBe("sticky"); + expect(pinnedHead?.style.left).toBe("0px"); + expect(normalHead?.style.position).toBe(""); + }); +}); + +describe("DataTable row click guard", () => { + it("fires onRowClick from a plain cell but not from interactive elements", async () => { + const user = userEvent.setup(); + const onRowClick = vi.fn(); + render(); + + await user.click(screen.getByTestId("name-cell")); + expect(onRowClick).toHaveBeenCalledTimes(1); + expect(onRowClick).toHaveBeenCalledWith(expect.objectContaining({ id: "a" })); + + await user.click(screen.getByTestId("row-button")); + expect(onRowClick).toHaveBeenCalledTimes(1); + + await user.click(screen.getByTestId("row-input")); + expect(onRowClick).toHaveBeenCalledTimes(1); + }); +}); + +describe("DataTable expansion", () => { + const subComponent = ({ row }: { row: { original: Person } }) => ( +
details for {row.original.name}
+ ); + + it("toggles the sub-row in uncontrolled mode", async () => { + const user = userEvent.setup(); + render( + row.id} + getRowCanExpand={() => true} + renderSubComponent={subComponent} + />, + ); + + expect(screen.queryByTestId("sub-row")).not.toBeInTheDocument(); + await user.click(screen.getByTestId("expand-a")); + expect(screen.getByTestId("sub-row")).toBeInTheDocument(); + await user.click(screen.getByTestId("expand-a")); + expect(screen.queryByTestId("sub-row")).not.toBeInTheDocument(); + }); + + it("toggles the sub-row in controlled mode driven by parent state", async () => { + const user = userEvent.setup(); + const Harness = () => { + const [expanded, setExpanded] = useState({}); + return ( + row.id} + expanded={expanded} + onExpandedChange={setExpanded} + getRowCanExpand={() => true} + renderSubComponent={subComponent} + /> + ); + }; + render(); + + expect(screen.queryByTestId("sub-row")).not.toBeInTheDocument(); + await user.click(screen.getByTestId("expand-a")); + expect(screen.getByTestId("sub-row")).toBeInTheDocument(); + await user.click(screen.getByTestId("expand-a")); + expect(screen.queryByTestId("sub-row")).not.toBeInTheDocument(); + }); + + it("stays collapsed in controlled mode when the parent ignores the change", async () => { + const user = userEvent.setup(); + const onExpandedChange = vi.fn(); + render( + row.id} + expanded={{}} + onExpandedChange={onExpandedChange} + getRowCanExpand={() => true} + renderSubComponent={subComponent} + />, + ); + + await user.click(screen.getByTestId("expand-a")); + expect(onExpandedChange).toHaveBeenCalledTimes(1); + expect(screen.queryByTestId("sub-row")).not.toBeInTheDocument(); + }); +}); + +describe("DataTable row styling and footer", () => { + it("applies rowClassName to the matching row only", () => { + const data = [person("a", "Alice", true), person("b", "Bob", false)]; + const { container } = render( + row.id} + rowClassName={(row) => (row.original.flagged ? "flagged-row" : "")} + />, + ); + + expect(container.querySelector('tr[data-row-id="a"]')?.className).toContain("flagged-row"); + expect(container.querySelector('tr[data-row-id="b"]')?.className).not.toContain("flagged-row"); + }); + + it("renders the footer slot inside a tfoot element", () => { + render( + ( + + Total: 3 + + )} + />, + ); + + expect(screen.getByTestId("footer-row").closest("tfoot")).not.toBeNull(); + }); +}); + +describe("DataTable layout", () => { + it("exposes resize handles with stable selectors only when resizing is enabled", () => { + const { container, rerender } = render( + , + ); + expect(container.querySelectorAll("[data-resizer][data-header-id]").length).toBe(2); + + rerender(); + expect(container.querySelectorAll("[data-resizer]").length).toBe(0); + }); + + it("makes the header sticky and constrains body height when maxBodyHeight is set", () => { + const { container } = render(); + expect(container.querySelector("thead")?.className).toContain("sticky"); + const scroller = container.querySelector('[data-slot="table-container"]')?.parentElement as HTMLElement; + expect(scroller.style.maxHeight).toBe("240px"); + }); +}); + +describe("DataTable misconfiguration guards", () => { + it("throws when server sorting is missing required props", () => { + const spy = vi.spyOn(console, "error").mockImplementation(() => {}); + expect(() => render()).toThrow( + /sortingMode='server'/, + ); + spy.mockRestore(); + }); + + it("throws when server pagination is missing required props", () => { + const spy = vi.spyOn(console, "error").mockImplementation(() => {}); + expect(() => render()).toThrow( + /paginationMode='server'/, + ); + spy.mockRestore(); + }); + + it("throws when both defaultSorting and sorting are provided", () => { + const spy = vi.spyOn(console, "error").mockImplementation(() => {}); + expect(() => + render( + , + ), + ).toThrow(/defaultSorting/); + spy.mockRestore(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx new file mode 100644 index 00000000000..2e95ee170fa --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx @@ -0,0 +1,495 @@ +"use client"; + +import { + type Cell, + type Column, + type ColumnDef, + type ColumnPinningState, + type ColumnSizingState, + type ExpandedState, + flexRender, + getCoreRowModel, + getExpandedRowModel, + getPaginationRowModel, + getSortedRowModel, + type Header, + type OnChangeFn, + type Row, + type RowData, + type Table, + type TableOptions, + useReactTable, + type VisibilityState, +} from "@tanstack/react-table"; +import * as React from "react"; +import { Fragment, useState } from "react"; + +import { + Table as TableRoot, + TableBody, + TableCell, + TableFooter, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { cn } from "@/lib/cva.config"; + +import "./columnMeta"; +import { DataTablePagination, DEFAULT_PAGE_SIZE_OPTIONS } from "./DataTablePagination"; +import type { ColumnPinnedSide, DataTableProps, DataTableSize, PaginationMode, SortingMode } from "./types"; + +const INTERACTIVE_SELECTOR = "button, a, input, select, textarea, [role=checkbox], [data-row-click-exempt]"; + +const noop = () => {}; + +export class DataTableConfigError extends Error { + constructor(messages: readonly string[]) { + super(`DataTable misconfiguration:\n- ${messages.join("\n- ")}`); + this.name = "DataTableConfigError"; + } +} + +export function validateDataTableConfig( + props: DataTableProps, +): readonly string[] { + const serverSortingIncomplete = + props.sortingMode === "server" && (props.sorting === undefined || props.onSortingChange === undefined); + + const serverPaginationPropsMissing = + props.pagination === undefined || props.onPaginationChange === undefined || props.rowCount === undefined; + const serverPaginationIncomplete = props.paginationMode === "server" && serverPaginationPropsMissing; + + const bothSortingSources = props.defaultSorting !== undefined && props.sorting !== undefined; + + return [ + serverSortingIncomplete ? "sortingMode='server' requires both `sorting` and `onSortingChange`." : null, + serverPaginationIncomplete + ? "paginationMode='server' requires `pagination`, `onPaginationChange`, and `rowCount`." + : null, + bothSortingSources ? "Provide either `defaultSorting` (uncontrolled) or `sorting` (controlled), not both." : null, + ].filter((message): message is string => message !== null); +} + +function columnDefId(column: ColumnDef): string | undefined { + if ("id" in column && typeof column.id === "string") { + return column.id; + } + if ("accessorKey" in column && column.accessorKey != null) { + return String(column.accessorKey); + } + return undefined; +} + +function derivePinning(columns: ColumnDef[]): ColumnPinningState { + const collect = (side: ColumnPinnedSide): string[] => + columns + .filter((column) => column.meta?.pinned === side) + .map(columnDefId) + .filter((id): id is string => id !== undefined); + return { left: collect("left"), right: collect("right") }; +} + +function buildRowModels( + sortingMode: SortingMode, + paginationMode: PaginationMode, + getRowCanExpand: ((row: Row) => boolean) | undefined, +): Partial> { + return { + ...(sortingMode === "client" ? { getSortedRowModel: getSortedRowModel() } : {}), + ...(paginationMode === "client" ? { getPaginationRowModel: getPaginationRowModel() } : {}), + ...(getRowCanExpand !== undefined ? { getRowCanExpand, getExpandedRowModel: getExpandedRowModel() } : {}), + }; +} + +function stickyZIndex(isPinned: boolean, isHeader: boolean): number { + if (isPinned && isHeader) { + return 30; + } + if (isHeader) { + return 20; + } + return 10; +} + +function pinnedShadow(pinned: false | ColumnPinnedSide): string { + if (pinned === "left") { + return "shadow-[inset_-1px_0_0_var(--color-border)]"; + } + if (pinned === "right") { + return "shadow-[inset_1px_0_0_var(--color-border)]"; + } + return ""; +} + +function computeStickyStyle( + column: Column, + isHeader: boolean, + stickyHeader: boolean, +): { style: React.CSSProperties; className: string } { + const pinned = column.getIsPinned(); + const stickyTop = isHeader && stickyHeader; + if (!pinned && !stickyTop) { + return { style: {}, className: "" }; + } + + const left = pinned === "left" ? column.getStart("left") : undefined; + const right = pinned === "right" ? column.getAfter("right") : undefined; + + const style: React.CSSProperties = { + position: "sticky", + zIndex: stickyZIndex(pinned !== false, isHeader), + ...(stickyTop ? { top: 0 } : {}), + ...(left !== undefined ? { left } : {}), + ...(right !== undefined ? { right } : {}), + }; + + return { style, className: cn(pinned ? "bg-background" : "", pinnedShadow(pinned)) }; +} + +function widthStyle( + column: Column, + enableColumnResizing: boolean, +): React.CSSProperties | undefined { + if (enableColumnResizing || column.columnDef.size !== undefined) { + return { width: column.getSize() }; + } + return undefined; +} + +interface HeadCellProps { + header: Header; + size: DataTableSize; + stickyHeader: boolean; + enableColumnResizing: boolean; +} + +function DataTableHeadCell({ header, size, stickyHeader, enableColumnResizing }: HeadCellProps) { + const { column } = header; + const meta = column.columnDef.meta; + const sticky = computeStickyStyle(column, true, stickyHeader); + const canResize = enableColumnResizing && column.getCanResize(); + + return ( + + {header.isPlaceholder ? null : ( +
+ {flexRender(column.columnDef.header, header.getContext())} +
+ )} + {canResize && ( +
column.resetSize()} + className={cn( + "absolute top-0 right-0 h-full w-1 cursor-col-resize touch-none select-none hover:bg-border", + column.getIsResizing() ? "bg-primary" : "", + )} + /> + )} + + ); +} + +interface BodyCellProps { + cell: Cell; + size: DataTableSize; + stickyHeader: boolean; + enableColumnResizing: boolean; +} + +function DataTableBodyCell({ cell, size, stickyHeader, enableColumnResizing }: BodyCellProps) { + const { column } = cell; + const meta = column.columnDef.meta; + const sticky = computeStickyStyle(column, false, stickyHeader); + + return ( + + {flexRender(column.columnDef.cell, cell.getContext())} + + ); +} + +interface BodyRowProps { + row: Row; + size: DataTableSize; + stickyHeader: boolean; + enableColumnResizing: boolean; + onRowClick?: (row: TData) => void; + rowClassName?: (row: Row) => string; + renderSubComponent?: (props: { row: Row }) => React.ReactElement; +} + +function DataTableBodyRow({ + row, + size, + stickyHeader, + enableColumnResizing, + onRowClick, + rowClassName, + renderSubComponent, +}: BodyRowProps) { + const clickable = onRowClick !== undefined; + const cells = row.getVisibleCells(); + + const handleClick = (event: React.MouseEvent) => { + if (onRowClick === undefined) { + return; + } + const target = event.target as HTMLElement | null; + if (target === null || !event.currentTarget.contains(target)) { + return; + } + if (target.closest(INTERACTIVE_SELECTOR) !== null) { + return; + } + onRowClick(row.original); + }; + + return ( + + + {cells.map((cell) => ( + + ))} + + {renderSubComponent !== undefined && row.getIsExpanded() && ( + + + {renderSubComponent({ row })} + + + )} + + ); +} + +function MessageRow({ colSpan, children }: { colSpan: number; children: React.ReactNode }) { + return ( + + + {children} + + + ); +} + +function useControllable( + controlled: T | undefined, + controlledOnChange: OnChangeFn | undefined, + initial: T, +): { value: T; onChange: OnChangeFn } { + const [internal, setInternal] = useState(initial); + if (controlled !== undefined) { + return { value: controlled, onChange: controlledOnChange ?? noop }; + } + return { value: internal, onChange: setInternal }; +} + +function useDataTableInstance(props: DataTableProps): Table { + const { + data, + columns, + getRowId, + sortingMode = "none", + sorting, + onSortingChange, + defaultSorting, + enableSortingRemoval = false, + paginationMode = "none", + pagination, + onPaginationChange, + rowCount, + pageSizeOptions = DEFAULT_PAGE_SIZE_OPTIONS, + enableColumnResizing = false, + columnResizeMode = "onEnd", + defaultColumnVisibility, + getRowCanExpand, + renderSubComponent, + expanded, + onExpandedChange, + } = props; + + const sortingState = useControllable(sorting, onSortingChange, defaultSorting ?? []); + const paginationState = useControllable(pagination, onPaginationChange, { + pageIndex: 0, + pageSize: pageSizeOptions[0] ?? 25, + }); + const expandedState = useControllable(expanded, onExpandedChange, {}); + const [columnVisibility, setColumnVisibility] = useState(defaultColumnVisibility ?? {}); + const [columnSizing, setColumnSizing] = useState({}); + const columnPinning = React.useMemo(() => derivePinning(columns), [columns]); + const expansionGuard = renderSubComponent !== undefined ? getRowCanExpand : undefined; + + const tableOptions: TableOptions = { + data, + columns, + state: { + sorting: sortingState.value, + pagination: paginationState.value, + expanded: expandedState.value, + columnVisibility, + columnSizing, + }, + initialState: { columnPinning }, + manualSorting: sortingMode === "server", + manualPagination: paginationMode === "server", + enableSortingRemoval, + enableColumnResizing, + columnResizeMode, + onSortingChange: sortingState.onChange, + onPaginationChange: paginationState.onChange, + onExpandedChange: expandedState.onChange, + onColumnVisibilityChange: setColumnVisibility, + onColumnSizingChange: setColumnSizing, + getCoreRowModel: getCoreRowModel(), + ...buildRowModels(sortingMode, paginationMode, expansionGuard), + ...(getRowId !== undefined ? { getRowId } : {}), + ...(paginationMode === "server" && rowCount !== undefined ? { rowCount } : {}), + }; + + return useReactTable(tableOptions); +} + +export function DataTable(props: DataTableProps) { + // Validate once at construction so a misconfig surfaces immediately instead of on every render. + useState(() => { + const errors = validateDataTableConfig(props); + if (errors.length > 0) { + throw new DataTableConfigError(errors); + } + return null; + }); + + const { + isLoading = false, + loadingMessage = "Loading…", + noDataMessage = "No results", + paginationMode = "none", + rowCount, + pageSizeOptions = DEFAULT_PAGE_SIZE_OPTIONS, + enableColumnResizing = false, + onRowClick, + rowClassName, + renderSubComponent, + maxBodyHeight, + size = "default", + toolbar, + paginationSlot, + footer, + } = props; + + const table = useDataTableInstance(props); + + const rows = table.getRowModel().rows; + const visibleColumnCount = table.getVisibleLeafColumns().length; + const stickyHeader = maxBodyHeight !== undefined; + const tableStyle = enableColumnResizing ? { width: table.getTotalSize() } : undefined; + + const renderPagination = (): React.ReactNode => { + if (paginationSlot !== undefined) { + return paginationSlot(table); + } + if (paginationMode === "none") { + return null; + } + const current = table.getState().pagination; + const total = paginationMode === "server" ? rowCount ?? 0 : table.getPrePaginationRowModel().rows.length; + return ( + table.setPageIndex(next)} + onPageSizeChange={(next) => table.setPageSize(next)} + pageSizeOptions={pageSizeOptions} + isLoading={isLoading} + /> + ); + }; + + const renderBody = (): React.ReactNode => { + if (isLoading) { + return {loadingMessage}; + } + if (rows.length === 0) { + return {noDataMessage}; + } + return rows.map((row) => ( + + )); + }; + + return ( +
+ {toolbar !== undefined &&
{toolbar(table)}
} +
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + ))} + + ))} + + {renderBody()} + {footer !== undefined && {footer(table)}} + +
+ {renderPagination()} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTablePagination.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTablePagination.test.tsx new file mode 100644 index 00000000000..e5bd4a55b53 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTablePagination.test.tsx @@ -0,0 +1,68 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; + +import { DataTablePagination } from "./DataTablePagination"; + +const baseProps = { + page: 0, + pageSize: 25, + rowCount: 100, + onPageChange: () => {}, + onPageSizeChange: () => {}, +}; + +describe("DataTablePagination", () => { + it("renders the current range from plain props", () => { + render(); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-25 of 100"); + }); + + it("computes the range for a middle page and clamps the end to rowCount", () => { + render(); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 91-100 of 100"); + }); + + it("disables the previous controls on the first page", () => { + render(); + expect(screen.getByTestId("pagination-first")).toBeDisabled(); + expect(screen.getByTestId("pagination-prev")).toBeDisabled(); + expect(screen.getByTestId("pagination-next")).toBeEnabled(); + }); + + it("disables the next controls on the last page", () => { + render(); + expect(screen.getByTestId("pagination-next")).toBeDisabled(); + expect(screen.getByTestId("pagination-last")).toBeDisabled(); + expect(screen.getByTestId("pagination-prev")).toBeEnabled(); + }); + + it("advances by one page when next is clicked", async () => { + const user = userEvent.setup(); + const onPageChange = vi.fn(); + render(); + await user.click(screen.getByTestId("pagination-next")); + expect(onPageChange).toHaveBeenCalledWith(2); + }); + + it("jumps to the last page index when last is clicked", async () => { + const user = userEvent.setup(); + const onPageChange = vi.fn(); + render(); + await user.click(screen.getByTestId("pagination-last")); + expect(onPageChange).toHaveBeenCalledWith(3); + }); + + it("shows an empty state and disables all navigation when there are no rows", () => { + render(); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("No results"); + expect(screen.getByTestId("pagination-next")).toBeDisabled(); + expect(screen.getByTestId("pagination-prev")).toBeDisabled(); + }); + + it("disables navigation while loading", () => { + render(); + expect(screen.getByTestId("pagination-next")).toBeDisabled(); + expect(screen.getByTestId("pagination-prev")).toBeDisabled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTablePagination.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTablePagination.tsx new file mode 100644 index 00000000000..5a30b12f27f --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTablePagination.tsx @@ -0,0 +1,113 @@ +"use client"; + +import { ChevronLeft, ChevronRight, ChevronsLeft, ChevronsRight } from "lucide-react"; + +import { Button } from "@/components/ui/button"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { cn } from "@/lib/cva.config"; + +export const DEFAULT_PAGE_SIZE_OPTIONS = [25, 50, 100]; + +export interface DataTablePaginationProps { + page: number; + pageSize: number; + rowCount: number; + onPageChange: (page: number) => void; + onPageSizeChange: (pageSize: number) => void; + pageSizeOptions?: number[]; + isLoading?: boolean; + className?: string; +} + +export function DataTablePagination({ + page, + pageSize, + rowCount, + onPageChange, + onPageSizeChange, + pageSizeOptions = DEFAULT_PAGE_SIZE_OPTIONS, + isLoading = false, + className, +}: DataTablePaginationProps) { + const pageCount = pageSize > 0 ? Math.ceil(rowCount / pageSize) : 0; + const start = rowCount === 0 ? 0 : page * pageSize + 1; + const end = Math.min((page + 1) * pageSize, rowCount); + const canPrev = page > 0 && !isLoading; + const canNext = page < pageCount - 1 && !isLoading; + const lastPage = Math.max(pageCount - 1, 0); + + return ( +
+
+ Rows per page + +
+ +
+ + {rowCount === 0 ? "No results" : `Showing ${start}-${end} of ${rowCount}`} + +
+ + + + +
+
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSortHeader.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSortHeader.test.tsx new file mode 100644 index 00000000000..a6164307a78 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSortHeader.test.tsx @@ -0,0 +1,112 @@ +import { + type ColumnDef, + flexRender, + getCoreRowModel, + getSortedRowModel, + type OnChangeFn, + type SortingState, + useReactTable, +} from "@tanstack/react-table"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; + +import { DataTableSortHeader, type DataTableSortVariant } from "./DataTableSortHeader"; + +interface Item { + name: string; +} + +interface HarnessProps { + variant: DataTableSortVariant; + canSort?: boolean; + onSortingChange?: OnChangeFn; +} + +function SortHeaderHarness({ variant, canSort = true, onSortingChange }: HarnessProps) { + const [sorting, setSorting] = useState([]); + const columns: ColumnDef[] = [ + { + accessorKey: "name", + enableSorting: canSort, + header: ({ column }) => , + }, + ]; + const options = { + data: [{ name: "x" }], + columns, + state: { sorting }, + onSortingChange: (updater: SortingState | ((prev: SortingState) => SortingState)) => { + setSorting(updater); + onSortingChange?.(updater); + }, + getCoreRowModel: getCoreRowModel(), + getSortedRowModel: getSortedRowModel(), + }; + const table = useReactTable(options); + + return ( + + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + ))} + + ))} + +
{flexRender(header.column.columnDef.header, header.getContext())}
+ ); +} + +describe("DataTableSortHeader", () => { + it("renders a plain label and no button when the column cannot sort", () => { + render(); + expect(screen.queryByTestId("sort-header-name")).toBeNull(); + expect(screen.getByText("Name")).toBeInTheDocument(); + }); + + it("header-cycle indicator advances none -> asc -> desc on click", async () => { + const user = userEvent.setup(); + render(); + const indicator = () => screen.getByTestId("sort-header-name").querySelector("[data-sort-indicator]"); + + expect(indicator()).toHaveAttribute("data-sort-indicator", "none"); + await user.click(screen.getByTestId("sort-header-name")); + expect(indicator()).toHaveAttribute("data-sort-indicator", "asc"); + await user.click(screen.getByTestId("sort-header-name")); + expect(indicator()).toHaveAttribute("data-sort-indicator", "desc"); + }); + + it("dropdown-tristate sets ascending, descending, and reset from the menu", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByTestId("sort-trigger-name")); + await user.click(await screen.findByText("Descending")); + expect(screen.getByTestId("sort-trigger-name").querySelector('[data-sort-indicator="desc"]')).not.toBeNull(); + + await user.click(screen.getByTestId("sort-trigger-name")); + await user.click(await screen.findByText("Ascending")); + expect(screen.getByTestId("sort-trigger-name").querySelector('[data-sort-indicator="asc"]')).not.toBeNull(); + + await user.click(screen.getByTestId("sort-trigger-name")); + await user.click(await screen.findByText("Reset")); + expect(screen.getByTestId("sort-trigger-name").querySelector('[data-sort-indicator="none"]')).not.toBeNull(); + }); + + it("dropdown-tristate trigger stops the click from reaching an outer handler", async () => { + const user = userEvent.setup(); + const onOuterClick = vi.fn(); + render( +
+ +
, + ); + + await user.click(screen.getByTestId("sort-trigger-name")); + expect(onOuterClick).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSortHeader.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSortHeader.tsx new file mode 100644 index 00000000000..1cf09ce4f47 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableSortHeader.tsx @@ -0,0 +1,96 @@ +"use client"; + +import { Menu } from "@base-ui/react/menu"; +import type { Column, SortDirection } from "@tanstack/react-table"; +import { ChevronDown, ChevronsUpDown, ChevronUp, X } from "lucide-react"; +import type * as React from "react"; + +import { cn } from "@/lib/cva.config"; + +export type DataTableSortVariant = "header-cycle" | "dropdown-tristate"; + +interface DataTableSortHeaderProps { + column: Column; + title: React.ReactNode; + variant?: DataTableSortVariant; + className?: string; +} + +function SortIndicator({ sorted }: { sorted: false | SortDirection }) { + if (sorted === "asc") { + return ; + } + if (sorted === "desc") { + return ; + } + return ; +} + +const MENU_ITEM_CLASS = + "flex cursor-default items-center gap-2 rounded-sm px-2 py-1.5 outline-hidden select-none data-highlighted:bg-accent data-highlighted:text-accent-foreground"; + +export function DataTableSortHeader({ + column, + title, + variant = "header-cycle", + className, +}: DataTableSortHeaderProps) { + const sorted = column.getIsSorted(); + + if (!column.getCanSort()) { + return {title}; + } + + if (variant === "dropdown-tristate") { + return ( +
+ {title} + + event.stopPropagation()} + className={cn( + "inline-flex size-6 items-center justify-center rounded-md hover:bg-muted", + sorted ? "text-primary" : "text-muted-foreground", + )} + > + + + } + /> + + + + column.toggleSorting(false)}> + Ascending + + column.toggleSorting(true)}> + Descending + + column.clearSorting()}> + Reset + + + + + +
+ ); + } + + return ( + + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableToolbar.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableToolbar.test.tsx new file mode 100644 index 00000000000..5d1f5340f5d --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableToolbar.test.tsx @@ -0,0 +1,35 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; + +import { DataTableToolbar } from "./DataTableToolbar"; + +describe("DataTableToolbar", () => { + it("renders slotted action children", () => { + render( + + + , + ); + expect(screen.getByTestId("toolbar-action")).toBeInTheDocument(); + }); + + it("shows the reset button only when there are active filters", async () => { + const user = userEvent.setup(); + const onResetFilters = vi.fn(); + const { rerender } = render(); + expect(screen.queryByText("Reset Filters")).toBeNull(); + + rerender(); + await user.click(screen.getByText("Reset Filters")); + expect(onResetFilters).toHaveBeenCalledTimes(1); + }); + + it("wires the filters toggle button", async () => { + const user = userEvent.setup(); + const onToggleFilters = vi.fn(); + render(); + await user.click(screen.getByText("Filters")); + expect(onToggleFilters).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableToolbar.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableToolbar.tsx new file mode 100644 index 00000000000..80b4ecc7edd --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableToolbar.tsx @@ -0,0 +1,55 @@ +"use client"; + +import { Search } from "lucide-react"; +import type * as React from "react"; + +import { FilterInput } from "@/components/common_components/Filters/FilterInput"; +import { FiltersButton } from "@/components/common_components/Filters/FiltersButton"; +import { ResetFiltersButton } from "@/components/common_components/Filters/ResetFiltersButton"; +import { cn } from "@/lib/cva.config"; + +interface DataTableToolbarProps { + searchValue?: string; + onSearchChange?: (value: string) => void; + searchPlaceholder?: string; + filtersActive?: boolean; + hasActiveFilters?: boolean; + onToggleFilters?: () => void; + onResetFilters?: () => void; + children?: React.ReactNode; + className?: string; +} + +export function DataTableToolbar({ + searchValue, + onSearchChange, + searchPlaceholder = "Search", + filtersActive = false, + hasActiveFilters = false, + onToggleFilters, + onResetFilters, + children, + className, +}: DataTableToolbarProps) { + const showReset = onResetFilters !== undefined && hasActiveFilters; + + return ( +
+
+ {onSearchChange !== undefined && ( + + )} + {onToggleFilters !== undefined && ( + + )} + {showReset && } +
+ {children !== undefined &&
{children}
} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTableViewOptions.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableViewOptions.tsx new file mode 100644 index 00000000000..ab56aafe7b5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTableViewOptions.tsx @@ -0,0 +1,55 @@ +"use client"; + +import { Menu } from "@base-ui/react/menu"; +import type { Table } from "@tanstack/react-table"; +import { Check, SlidersHorizontal } from "lucide-react"; + +import { Button } from "@/components/ui/button"; + +interface DataTableViewOptionsProps { + table: Table; + label?: string; + className?: string; +} + +export function DataTableViewOptions({ table, label = "View", className }: DataTableViewOptionsProps) { + const hideableColumns = table.getAllLeafColumns().filter((column) => column.getCanHide()); + + if (hideableColumns.length === 0) { + return null; + } + + return ( + + + + {label} + + } + /> + + + + {hideableColumns.map((column) => ( + column.toggleVisibility(checked)} + closeOnClick={false} + data-testid={`view-option-${column.id}`} + className="relative flex cursor-default items-center gap-2 rounded-sm py-1.5 pr-2 pl-7 capitalize outline-hidden select-none data-highlighted:bg-accent data-highlighted:text-accent-foreground" + > + + + + {column.columnDef.meta?.title ?? column.id} + + ))} + + + + + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/columnMeta.ts b/ui/litellm-dashboard/src/components/shared/DataTable/columnMeta.ts new file mode 100644 index 00000000000..46e72226038 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/columnMeta.ts @@ -0,0 +1,13 @@ +import type { RowData } from "@tanstack/react-table"; + +import type { ColumnPinnedSide } from "./types"; + +declare module "@tanstack/react-table" { + interface ColumnMeta { + numeric?: boolean; + className?: string; + headerClassName?: string; + title?: string; + pinned?: ColumnPinnedSide; + } +} diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/index.ts b/ui/litellm-dashboard/src/components/shared/DataTable/index.ts new file mode 100644 index 00000000000..49a4430bbee --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/index.ts @@ -0,0 +1,16 @@ +import "./columnMeta"; + +export { DataTable, DataTableConfigError, validateDataTableConfig } from "./DataTable"; +export { DataTablePagination, DEFAULT_PAGE_SIZE_OPTIONS } from "./DataTablePagination"; +export { DataTableToolbar } from "./DataTableToolbar"; +export { DataTableViewOptions } from "./DataTableViewOptions"; +export { DataTableSortHeader, type DataTableSortVariant } from "./DataTableSortHeader"; +export type { DataTablePaginationProps } from "./DataTablePagination"; +export type { + ColumnPinnedSide, + ColumnResizeMode, + DataTableProps, + DataTableSize, + PaginationMode, + SortingMode, +} from "./types"; diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/types.ts b/ui/litellm-dashboard/src/components/shared/DataTable/types.ts new file mode 100644 index 00000000000..8fa6f21c4d3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DataTable/types.ts @@ -0,0 +1,60 @@ +import type { + ColumnDef, + ExpandedState, + OnChangeFn, + PaginationState, + Row, + RowData, + SortingState, + Table, + VisibilityState, +} from "@tanstack/react-table"; +import type * as React from "react"; + +export type SortingMode = "none" | "client" | "server"; +export type PaginationMode = "none" | "client" | "server"; +export type ColumnResizeMode = "onEnd" | "onChange"; +export type DataTableSize = "compact" | "default"; +export type ColumnPinnedSide = "left" | "right"; + +export interface DataTableProps { + data: TData[]; + columns: ColumnDef[]; + getRowId?: (row: TData, index: number, parent?: Row) => string; + + isLoading?: boolean; + loadingMessage?: string; + noDataMessage?: React.ReactNode; + + sortingMode?: SortingMode; + sorting?: SortingState; + onSortingChange?: OnChangeFn; + defaultSorting?: SortingState; + enableSortingRemoval?: boolean; + + paginationMode?: PaginationMode; + pagination?: PaginationState; + onPaginationChange?: OnChangeFn; + rowCount?: number; + pageSizeOptions?: number[]; + + enableColumnResizing?: boolean; + columnResizeMode?: ColumnResizeMode; + defaultColumnVisibility?: VisibilityState; + + getRowCanExpand?: (row: Row) => boolean; + renderSubComponent?: (props: { row: Row }) => React.ReactElement; + expanded?: ExpandedState; + onExpandedChange?: OnChangeFn; + + onRowClick?: (row: TData) => void; + + rowClassName?: (row: Row) => string; + + maxBodyHeight?: number | string; + size?: DataTableSize; + + toolbar?: (table: Table) => React.ReactNode; + paginationSlot?: (table: Table) => React.ReactNode; + footer?: (table: Table) => React.ReactNode; +} diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index 46eac1c5772..4ccfb891417 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -536,7 +536,7 @@ describe("TeamInfoView", () => { await user.click(virtualKeysTab); await waitFor(() => { - expect(screen.getByText("Page 1 of 1")).toBeInTheDocument(); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-5 of 5"); }); }); @@ -584,9 +584,9 @@ describe("TeamInfoView", () => { expect(screen.getByRole("button", { name: "Filters" })).toBeInTheDocument(); }); expect(screen.getByRole("button", { name: "Reset Filters" })).toBeInTheDocument(); - expect(screen.getByText("Page 1 of 1")).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Previous" })).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Next" })).toBeInTheDocument(); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-1 of 1"); + expect(screen.getByTestId("pagination-prev")).toBeInTheDocument(); + expect(screen.getByTestId("pagination-next")).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx index 954f5ea98c5..fd81aaaad99 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx @@ -163,7 +163,7 @@ describe("TeamVirtualKeysTable", () => { expect(screen.getByText("bob_key_team1")).toBeInTheDocument(); }); - it("should show Page X of Y when multiple pages exist", async () => { + it("should show the current range from total_count when multiple pages exist", async () => { mockUseKeys.mockReturnValue({ data: { keys: [createMockKey()], @@ -179,7 +179,7 @@ describe("TeamVirtualKeysTable", () => { renderWithProviders(); await waitFor(() => { - expect(screen.getByText("Page 1 of 3")).toBeInTheDocument(); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-50 of 100"); }); }); @@ -203,17 +203,92 @@ describe("TeamVirtualKeysTable", () => { renderWithProviders(); await waitFor(() => { - expect(screen.getByText("Page 1 of 3")).toBeInTheDocument(); + expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-50 of 100"); }); - const nextButton = screen.getByRole("button", { name: "Next" }); - await user.click(nextButton); + await user.click(screen.getByTestId("pagination-next")); await waitFor(() => { expect(mockUseKeys).toHaveBeenLastCalledWith(2, 50, expect.objectContaining({ teamID: "team-1" })); }); }); + it("routes a sort-header click to useKeys as a server-side sort", async () => { + const user = userEvent.setup(); + mockUseKeys.mockReturnValue({ + data: { keys: [createMockKey()], total_count: 1, current_page: 1, total_pages: 1 } as KeysResponse, + isPending: false, + isFetching: false, + refetch: vi.fn(), + } as unknown as ReturnType); + + renderWithProviders(); + + await waitFor(() => expect(screen.getByTestId("sort-header-created_at")).toBeInTheDocument()); + await user.click(screen.getByTestId("sort-header-created_at")); + + await waitFor(() => + expect(mockUseKeys).toHaveBeenLastCalledWith( + 1, + 50, + expect.objectContaining({ sortBy: "created_at", sortOrder: "asc" }), + ), + ); + }); + + it("resets to the first page when the sort changes", async () => { + const user = userEvent.setup(); + mockUseKeys.mockImplementation( + (page: number) => + ({ + data: { + keys: [createMockKey({ token: `sk-p${page}`, key_alias: `page${page}_key` })], + total_count: 100, + current_page: page, + total_pages: 2, + }, + isPending: false, + isFetching: false, + refetch: vi.fn(), + }) as unknown as ReturnType, + ); + + renderWithProviders(); + + await user.click(await screen.findByTestId("pagination-next")); + await waitFor(() => expect(mockUseKeys).toHaveBeenLastCalledWith(2, 50, expect.anything())); + + await user.click(screen.getByTestId("sort-header-created_at")); + await waitFor(() => expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.anything())); + }); + + it("resets the sort order to the default when filters are reset", async () => { + const user = userEvent.setup(); + const result = { + data: { keys: [createMockKey()], total_count: 1, current_page: 1, total_pages: 1 }, + isPending: false, + isFetching: false, + refetch: vi.fn(), + } as unknown as ReturnType; + mockUseKeys.mockReturnValue(result); + + renderWithProviders(); + + await user.click(await screen.findByTestId("sort-header-created_at")); + await waitFor(() => + expect(mockUseKeys).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ sortOrder: "asc" })), + ); + + await user.click(screen.getByRole("button", { name: "Reset Filters" })); + await waitFor(() => + expect(mockUseKeys).toHaveBeenLastCalledWith( + 1, + 50, + expect.objectContaining({ sortBy: "created_at", sortOrder: "desc" }), + ), + ); + }); + it("should show Loading keys when isPending", async () => { mockUseKeys.mockReturnValue({ data: undefined, diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx index b7128e642a5..73f524e13f8 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx @@ -1,20 +1,11 @@ -// TO-DO: Standardize tables eventually - "use client"; import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; -import { ChevronDownIcon, ChevronRightIcon, ChevronUpIcon, SwitchVerticalIcon } from "@heroicons/react/outline"; -import { - ColumnDef, - flexRender, - getCoreRowModel, - PaginationState, - SortingState, - useReactTable, -} from "@tanstack/react-table"; -import { Badge, Icon, Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Text } from "@tremor/react"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { Popover, Skeleton, Tooltip, Typography } from "antd"; +import { DataTable, DataTablePagination, DataTableSortHeader } from "@/components/shared/DataTable"; +import { ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline"; +import { ColumnDef, PaginationState, SortingState } from "@tanstack/react-table"; +import { Badge, Icon, Text } from "@tremor/react"; +import { Popover, Tooltip, Typography } from "antd"; import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; @@ -36,10 +27,12 @@ interface TeamVirtualKeysTableProps { * TeamVirtualKeysTable – variant of VirtualKeysTable scoped to a single team. * Displays all virtual keys belonging to the team with same format and styling. */ +const DEFAULT_SORTING: SortingState = [{ id: "created_at", desc: true }]; + export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVirtualKeysTableProps) { const { accessToken } = useAuthorized(); const [selectedKey, setSelectedKey] = useState(null); - const [sorting, setSorting] = useState([{ id: "created_at", desc: true }]); + const [sorting, setSorting] = useState(DEFAULT_SORTING); const [tablePagination, setTablePagination] = useState({ pageIndex: 0, pageSize: 50, @@ -48,8 +41,6 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi "Organization ID": "", "Key Alias": "", "User ID": "", - "Sort By": "created_at", - "Sort Order": "desc", }); const sortBy = sorting.length > 0 ? sorting[0].id : "created_at"; @@ -83,7 +74,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi })); }, [keys?.keys, organization?.organization_id]); - const pageCount = keys?.total_pages ?? 0; + const rowCount = keys?.total_count ?? 0; const [expandedAccordions, setExpandedAccordions] = useState>({}); const currentTeam: Team = useMemo( @@ -125,18 +116,14 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi return () => window.removeEventListener("storage", handleStorageChange); }, [handleStorageChange]); - const handleFilterChange = useCallback((newFilters: Record, skipDebounce = false) => { + const handleFilterChange = useCallback((newFilters: Record) => { setFilters((prev) => ({ ...prev, "Organization ID": newFilters["Organization ID"] ?? prev["Organization ID"], "Key Alias": newFilters["Key Alias"] ?? prev["Key Alias"], "User ID": newFilters["User ID"] ?? prev["User ID"], - "Sort By": newFilters["Sort By"] ?? prev["Sort By"] ?? "created_at", - "Sort Order": newFilters["Sort Order"] ?? prev["Sort Order"] ?? "desc", })); - if (!skipDebounce) { - setTablePagination((prev) => ({ ...prev, pageIndex: 0 })); - } + setTablePagination((prev) => ({ ...prev, pageIndex: 0 })); }, []); const handleFilterReset = useCallback(() => { @@ -144,9 +131,8 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi "Organization ID": "", "Key Alias": "", "User ID": "", - "Sort By": "created_at", - "Sort Order": "desc", }); + setSorting(DEFAULT_SORTING); setTablePagination((prev) => ({ ...prev, pageIndex: 0 })); }, []); @@ -200,8 +186,8 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "token", accessorKey: "token", - header: "Key ID", - size: 100, + header: ({ column }) => , + size: 120, enableSorting: true, cell: (info) => ( setSelectedKey(info.row.original)} /> @@ -210,7 +196,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "key_alias", accessorKey: "key_alias", - header: "Key Alias", + header: ({ column }) => , size: 150, enableSorting: true, cell: (info) => { @@ -282,7 +268,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "created_at", accessorKey: "created_at", - header: "Created At", + header: ({ column }) => , size: 120, enableSorting: true, cell: (info) => , @@ -291,7 +277,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi id: "created_by", accessorKey: "created_by", header: "Created By", - size: 70, + size: 130, enableSorting: false, cell: (info) => { const userId = info.getValue() as string | null; @@ -349,7 +335,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "updated_at", accessorKey: "updated_at", - header: "Updated At", + header: ({ column }) => , size: 120, enableSorting: true, cell: (info) => , @@ -357,17 +343,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "last_active", accessorKey: "last_active", - header: () => ( - - Last Active - - - - - ), + header: "Last Active", size: 130, enableSorting: false, cell: (info) => , @@ -383,7 +359,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "spend", accessorKey: "spend", - header: "Spend (USD)", + header: ({ column }) => , size: 100, enableSorting: true, cell: (info) => , @@ -391,7 +367,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi { id: "max_budget", accessorKey: "max_budget", - header: "Budget (USD)", + header: ({ column }) => , size: 110, enableSorting: true, cell: (info) => ( @@ -511,39 +487,10 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi [expandedAccordions], ); - const handleSortingChange = useCallback( - (updaterOrValue: React.SetStateAction) => { - const newSorting = typeof updaterOrValue === "function" ? updaterOrValue(sorting) : updaterOrValue; - setSorting(newSorting); - if (newSorting?.length > 0) { - const sortState = newSorting[0]; - handleFilterChange( - { - "Sort By": sortState.id, - "Sort Order": sortState.desc ? "desc" : "asc", - }, - true, - ); - } - }, - [sorting, handleFilterChange], - ); - - const table = useReactTable({ - data: displayKeys, - columns, - columnResizeMode: "onChange", - columnResizeDirection: "ltr", - state: { sorting, pagination: tablePagination }, - onSortingChange: handleSortingChange, - onPaginationChange: setTablePagination, - getCoreRowModel: getCoreRowModel(), - // getSortedRowModel not needed — manualSorting: true delegates sorting to the server - enableSorting: true, - manualSorting: true, // Server sorts via useKeys. Avoid redundant client-side sort - manualPagination: true, - pageCount: pageCount, - }); + const handleSortingChange = useCallback((updaterOrValue: React.SetStateAction) => { + setSorting(updaterOrValue); + setTablePagination((prev) => ({ ...prev, pageIndex: 0 })); + }, []); return (
@@ -566,165 +513,36 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi />
-
-
- {isLoading || isFetching ? ( - - ) : ( - - Page {pageIndex + 1} of {table.getPageCount()} - - )} - - {isLoading || isFetching ? ( - - ) : ( - - )} - - {isLoading || isFetching ? ( - - ) : ( - - )} -
-
-
-
-
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - { - const resizer = document.querySelector(`[data-header-id="${header.id}"] .resizer`); - if (resizer) (resizer as HTMLElement).style.opacity = "0.5"; - }} - onMouseLeave={() => { - const resizer = document.querySelector(`[data-header-id="${header.id}"] .resizer`); - if (resizer && !header.column.getIsResizing()) - (resizer as HTMLElement).style.opacity = "0"; - }} - onClick={header.column.getCanSort() ? header.column.getToggleSortingHandler() : undefined} - > -
-
- {header.isPlaceholder - ? null - : flexRender(header.column.columnDef.header, header.getContext())} -
- {header.id !== "actions" && header.column.getCanSort() && ( -
- {header.column.getIsSorted() ? ( - { - asc: , - desc: , - }[header.column.getIsSorted() as string] - ) : ( - - )} -
- )} -
header.column.resetSize()} - onMouseDown={header.getResizeHandler()} - onTouchStart={header.getResizeHandler()} - className={`resizer ${table.options.columnResizeDirection} ${ - header.column.getIsResizing() ? "isResizing" : "" - }`} - style={{ - position: "absolute", - right: 0, - top: 0, - height: "100%", - width: "5px", - background: header.column.getIsResizing() ? "#3b82f6" : "transparent", - cursor: "col-resize", - userSelect: "none", - touchAction: "none", - opacity: header.column.getIsResizing() ? 1 : 0, - }} - /> -
- - ))} - - ))} - - - {isLoading || isFetching ? ( - - -
-

Loading keys...

-
- - - ) : displayKeys.length > 0 ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - 3 - ? "px-0" - : "" - }`} - > - {flexRender(cell.column.columnDef.cell, cell.getContext())} - - ))} - - )) - ) : ( - - -
-

No keys found

-
- - - )} - -
-
-
+
+ setTablePagination((prev) => ({ ...prev, pageIndex: nextPage }))} + onPageSizeChange={(nextSize) => setTablePagination({ pageIndex: 0, pageSize: nextSize })} + isLoading={isLoading || isFetching} + />
+ + null} + enableColumnResizing + columnResizeMode="onChange" + isLoading={isLoading || isFetching} + loadingMessage="Loading keys..." + noDataMessage="No keys found" + maxBodyHeight="75vh" + size="compact" + />
)}
diff --git a/ui/litellm-dashboard/src/components/ui/avatar.test.tsx b/ui/litellm-dashboard/src/components/ui/avatar.test.tsx new file mode 100644 index 00000000000..7e13593b767 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ui/avatar.test.tsx @@ -0,0 +1,15 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { Avatar, AvatarFallback } from "./avatar"; + +describe("Avatar", () => { + it("renders the fallback initials when no image is provided", () => { + render( + + AB + , + ); + const fallback = screen.getByText("AB"); + expect(fallback).toHaveAttribute("data-slot", "avatar-fallback"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ui/avatar.tsx b/ui/litellm-dashboard/src/components/ui/avatar.tsx new file mode 100644 index 00000000000..e257493a030 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ui/avatar.tsx @@ -0,0 +1,48 @@ +"use client"; + +import { Avatar as AvatarPrimitive } from "@base-ui/react/avatar"; +import * as React from "react"; + +import { cn } from "@/lib/cva.config"; + +const Avatar = React.forwardRef, AvatarPrimitive.Root.Props>( + ({ className, ...props }, ref) => ( + + ), +); +Avatar.displayName = "Avatar"; + +const AvatarImage = React.forwardRef, AvatarPrimitive.Image.Props>( + ({ className, ...props }, ref) => ( + + ), +); +AvatarImage.displayName = "AvatarImage"; + +const AvatarFallback = React.forwardRef< + React.ComponentRef, + AvatarPrimitive.Fallback.Props +>(({ className, ...props }, ref) => ( + +)); +AvatarFallback.displayName = "AvatarFallback"; + +export { Avatar, AvatarImage, AvatarFallback }; diff --git a/ui/litellm-dashboard/src/components/ui/badge.test.tsx b/ui/litellm-dashboard/src/components/ui/badge.test.tsx new file mode 100644 index 00000000000..2f380694eae --- /dev/null +++ b/ui/litellm-dashboard/src/components/ui/badge.test.tsx @@ -0,0 +1,32 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { Badge } from "./badge"; + +describe("Badge", () => { + it("renders a span carrying the badge slot and variant classes by default", () => { + render(v1.2.3); + const badge = screen.getByText("v1.2.3"); + expect(badge.tagName).toBe("SPAN"); + expect(badge).toHaveAttribute("data-slot", "badge"); + expect(badge).toHaveAttribute("data-variant", "outline"); + expect(badge).toHaveClass("border-border"); + }); + + it("renders as an anchor via the render prop while keeping badge styling", () => { + render( + }> + v1.2.3 + , + ); + const link = screen.getByRole("link", { name: "v1.2.3" }); + expect(link.tagName).toBe("A"); + expect(link).toHaveAttribute("href", "https://docs.litellm.ai/release_notes"); + expect(link).toHaveAttribute("data-slot", "badge"); + expect(link).toHaveClass("border-border"); + }); + + it("lets className win over variant classes through twMerge", () => { + render(x); + expect(screen.getByText("x")).toHaveClass("text-muted-foreground"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ui/badge.tsx b/ui/litellm-dashboard/src/components/ui/badge.tsx index 2e1ebffa109..f64de004b52 100644 --- a/ui/litellm-dashboard/src/components/ui/badge.tsx +++ b/ui/litellm-dashboard/src/components/ui/badge.tsx @@ -1,5 +1,8 @@ +"use client"; + import * as React from "react"; import { type VariantProps } from "cva"; +import { useRender } from "@base-ui/react/use-render"; import { cn, cva } from "@/lib/cva.config"; @@ -21,18 +24,24 @@ const badgeVariants = cva({ }, }); -const Badge = React.forwardRef< - HTMLSpanElement, - React.ComponentPropsWithoutRef<"span"> & VariantProps ->(({ className, variant = "default", ...props }, ref) => ( - -)); +type BadgeProps = React.ComponentPropsWithoutRef<"span"> & + VariantProps & { + render?: useRender.RenderProp; + }; + +const Badge = React.forwardRef( + ({ className, variant = "default", render, ...props }, ref) => + useRender({ + render: render ?? , + ref, + props: { + "data-slot": "badge", + "data-variant": variant, + className: cn(badgeVariants({ variant }), className), + ...props, + }, + }), +); Badge.displayName = "Badge"; export { Badge, badgeVariants }; diff --git a/ui/litellm-dashboard/src/components/ui/breadcrumb.test.tsx b/ui/litellm-dashboard/src/components/ui/breadcrumb.test.tsx new file mode 100644 index 00000000000..b43eda1116d --- /dev/null +++ b/ui/litellm-dashboard/src/components/ui/breadcrumb.test.tsx @@ -0,0 +1,38 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { Breadcrumb, BreadcrumbItem, BreadcrumbList, BreadcrumbPage, BreadcrumbSeparator } from "./breadcrumb"; + +describe("Breadcrumb", () => { + it("renders a labelled nav and marks the current page", () => { + render( + + + Observability + + + Logs + + + , + ); + expect(screen.getByRole("navigation", { name: "breadcrumb" })).toBeInTheDocument(); + const page = screen.getByText("Logs"); + expect(page).toHaveAttribute("aria-current", "page"); + expect(page).toHaveAttribute("data-slot", "breadcrumb-page"); + }); + + it("renders a presentational separator", () => { + const { container } = render( + + + A + + B + + , + ); + const sep = container.querySelector('[data-slot="breadcrumb-separator"]'); + expect(sep).toHaveAttribute("aria-hidden", "true"); + expect(sep?.querySelector("svg")).not.toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ui/breadcrumb.tsx b/ui/litellm-dashboard/src/components/ui/breadcrumb.tsx new file mode 100644 index 00000000000..731a014362d --- /dev/null +++ b/ui/litellm-dashboard/src/components/ui/breadcrumb.tsx @@ -0,0 +1,78 @@ +import * as React from "react"; +import { ChevronRight } from "lucide-react"; + +import { cn } from "@/lib/cva.config"; + +const Breadcrumb = React.forwardRef>(({ ...props }, ref) => ( +